From 716837748483edde217676fd04c5c5274637827b Mon Sep 17 00:00:00 2001 From: Tihomir Surdilovic Date: Tue, 18 Aug 2026 23:56:36 -0400 Subject: [PATCH 001/139] Register archival task category in tdbg (#11587) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # What changed? `tdbg` now registers the **archival** task category (`5`) in its task-category registry (`tools/tdbg/app.go`), so `tdbg dlq list/merge/purge --dlq-type 5` works. Previously `merge`/`purge` rejected it with `unknown dlq category 5`. Because the `--dlq-type` help text is generated from the registry, archival now appears there too. Adds a regression test to `TestDLQCommand_V2`. ## Why? `tdbg` built its registry with `tasks.NewDefaultTaskCategoryRegistry()`, which omits `CategoryArchival`. The server registers archival conditionally in `TaskCategoryRegistryProvider` (`temporal/fx.go`) when archival is enabled, but `tdbg` is a client tool with no access to the cluster's archival config, so it rejected `--dlq-type 5` client-side — even though archival tasks are written to the history task DLQ and the server's `AdminService.MergeDLQTasks` accepts the category. That left archival DLQ tasks unrecoverable via the standard tool (operators had to call the RPC directly). Registering the category unconditionally in `tdbg` is safe: with archival disabled there is simply no archival DLQ to operate on. Fixes #11586. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --------- Signed-off-by: Tihomir Surdilovic Co-authored-by: Prathyush PV --- tools/tdbg/app.go | 4 +++- tools/tdbg/dlq_service_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/tools/tdbg/app.go b/tools/tdbg/app.go index 1b9c8efc4c3..d4f6fa639ec 100644 --- a/tools/tdbg/app.go +++ b/tools/tdbg/app.go @@ -32,9 +32,11 @@ type ( // NewCliApp instantiates a new instance of the CLI application. func NewCliApp(opts ...Option) *cli.App { + taskCategoryRegistry := tasks.NewDefaultTaskCategoryRegistry() + taskCategoryRegistry.AddCategory(tasks.CategoryArchival) params := Params{ ClientFactory: NewClientFactory(), - TaskCategoryRegistry: tasks.NewDefaultTaskCategoryRegistry(), + TaskCategoryRegistry: taskCategoryRegistry, Writer: os.Stdout, ErrWriter: os.Stderr, TaskBlobEncoder: NewProtoTaskBlobEncoder(NewPredefinedTaskBlobDeserializer()), diff --git a/tools/tdbg/dlq_service_test.go b/tools/tdbg/dlq_service_test.go index 08480f4f66b..e63c85723eb 100644 --- a/tools/tdbg/dlq_service_test.go +++ b/tools/tdbg/dlq_service_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/urfave/cli/v2" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" @@ -363,6 +364,32 @@ func TestDLQCommand_V2(t *testing.T) { p.expectedErrSubstrings = []string{"failed to find last message ID", "connection failed"} }, }, + { + name: "merge archival DLQ (category 5) is supported", + override: func(p *dlqTestParams) { + p.command = "merge" + p.dlqType = strconv.Itoa(tasks.CategoryArchival.ID()) + p.lastMessageID = "" // No last message ID provided + p.adminClient.err = nil + queueName := persistence.GetHistoryTaskQueueName( + tasks.CategoryArchival.ID(), "test-source-cluster", "test-target-cluster") + p.adminClient.listQueueResponses = []*adminservice.ListQueuesResponse{ + { + Queues: []*adminservice.ListQueuesResponse_QueueInfo{ + { + QueueName: queueName, + MessageCount: 10, + LastMessageId: 150, + }, + }, + NextPageToken: nil, + }, + } + }, + validateStdout: func(t *testing.T, b *bytes.Buffer) { + require.Contains(t, b.String(), "Found last message ID: 150") + }, + }, } { t.Run(tc.name, func(t *testing.T) { tc.version = "v2" From 732f7b412cf70f6f72a9b748d5e6dc59dbf2e386 Mon Sep 17 00:00:00 2001 From: Yaniv Kaul Date: Wed, 19 Aug 2026 07:00:10 +0300 Subject: [PATCH 002/139] Guard against nil History in AppendRawHistoryNodes (#10309) Add nil check for `request.History` before accessing `request.History.Data` in `serializeAppendRawHistoryNodesRequest` to prevent a panic when the caller passes a nil DataBlob. Co-authored-by: Prathyush PV --- common/persistence/history_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/persistence/history_manager.go b/common/persistence/history_manager.go index 59e0db63291..705c333f9cd 100644 --- a/common/persistence/history_manager.go +++ b/common/persistence/history_manager.go @@ -418,7 +418,7 @@ func (m *executionManagerImpl) serializeAppendRawHistoryNodesRequest( return nil, serviceerror.NewInvalidArgument(fmt.Sprintf("unable to parse branch token: %v", err)) } - if len(request.History.Data) == 0 { + if request.History == nil || len(request.History.Data) == 0 { return nil, &InvalidPersistenceRequestError{ Msg: "events to be appended cannot be empty", } From fb8894cfcc0a511b06146d291f673f715df3b51a Mon Sep 17 00:00:00 2001 From: Yaniv Kaul Date: Wed, 19 Aug 2026 07:16:25 +0300 Subject: [PATCH 003/139] perf: return TimerSequenceID by value instead of pointer (#11177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `TimerSequenceID` (~48 bytes) is heap-allocated and returned as a `*TimerSequenceID` from 5 getter functions: `getUserTimerTimeout`, `getActivityScheduleToStartTimeout`, `getActivityScheduleToCloseTimeout`, `getActivityStartToCloseTimeout`, and `getActivityHeartbeatTimeout`. These are called for every pending timer/activity during `LoadAndSortUserTimers()` and `LoadAndSortActivityTimers()` — a hot path in every workflow task. Callers were already dereferencing the pointer before appending to the value-type `[]TimerSequenceID` slice, so the existing code was allocating on the heap only to immediately copy to the stack. ## Changes - `TimerSequenceID` is now returned by value with a `(TimerSequenceID, bool)` tuple - Idiomatic Go pattern (same as map access) replaces nil-check sentinel - All 5 getter methods updated + call sites and tests adapted ## Impact Eliminates one heap allocation per getter call on the timer-sorting hot path. ## Tests - `service/history/workflow` (1075 tests): ✅ passed --------- Co-authored-by: Prathyush PV --- service/history/workflow/timer_sequence.go | 103 +++++++------- .../history/workflow/timer_sequence_test.go | 130 ++++++++++-------- 2 files changed, 120 insertions(+), 113 deletions(-) diff --git a/service/history/workflow/timer_sequence.go b/service/history/workflow/timer_sequence.go index ea261ddccff..20c3be25e19 100644 --- a/service/history/workflow/timer_sequence.go +++ b/service/history/workflow/timer_sequence.go @@ -171,11 +171,7 @@ func (t *timerSequenceImpl) LoadAndSortUserTimers() []TimerSequenceID { for _, timerInfo := range pendingTimers { - if sequenceID := t.getUserTimerTimeout( - timerInfo, - ); sequenceID != nil { - timers = append(timers, *sequenceID) - } + timers = append(timers, t.getUserTimerTimeout(timerInfo)) } sort.Sort(timers) @@ -193,28 +189,28 @@ func (t *timerSequenceImpl) LoadAndSortActivityTimers() []TimerSequenceID { if activityInfo.Paused { continue } - if sequenceID := t.getActivityScheduleToCloseTimeout( + if sequenceID, ok := t.getActivityScheduleToCloseTimeout( activityInfo, - ); sequenceID != nil { - activityTimers = append(activityTimers, *sequenceID) + ); ok { + activityTimers = append(activityTimers, sequenceID) } - if sequenceID := t.getActivityScheduleToStartTimeout( + if sequenceID, ok := t.getActivityScheduleToStartTimeout( activityInfo, - ); sequenceID != nil { - activityTimers = append(activityTimers, *sequenceID) + ); ok { + activityTimers = append(activityTimers, sequenceID) } - if sequenceID := t.getActivityStartToCloseTimeout( + if sequenceID, ok := t.getActivityStartToCloseTimeout( activityInfo, - ); sequenceID != nil { - activityTimers = append(activityTimers, *sequenceID) + ); ok { + activityTimers = append(activityTimers, sequenceID) } - if sequenceID := t.getActivityHeartbeatTimeout( + if sequenceID, ok := t.getActivityHeartbeatTimeout( activityInfo, - ); sequenceID != nil { - activityTimers = append(activityTimers, *sequenceID) + ); ok { + activityTimers = append(activityTimers, sequenceID) } } @@ -224,11 +220,11 @@ func (t *timerSequenceImpl) LoadAndSortActivityTimers() []TimerSequenceID { func (t *timerSequenceImpl) getUserTimerTimeout( timerInfo *persistencespb.TimerInfo, -) *TimerSequenceID { +) TimerSequenceID { expiryTime := timerInfo.ExpiryTime - return &TimerSequenceID{ + return TimerSequenceID{ EventID: timerInfo.GetStartedEventId(), Timestamp: timestamp.TimeValue(expiryTime), TimerType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, @@ -256,14 +252,15 @@ type activityTimerDeadlines [4]*time.Time // definition of each deadline. func getActivityTimerDeadlines(activityInfo *persistencespb.ActivityInfo) activityTimerDeadlines { var deadlines activityTimerDeadlines - for i, sequenceID := range [4]*TimerSequenceID{ - getActivityScheduleToCloseTimeout(activityInfo), - getActivityScheduleToStartTimeout(activityInfo), - getActivityStartToCloseTimeout(activityInfo), - getActivityHeartbeatTimeout(activityInfo), + for i, getTimeout := range [4]func(*persistencespb.ActivityInfo) (TimerSequenceID, bool){ + getActivityScheduleToCloseTimeout, + getActivityScheduleToStartTimeout, + getActivityStartToCloseTimeout, + getActivityHeartbeatTimeout, } { - if sequenceID != nil { - deadlines[i] = &sequenceID.Timestamp + if sequenceID, ok := getTimeout(activityInfo); ok { + deadline := sequenceID.Timestamp + deadlines[i] = &deadline } } return deadlines @@ -291,58 +288,58 @@ func (d activityTimerDeadlines) changedMask(other activityTimerDeadlines) int32 func (t *timerSequenceImpl) getActivityScheduleToStartTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { return getActivityScheduleToStartTimeout(activityInfo) } func getActivityScheduleToStartTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { // activity is not scheduled yet, probably due to retry & backoff if activityInfo.ScheduledEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } // activity is already started if activityInfo.StartedEventId != common.EmptyEventID { - return nil + return TimerSequenceID{}, false } scheduleToStartDuration := timestamp.DurationValue(activityInfo.ScheduleToStartTimeout) if scheduleToStartDuration == 0 { - return nil + return TimerSequenceID{}, false } timeoutTime := timestamp.TimeValue(activityInfo.ScheduledTime).Add(scheduleToStartDuration) - return &TimerSequenceID{ + return TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: timeoutTime, TimerType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, TimerCreated: (activityInfo.TimerTaskStatus & TimerTaskStatusCreatedScheduleToStart) > 0, Attempt: activityInfo.Attempt, - } + }, true } func (t *timerSequenceImpl) getActivityScheduleToCloseTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { return getActivityScheduleToCloseTimeout(activityInfo) } func getActivityScheduleToCloseTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { // activity is not scheduled yet, probably due to retry & backoff if activityInfo.ScheduledEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } scheduleToCloseDuration := timestamp.DurationValue(activityInfo.ScheduleToCloseTimeout) if scheduleToCloseDuration == 0 { - return nil + return TimerSequenceID{}, false } var timeoutTime time.Time @@ -354,75 +351,75 @@ func getActivityScheduleToCloseTimeout( timeoutTime = timestamp.TimeValue(activityInfo.ScheduledTime).Add(scheduleToCloseDuration) } - return &TimerSequenceID{ + return TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: timeoutTime, TimerType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, TimerCreated: (activityInfo.TimerTaskStatus & TimerTaskStatusCreatedScheduleToClose) > 0, Attempt: activityInfo.Attempt, - } + }, true } func (t *timerSequenceImpl) getActivityStartToCloseTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { return getActivityStartToCloseTimeout(activityInfo) } func getActivityStartToCloseTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { // activity is not scheduled yet, probably due to retry & backoff if activityInfo.ScheduledEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } // activity is not started yet if activityInfo.StartedEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } startToCloseDuration := timestamp.DurationValue(activityInfo.StartToCloseTimeout) if startToCloseDuration == 0 { - return nil + return TimerSequenceID{}, false } timeoutTime := timestamp.TimeValue(activityInfo.StartedTime).Add(startToCloseDuration) - return &TimerSequenceID{ + return TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: timeoutTime, TimerType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, TimerCreated: (activityInfo.TimerTaskStatus & TimerTaskStatusCreatedStartToClose) > 0, Attempt: activityInfo.Attempt, - } + }, true } func (t *timerSequenceImpl) getActivityHeartbeatTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { return getActivityHeartbeatTimeout(activityInfo) } func getActivityHeartbeatTimeout( activityInfo *persistencespb.ActivityInfo, -) *TimerSequenceID { +) (TimerSequenceID, bool) { // activity is not scheduled yet, probably due to retry & backoff if activityInfo.ScheduledEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } // activity is not started yet if activityInfo.StartedEventId == common.EmptyEventID { - return nil + return TimerSequenceID{}, false } // not heartbeat timeout configured heartbeatDuration := timestamp.DurationValue(activityInfo.HeartbeatTimeout) if heartbeatDuration == 0 { - return nil + return TimerSequenceID{}, false } // use the latest time as last heartbeat time @@ -437,13 +434,13 @@ func getActivityHeartbeatTimeout( heartbeatTimeout := lastHeartbeat.Add(heartbeatDuration) - return &TimerSequenceID{ + return TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: heartbeatTimeout, TimerType: enumspb.TIMEOUT_TYPE_HEARTBEAT, TimerCreated: (activityInfo.TimerTaskStatus & TimerTaskStatusCreatedHeartbeat) > 0, Attempt: activityInfo.Attempt, - } + }, true } func timerTypeToTimerMask( diff --git a/service/history/workflow/timer_sequence_test.go b/service/history/workflow/timer_sequence_test.go index 9a90b1ea3a0..ebf0da8b2b7 100644 --- a/service/history/workflow/timer_sequence_test.go +++ b/service/history/workflow/timer_sequence_test.go @@ -927,7 +927,7 @@ func (s *timerSequenceSuite) TestGetUserTimerTimeout() { TaskStatus: TimerTaskStatusCreated, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: timerInfo.StartedEventId, Timestamp: timerExpiry.AsTime(), TimerType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, @@ -962,8 +962,8 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithTimeout_N Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithTimeout_Scheduled_NotStarted() { @@ -984,7 +984,7 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithTimeout_S Attempt: 12, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: activityInfo.ScheduledTime.AsTime().Add(activityInfo.ScheduleToStartTimeout.AsDuration()), TimerType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, @@ -992,12 +992,14 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithTimeout_S Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + timerSequence, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) activityInfo.TimerTaskStatus = TimerTaskStatusNone expectedTimerSequence.TimerCreated = false - timerSequence = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + timerSequence, ok = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) } @@ -1019,12 +1021,12 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithTimeout_S Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeout_NotScheduled() { @@ -1045,8 +1047,8 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeou Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeout_Scheduled_NotStarted() { @@ -1067,12 +1069,12 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeou Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeout_Scheduled_Started() { @@ -1093,12 +1095,12 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToStartTimeout_WithoutTimeou Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityScheduleToStartTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithTimeout_NotScheduled() { @@ -1119,8 +1121,8 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithTimeout_N Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithTimeout_Scheduled() { @@ -1142,7 +1144,7 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithTimeout_S Attempt: 12, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: activityInfo.ScheduledTime.AsTime().Add(activityInfo.ScheduleToCloseTimeout.AsDuration()), TimerType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, @@ -1150,12 +1152,14 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithTimeout_S Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + timerSequence, ok := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) activityInfo.TimerTaskStatus = TimerTaskStatusNone expectedTimerSequence.TimerCreated = false - timerSequence = s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + timerSequence, ok = s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) } @@ -1177,8 +1181,8 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithoutTimeou Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithoutTimeout_Scheduled() { @@ -1199,12 +1203,12 @@ func (s *timerSequenceSuite) TestGetActivityScheduleToCloseTimeout_WithoutTimeou Attempt: 12, } - timerSequence := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityScheduleToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithTimeout_NotStarted() { @@ -1225,8 +1229,8 @@ func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithTimeout_NotS Attempt: 12, } - timerSequence := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithTimeout_Started() { @@ -1247,7 +1251,7 @@ func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithTimeout_Star Attempt: 12, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: activityInfo.StartedTime.AsTime().Add(activityInfo.StartToCloseTimeout.AsDuration()), TimerType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, @@ -1255,12 +1259,14 @@ func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithTimeout_Star Attempt: 12, } - timerSequence := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + timerSequence, ok := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) activityInfo.TimerTaskStatus = TimerTaskStatusNone expectedTimerSequence.TimerCreated = false - timerSequence = s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + timerSequence, ok = s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) } @@ -1282,8 +1288,8 @@ func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithoutTimeout_N Attempt: 12, } - timerSequence := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithoutTimeout_Started() { @@ -1304,12 +1310,12 @@ func (s *timerSequenceSuite) TestGetActivityStartToCloseTimeout_WithoutTimeout_S Attempt: 12, } - timerSequence := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityStartToCloseTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityStartToCloseTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_NotStarted() { @@ -1330,8 +1336,8 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_NotSt Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_Started_NoHeartbeat() { @@ -1352,7 +1358,7 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_Start Attempt: 12, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: activityInfo.StartedTime.AsTime().Add(activityInfo.HeartbeatTimeout.AsDuration()), TimerType: enumspb.TIMEOUT_TYPE_HEARTBEAT, @@ -1360,12 +1366,14 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_Start Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + timerSequence, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) activityInfo.TimerTaskStatus = TimerTaskStatusNone expectedTimerSequence.TimerCreated = false - timerSequence = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + timerSequence, ok = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) } @@ -1387,7 +1395,7 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_Start Attempt: 12, } - expectedTimerSequence := &TimerSequenceID{ + expectedTimerSequence := TimerSequenceID{ EventID: activityInfo.ScheduledEventId, Timestamp: activityInfo.LastHeartbeatUpdateTime.AsTime().Add(activityInfo.HeartbeatTimeout.AsDuration()), TimerType: enumspb.TIMEOUT_TYPE_HEARTBEAT, @@ -1395,12 +1403,14 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithHeartbeat_Start Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + timerSequence, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) activityInfo.TimerTaskStatus = TimerTaskStatusNone expectedTimerSequence.TimerCreated = false - timerSequence = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + timerSequence, ok = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.True(ok) s.Equal(expectedTimerSequence, timerSequence) } @@ -1422,8 +1432,8 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithoutHeartbeat_No Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithoutHeartbeat_Started_NoHeartbeat() { @@ -1444,12 +1454,12 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithoutHeartbeat_St Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithoutHeartbeat_Started_Heartbeated() { @@ -1470,12 +1480,12 @@ func (s *timerSequenceSuite) TestGetActivityHeartbeatTimeout_WithoutHeartbeat_St Attempt: 12, } - timerSequence := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok := s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) activityInfo.TimerTaskStatus = TimerTaskStatusNone - timerSequence = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) - s.Empty(timerSequence) + _, ok = s.timerSequence.getActivityHeartbeatTimeout(activityInfo) + s.False(ok) } func (s *timerSequenceSuite) TestConversion() { From 7ab5df336c86ff8be5cfe63b5a0a54f560092684 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 19 Aug 2026 10:09:51 -0700 Subject: [PATCH 004/139] Trace outbound Nexus HTTP requests (#11559) ## What changed? Instrumented local frontend, CHASM callback, and external Nexus operation HTTP clients with the shared OpenTelemetry transport. Legacy HSM callbacks and cross-cluster forwarding are intentionally out of scope. ## Why? Outbound Nexus HTTP calls need to carry trace context so callbacks and internal frontend calls remain connected to their originating spans. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [x] added new functional test(s) --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- chasm/lib/callback/fx.go | 10 +- chasm/lib/callback/request.go | 4 +- chasm/lib/nexusoperation/fx.go | 4 +- common/resource/fx.go | 2 + common/rpc/connection_cache_test.go | 2 +- common/rpc/rpc.go | 55 ++-- common/rpc/rpc_test.go | 6 +- common/rpc/test/http_test.go | 3 + common/rpc/test/rpc_localstore_tls_test.go | 24 +- common/rpc/test/rpc_token_auth_test.go | 8 +- common/telemetry/http.go | 21 ++ common/telemetry/http_test.go | 12 + temporal/fx.go | 2 + tests/nexus_otel_test.go | 333 +++++++++++++++++++++ tests/nexus_test_base.go | 12 +- tests/testcore/clients.go | 1 + tests/testcore/functional_test_base.go | 21 +- tests/testcore/test_env.go | 10 + 18 files changed, 474 insertions(+), 56 deletions(-) create mode 100644 tests/nexus_otel_test.go diff --git a/chasm/lib/callback/fx.go b/chasm/lib/callback/fx.go index 912302789fd..0f7271f22fa 100644 --- a/chasm/lib/callback/fx.go +++ b/chasm/lib/callback/fx.go @@ -11,6 +11,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/telemetry" queuescommon "go.temporal.io/server/service/history/queues/common" "go.uber.org/fx" ) @@ -29,16 +30,19 @@ func httpCallerProviderProvider( rpcFactory common.RPCFactory, httpClientCache *cluster.FrontendHTTPClientCache, logger log.Logger, + httpClientTransportInstrumenter telemetry.HTTPClientTransportInstrumenter, ) (HTTPCallerProvider, error) { localClient, err := rpcFactory.CreateLocalFrontendHTTPClient() if err != nil { return nil, fmt.Errorf("cannot create local frontend HTTP client: %w", err) } - defaultTransport, err := common.NewHTTPTransport(nil) + externalTransport, err := common.NewHTTPTransport(nil) if err != nil { return nil, err } - defaultClient := &http.Client{Transport: defaultTransport} + externalClient := &http.Client{ + Transport: httpClientTransportInstrumenter.Instrument(externalTransport), + } callbackTokenGenerator := commonnexus.NewCallbackTokenGenerator() m := collection.NewOnceMap(func(queuescommon.NamespaceIDAndDestination) HTTPCaller { @@ -48,7 +52,7 @@ func httpCallerProviderProvider( namespaceRegistry, httpClientCache, callbackTokenGenerator, - defaultClient, + externalClient, localClient, logger, ) diff --git a/chasm/lib/callback/request.go b/chasm/lib/callback/request.go index 43522c04a66..88747a8d33e 100644 --- a/chasm/lib/callback/request.go +++ b/chasm/lib/callback/request.go @@ -109,7 +109,7 @@ func routeRequest( namespaceRegistry namespace.Registry, httpClientCache *cluster.FrontendHTTPClientCache, callbackTokenGenerator *commonnexus.CallbackTokenGenerator, - defaultClient *http.Client, + externalClient *http.Client, localClient *common.FrontendHTTPClient, logger log.Logger, ) (*http.Response, error) { @@ -119,7 +119,7 @@ func routeRequest( // This source header is populated in nexusoperations/tasks (via the ClientProvider) for worker targets // if this header is not populated then we assume it's an external target. if r.Header == nil || r.Header.Get(callbackSourceHeader) == "" { - return defaultClient.Do(r) + return externalClient.Do(r) } // If we got here, we assume that the endpoint in the original call was a worker target, and we should route // internally, either to a local frontend, or one of the other connected clusters' frontends. diff --git a/chasm/lib/nexusoperation/fx.go b/chasm/lib/nexusoperation/fx.go index 4d122cd3ba0..364451fab8d 100644 --- a/chasm/lib/nexusoperation/fx.go +++ b/chasm/lib/nexusoperation/fx.go @@ -20,6 +20,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc" + "go.temporal.io/server/common/telemetry" "go.uber.org/fx" ) @@ -122,6 +123,7 @@ func clientProviderFactory( httpTransportProvider NexusTransportProvider, clusterMetadata cluster.Metadata, rpcFactory common.RPCFactory, + httpClientTransportInstrumenter telemetry.HTTPClientTransportInstrumenter, ) (ClientProvider, error) { cl, err := rpcFactory.CreateLocalFrontendHTTPClient() if err != nil { @@ -135,7 +137,7 @@ func clientProviderFactory( m := collection.NewFallibleOnceMap(func(key clientProviderCacheKey) (*http.Client, error) { transport := httpTransportProvider(key.namespaceID, key.endpointID) return &http.Client{ - Transport: responseSizeLimiter{transport}, + Transport: httpClientTransportInstrumenter.Instrument(responseSizeLimiter{transport}), }, nil }) diff --git a/common/resource/fx.go b/common/resource/fx.go index 78affab8a2d..dfb75a4d0b2 100644 --- a/common/resource/fx.go +++ b/common/resource/fx.go @@ -436,6 +436,7 @@ func RPCFactoryProvider( monitor membership.Monitor, dc *dynamicconfig.Collection, tokenProvider auth.TokenProvider, + frontendHTTPTransportInstrumenter telemetry.HTTPClientTransportInstrumenter, ) (common.RPCFactory, error) { frontendURL, frontendHTTPURL, frontendHTTPPort, frontendTLSConfig, err := getFrontendConnectionDetails(cfg, tlsConfigProvider, resolver) if err != nil { @@ -458,6 +459,7 @@ func RPCFactoryProvider( frontendHTTPURL, frontendHTTPPort, frontendTLSConfig, + frontendHTTPTransportInstrumenter, options, perServiceDialOptions, monitor, diff --git a/common/rpc/connection_cache_test.go b/common/rpc/connection_cache_test.go index 3208740fabf..5e02a6ba2b2 100644 --- a/common/rpc/connection_cache_test.go +++ b/common/rpc/connection_cache_test.go @@ -20,7 +20,7 @@ const ( ) func newCacheTestFactory() *RPCFactory { - return NewFactory(nil, "tester", log.NewNoopLogger(), metrics.NoopMetricsHandler, nil, "", "", 0, nil, nil, nil, nil, nil) + return NewFactory(nil, "tester", log.NewNoopLogger(), metrics.NoopMetricsHandler, nil, "", "", 0, nil, nil, nil, nil, nil, nil) } func cachedConn(f *RPCFactory, hostName string) (*grpc.ClientConn, bool) { diff --git a/common/rpc/rpc.go b/common/rpc/rpc.go index 1ffdd0820aa..40062a8e335 100644 --- a/common/rpc/rpc.go +++ b/common/rpc/rpc.go @@ -24,6 +24,7 @@ import ( "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" + "go.temporal.io/server/common/telemetry" "go.temporal.io/server/temporal/environment" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -45,10 +46,11 @@ type RPCFactory struct { logger log.Logger metricsHandler metrics.Handler - frontendURL string - frontendHTTPURL string - frontendHTTPPort int - frontendTLSConfig *tls.Config + frontendURL string + frontendHTTPURL string + frontendHTTPPort int + frontendTLSConfig *tls.Config + frontendHTTPTransportInstrumenter telemetry.HTTPClientTransportInstrumenter grpcListener func() net.Listener tlsFactory encryption.TLSConfigProvider @@ -77,8 +79,7 @@ type RPCFactory struct { EnableInternodeClientKeepalive bool } -// NewFactory builds a new RPCFactory -// conforming to the underlying configuration +// NewFactory builds a new RPCFactory conforming to the underlying configuration. func NewFactory( cfg *config.Config, sName primitives.ServiceName, @@ -89,6 +90,7 @@ func NewFactory( frontendHTTPURL string, frontendHTTPPort int, frontendTLSConfig *tls.Config, + frontendHTTPTransportInstrumenter telemetry.HTTPClientTransportInstrumenter, commonDialOptions []grpc.DialOption, perServiceDialOptions map[primitives.ServiceName][]grpc.DialOption, monitor membership.Monitor, @@ -101,21 +103,22 @@ func NewFactory( requireRemoteClusterAuth = cfg.Global.Authorization.RemoteClusterAuth.Require } f := &RPCFactory{ - config: cfg, - serviceName: sName, - logger: logger, - metricsHandler: metricsHandler, - frontendURL: frontendURL, - frontendHTTPURL: frontendHTTPURL, - frontendHTTPPort: frontendHTTPPort, - frontendTLSConfig: frontendTLSConfig, - tlsFactory: tlsProvider, - commonDialOptions: commonDialOptions, - perServiceDialOptions: perServiceDialOptions, - tokenProvider: tokenProvider, - authHeaderName: authHeaderName, - requireRemoteClusterAuth: requireRemoteClusterAuth, - monitor: monitor, + config: cfg, + serviceName: sName, + logger: logger, + metricsHandler: metricsHandler, + frontendURL: frontendURL, + frontendHTTPURL: frontendHTTPURL, + frontendHTTPPort: frontendHTTPPort, + frontendTLSConfig: frontendTLSConfig, + frontendHTTPTransportInstrumenter: frontendHTTPTransportInstrumenter, + tlsFactory: tlsProvider, + commonDialOptions: commonDialOptions, + perServiceDialOptions: perServiceDialOptions, + tokenProvider: tokenProvider, + authHeaderName: authHeaderName, + requireRemoteClusterAuth: requireRemoteClusterAuth, + monitor: monitor, } f.grpcListener = sync.OnceValue(f.createGRPCListener) f.localFrontendClient = sync.OnceValues(f.createLocalFrontendHTTPClient) @@ -436,8 +439,6 @@ func (d *RPCFactory) createLocalFrontendHTTPClient() (*common.FrontendHTTPClient if err != nil { return nil, err } - client := http.Client{} - // Default to http unless TLS is configured. scheme := "http" if d.frontendTLSConfig != nil { @@ -445,8 +446,9 @@ func (d *RPCFactory) createLocalFrontendHTTPClient() (*common.FrontendHTTPClient } var address string + var clientTransport http.RoundTripper if r := serviceResolverFromGRPCURL(d.frontendHTTPURL); r != nil { - client.Transport = &roundTripper{ + clientTransport = &roundTripper{ resolver: r, underlying: transport, httpPort: d.frontendHTTPPort, @@ -454,9 +456,12 @@ func (d *RPCFactory) createLocalFrontendHTTPClient() (*common.FrontendHTTPClient address = "internal" // This will be replaced by the roundTripper } else { // Use the URL as-is and leave the transport unmodified. - client.Transport = transport + clientTransport = transport address = d.frontendHTTPURL } + client := http.Client{ + Transport: d.frontendHTTPTransportInstrumenter.Instrument(clientTransport), + } return &common.FrontendHTTPClient{ Client: client, diff --git a/common/rpc/rpc_test.go b/common/rpc/rpc_test.go index 0983e1ecd11..6608b53ac1e 100644 --- a/common/rpc/rpc_test.go +++ b/common/rpc/rpc_test.go @@ -13,7 +13,7 @@ import ( func TestCreateLocalFrontendGRPCConnectionIsCached(t *testing.T) { f := NewFactory(nil, primitives.FrontendService, log.NewNoopLogger(), - metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil) + metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil, nil) first := f.CreateLocalFrontendGRPCConnection() second := f.CreateLocalFrontendGRPCConnection() require.NotNil(t, first) @@ -23,7 +23,7 @@ func TestCreateLocalFrontendGRPCConnectionIsCached(t *testing.T) { func TestCreateRemoteFrontendGRPCConnection_CachedPerAddress(t *testing.T) { f := NewFactory(nil, primitives.FrontendService, log.NewNoopLogger(), - metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil) + metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil, nil) firstA := f.CreateRemoteFrontendGRPCConnection("remote-a:7233") secondA := f.CreateRemoteFrontendGRPCConnection("remote-a:7233") @@ -43,7 +43,7 @@ func TestCreateRemoteFrontendGRPCConnection_CachedPerAddress(t *testing.T) { // Concurrent first callers for one rpcAddress converge on a single connection. func TestCreateRemoteFrontendGRPCConnection_ConcurrentSameAddressConverges(t *testing.T) { f := NewFactory(nil, primitives.FrontendService, log.NewNoopLogger(), - metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil) + metrics.NoopMetricsHandler, nil, "localhost:7233", "", 0, nil, nil, nil, nil, nil, nil) const goroutines = 20 conns := make([]*grpc.ClientConn, goroutines) diff --git a/common/rpc/test/http_test.go b/common/rpc/test/http_test.go index ec6e1bd8701..4e107b94bb5 100644 --- a/common/rpc/test/http_test.go +++ b/common/rpc/test/http_test.go @@ -42,6 +42,7 @@ func TestCreateLocalFrontendHTTPClient_UsingMembership(t *testing.T) { int(port), nil, // No TLS nil, + nil, map[primitives.ServiceName][]grpc.DialOption{}, monitor, nil, @@ -75,6 +76,7 @@ func TestCreateLocalFrontendHTTPClient_UsingFixedHostPort(t *testing.T) { 0, // Port is unused nil, // No TLS nil, + nil, map[primitives.ServiceName][]grpc.DialOption{}, nil, // monitor should not be used nil, @@ -109,6 +111,7 @@ func TestCreateLocalFrontendHTTPClient_UsingFixedHostPort_AndTLS(t *testing.T) { 0, // Port is unused tlsConfig, nil, + nil, map[primitives.ServiceName][]grpc.DialOption{}, nil, // monitor should not be used nil, diff --git a/common/rpc/test/rpc_localstore_tls_test.go b/common/rpc/test/rpc_localstore_tls_test.go index 360da89d602..734fd66edb2 100644 --- a/common/rpc/test/rpc_localstore_tls_test.go +++ b/common/rpc/test/rpc_localstore_tls_test.go @@ -110,7 +110,7 @@ func (s *localStoreRPCSuite) SetupSuite() { provider, err := encryption.NewTLSConfigProviderFromConfig(serverCfgInsecure.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) - insecureFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, nil, nil, nil, nil, nil) + insecureFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, nil, nil, nil, nil, nil, nil) s.NotNil(insecureFactory) s.insecureRPCFactory = i(insecureFactory) @@ -320,26 +320,26 @@ func (s *localStoreRPCSuite) setupFrontend() { s.NoError(err) tlsConfig, err := provider.GetFrontendClientConfig() s.NoError(err) - frontendMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + frontendMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(frontendMutualTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreServerTLS.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) - frontendServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, nil, nil, nil, nil, nil) + frontendServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, nil, nil, nil, nil, nil, nil) s.NotNil(frontendServerTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreMutualTLSSystemWorker.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - frontendSystemWorkerMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + frontendSystemWorkerMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(frontendSystemWorkerMutualTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreMutualTLSWithRefresh.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - frontendMutualTLSRefreshFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + frontendMutualTLSRefreshFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(frontendMutualTLSRefreshFactory) s.frontendMutualTLSRPCFactory = f(frontendMutualTLSFactory) @@ -356,7 +356,7 @@ func (s *localStoreRPCSuite) setupFrontend() { s.NoError(err) tlsConfig, err = s.dynamicConfigProvider.GetFrontendClientConfig() s.NoError(err) - dynamicServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, s.dynamicConfigProvider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + dynamicServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, s.dynamicConfigProvider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.frontendDynamicTLSFactory = f(dynamicServerTLSFactory) s.internodeDynamicTLSFactory = i(dynamicServerTLSFactory) @@ -366,7 +366,7 @@ func (s *localStoreRPCSuite) setupFrontend() { s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - frontendRootCAForceTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + frontendRootCAForceTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(frontendServerTLSFactory) s.frontendConfigRootCAForceTLSFactory = f(frontendRootCAForceTLSFactory) @@ -374,7 +374,7 @@ func (s *localStoreRPCSuite) setupFrontend() { s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - remoteClusterMutualTLSRPCFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + remoteClusterMutualTLSRPCFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(remoteClusterMutualTLSRPCFactory) s.remoteClusterMutualTLSRPCFactory = r(remoteClusterMutualTLSRPCFactory) } @@ -412,28 +412,28 @@ func (s *localStoreRPCSuite) setupInternode() { s.NoError(err) tlsConfig, err := provider.GetFrontendClientConfig() s.NoError(err) - internodeMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + internodeMutualTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(internodeMutualTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreServerTLS.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - internodeServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + internodeServerTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(internodeServerTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreAltMutualTLS.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - internodeMutualAltTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + internodeMutualAltTLSFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(internodeMutualAltTLSFactory) provider, err = encryption.NewTLSConfigProviderFromConfig(localStoreMutualTLSWithRefresh.TLS, metrics.NoopMetricsHandler, s.logger, nil) s.NoError(err) tlsConfig, err = provider.GetFrontendClientConfig() s.NoError(err) - internodeMutualTLSRefreshFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil) + internodeMutualTLSRefreshFactory := rpc.NewFactory(cfg, "tester", s.logger, nil, provider, frontendURL, frontendHTTPURL, 0, tlsConfig, nil, nil, nil, nil, nil) s.NotNil(internodeMutualTLSRefreshFactory) s.internodeMutualTLSRPCFactory = i(internodeMutualTLSFactory) diff --git a/common/rpc/test/rpc_token_auth_test.go b/common/rpc/test/rpc_token_auth_test.go index 178356d3d30..e4ff61a50c1 100644 --- a/common/rpc/test/rpc_token_auth_test.go +++ b/common/rpc/test/rpc_token_auth_test.go @@ -99,7 +99,7 @@ func TestTokenAuthHeader_SentOnRemoteConnection(t *testing.T) { log.NewTestLogger(), metrics.NoopMetricsHandler, env.provider, "dns:///127.0.0.1:0", "dns:///127.0.0.1:0", 0, nil, - nil, nil, nil, + nil, nil, nil, nil, tokenProvider, ) @@ -151,7 +151,7 @@ func TestTokenAuthHeader_NotSentWhenNoProvider(t *testing.T) { log.NewTestLogger(), metrics.NoopMetricsHandler, env.provider, "dns:///127.0.0.1:0", "dns:///127.0.0.1:0", 0, nil, - nil, nil, nil, + nil, nil, nil, nil, nil, ) @@ -209,7 +209,7 @@ func TestTokenAuthHeader_ReceiverRejectsWrongToken(t *testing.T) { log.NewTestLogger(), metrics.NoopMetricsHandler, env.provider, "dns:///127.0.0.1:0", "dns:///127.0.0.1:0", 0, nil, - nil, nil, nil, + nil, nil, nil, nil, tokenProvider, ) @@ -256,7 +256,7 @@ func TestTokenAuthHeader_StrictModeRejectsEmptyToken(t *testing.T) { log.NewTestLogger(), metrics.NoopMetricsHandler, env.provider, "dns:///127.0.0.1:0", "dns:///127.0.0.1:0", 0, nil, - nil, nil, nil, + nil, nil, nil, nil, tokenProvider, ) diff --git a/common/telemetry/http.go b/common/telemetry/http.go index 1f7b4aab441..c825e49a0aa 100644 --- a/common/telemetry/http.go +++ b/common/telemetry/http.go @@ -14,6 +14,27 @@ import ( "go.opentelemetry.io/otel/trace" ) +// HTTPClientTransportInstrumenter instruments HTTP transports with the service's tracing configuration. +type HTTPClientTransportInstrumenter func(http.RoundTripper) http.RoundTripper + +// NewHTTPClientTransportInstrumenter binds a tracer provider and propagator for reuse across HTTP transports. +func NewHTTPClientTransportInstrumenter( + tracerProvider trace.TracerProvider, + propagator propagation.TextMapPropagator, +) HTTPClientTransportInstrumenter { + return func(rt http.RoundTripper) http.RoundTripper { + return NewHTTPClientTransport(rt, tracerProvider, propagator) + } +} + +// Instrument instruments rt, or returns it unchanged when the instrumenter is unset. +func (i HTTPClientTransportInstrumenter) Instrument(rt http.RoundTripper) http.RoundTripper { + if i == nil { + return rt + } + return i(rt) +} + // NewHTTPClientTransport instruments outbound HTTP requests with OpenTelemetry client spans // and injects trace context using propagator. If propagator is nil, it defaults to W3C Trace Context. func NewHTTPClientTransport( diff --git a/common/telemetry/http_test.go b/common/telemetry/http_test.go index 853334ad878..3aafffeea72 100644 --- a/common/telemetry/http_test.go +++ b/common/telemetry/http_test.go @@ -125,6 +125,18 @@ var errTestBodyRead = errors.New("body read failed") // Verifies client transport construction, propagation, and debug body handling. func TestNewHTTPClientTransport(t *testing.T) { + // The instrumenter carries HTTP tracing configuration without exposing it to transport owners. + t.Run("TransportInstrumenter", func(t *testing.T) { + t.Parallel() + + rt := http.DefaultTransport + var disabled HTTPClientTransportInstrumenter + require.Same(t, rt, disabled.Instrument(rt)) + + instrumenter := NewHTTPClientTransportInstrumenter(trace.NewTracerProvider(), nil) + require.NotSame(t, rt, instrumenter.Instrument(rt)) + }) + // A nil tracer provider disables instrumentation without changing the transport identity. t.Run("Disabled", func(t *testing.T) { t.Parallel() diff --git a/temporal/fx.go b/temporal/fx.go index 667dfb1673d..44630f5e700 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -1036,6 +1036,7 @@ var TraceExportModule = fx.Options( // default: propagation.TraceContext{} // - telemetry.ServerStatsHandler // - telemetry.ClientStatsHandler +// - telemetry.HTTPClientTransportInstrumenter var ServiceTracingModule = fx.Options( fx.Supply([]otelsdktrace.BatchSpanProcessorOption{}), fx.Provide( @@ -1105,6 +1106,7 @@ var ServiceTracingModule = fx.Options( fx.Provide(func() propagation.TextMapPropagator { return propagation.TraceContext{} }), fx.Provide(telemetry.NewServerStatsHandler), fx.Provide(telemetry.NewClientStatsHandler), + fx.Provide(telemetry.NewHTTPClientTransportInstrumenter), fx.Provide(metrics.NewServerStatsHandler), ) diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go new file mode 100644 index 00000000000..0de5a4604e5 --- /dev/null +++ b/tests/nexus_otel_test.go @@ -0,0 +1,333 @@ +package tests + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "slices" + "testing" + "time" + + "github.com/nexus-rpc/sdk-go/nexus" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/propagation" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.37.0" + oteltrace "go.opentelemetry.io/otel/trace" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporalnexus" + "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +type nexusHTTPSpan struct { + TraceID int + SpanID int + Name string + ServiceName string + Kind oteltrace.SpanKind + URLPath string +} + +type NexusOTELSuite struct { + parallelsuite.Suite[*NexusOTELSuite] +} + +func TestNexusOTELSuite(t *testing.T) { + parallelsuite.Run(t, &NexusOTELSuite{}) +} + +func (s *NexusOTELSuite) newTestEnv(exporter sdktrace.SpanExporter) *NexusTestEnv { + return newNexusTestEnv(s.T(), true, + testcore.WithSpanExporter(exporter), + testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true), + testcore.WithDynamicConfig(dynamicconfig.EnableCHASMCallbacks, true), + testcore.WithDynamicConfig(nexusoperation.Enabled, true), + testcore.WithDynamicConfig( + callback.AllowedAddresses, + []any{map[string]any{"Pattern": "*", "AllowInsecure": true}}, + ), + ) +} + +// Verifies production callback wiring propagates trace context and stored headers end to end. +func (s *NexusOTELSuite) TestCallback() { + exporter := tracetest.NewInMemoryExporter() + env := s.newTestEnv(exporter) + + requestHeaders := make(chan http.Header, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestHeaders <- r.Header.Clone() + w.WriteHeader(http.StatusOK) + })) + s.T().Cleanup(server.Close) + + callbackWorker := worker.New(env.SdkClient(), env.Tv().TaskQueue().GetName(), worker.Options{}) + callbackWorker.RegisterWorkflowWithOptions( + func(workflow.Context) error { return nil }, + workflow.RegisterOptions{Name: env.Tv().WorkflowType().GetName()}, + ) + s.NoError(callbackWorker.Start()) + s.T().Cleanup(callbackWorker.Stop) + + callbackHeaderValue := env.Tv().Any().String() + startResponse, err := env.FrontendClient().StartWorkflowExecution(s.Context(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: env.Tv().RequestID(), + Namespace: env.Namespace().String(), + WorkflowId: env.Tv().WorkflowID(), + WorkflowType: env.Tv().WorkflowType(), + TaskQueue: env.Tv().TaskQueue(), + WorkflowRunTimeout: durationpb.New(time.Minute), + Identity: env.Tv().Any().String(), + CompletionCallbacks: []*commonpb.Callback{{ + Variant: &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: server.URL, + Header: map[string]string{ + "X-Callback-Header": callbackHeaderValue, + }, + }, + }, + }}, + }) + s.NoError(err) + s.NoError(env.SdkClient().GetWorkflow(s.Context(), env.Tv().WorkflowID(), startResponse.RunId).Get(s.Context(), nil)) + + var headers http.Header + select { + case headers = <-requestHeaders: + case <-s.Context().Done(): + s.FailNow("timed out waiting for Nexus callback", s.Context().Err().Error()) + } + s.Equal(callbackHeaderValue, headers.Get("X-Callback-Header")) + httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ + TraceID: 1, + SpanID: 1, + Name: "HTTP POST", + ServiceName: "io.temporal.history", + Kind: oteltrace.SpanKindClient, + }}) + spanContext := oteltrace.SpanContextFromContext( + propagation.TraceContext{}.Extract(s.Context(), propagation.HeaderCarrier(headers)), + ) + s.Require().True(spanContext.IsValid()) + s.Require().Equal(spanContext.TraceID(), httpSpans[0].SpanContext.TraceID()) + s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) +} + +// Verifies asynchronous start and cancellation use the instrumented external Nexus HTTP client. +func (s *NexusOTELSuite) TestOperation() { + exporter := tracetest.NewInMemoryExporter() + callerEnv := s.newTestEnv(exporter) + handlerEnv := s.newTestEnv(exporter) + tv := callerEnv.Tv() + handlerTaskQueue := handlerEnv.Tv().TaskQueue().GetName() + + handlerWorkflow := func(ctx workflow.Context, _ nexus.NoValue) (nexus.NoValue, error) { + workflow.GetSignalChannel(ctx, "complete").Receive(ctx, nil) + return nil, nil + } + operation := temporalnexus.NewWorkflowRunOperation( + "test-operation", + handlerWorkflow, + func(_ context.Context, _ nexus.NoValue, options nexus.StartOperationOptions) (client.StartWorkflowOptions, error) { + return client.StartWorkflowOptions{ + ID: options.RequestID, + TaskQueue: handlerTaskQueue, + }, nil + }, + ) + service := nexus.NewService("test-service") + service.MustRegister(operation) + handlerWorker := worker.New(handlerEnv.SdkClient(), handlerTaskQueue, worker.Options{}) + handlerWorker.RegisterWorkflow(handlerWorkflow) + handlerWorker.RegisterNexusService(service) + s.NoError(handlerWorker.Start()) + s.T().Cleanup(handlerWorker.Stop) + + handlerWorkerEndpoint := handlerEnv.createNexusEndpoint(s.Context(), s.T(), testcore.RandomizedNexusEndpoint(s.T().Name()), handlerTaskQueue) + callerExternalEndpoint := callerEnv.createExternalNexusEndpoint(s.Context(), s.T(), handlerEnv.dispatchByEndpointURL(handlerWorkerEndpoint.Id)) + operationID := tv.Any().String() + startResponse, err := callerEnv.FrontendClient().StartNexusOperationExecution(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + Namespace: callerEnv.Namespace().String(), + OperationId: operationID, + Endpoint: callerExternalEndpoint, + Service: service.Name, + Operation: operation.Name(), + RequestId: tv.RequestID(), + ScheduleToCloseTimeout: durationpb.New(time.Minute), + }) + s.NoError(err) + + pollResponse, err := callerEnv.FrontendClient().PollNexusOperationExecution(s.Context(), &workflowservice.PollNexusOperationExecutionRequest{ + Namespace: callerEnv.Namespace().String(), + OperationId: operationID, + RunId: startResponse.RunId, + WaitStage: enumspb.NEXUS_OPERATION_WAIT_STAGE_STARTED, + }) + s.NoError(err) + + s.Require().Equal(enumspb.NEXUS_OPERATION_WAIT_STAGE_STARTED, pollResponse.GetWaitStage()) + _, err = callerEnv.FrontendClient().RequestCancelNexusOperationExecution(s.Context(), &workflowservice.RequestCancelNexusOperationExecutionRequest{ + Namespace: callerEnv.Namespace().String(), + OperationId: operationID, + RunId: startResponse.RunId, + Reason: tv.Any().String(), + }) + s.NoError(err) + + operationURLPath := "/nexus/endpoints/" + handlerWorkerEndpoint.Id + "/services/" + service.Name + "/" + operation.Name() + s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{ + { + TraceID: 1, + SpanID: 1, + Name: "HTTP POST", + ServiceName: "io.temporal.history", + Kind: oteltrace.SpanKindClient, + URLPath: operationURLPath, + }, + { + TraceID: 2, + SpanID: 1, + Name: "HTTP POST", + ServiceName: "io.temporal.history", + Kind: oteltrace.SpanKindClient, + URLPath: operationURLPath + "/cancel", + }, + }) +} + +// Verifies worker-target Nexus operations trace requests routed through the local frontend client. +func (s *NexusOTELSuite) TestWorkerOperation() { + exporter := tracetest.NewInMemoryExporter() + env := s.newTestEnv(exporter) + tv := env.Tv().WithTaskQueue(env.WorkerTaskQueue()) + + requestHeaders := make(chan nexus.Header, 1) + service := nexus.NewService("test-service") + operation := nexus.NewSyncOperation("test-operation", func(_ context.Context, _ nexus.NoValue, options nexus.StartOperationOptions) (string, error) { + requestHeaders <- options.Header + return tv.Any().String(), nil + }) + service.MustRegister(operation) + + nexusWorker := worker.New(env.SdkClient(), tv.TaskQueue().GetName(), worker.Options{}) + nexusWorker.RegisterNexusService(service) + s.NoError(nexusWorker.Start()) + s.T().Cleanup(nexusWorker.Stop) + + endpoint := env.createNexusEndpoint(s.Context(), s.T(), testcore.RandomizedNexusEndpoint(s.T().Name()), tv.TaskQueue().GetName()) + _, err := env.FrontendClient().StartNexusOperationExecution(s.Context(), &workflowservice.StartNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: tv.Any().String(), + Endpoint: endpoint.GetSpec().GetName(), + Service: service.Name, + Operation: operation.Name(), + RequestId: tv.RequestID(), + ScheduleToCloseTimeout: durationpb.New(time.Minute), + }) + s.NoError(err) + + var headers nexus.Header + select { + case headers = <-requestHeaders: + case <-s.Context().Done(): + s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) + } + operationURLPath := "/nexus/endpoints/" + endpoint.Id + "/services/" + service.Name + "/" + operation.Name() + httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ + TraceID: 1, + SpanID: 1, + Name: "HTTP POST", + ServiceName: "io.temporal.history", + Kind: oteltrace.SpanKindClient, + URLPath: operationURLPath, + }}) + spanContext := oteltrace.SpanContextFromContext( + propagation.TraceContext{}.Extract(s.Context(), propagation.MapCarrier(headers)), + ) + s.Require().True(spanContext.IsValid()) + s.Require().Equal(spanContext.TraceID(), httpSpans[0].SpanContext.TraceID()) + s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) +} + +// requireNexusHTTPSpans compares all exported HTTP spans after assigning stable local IDs +// and returns the matching raw spans for context propagation assertions. +func (s *NexusOTELSuite) requireNexusHTTPSpans( + exporter *tracetest.InMemoryExporter, + expected []nexusHTTPSpan, +) tracetest.SpanStubs { + s.T().Helper() + var httpSpans tracetest.SpanStubs + s.Await(func(s *NexusOTELSuite) { + var actual []nexusHTTPSpan + actual, httpSpans = s.nexusHTTPSpans(exporter.GetSpans()) + s.Require().Equal(expected, actual) + }, 10*time.Second, 100*time.Millisecond) + return httpSpans +} + +func (s *NexusOTELSuite) nexusHTTPSpans( + spans tracetest.SpanStubs, +) ([]nexusHTTPSpan, tracetest.SpanStubs) { + httpSpans := slices.DeleteFunc(spans, func(span tracetest.SpanStub) bool { + return span.InstrumentationScope.Name != otelhttp.ScopeName + }) + slices.SortFunc(httpSpans, func(a, b tracetest.SpanStub) int { + return a.StartTime.Compare(b.StartTime) + }) + + traceIDs := make(map[oteltrace.TraceID]int) + spanIDs := make(map[oteltrace.TraceID]map[oteltrace.SpanID]int) + for _, span := range httpSpans { + traceID := span.SpanContext.TraceID() + if _, ok := traceIDs[traceID]; !ok { + traceIDs[traceID] = len(traceIDs) + 1 + spanIDs[traceID] = make(map[oteltrace.SpanID]int) + } + spanIDs[traceID][span.SpanContext.SpanID()] = len(spanIDs[traceID]) + 1 + } + + result := make([]nexusHTTPSpan, 0, len(httpSpans)) + for _, span := range httpSpans { + traceID := span.SpanContext.TraceID() + var serviceName string + if span.Resource != nil { + if value, ok := span.Resource.Set().Value(semconv.ServiceNameKey); ok { + serviceName = value.AsString() + } + } + var urlPath string + for _, attr := range span.Attributes { + if attr.Key == semconv.URLPathKey { + urlPath = attr.Value.AsString() + break + } + if attr.Key == semconv.URLFullKey { + if parsedURL, err := url.Parse(attr.Value.AsString()); err == nil { + urlPath = parsedURL.Path + } + } + } + result = append(result, nexusHTTPSpan{ + TraceID: traceIDs[traceID], + SpanID: spanIDs[traceID][span.SpanContext.SpanID()], + Name: span.Name, + ServiceName: serviceName, + Kind: span.SpanKind, + URLPath: urlPath, + }) + } + return result, httpSpans +} diff --git a/tests/nexus_test_base.go b/tests/nexus_test_base.go index 38ce6dc13cb..0c3100fe61e 100644 --- a/tests/nexus_test_base.go +++ b/tests/nexus_test_base.go @@ -67,16 +67,20 @@ func (env *NexusTestEnv) createRandomNexusEndpoint(ctx context.Context, t *testi // createRandomExternalNexusServer creates a mock nexus server that listens via a randomized endpointName and return this name to the caller. func (env *NexusTestEnv) createRandomExternalNexusServer(ctx context.Context, t *testing.T, handler nexustest.Handler) string { - endpointName := testcore.RandomizedNexusEndpoint(t.Name()) listenAddr := nexustest.AllocListenAddress() nexustest.NewNexusServer(t, listenAddr, handler) + return env.createExternalNexusEndpoint(ctx, t, "http://"+listenAddr) +} + +func (env *NexusTestEnv) createExternalNexusEndpoint(ctx context.Context, t *testing.T, url string) string { + endpointName := testcore.RandomizedNexusEndpoint(t.Name()) resp, err := env.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ Spec: &nexuspb.EndpointSpec{ Name: endpointName, Target: &nexuspb.EndpointTarget{ Variant: &nexuspb.EndpointTarget_External_{ External: &nexuspb.EndpointTarget_External{ - Url: "http://" + listenAddr, + Url: url, }, }, }, @@ -95,6 +99,10 @@ func (env *NexusTestEnv) createRandomExternalNexusServer(ctx context.Context, t return endpointName } +func (env *NexusTestEnv) dispatchByEndpointURL(endpoint string) string { + return "http://" + env.HttpAPIAddress() + "/" + cnexus.RouteDispatchNexusTaskByEndpoint.Path(endpoint) +} + // nexusTaskResponse represents a successful response from a nexus task handler. // A nil response indicates no response should be sent (e.g., handler timed out). type nexusTaskResponse struct { diff --git a/tests/testcore/clients.go b/tests/testcore/clients.go index 3af9a14ca2e..376b92a8cb8 100644 --- a/tests/testcore/clients.go +++ b/tests/testcore/clients.go @@ -177,6 +177,7 @@ func (c *clients) ensureMatching() { frontendTLSConfig, nil, nil, + nil, monitor, c.tokenProvider, ) diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index fa7c3615e33..7af807f6dc3 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -102,6 +102,7 @@ type ( EnableHistoryTaskRecorder bool EnableReplicationRecorder bool EnableArchival bool + SpanExporter sdktrace.SpanExporter AdditionalServerOptions []temporal.ServerOption } TestClusterOption func(params *testClusterParams) @@ -180,6 +181,12 @@ func WithReplicationStreamRecorder() TestClusterOption { } } +func withSpanExporter(exporter sdktrace.SpanExporter) TestClusterOption { + return func(params *testClusterParams) { + params.SpanExporter = exporter + } +} + func WithSharedCluster() TestClusterOption { return func(params *testClusterParams) { params.SharedCluster = true @@ -309,6 +316,9 @@ func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { AdditionalServerOptions: params.AdditionalServerOptions, WorkerConfig: WorkerConfig{DisableWorker: !params.EnableWorkerService}, } + if params.SpanExporter != nil { + setSpanExporter(s.testClusterConfig, "test", params.SpanExporter) + } // Apply configuration for shared clusters. if params.SharedCluster { @@ -323,9 +333,7 @@ func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { s.otelExporter = testtelemetry.NewFileExporter(otelOutputDir) // Direct the OTEL exporter to the collector. - s.testClusterConfig.SpanExporters = map[telemetry.SpanExporterType]sdktrace.SpanExporter{ - telemetry.OtelTracesOtlpExporterType: s.otelExporter, - } + setSpanExporter(s.testClusterConfig, telemetry.OtelTracesOtlpExporterType, s.otelExporter) } var err error @@ -343,6 +351,13 @@ func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { s.Require().NoError(err) } +func setSpanExporter(clusterConfig *TestClusterConfig, exporterType telemetry.SpanExporterType, exporter sdktrace.SpanExporter) { + if clusterConfig.SpanExporters == nil { + clusterConfig.SpanExporters = make(map[telemetry.SpanExporterType]sdktrace.SpanExporter) + } + clusterConfig.SpanExporters[exporterType] = exporter +} + func sharedClusterPersistence(defaults persistencetests.TestBaseOptions) persistencetests.TestBaseOptions { if defaults.StoreType == config.StoreTypeSQL && defaults.SQLDBPluginName == sqlite.PluginName { // Use file-based SQLite for shared clusters to support parallel test access. diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index 8be1d16394d..b0d68cafe1e 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -13,6 +13,7 @@ import ( "github.com/dgryski/go-farm" "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/workflowservice/v1" sdkclient "go.temporal.io/sdk/client" @@ -114,6 +115,15 @@ func WithDedicatedCluster() TestOption { } } +// WithSpanExporter enables OpenTelemetry tracing with exporter on a dedicated test cluster. +func WithSpanExporter(exporter sdktrace.SpanExporter) TestOption { + return func(o *testOptions) { + o.dedicatedCluster = true + o.clusterOptions = append(o.clusterOptions, withSpanExporter(exporter)) + o.dedicatedReason = "span exporter configured" + } +} + // WithDisableTestloggerFailure disables the test logger's behavior of failing // the test when an error log matches a registered expectation (e.g. soft-assert // errors tagged with tag.FailedAssertion). Use for tests that intentionally From 438847b1425e7ccf3477527ad51cabb41b00d670 Mon Sep 17 00:00:00 2001 From: michaely520 Date: Wed, 19 Aug 2026 10:27:10 -0700 Subject: [PATCH 005/139] replication: emit lifecycle error events (#11595) ## What changed? - Adds `phase=error` to the existing `replication_lifecycle` wide event; no new event type or table. - Covers state-based replication (`SyncVersionedTransition`, `VerifyVersionedTransition`, and `SyncWorkflowState`) plus standby transfer, timer, and outbound queue failures. - Captures sender, passive execution/apply, verification, recovery/refetch, namespace refresh, Nack, DLQ, and history-branch cleanup boundaries. - Uses one shared error builder with small sender, executable-task, NDC, and standby-queue adapters. - Records workflow identity, source task identity, target context, operation, error, attempt/priority, disposition/recovery, and extensible diagnostics in `details`. - Identifies apply provenance as `apply_artifact_source=task_payload` or `sync_state_refetch`. - Remains gated by `history.emitReplicationLifecycleEvents` (default off). ## Why? Replication failures span the sender, passive executor, recovery loop, and apply layer. Recording these boundaries in the existing lifecycle event makes the path of a workflow or replication task directly traceable without adding another event schema. ## Example traces The examples below are abridged records captured from the two-cluster XDC test. Events for the same task correlate on `source_cluster`, `source_shard`, and `source_task_id`; workflow identity is present on every record. A state task that fails on the passive cluster and is written to the DLQ: ```json {"phase":"sent","task_type":"sync_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048581,"priority":"High"} {"phase":"error","task_type":"sync_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048581,"details":{"operation":"passive_task_execution","error":"failed to apply replication task","error_type":"serviceerror.InvalidArgument","apply_artifact_source":"task_payload","attempt":1,"priority":"High","target_cluster":"standby"}} {"phase":"error","task_type":"sync_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048581,"details":{"operation":"task_execution","error":"failed to apply replication task","terminal":true,"priority":"High","target_cluster":"standby"}} {"phase":"error","task_type":"sync_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048581,"details":{"operation":"dlq_write","disposition":"dlq","terminal":true,"priority":"High","target_cluster":"standby","target_shard":1}} ``` A verification task that detects missing state, refetches it, and then verifies successfully: ```json {"phase":"sent","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"priority":"High"} {"phase":"executing","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"attempt":1} {"phase":"applied","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"outcome":"resend_needed"} {"phase":"error","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"details":{"operation":"standby_verification","error":"missing mutable state, resend","error_type":"serviceerror.SyncState","recovery_action":"sync_state","priority":"High","target_cluster":"standby"}} {"phase":"applied","task_type":"sync_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"outcome":"applied","details":{"apply_artifact_source":"sync_state_refetch"}} {"phase":"executing","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"attempt":1} {"phase":"applied","task_type":"verify_versioned_transition","workflow_id":"example-workflow","source_cluster":"active","source_shard":1,"source_task_id":1048595,"outcome":"verified"} ``` ## How was it tested? - `go test -tags test_dep ./common/wideevents ./service/history/replication ./service/history/ndc ./service/history ./tests/testcore` - Changed-lines `golangci-lint`: 0 issues. - Temporary, uncommitted two-cluster XDC tests forced passive task failures, DLQ handling, standby verification, and SyncState refetch/recovery with lifecycle events both enabled and disabled. - A tiered-processing run confirmed concrete `High` priority on sent and error records. ## Risks - Enabling the dynamic config increases event volume; retries and recovery can produce several error phases for one source task. - `details.operation`, `details.disposition`, and `details.recovery_action` distinguish those boundaries. - Emission is best effort and does not change replication error propagation, retry, or recovery behavior. --- common/dynamicconfig/constants.go | 2 +- common/wideevents/context.go | 9 +- common/wideevents/replication_events.go | 91 ++++++++- .../history/ndc/workflow_state_replicator.go | 113 ++++++++++- .../outbound_queue_standby_task_executor.go | 1 + ...ecutable_sync_versioned_transition_task.go | 11 +- .../history/replication/executable_task.go | 52 ++++- ...utable_verify_versioned_transition_task.go | 6 + .../executable_workflow_state_task.go | 17 ++ .../history/replication/replication_events.go | 181 ++++++++++++++++++ service/history/replication/stream_sender.go | 20 +- service/history/standby_task_events.go | 66 +++++++ .../timer_queue_standby_task_executor.go | 1 + .../transfer_queue_standby_task_executor.go | 1 + 14 files changed, 548 insertions(+), 23 deletions(-) create mode 100644 service/history/standby_task_events.go diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 12040a88ada..1c417249ff4 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1725,7 +1725,7 @@ leaves the membership ring, giving in-flight long-polls time to drain before the EmitReplicationLifecycleEvents = NewGlobalBoolSetting( "history.emitReplicationLifecycleEvents", false, - `EmitReplicationLifecycleEvents controls whether the history service emits ReplicationLifecycle wide events (sent/executing/applied phases). Cluster-level; default off.`, + `EmitReplicationLifecycleEvents controls whether the history service emits ReplicationLifecycle wide events (sent/executing/applied/skipped/error phases). Cluster-level; default off.`, ) EnableCloseInboundReplicationStreamOnShutdown = NewGlobalBoolSetting( "history.enableCloseInboundReplicationStreamOnShutdown", diff --git a/common/wideevents/context.go b/common/wideevents/context.go index 35d89a0a898..8eba7187af3 100644 --- a/common/wideevents/context.go +++ b/common/wideevents/context.go @@ -4,11 +4,12 @@ import "context" type replicationTaskOriginCtxKey struct{} -// ReplicationTaskOrigin identifies where a replicated artifact came from; diagnostic only. TaskID is -// unset when the applied artifact was re-fetched rather than being that queue entry's own payload. +// ReplicationTaskOrigin identifies the source task whose processing produced an event. type ReplicationTaskOrigin struct { - ShardID int32 - TaskID int64 + ClusterName string + ShardID int32 + TaskID int64 + ApplyArtifactSource ReplicationApplyArtifactSource } // SetReplicationTaskOrigin stamps origin onto ctx. diff --git a/common/wideevents/replication_events.go b/common/wideevents/replication_events.go index 58794d4d470..f8a425b0c5e 100644 --- a/common/wideevents/replication_events.go +++ b/common/wideevents/replication_events.go @@ -1,16 +1,32 @@ package wideevents import ( + "maps" + "go.opentelemetry.io/otel/log" persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/util" ) // ReplicationLifecycleEventName is the stable event name for the ReplicationLifecycle wide event, -// which traces a replication task sent -> executing -> applied. +// which traces a replication task through sent, executing, applied, skipped, and error phases. const ReplicationLifecycleEventName = "replication_lifecycle" type ReplicationPhase string +// ReplicationDisposition records what the task owner will do with the original replication task +// after the error. It does not describe where the error occurred, its cause, or a recovery action. +type ReplicationDisposition string + +// ReplicationRecoveryAction records a compensating action selected in response to an error. +type ReplicationRecoveryAction string + +// ReplicationErrorClassification adds a queryable semantic classification to an error. +type ReplicationErrorClassification string + +// ReplicationApplyArtifactSource records how the state artifact supplied to an apply was obtained. +type ReplicationApplyArtifactSource string + const ( ReplicationSent ReplicationPhase = "sent" ReplicationExecuting ReplicationPhase = "executing" @@ -20,6 +36,28 @@ const ( // wedging the stream. Such a task never reaches executing/applied, so this is where its trace // ends. See StreamSenderImpl.recordStuckTaskSkipped. ReplicationSkipped ReplicationPhase = "skipped" + ReplicationError ReplicationPhase = "error" +) + +const ( + ReplDispositionRetry ReplicationDisposition = "retry" + ReplDispositionDiscarded ReplicationDisposition = "discarded" + ReplDispositionDLQ ReplicationDisposition = "dlq" +) + +const ( + ReplRecoveryActionCleanup ReplicationRecoveryAction = "cleanup" + ReplRecoveryActionSyncState ReplicationRecoveryAction = "sync_state" + ReplRecoveryActionResendHistory ReplicationRecoveryAction = "resend_history" +) + +const ( + ReplErrorClassificationDuplicate ReplicationErrorClassification = "duplicate" +) + +const ( + ReplApplyArtifactSourceTaskPayload ReplicationApplyArtifactSource = "task_payload" + ReplApplyArtifactSourceSyncStateRefetch ReplicationApplyArtifactSource = "sync_state_refetch" ) const ( @@ -42,8 +80,57 @@ const ( ArtifactKindMutation = "mutation" ) +// Replication operation identifies the stage or action that encountered the error. Disposition, +// recovery_action, and classification describe what happens next and how to interpret the error; +// they are independent of operation and remain optional fields in details. +const ( + ReplOperationTaskExecution = "task_execution" + ReplOperationTaskConversion = "task_conversion" + ReplOperationRateLimit = "rate_limit" + ReplOperationStreamSend = "stream_send" + ReplOperationPassiveTaskExecution = "passive_task_execution" + ReplOperationPassiveApply = "passive_apply" + ReplOperationStandbyVerification = "standby_verification" + ReplOperationStandbyTaskExecution = "standby_task_execution" + ReplOperationSyncState = "sync_state" + ReplOperationSyncStateApply = "sync_state_apply" + ReplOperationSyncVersionedTransitionSyncState = "sync_versioned_transition_sync_state" + ReplOperationSyncWorkflowStateSyncState = "sync_workflow_state_sync_state" + ReplOperationStandbyVerificationSyncState = "standby_verification_sync_state" + ReplOperationHistoryBackfill = "history_backfill" + ReplOperationHistoryResend = "history_resend" + ReplOperationHistoryBranchCleanup = "history_branch_cleanup" + ReplOperationNamespaceSync = "namespace_sync" + ReplOperationDLQWrite = "dlq_write" +) + +// EmitReplicationError normalizes the common error envelope and emits it into the existing +// replication_lifecycle event, not a separate event table. +func EmitReplicationError( + logger log.Logger, + payload ReplicationLifecyclePayload, + operation string, + message string, + err error, + extraDetails map[string]any, +) { + details := make(map[string]any, len(payload.Details)+len(extraDetails)+4) + maps.Copy(details, payload.Details) + maps.Copy(details, extraDetails) + details["operation"] = operation + details["message"] = message + if err != nil { + details["error"] = err.Error() + details["error_type"] = util.ErrorType(err) + } + + payload.Phase = ReplicationError + payload.Details = details + Emit(logger, payload) +} + type ReplicationLifecyclePayload struct { - // Phase is sent, executing, applied or skipped. + // Phase is sent, executing, applied, skipped or error. Phase ReplicationPhase // TaskType is which replication task this is. TaskType string diff --git a/service/history/ndc/workflow_state_replicator.go b/service/history/ndc/workflow_state_replicator.go index 8a524e0a92d..d63f97dd6d9 100644 --- a/service/history/ndc/workflow_state_replicator.go +++ b/service/history/ndc/workflow_state_replicator.go @@ -117,6 +117,13 @@ func (r *WorkflowStateReplicatorImpl) SyncWorkflowState( namespaceID := namespace.ID(executionInfo.GetNamespaceId()) wid := executionInfo.GetWorkflowId() rid := executionState.GetRunId() + defer func() { + if retError != nil { + r.emitReplicationApplyError( + ctx, wideevents.ReplTaskSyncWorkflowState, request.GetRemoteCluster(), executionInfo, executionState, + "snapshot", "Failed to apply replicated workflow state", retError) + } + }() if executionState.State != enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED { return serviceerror.NewInternal("Replicate non completed workflow state is not supported.") } @@ -240,9 +247,9 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( wid := executionInfo.GetWorkflowId() rid := executionState.GetRunId() - // emitApplied gates the best-effort "applied" lifecycle event; computed once here so the gate - // lives at the call site, like the other replication lifecycle emitters. - emitApplied := r.eventLogger != nil && r.shardContext.GetConfig().EmitReplicationLifecycleEvents() + // emitLifecycle gates the best-effort applied/error lifecycle event; computed once here so the + // gate lives at the call site, like the other replication lifecycle emitters. + emitLifecycle := r.eventLogger != nil && r.shardContext.GetConfig().EmitReplicationLifecycleEvents() // ms is the mutable state being applied; appliedMS is a snapshot of its post-apply state taken // under the workflow lock (see the releaseFn wrapper below) for the deferred emit. Reading the // live ms after the lock is released would race with the next writer. @@ -250,8 +257,12 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( var appliedMS *persistencespb.WorkflowMutableState origin := wideevents.ReplicationTaskOriginFromContext(ctx) defer func() { - if emitApplied && retError == nil { + if emitLifecycle && retError == nil { r.emitReplicationVersionedTransitionApplied(namespaceID, wid, rid, appliedMS, sourceClusterName, origin) + } else if emitLifecycle { + r.emitReplicationVersionedTransitionApplyError( + ctx, wideevents.ReplTaskSyncVersionedTransition, sourceClusterName, executionInfo, executionState, + versionedTransitionArtifact, "Failed to apply replicated versioned transition", retError) } }() @@ -269,7 +280,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( if err != nil { return err } - if emitApplied { + if emitLifecycle { // Snapshot the post-apply mutable state at release time: the only point that is both after // the apply (which mutates ms in place) and still under the workflow lock, since the apply // releases via the transaction manager. @@ -403,6 +414,9 @@ func (r *WorkflowStateReplicatorImpl) emitReplicationVersionedTransitionApplied( SourceShard: origin.ShardID, SourceTaskID: origin.TaskID, } + if origin.ApplyArtifactSource != "" { + payload.Details = map[string]any{"apply_artifact_source": origin.ApplyArtifactSource} + } if ms != nil { info := ms.GetExecutionInfo() state := ms.GetExecutionState() @@ -444,6 +458,87 @@ func (r *WorkflowStateReplicatorImpl) emitReplicationVersionedTransitionApplied( wideevents.Emit(r.eventLogger, payload) } +func (r *WorkflowStateReplicatorImpl) emitReplicationVersionedTransitionApplyError( + ctx context.Context, + taskType string, + sourceClusterName string, + info *persistencespb.WorkflowExecutionInfo, + state *persistencespb.WorkflowExecutionState, + artifact *replicationspb.VersionedTransitionArtifact, + message string, + err error, +) { + artifactKind := wideevents.ArtifactKindMutation + if artifact.GetSyncWorkflowStateSnapshotAttributes() != nil { + artifactKind = wideevents.ArtifactKindSnapshot + } + r.emitReplicationApplyError(ctx, taskType, sourceClusterName, info, state, artifactKind, message, err) +} + +// emitReplicationApplyError extracts artifact identity before delegating to the apply-side adapter. +func (r *WorkflowStateReplicatorImpl) emitReplicationApplyError( + ctx context.Context, + taskType string, + sourceClusterName string, + info *persistencespb.WorkflowExecutionInfo, + state *persistencespb.WorkflowExecutionState, + artifactKind string, + message string, + err error, +) { + payload := wideevents.ReplicationLifecyclePayload{ + TaskType: taskType, + NamespaceID: info.GetNamespaceId(), + WorkflowID: info.GetWorkflowId(), + RunID: state.GetRunId(), + } + if vt := transitionhistory.LastVersionedTransition(info.GetTransitionHistory()); vt != nil { + payload.FailoverVersion = vt.GetNamespaceFailoverVersion() + payload.TransitionCount = vt.GetTransitionCount() + } + details := map[string]any{ + "artifact_kind": artifactKind, + } + if errors.Is(err, consts.ErrDuplicate) { + details["classification"] = wideevents.ReplErrorClassificationDuplicate + } + r.emitReplicationError(ctx, payload, sourceClusterName, wideevents.ReplOperationPassiveApply, message, err, details) +} + +// emitReplicationError supplies fields authoritative on the applying cluster. +func (r *WorkflowStateReplicatorImpl) emitReplicationError( + ctx context.Context, + payload wideevents.ReplicationLifecyclePayload, + sourceClusterName string, + operation string, + message string, + err error, + details map[string]any, +) { + if r.eventLogger == nil || !r.shardContext.GetConfig().EmitReplicationLifecycleEvents() { + return + } + + origin := wideevents.ReplicationTaskOriginFromContext(ctx) + if name, nsErr := r.namespaceRegistry.GetNamespaceName(namespace.ID(payload.NamespaceID)); nsErr == nil { + payload.Namespace = name.String() + } + payload.Shard = r.shardContext.GetShardID() + if sourceClusterName == "" { + sourceClusterName = origin.ClusterName + } + payload.SourceCluster = sourceClusterName + payload.SourceShard = origin.ShardID + payload.SourceTaskID = origin.TaskID + payload.Details = map[string]any{ + "target_cluster": r.clusterMetadata.GetCurrentClusterName(), + } + if origin.ApplyArtifactSource != "" { + payload.Details["apply_artifact_source"] = origin.ApplyArtifactSource + } + wideevents.EmitReplicationError(r.eventLogger, payload, operation, message, err, details) +} + // populateAppliedExecutionSummary fills the post-apply mutable-state summary fields shared across // applied lifecycle hooks. It is best-effort and nil-safe. func populateAppliedExecutionSummary( @@ -776,6 +871,14 @@ func (r *WorkflowStateReplicatorImpl) deleteNewBranchWhenError( tag.WorkflowID(workflowID), tag.WorkflowRunID(runID), tag.ShardID(r.shardContext.GetShardID())) + r.emitReplicationError(ctx, wideevents.ReplicationLifecyclePayload{ + TaskType: wideevents.ReplTaskSyncVersionedTransition, + NamespaceID: namespaceID.String(), + WorkflowID: workflowID, + RunID: runID, + }, "", wideevents.ReplOperationHistoryBranchCleanup, "History branch cleanup skipped after apply error; branch may be orphaned", err, map[string]any{ + "archetype_id": archetypeID, + }) } } diff --git a/service/history/outbound_queue_standby_task_executor.go b/service/history/outbound_queue_standby_task_executor.go index 16a5f4fd7f8..2d953deb5c5 100644 --- a/service/history/outbound_queue_standby_task_executor.go +++ b/service/history/outbound_queue_standby_task_executor.go @@ -69,6 +69,7 @@ func (e *outboundQueueStandbyTaskExecutor) Execute( executable.GetWorkflowID(), ) respond := func(err error) queues.ExecuteResponse { + emitStandbyTaskError(e.shardContext, executable, taskType, err) metricsTags := []metrics.Tag{ namespaceTag, metrics.TaskTypeTag(taskType), diff --git a/service/history/replication/executable_sync_versioned_transition_task.go b/service/history/replication/executable_sync_versioned_transition_task.go index 89ea00974ee..110bf1e3e65 100644 --- a/service/history/replication/executable_sync_versioned_transition_task.go +++ b/service/history/replication/executable_sync_versioned_transition_task.go @@ -103,10 +103,7 @@ func (e *ExecutableSyncVersionedTransitionTask) Execute() error { ctx, cancel := newTaskContext(namespaceName, e.Config.ReplicationTaskApplyTimeout(), callerInfo) defer cancel() - ctx = wideevents.SetReplicationTaskOrigin(ctx, wideevents.ReplicationTaskOrigin{ - ShardID: e.SourceShardKey().ShardID, - TaskID: e.TaskID(), - }) + ctx = setReplicationTaskOrigin(ctx, e.ExecutableTask, wideevents.ReplApplyArtifactSourceTaskPayload) shardContext, err := e.ShardController.GetShardByNamespaceWorkflow( namespace.ID(e.NamespaceID), e.WorkflowID, @@ -144,6 +141,7 @@ func (e *ExecutableSyncVersionedTransitionTask) HandleErr(err error) error { tag.TaskID(e.TaskID()), tag.Error(err), ) + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationPassiveTaskExecution, "SyncVersionedTransition replication task encountered error", err, nil) callerInfo := getReplicaitonCallerInfo(e.GetPriority()) switch taskErr := err.(type) { case *serviceerrors.SyncState: @@ -170,6 +168,7 @@ func (e *ExecutableSyncVersionedTransitionTask) HandleErr(err error) error { tag.TaskID(e.TaskID()), tag.Error(syncStateErr), ) + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationSyncVersionedTransitionSyncState, "SyncVersionedTransition recovery failed during sync state", syncStateErr, nil) return err } return nil @@ -226,6 +225,10 @@ func (e *ExecutableSyncVersionedTransitionTask) HandleErr(err error) error { endEventVersion, "", ); resendErr != nil { + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationHistoryBackfill, "SyncVersionedTransition history backfill failed", resendErr, map[string]any{ + "first_event_id": startEvent, + "last_event_id": endEvent, + }) return err } return e.Execute() diff --git a/service/history/replication/executable_task.go b/service/history/replication/executable_task.go index d645aa68c62..f8ee4ca7d67 100644 --- a/service/history/replication/executable_task.go +++ b/service/history/replication/executable_task.go @@ -198,6 +198,9 @@ func (e *ExecutableTaskImpl) Nack(err error) { "replication task: %v encountered nack event", e.taskID, ), tag.Error(err)) + e.emitReplicationTaskError(wideevents.ReplOperationTaskExecution, "Replication task encountered nack event", err, map[string]any{ + "terminal": true, + }) now := time.Now().UTC() e.emitFinishMetrics(now) @@ -733,6 +736,9 @@ func (e *ExecutableTaskImpl) SyncState( if err != nil { var resourceExhaustedError *serviceerror.ResourceExhausted if errors.As(err, &resourceExhaustedError) { + e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Sync workflow state request exceeded resource limits", err, map[string]any{ + "request_payload_size": req.Size(), + }) return false, serviceerror.NewInvalidArgumentf("sync workflow state failed due to resource exhausted: %v, request payload size: %v", err, req.Size()) } logger := log.With(e.Logger, @@ -745,12 +751,18 @@ func (e *ExecutableTaskImpl) SyncState( var workflowNotReady *serviceerror.WorkflowNotReady if errors.As(err, &workflowNotReady) { logger.Info("Dropped replication task as source mutable state has buffered events.", tag.Error(err)) + e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Dropped replication task because source mutable state has buffered events", err, map[string]any{ + "disposition": wideevents.ReplDispositionDiscarded, + }) return false, nil } var notFoundErr *serviceerror.NotFound if errors.As(err, ¬FoundErr) { logger.Error( "workflow not found in source cluster, proceed to cleanup") + e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Workflow not found in source cluster; cleaning up target", err, map[string]any{ + "recovery_action": wideevents.ReplRecoveryActionCleanup, + }) // workflow is not found in source cluster, cleanup workflow in target cluster return false, e.DeleteWorkflow( ctx, @@ -772,6 +784,9 @@ func (e *ExecutableTaskImpl) SyncState( if len(taskEquivalents) == 0 { // Just drop the task since there's nothing to replicate in event-based stack. logger.Info("Dropped replication task as there's no event-based replication task equivalent.") + e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Dropped replication task because no event-based equivalent exists", err, map[string]any{ + "disposition": wideevents.ReplDispositionDiscarded, + }) return false, nil } @@ -792,6 +807,11 @@ func (e *ExecutableTaskImpl) SyncState( ShardId: e.sourceShardKey.ShardID, Tasks: tasksToAdd, }) + if err != nil { + e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Failed to enqueue event-based replication task equivalents", err, map[string]any{ + "equivalent_count": len(tasksToAdd), + }) + } return false, err } @@ -806,12 +826,14 @@ func (e *ExecutableTaskImpl) SyncState( if err != nil { return false, err } - // No TaskID: this artifact was re-fetched from the source, so it is not this task's payload. - ctx = wideevents.SetReplicationTaskOrigin(ctx, wideevents.ReplicationTaskOrigin{ShardID: e.sourceShardKey.ShardID}) + ctx = setReplicationTaskOrigin(ctx, e, wideevents.ReplApplyArtifactSourceSyncStateRefetch) err = engine.ReplicateVersionedTransition(ctx, syncStateErr.ArchetypeId, resp.VersionedTransitionArtifact, e.SourceClusterName()) if err == nil || errors.Is(err, consts.ErrDuplicate) { return true, nil } + e.emitReplicationTaskError(wideevents.ReplOperationSyncStateApply, "Failed to apply mutable state refetched from source", err, map[string]any{ + "apply_artifact_source": wideevents.ReplApplyArtifactSourceSyncStateRefetch, + }) return false, err } @@ -863,6 +885,9 @@ func (e *ExecutableTaskImpl) GetNamespaceInfo( _, err = e.EagerNamespaceRefresher.SyncNamespaceFromSourceCluster(ctx, namespace.ID(namespaceID), e.sourceClusterName) if err != nil { e.ThrottledLogger.Error("Failed to SyncNamespaceFromSourceCluster", tag.Error(err)) + e.emitReplicationTaskError(wideevents.ReplOperationNamespaceSync, "Failed to refresh namespace from source cluster; replication task skipped", err, map[string]any{ + "disposition": wideevents.ReplDispositionDiscarded, + }) return "", false, nil } default: @@ -901,6 +926,11 @@ func (e *ExecutableTaskImpl) MarkPoisonPill() error { tag.SourceCluster(e.SourceClusterName()), tag.ReplicationTask(taskInfo), ) + e.emitReplicationTaskError(wideevents.ReplOperationDLQWrite, "Writing replication task to DLQ reached maximum attempts", nil, map[string]any{ + "dlq_attempt": e.markPoisonPillAttempts, + "disposition": wideevents.ReplDispositionDiscarded, + "terminal": true, + }) return nil } e.markPoisonPillAttempts++ @@ -913,7 +943,7 @@ func (e *ExecutableTaskImpl) MarkPoisonPill() error { return err } - e.Logger.Error("Enqueued replication task to DLQ", + dlqLogger := log.With(e.Logger, tag.TargetShardID(shardContext.GetShardID()), tag.SourceShardID(e.sourceShardKey.ShardID), tag.WorkflowNamespaceID(e.replicationTask.RawTaskInfo.NamespaceId), @@ -931,7 +961,21 @@ func (e *ExecutableTaskImpl) MarkPoisonPill() error { ) defer cancel() - return writeTaskToDLQ(ctx, e.DLQWriter, e.sourceShardKey.ShardID, e.SourceClusterName(), shardContext.GetShardID(), taskInfo) + err = writeTaskToDLQ(ctx, e.DLQWriter, e.sourceShardKey.ShardID, e.SourceClusterName(), shardContext.GetShardID(), taskInfo) + if err != nil { + dlqLogger.Error("Failed to enqueue replication task to DLQ", tag.Error(err)) + e.emitReplicationTaskError(wideevents.ReplOperationDLQWrite, "Failed to enqueue replication task to DLQ", err, map[string]any{ + "target_shard": shardContext.GetShardID(), + }) + return err + } + dlqLogger.Error("Enqueued replication task to DLQ") + e.emitReplicationTaskError(wideevents.ReplOperationDLQWrite, "Enqueued replication task to DLQ", nil, map[string]any{ + "target_shard": shardContext.GetShardID(), + "disposition": wideevents.ReplDispositionDLQ, + "terminal": true, + }) + return nil } func newTaskContext( diff --git a/service/history/replication/executable_verify_versioned_transition_task.go b/service/history/replication/executable_verify_versioned_transition_task.go index 78fe5bac9fe..6aefd18da03 100644 --- a/service/history/replication/executable_verify_versioned_transition_task.go +++ b/service/history/replication/executable_verify_versioned_transition_task.go @@ -308,6 +308,11 @@ func (e *ExecutableVerifyVersionedTransitionTask) HandleErr(err error) error { tag.TaskID(e.TaskID()), tag.Error(err), ) + details := map[string]any{} + if _, ok := err.(*serviceerrors.SyncState); ok { + details["recovery_action"] = wideevents.ReplRecoveryActionSyncState + } + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationStandbyVerification, "Standby versioned transition verification failed", err, details) switch taskErr := err.(type) { case *serviceerrors.SyncState: callerInfo := getReplicaitonCallerInfo(e.GetPriority()) @@ -334,6 +339,7 @@ func (e *ExecutableVerifyVersionedTransitionTask) HandleErr(err error) error { tag.TaskID(e.TaskID()), tag.Error(syncStateErr), ) + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationStandbyVerificationSyncState, "Standby verification recovery failed during sync state", syncStateErr, nil) return err } return nil diff --git a/service/history/replication/executable_workflow_state_task.go b/service/history/replication/executable_workflow_state_task.go index f04cb902af5..9f3640d8ee6 100644 --- a/service/history/replication/executable_workflow_state_task.go +++ b/service/history/replication/executable_workflow_state_task.go @@ -109,6 +109,7 @@ func (e *ExecutableWorkflowStateTask) Execute() error { } ctx, cancel := newTaskContext(namespaceName, e.Config.ReplicationTaskApplyTimeout(), callerInfo) defer cancel() + ctx = setReplicationTaskOrigin(ctx, e.ExecutableTask, wideevents.ReplApplyArtifactSourceTaskPayload) shardContext, err := e.ShardController.GetShardByNamespaceWorkflow( namespace.ID(e.NamespaceID), @@ -135,6 +136,18 @@ func (e *ExecutableWorkflowStateTask) HandleErr(err error) error { e.MarkTaskDuplicated() return nil } + var notFoundErr *serviceerror.NotFound + if err != nil && !errors.As(err, ¬FoundErr) { + details := map[string]any{} + switch err.(type) { + case *serviceerrors.SyncState: + details["recovery_action"] = wideevents.ReplRecoveryActionSyncState + case *serviceerrors.RetryReplication: + details["recovery_action"] = wideevents.ReplRecoveryActionResendHistory + default: + } + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationPassiveTaskExecution, "SyncWorkflowState replication task encountered error", err, details) + } callerInfo := getReplicaitonCallerInfo(e.GetPriority()) switch retryErr := err.(type) { case *serviceerrors.SyncState: @@ -161,6 +174,7 @@ func (e *ExecutableWorkflowStateTask) HandleErr(err error) error { tag.TaskID(e.TaskID()), tag.Error(syncStateErr), ) + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationSyncWorkflowStateSyncState, "SyncWorkflowState recovery failed during sync state", syncStateErr, nil) return err } return nil @@ -185,6 +199,9 @@ func (e *ExecutableWorkflowStateTask) HandleErr(err error) error { retryErr, ResendAttempt, ); resendErr != nil || !doContinue { + if resendErr != nil { + emitExecutableTaskError(e.ExecutableTask, wideevents.ReplOperationHistoryResend, "SyncWorkflowState history resend failed", resendErr, nil) + } return err } return e.Execute() diff --git a/service/history/replication/replication_events.go b/service/history/replication/replication_events.go index 9d5986993f3..42f2af70a36 100644 --- a/service/history/replication/replication_events.go +++ b/service/history/replication/replication_events.go @@ -1,7 +1,9 @@ package replication import ( + "context" "errors" + "maps" commonpb "go.temporal.io/api/common/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -20,6 +22,185 @@ import ( "go.temporal.io/server/service/history/tasks" ) +func replicationLifecycleTaskType(taskType enumsspb.ReplicationTaskType) (string, bool) { + switch taskType { + case enumsspb.REPLICATION_TASK_TYPE_SYNC_WORKFLOW_STATE_TASK: + return wideevents.ReplTaskSyncWorkflowState, true + case enumsspb.REPLICATION_TASK_TYPE_SYNC_VERSIONED_TRANSITION_TASK: + return wideevents.ReplTaskSyncVersionedTransition, true + case enumsspb.REPLICATION_TASK_TYPE_VERIFY_VERSIONED_TRANSITION_TASK: + return wideevents.ReplTaskVerifyVersionedTransition, true + default: + return "", false + } +} + +func stateBasedTaskTypeFromHistoryTask(taskType enumsspb.TaskType) (string, bool) { + switch taskType { + case enumsspb.TASK_TYPE_REPLICATION_SYNC_WORKFLOW_STATE: + return wideevents.ReplTaskSyncWorkflowState, true + case enumsspb.TASK_TYPE_REPLICATION_SYNC_VERSIONED_TRANSITION: + return wideevents.ReplTaskSyncVersionedTransition, true + default: + return "", false + } +} + +// emitReplicationTaskError emits a context-rich error phase for an executable task. Identity is +// resolved here so individual error sites cannot accidentally omit source or target context. +func (e *ExecutableTaskImpl) emitReplicationTaskError( + operation string, + message string, + err error, + details map[string]any, +) { + if !e.Config.EmitReplicationLifecycleEvents() { + return + } + if e.replicationTask == nil { + return + } + taskType, ok := replicationLifecycleTaskType(e.replicationTask.GetTaskType()) + if !ok { + return + } + + namespaceID, workflowID, runID := e.workflowKeyFromTask() + if namespaceID == "" || workflowID == "" { + return + } + shardContext, shardErr := e.ShardController.GetShardByNamespaceWorkflow( + namespace.ID(namespaceID), + workflowID, + ) + if shardErr != nil || shardContext.GetEventLogger() == nil { + return + } + + namespaceName := e.NamespaceName() + if namespaceName == "" { + if name, nsErr := e.NamespaceCache.GetNamespaceName(namespace.ID(namespaceID)); nsErr == nil { + namespaceName = name.String() + } + } + if details == nil { + details = make(map[string]any, 4) + } else { + details = maps.Clone(details) + } + details["attempt"] = e.Attempt() + details["priority"] = e.GetPriority().String() + details["target_cluster"] = e.ClusterMetadata.GetCurrentClusterName() + if _, ok := details["apply_artifact_source"]; !ok { + details["apply_artifact_source"] = wideevents.ReplApplyArtifactSourceTaskPayload + } + sourceTaskID := e.replicationTask.GetSourceTaskId() + if sourceTaskID == 0 { + sourceTaskID = e.TaskID() + } + + payload := wideevents.ReplicationLifecyclePayload{ + TaskType: taskType, + Shard: shardContext.GetShardID(), + Namespace: namespaceName, + NamespaceID: namespaceID, + WorkflowID: workflowID, + RunID: runID, + SourceCluster: e.SourceClusterName(), + SourceShard: e.SourceShardKey().ShardID, + SourceTaskID: sourceTaskID, + } + if vt := e.replicationTask.GetVersionedTransition(); vt != nil { + payload.FailoverVersion = vt.GetNamespaceFailoverVersion() + payload.TransitionCount = vt.GetTransitionCount() + } + wideevents.EmitReplicationError(shardContext.GetEventLogger(), payload, operation, message, err, details) +} + +type replicationErrorEmitter interface { + emitReplicationTaskError(string, string, error, map[string]any) +} + +func setReplicationTaskOrigin( + ctx context.Context, + task ExecutableTask, + applyArtifactSource wideevents.ReplicationApplyArtifactSource, +) context.Context { + taskImpl, ok := task.(*ExecutableTaskImpl) + if !ok { + return ctx + } + return wideevents.SetReplicationTaskOrigin(ctx, wideevents.ReplicationTaskOrigin{ + ClusterName: taskImpl.sourceClusterName, + ShardID: taskImpl.sourceShardKey.ShardID, + TaskID: taskImpl.taskID, + ApplyArtifactSource: applyArtifactSource, + }) +} + +func emitExecutableTaskError( + task ExecutableTask, + operation string, + message string, + err error, + details map[string]any, +) { + if emitter, ok := task.(replicationErrorEmitter); ok { + emitter.emitReplicationTaskError(operation, message, err, details) + } +} + +func (s *StreamSenderImpl) emitReplicationSenderError( + item tasks.Task, + replicationTaskType enumsspb.ReplicationTaskType, + priority enumsspb.TaskPriority, + attempt int64, + operation string, + message string, + err error, +) { + taskType, ok := stateBasedTaskTypeFromHistoryTask(item.GetType()) + if !ok { + return + } + if convertedTaskType, ok := replicationLifecycleTaskType(replicationTaskType); ok { + taskType = convertedTaskType + } + logger := s.shardContext.GetEventLogger() + if logger == nil { + return + } + namespaceID := item.GetNamespaceID() + namespaceName := "" + if name, nsErr := s.shardContext.GetNamespaceRegistry().GetNamespaceName(namespace.ID(namespaceID)); nsErr == nil { + namespaceName = name.String() + } + details := map[string]any{ + "attempt": attempt, + "priority": priority.String(), + "target_cluster": s.clientClusterName, + "target_shard": s.clientShardKey.ShardID, + } + wideevents.EmitReplicationError( + logger, + wideevents.ReplicationLifecyclePayload{ + TaskType: taskType, + Shard: s.serverShardKey.ShardID, + Namespace: namespaceName, + NamespaceID: namespaceID, + WorkflowID: item.GetWorkflowID(), + RunID: item.GetRunID(), + SourceCluster: s.shardContext.GetClusterMetadata().GetCurrentClusterName(), + SourceShard: s.serverShardKey.ShardID, + SourceTaskID: item.GetTaskID(), + }, + operation, + message, + err, + details, + ) +} + // emitReplicationExecuting emits a best-effort "executing" ReplicationLifecycle event when an // executable replication task is picked up to execute on the target cluster. It never affects // control flow: a nil logger / unresolved shard is a no-op and namespace resolution failures diff --git a/service/history/replication/stream_sender.go b/service/history/replication/stream_sender.go index d0311937405..62a4f5ab648 100644 --- a/service/history/replication/stream_sender.go +++ b/service/history/replication/stream_sender.go @@ -28,6 +28,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/quotas" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/configs" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" @@ -591,7 +592,14 @@ Loop: // Wrap as convertError so isSkippable can tell "the task could not be built" // (its source info is corrupt/unusable) apart from transient send/rate-limit // failures, which must not be skipped. - return s.recordRetry(item, attempt, &convertError{err: fmt.Errorf("convert: %w", err)}) + return s.recordRetry( + item, + enumsspb.REPLICATION_TASK_TYPE_UNSPECIFIED, + priority, + attempt, + wideevents.ReplOperationTaskConversion, + &convertError{err: fmt.Errorf("convert: %w", err)}, + ) } if task == nil { return nil @@ -622,7 +630,7 @@ Loop: 0, "", )); err != nil { - return s.recordRetry(item, attempt, fmt.Errorf("rate_limit: %w", err)) + return s.recordRetry(item, task.GetTaskType(), priority, attempt, wideevents.ReplOperationRateLimit, fmt.Errorf("rate_limit: %w", err)) } metrics.ReplicationRateLimitLatency.With(s.metrics).Record(time.Since(rlStartTime), metrics.OperationTag(TaskOperationTag(task))) } @@ -639,7 +647,7 @@ Loop: }, }, }); err != nil { - return s.recordRetry(item, attempt, fmt.Errorf("send: %w", err)) + return s.recordRetry(item, task.GetTaskType(), priority, attempt, wideevents.ReplOperationStreamSend, fmt.Errorf("send: %w", err)) } skipCount = 0 metrics.ReplicationTasksSend.With(s.metrics).Record( @@ -792,7 +800,10 @@ func (s *StreamSenderImpl) getTaskTargetCluster(task tasks.Task) []string { func (s *StreamSenderImpl) recordRetry( item tasks.Task, + replicationTaskType enumsspb.ReplicationTaskType, + priority enumsspb.TaskPriority, attempt int64, + operation string, err error, ) error { s.shardContext.GetThrottledLogger().Warn("Replication task send retry", @@ -802,6 +813,9 @@ func (s *StreamSenderImpl) recordRetry( tag.Counter(int(attempt)), tag.Error(err), ) + if s.config.EmitReplicationLifecycleEvents() { + s.emitReplicationSenderError(item, replicationTaskType, priority, attempt, operation, "Replication task send retry", err) + } return err } diff --git a/service/history/standby_task_events.go b/service/history/standby_task_events.go new file mode 100644 index 00000000000..a216c627f84 --- /dev/null +++ b/service/history/standby_task_events.go @@ -0,0 +1,66 @@ +package history + +import ( + "errors" + "time" + + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/wideevents" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/queues" + "go.temporal.io/server/service/history/tasks" +) + +func emitStandbyTaskError( + shardContext historyi.ShardContext, + executable queues.Executable, + taskType string, + err error, +) { + if err == nil || !shardContext.GetConfig().EmitReplicationLifecycleEvents() { + return + } + eventLogger := shardContext.GetEventLogger() + if eventLogger == nil { + return + } + + task := executable.GetTask() + namespaceName := "" + activeCluster := "" + if entry, nsErr := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(task.GetNamespaceID())); nsErr == nil { + namespaceName = entry.Name().String() + activeCluster = entry.ActiveClusterName(namespace.RoutingKey{ID: task.GetWorkflowID()}) + } + + details := map[string]any{ + "active_cluster": activeCluster, + "attempt": executable.Attempt(), + "category": task.GetCategory().Name(), + "local_task_id": task.GetTaskID(), + "local_task_type": task.GetType().String(), + "target_cluster": shardContext.GetClusterMetadata().GetCurrentClusterName(), + "visibility_time": task.GetVisibilityTime().Format(time.RFC3339Nano), + } + if errors.Is(err, consts.ErrTaskRetry) { + details["disposition"] = wideevents.ReplDispositionRetry + } else if errors.Is(err, consts.ErrTaskDiscarded) { + details["disposition"] = wideevents.ReplDispositionDiscarded + } + if versionedTask, ok := task.(tasks.HasVersion); ok { + details["version"] = versionedTask.GetVersion() + } + if destinationTask, ok := task.(tasks.HasDestination); ok { + details["destination"] = destinationTask.GetDestination() + } + + wideevents.EmitReplicationError(eventLogger, wideevents.ReplicationLifecyclePayload{ + TaskType: taskType, + Shard: shardContext.GetShardID(), + Namespace: namespaceName, + NamespaceID: task.GetNamespaceID(), + WorkflowID: task.GetWorkflowID(), + RunID: task.GetRunID(), + }, wideevents.ReplOperationStandbyTaskExecution, "Standby queue task execution failed", err, details) +} diff --git a/service/history/timer_queue_standby_task_executor.go b/service/history/timer_queue_standby_task_executor.go index 35da61b9c9b..733284b3bd2 100644 --- a/service/history/timer_queue_standby_task_executor.go +++ b/service/history/timer_queue_standby_task_executor.go @@ -115,6 +115,7 @@ func (t *timerQueueStandbyTaskExecutor) Execute( default: err = queueserrors.NewUnprocessableTaskError("unknown task type") } + emitStandbyTaskError(t.shardContext, executable, taskTypeTagValue, err) return queues.ExecuteResponse{ ExecutionMetricTags: metricsTags, diff --git a/service/history/transfer_queue_standby_task_executor.go b/service/history/transfer_queue_standby_task_executor.go index a6fc09d831b..9e0f1df07ff 100644 --- a/service/history/transfer_queue_standby_task_executor.go +++ b/service/history/transfer_queue_standby_task_executor.go @@ -112,6 +112,7 @@ func (t *transferQueueStandbyTaskExecutor) Execute( default: err = errUnknownTransferTask } + emitStandbyTaskError(t.shardContext, executable, taskType, err) return queues.ExecuteResponse{ ExecutionMetricTags: metricsTags, From 462de7718b212953a0783d770d122b468c0c6330 Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 19 Aug 2026 11:16:30 -0700 Subject: [PATCH 006/139] tdbg: add per-schedule V1/V2/sentinel status check to schedule migrate status (#11636) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Adds a `--schedule-id` flag to `tdbg schedule migrate status`. When set, it looks up one specific schedule instead of the namespace-wide V1-vs-V2 counts, and reports whether that schedule is currently V1 (workflow-backed), V2 (CHASM), or caught in a migration sentinel state. ## Why During schedule migration triage, an operator often needs to know the status of *one specific* schedule, not aggregate counts. There was no way to answer that without manually poking `execution describe` and reasoning about CHASM node internals by hand. ## How it works It always probes **both** sides — the V1 workflow ID (`temporal-sys-scheduler:`) and the V2/CHASM business ID (``) — via the same `DescribeMutableState` RPC that `tdbg execution describe` uses, regardless of which ID form was passed in. It deliberately does not short-circuit based on the input's shape (e.g. its prefix) without confirming against the server — an unconfirmed inference is confusing/untrustworthy in a diagnostic tool. This lets it flag both kinds of migration sentinel: - a **CHASM-side sentinel** `Scheduler` component reserving the ID during a V1→V2 migration - a **V1-side `DummyWorkflow`** reserving the workflow ID during a V2→V1 rollback Output leads with a plain-language headline describing the schedule's current, authoritative form (written for someone with no prior knowledge of the V1/V2 migration internals), followed by a note about any sentinel found on the other side, a details table of both sides' raw status, and the exact `execution describe` invocations to inspect each side further. ### Example output (V2→V1 rollback in flight) ``` Schedule "foo" is a V2 (CHASM) schedule. Additionally, a placeholder ("sentinel") V1 workflow exists at workflow ID "temporal-sys-scheduler:foo", reserving that ID while a V2→V1 rollback is in progress. This is expected during rollback and requires no action. Details: V1 (workflow-backed) [workflow ID temporal-sys-scheduler:foo]: sentinel (V2→V1 rollback placeholder) V2 (CHASM) [business ID foo]: genuine Inspect further: V1: tdbg execution describe --workflow-id temporal-sys-scheduler:foo -n V2: tdbg execution describe --workflow-id foo --archetype scheduler.scheduler -n ``` ## Testing - `go test ./tools/tdbg/... -run TestScheduleStatus -v` — new tests cover: prefixed input, genuine V1, genuine V2, V1 genuine + CHASM sentinel, V1 sentinel + genuine V2, not-found on both sides, and an unexpected/inconsistent-state fallback. Existing aggregate-count tests are unaffected. - `go build ./tools/...`, `go vet ./tools/tdbg/...` - Manually checked `tdbg schedule migrate status --help` output. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- tools/tdbg/commands.go | 176 ++++++++++++++++++++++++++++- tools/tdbg/schedule_status_test.go | 162 ++++++++++++++++++++++++++ tools/tdbg/tdbg_commands.go | 12 +- 3 files changed, 347 insertions(+), 3 deletions(-) diff --git a/tools/tdbg/commands.go b/tools/tdbg/commands.go index 95752fc0a96..d65e429e446 100644 --- a/tools/tdbg/commands.go +++ b/tools/tdbg/commands.go @@ -22,6 +22,7 @@ import ( historyspb "go.temporal.io/server/api/history/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/codec" "go.temporal.io/server/common/log" @@ -31,6 +32,7 @@ import ( "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/service/history/tasks" + "go.temporal.io/server/service/worker/dummy" "go.temporal.io/server/service/worker/scheduler" "google.golang.org/protobuf/encoding/prototext" "google.golang.org/protobuf/types/known/durationpb" @@ -951,8 +953,13 @@ func v2ScheduleVisibilityQuery() string { // AdminScheduleStatus reports how many schedules in --namespace are currently V1 // (workflow-backed) vs V2 (CHASM), using the same default visibility queries as -// `schedule migrate --from-visibility`. +// `schedule migrate --from-visibility`. With --schedule-id, it instead reports the migration +// status of that one schedule. func AdminScheduleStatus(c *cli.Context, clientFactory ClientFactory) error { + if scheduleID := c.String(FlagScheduleID); scheduleID != "" { + return describeScheduleMigrationStatus(c, clientFactory, scheduleID) + } + ns, err := getRequiredOption(c, FlagNamespace) if err != nil { return err @@ -989,6 +996,173 @@ func countScheduleVisibility(c *cli.Context, wfClient workflowservice.WorkflowSe return resp.GetCount(), nil } +// v2ScheduleStatus is what was found, if anything, on the CHASM (V2) side of a schedule ID. +type v2ScheduleStatus struct { + found bool + sentinel bool // only meaningful when found +} + +// isGenuine reports whether this is a real V2 schedule, as opposed to a migration sentinel +// reserving the ID. +func (s v2ScheduleStatus) isGenuine() bool { return s.found && !s.sentinel } + +func (s v2ScheduleStatus) describe() string { + switch { + case !s.found: + return "not found" + case s.sentinel: + return "sentinel (V1→V2 migration placeholder)" + default: + return "genuine" + } +} + +// v1ScheduleStatus is what was found, if anything, on the workflow (V1) side of a schedule ID. +type v1ScheduleStatus struct { + found bool + workflowType string // only meaningful when found +} + +// isGenuine reports whether this is a real V1 scheduler workflow. +func (s v1ScheduleStatus) isGenuine() bool { + return s.found && s.workflowType == scheduler.WorkflowType +} + +// isSentinel reports whether this is a placeholder DummyWorkflow reserving the V1 workflow ID +// during a V2->V1 rollback. +func (s v1ScheduleStatus) isSentinel() bool { + return s.found && s.workflowType == dummy.DummyWFTypeName +} + +func (s v1ScheduleStatus) describe() string { + switch { + case !s.found: + return "not found" + case s.isGenuine(): + return "genuine" + case s.isSentinel(): + return "sentinel (V2→V1 rollback placeholder)" + default: + return fmt.Sprintf("unexpected workflow type %q", s.workflowType) + } +} + +// describeScheduleMigrationStatus reports whether a single schedule is currently V1 +// (workflow-backed), V2 (CHASM), or caught in a migration sentinel state, using the same +// DescribeMutableState RPC as `execution describe`. It always probes both the V1 workflow ID and +// the V2 (CHASM) business ID -- regardless of which form --schedule-id was given in -- so the +// report reflects a confirmed lookup rather than an inference from the ID's shape alone. +func describeScheduleMigrationStatus(c *cli.Context, clientFactory ClientFactory, scheduleID string) error { + ns, err := getRequiredOption(c, FlagNamespace) + if err != nil { + return err + } + + rawID := strings.TrimPrefix(scheduleID, primitives.ScheduleWorkflowIDPrefix) + v1ID := scheduleID + if rawID == scheduleID { + v1ID = primitives.ScheduleWorkflowIDPrefix + scheduleID + } + + adminClient := clientFactory.AdminClient(c) + + v2, err := describeSchedulerChasmState(c, adminClient, ns, rawID) + if err != nil { + return fmt.Errorf("unable to describe schedule %q: %w", scheduleID, err) + } + v1, err := describeWorkflowTypeName(c, adminClient, ns, v1ID) + if err != nil { + return fmt.Errorf("unable to describe schedule %q: %w", scheduleID, err) + } + + w := c.App.Writer + switch { + case v1.isGenuine() && !v2.found: + _, _ = fmt.Fprintf(w, "Schedule %q is a V1 (workflow-backed) schedule.\n", scheduleID) + case !v1.found && v2.isGenuine(): + _, _ = fmt.Fprintf(w, "Schedule %q is a V2 (CHASM) schedule.\n", scheduleID) + case v1.isGenuine() && v2.sentinel: + _, _ = fmt.Fprintf(w, "Schedule %q is a V1 (workflow-backed) schedule.\n", scheduleID) + _, _ = fmt.Fprintf(w, "Additionally, a placeholder (\"sentinel\") V2 entity exists at business ID %q, "+ + "reserving that ID while a V1→V2 migration is in progress. This is expected during migration and "+ + "requires no action — the V1 schedule above remains authoritative until the migration completes.\n", + rawID) + case v1.isSentinel() && v2.isGenuine(): + _, _ = fmt.Fprintf(w, "Schedule %q is a V2 (CHASM) schedule.\n", scheduleID) + _, _ = fmt.Fprintf(w, "Additionally, a placeholder (\"sentinel\") V1 workflow exists at workflow ID %q, "+ + "reserving that ID while a V2→V1 rollback is in progress. This is expected during rollback and "+ + "requires no action.\n", v1ID) + case !v1.found && !v2.found: + _, _ = fmt.Fprintf(w, "Schedule %q was not found as either a V1 (workflow-backed) or V2 (CHASM) schedule. "+ + "It may not exist, may have been deleted, or the ID may be mistyped.\n", scheduleID) + default: + _, _ = fmt.Fprintf(w, "Schedule %q is in an unexpected state — both sides below should not normally be "+ + "true at the same time. This needs manual investigation.\n", scheduleID) + } + + _, _ = fmt.Fprint(w, "\nDetails:\n") + _, _ = fmt.Fprintf(w, " %-22s[workflow ID %s]: %s\n", "V1 (workflow-backed)", v1ID, v1.describe()) + _, _ = fmt.Fprintf(w, " %-22s[business ID %s]: %s\n", "V2 (CHASM)", rawID, v2.describe()) + + _, _ = fmt.Fprint(w, "\nInspect further:\n") + _, _ = fmt.Fprintf(w, " V1: tdbg execution describe --workflow-id %s -n %s\n", v1ID, ns) + _, _ = fmt.Fprintf(w, " V2: tdbg execution describe --workflow-id %s --archetype %s -n %s\n", rawID, chasm.SchedulerArchetype, ns) + + return nil +} + +// describeSchedulerChasmState looks up the CHASM (V2) side of scheduleID and reports whether a +// genuine schedule, a migration sentinel, or nothing exists there. +func describeSchedulerChasmState(c *cli.Context, adminClient adminservice.AdminServiceClient, ns, scheduleID string) (v2ScheduleStatus, error) { + ctx, cancel := newContext(c) + defer cancel() + resp, err := adminClient.DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: scheduleID}, + Archetype: chasm.SchedulerArchetype, + }) + if common.IsNotFoundError(err) { + return v2ScheduleStatus{}, nil + } + if err != nil { + return v2ScheduleStatus{}, err + } + + // The CHASM tree's root node -- the Scheduler component itself -- always encodes to the + // empty path. + node, ok := resp.GetDatabaseMutableState().GetChasmNodes()[""] + if !ok { + return v2ScheduleStatus{}, nil + } + var state schedulerpb.SchedulerState + if err := serialization.Decode(node.GetData(), &state); err != nil { + return v2ScheduleStatus{}, fmt.Errorf("failed to decode scheduler state: %w", err) + } + return v2ScheduleStatus{found: true, sentinel: state.GetSentinel()}, nil +} + +// describeWorkflowTypeName looks up the workflow (V1) side of workflowID and reports its +// workflow type, if it exists. +func describeWorkflowTypeName(c *cli.Context, adminClient adminservice.AdminServiceClient, ns, workflowID string) (v1ScheduleStatus, error) { + ctx, cancel := newContext(c) + defer cancel() + resp, err := adminClient.DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID}, + Archetype: chasm.WorkflowArchetype, + }) + if common.IsNotFoundError(err) { + return v1ScheduleStatus{}, nil + } + if err != nil { + return v1ScheduleStatus{}, err + } + return v1ScheduleStatus{ + found: true, + workflowType: resp.GetDatabaseMutableState().GetExecutionInfo().GetWorkflowTypeName(), + }, nil +} + // migrateSingleSchedule migrates one schedule and performs the migration immediately. func migrateSingleSchedule( c *cli.Context, diff --git a/tools/tdbg/schedule_status_test.go b/tools/tdbg/schedule_status_test.go index 5aecbc75d00..6f34526684f 100644 --- a/tools/tdbg/schedule_status_test.go +++ b/tools/tdbg/schedule_status_test.go @@ -8,8 +8,15 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/adminservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/primitives" + "go.temporal.io/server/service/worker/dummy" "go.temporal.io/server/service/worker/scheduler" "go.temporal.io/server/tools/tdbg" "go.temporal.io/server/tools/tdbg/tdbgtest" @@ -95,3 +102,158 @@ func TestScheduleStatus_CountError(t *testing.T) { require.Contains(t, err.Error(), "unable to count") require.Contains(t, err.Error(), "boom") } + +// describeMutableStateAdminClient fakes AdminService.DescribeMutableState for the single-schedule +// `status --schedule-id` path, keyed by the requested WorkflowId. Any WorkflowId with no fixture +// registered returns NotFound, matching a real server's response for a nonexistent execution. +type describeMutableStateAdminClient struct { + adminservice.AdminServiceClient + responses map[string]*adminservice.DescribeMutableStateResponse // workflowID -> response + + requests []*adminservice.DescribeMutableStateRequest +} + +func (c *describeMutableStateAdminClient) DescribeMutableState( + _ context.Context, + req *adminservice.DescribeMutableStateRequest, + _ ...grpc.CallOption, +) (*adminservice.DescribeMutableStateResponse, error) { + c.requests = append(c.requests, req) + if resp, ok := c.responses[req.GetExecution().GetWorkflowId()]; ok { + return resp, nil + } + return nil, serviceerror.NewNotFound("not found") +} + +// chasmSchedulerStateResponse builds a DescribeMutableState response for a CHASM Scheduler +// component at the tree root, as produced by the real server for a V2 schedule or CHASM-side +// sentinel. +func chasmSchedulerStateResponse(t *testing.T, sentinel bool) *adminservice.DescribeMutableStateResponse { + t.Helper() + blob, err := serialization.Encode(&schedulerpb.SchedulerState{Sentinel: sentinel}) + require.NoError(t, err) + return &adminservice.DescribeMutableStateResponse{ + DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ChasmNodes: map[string]*persistencespb.ChasmNode{ + "": {Data: blob}, + }, + }, + } +} + +// workflowTypeResponse builds a DescribeMutableState response for a plain workflow execution of +// the given type, as produced by the real server for a V1 scheduler workflow or a V1-side +// (dummy workflow) sentinel. +func workflowTypeResponse(typeName string) *adminservice.DescribeMutableStateResponse { + return &adminservice.DescribeMutableStateResponse{ + DatabaseMutableState: &persistencespb.WorkflowMutableState{ + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{WorkflowTypeName: typeName}, + }, + } +} + +func runScheduleStatusForSchedule(t *testing.T, admin adminservice.AdminServiceClient, args ...string) (stdoutStr, stderrStr string, err error) { + t.Helper() + var stdout, stderr bytes.Buffer + factory := migrateClientFactory{admin: admin} + app := tdbgtest.NewCliApp(func(params *tdbg.Params) { + params.ClientFactory = factory + params.Writer = &stdout + params.ErrWriter = &stderr + }) + runArgs := append([]string{"tdbg"}, args...) + err = app.Run(runArgs) + return stdout.String(), stderr.String(), err +} + +func TestScheduleStatus_SingleSchedule_PrefixedInputStillDoesBothLookups(t *testing.T) { + v1ID := primitives.ScheduleWorkflowIDPrefix + "foo" + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + v1ID: workflowTypeResponse(scheduler.WorkflowType), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", v1ID) + require.NoError(t, err) + + require.Len(t, admin.requests, 2) + gotIDs := []string{admin.requests[0].GetExecution().GetWorkflowId(), admin.requests[1].GetExecution().GetWorkflowId()} + require.ElementsMatch(t, []string{v1ID, "foo"}, gotIDs) + + require.Contains(t, stdout, fmt.Sprintf("Schedule %q is a V1 (workflow-backed) schedule.\n", v1ID)) + require.NotContains(t, stdout, "Additionally") +} + +func TestScheduleStatus_SingleSchedule_GenuineV1(t *testing.T) { + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + primitives.ScheduleWorkflowIDPrefix + "foo": workflowTypeResponse(scheduler.WorkflowType), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" is a V1 (workflow-backed) schedule.`+"\n") + require.NotContains(t, stdout, "Additionally") + require.Contains(t, stdout, "[workflow ID temporal-sys-scheduler:foo]: genuine") + require.Contains(t, stdout, "[business ID foo]: not found") +} + +func TestScheduleStatus_SingleSchedule_GenuineV2(t *testing.T) { + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + "foo": chasmSchedulerStateResponse(t, false), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" is a V2 (CHASM) schedule.`+"\n") + require.NotContains(t, stdout, "Additionally") +} + +func TestScheduleStatus_SingleSchedule_V1GenuineWithV2Sentinel(t *testing.T) { + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + primitives.ScheduleWorkflowIDPrefix + "foo": workflowTypeResponse(scheduler.WorkflowType), + "foo": chasmSchedulerStateResponse(t, true), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" is a V1 (workflow-backed) schedule.`+"\n") + require.Contains(t, stdout, "Additionally, a placeholder (\"sentinel\") V2 entity exists") + require.Contains(t, stdout, "V1→V2 migration is in progress") +} + +func TestScheduleStatus_SingleSchedule_V1SentinelWithV2Genuine(t *testing.T) { + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + primitives.ScheduleWorkflowIDPrefix + "foo": workflowTypeResponse(dummy.DummyWFTypeName), + "foo": chasmSchedulerStateResponse(t, false), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" is a V2 (CHASM) schedule.`+"\n") + require.Contains(t, stdout, "Additionally, a placeholder (\"sentinel\") V1 workflow exists") + require.Contains(t, stdout, "V2→V1 rollback is in progress") +} + +func TestScheduleStatus_SingleSchedule_NotFoundBothSides(t *testing.T) { + admin := &describeMutableStateAdminClient{} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" was not found as either a V1 (workflow-backed) or V2 (CHASM) schedule.`) +} + +func TestScheduleStatus_SingleSchedule_UnexpectedBothGenuine(t *testing.T) { + admin := &describeMutableStateAdminClient{responses: map[string]*adminservice.DescribeMutableStateResponse{ + primitives.ScheduleWorkflowIDPrefix + "foo": workflowTypeResponse(scheduler.WorkflowType), + "foo": chasmSchedulerStateResponse(t, false), + }} + + stdout, _, err := runScheduleStatusForSchedule(t, admin, "-n", "my-ns", "schedule", "migrate", "status", "--schedule-id", "foo") + require.NoError(t, err) + + require.Contains(t, stdout, `Schedule "foo" is in an unexpected state`) +} diff --git a/tools/tdbg/tdbg_commands.go b/tools/tdbg/tdbg_commands.go index e987600e35f..b7150b44b2a 100644 --- a/tools/tdbg/tdbg_commands.go +++ b/tools/tdbg/tdbg_commands.go @@ -347,8 +347,16 @@ func newAdminScheduleCommands(clientFactory ClientFactory) []*cli.Command { }, Subcommands: []*cli.Command{ { - Name: "status", - Usage: "Show counts of V1 (workflow-backed) and V2 (CHASM) schedules in --namespace, from visibility", + Name: "status", + Usage: "Show counts of V1 (workflow-backed) and V2 (CHASM) schedules in --namespace, from visibility. " + + "With --schedule-id, instead reports the migration status of that one schedule (V1, V2, or sentinel).", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagScheduleID, + Aliases: FlagScheduleIDAlias, + Usage: "Schedule ID to check the migration status of a single schedule, instead of namespace-wide counts", + }, + }, Action: func(c *cli.Context) error { return AdminScheduleStatus(c, clientFactory) }, From 5b410facf19bf26f940eeac43331640cdcb0394d Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 19 Aug 2026 11:21:18 -0700 Subject: [PATCH 007/139] removed blanket timestamp wrapping (time-skipping fix) (#11635) ## What changed? - Removed blanket timestamp wrapping - cron/retry/start child wf use virtual time directly ## Why? there was a double-shifting for CaN but not for other cases it shall be fixed unifying the clock used by all virtual time propagating cases, and this PR chooses to unify in a way that all cases propagate use virtual time ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- service/history/api/create_workflow_util.go | 12 +- .../workflow_task_completed_handler.go | 4 +- .../transfer_queue_active_task_executor.go | 14 +- service/history/workflow/timeskipping.go | 32 +--- service/history/workflow/timeskipping_test.go | 72 ++++---- tests/timeskipping_propagation_test.go | 154 +++++++++++++++--- tests/timeskipping_test.go | 21 +++ tests/xdc/timeskipping_replication_test.go | 10 +- 8 files changed, 227 insertions(+), 92 deletions(-) diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index c62b3be4df4..af62e3f2262 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -49,13 +49,18 @@ func NewWorkflowWithSignal( startRequest *historyservice.StartWorkflowExecutionRequest, signalWithStartRequest *workflowservice.SignalWithStartWorkflowExecutionRequest, ) (historyi.MutableState, error) { - newMutableState, err := CreateMutableState( + startTime := workflow.AdjustNowWithTimeSkipping( + shard.GetTimeSource().Now(), + startRequest.GetTimeSkippingStatePropagation(), + ) + newMutableState, err := createMutableState( shard, namespaceEntry, startRequest.StartRequest.WorkflowExecutionTimeout, startRequest.StartRequest.WorkflowRunTimeout, workflowID, runID, + startTime, ) if err != nil { return nil, err @@ -154,13 +159,14 @@ func NewWorkflowLeaseAndContext( ), nil } -func CreateMutableState( +func createMutableState( shard historyi.ShardContext, namespaceEntry *namespace.Namespace, executionTimeout *durationpb.Duration, runTimeout *durationpb.Duration, workflowID string, runID string, + startTime time.Time, ) (historyi.MutableState, error) { newMutableState := workflow.NewMutableState( shard, @@ -169,7 +175,7 @@ func CreateMutableState( namespaceEntry, workflowID, runID, - shard.GetTimeSource().Now(), + startTime, ) if err := newMutableState.SetHistoryTree(executionTimeout, runTimeout, runID); err != nil { return nil, err diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index 87001105728..a4b5fd8c04a 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -1456,7 +1456,7 @@ func (handler *workflowTaskCompletedHandler) handleRetry( handler.mutableState.GetNamespaceEntry(), handler.mutableState.GetWorkflowKey().WorkflowID, newRunID, - handler.shard.GetTimeSource().Now(), + handler.mutableState.Now(), handler.mutableState, ) if err != nil { @@ -1516,7 +1516,7 @@ func (handler *workflowTaskCompletedHandler) handleCron( handler.mutableState.GetNamespaceEntry(), handler.mutableState.GetWorkflowKey().WorkflowID, newRunID, - handler.shard.GetTimeSource().Now(), + handler.mutableState.Now(), handler.mutableState, ) if err != nil { diff --git a/service/history/transfer_queue_active_task_executor.go b/service/history/transfer_queue_active_task_executor.go index 96f51a337c8..0b39f89e193 100644 --- a/service/history/transfer_queue_active_task_executor.go +++ b/service/history/transfer_queue_active_task_executor.go @@ -1092,7 +1092,7 @@ func (t *transferQueueActiveTaskExecutor) processStartChildExecution( }, } - childRunID, childClock, err := t.startWorkflow( + childRunID, childClock, err := t.startChildWorkflow( ctx, task, parentNamespaceName, @@ -1671,7 +1671,7 @@ func (t *transferQueueActiveTaskExecutor) signalExternalExecution( return err } -func (t *transferQueueActiveTaskExecutor) startWorkflow( +func (t *transferQueueActiveTaskExecutor) startChildWorkflow( ctx context.Context, task *tasks.StartChildExecutionTask, namespace namespace.Name, @@ -1719,6 +1719,12 @@ func (t *transferQueueActiveTaskExecutor) startWorkflow( TimeSkippingConfig: attributes.GetTimeSkippingConfig(), } + statePropagation := attributes.GetTimeSkippingStatePropagation() + nowForExpirationAndBackoff := workflow.AdjustNowWithTimeSkipping( + t.shardContext.GetTimeSource().Now(), + statePropagation, + ) + request := common.CreateHistoryStartWorkflowRequest( targetNamespaceID.String(), startRequest, @@ -1734,13 +1740,13 @@ func (t *transferQueueActiveTaskExecutor) startWorkflow( Clock: vclock.NewVectorClock(t.shardContext.GetClusterMetadata().GetClusterID(), t.shardContext.GetShardID(), task.TaskID), }, rootExecutionInfo, - t.shardContext.GetTimeSource().Now(), + nowForExpirationAndBackoff, ) request.SourceVersionStamp = sourceVersionStamp request.InheritedBuildId = inheritedBuildId request.InheritedPinnedVersion = inheritedPinnedVersion - request.TimeSkippingStatePropagation = attributes.GetTimeSkippingStatePropagation() + request.TimeSkippingStatePropagation = statePropagation // Only set the AutoUpgrade info if the Pinned version is not set. if request.InheritedPinnedVersion == nil { diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 3eeaabab210..710c720450f 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -37,7 +37,6 @@ func (ms *MutableStateImpl) initTimeSkippingInfo( SessionSkipCount: timeSkippingStatePropagation.GetInitialSkipCount(), } ms.wrapTimeSourceWithTimeSkipping() - ms.wrapExecutionTimes(initialSkip) ms.applyFastForward(timeSkippingStatePropagation.GetFastForwardTargetTime()) ms.timeSkippingInfoUpdated = true } @@ -105,28 +104,6 @@ func (ms *MutableStateImpl) setAndStampFastForwardInfo( return tsi.FastForwardInfoLastUpdateVersionedTransition } -func (ms *MutableStateImpl) wrapExecutionTimes(initialSkippedDuration *durationpb.Duration) { - if initialSkippedDuration == nil || initialSkippedDuration.AsDuration() == 0 { - return - } - accum := initialSkippedDuration.AsDuration() - if !timeNotSet(ms.executionState.StartTime) { - ms.executionState.StartTime = timestamppb.New(ms.executionState.StartTime.AsTime().Add(accum)) - } - if !timeNotSet(ms.executionInfo.StartTime) { - ms.executionInfo.StartTime = timestamppb.New(ms.executionInfo.StartTime.AsTime().Add(accum)) - } - if !timeNotSet(ms.executionInfo.ExecutionTime) { - ms.executionInfo.ExecutionTime = timestamppb.New(ms.executionInfo.ExecutionTime.AsTime().Add(accum)) - } - if !timeNotSet(ms.executionInfo.WorkflowRunExpirationTime) { - ms.executionInfo.WorkflowRunExpirationTime = timestamppb.New(ms.executionInfo.WorkflowRunExpirationTime.AsTime().Add(accum)) - } - if !timeNotSet(ms.executionInfo.WorkflowExecutionExpirationTime) { - ms.executionInfo.WorkflowExecutionExpirationTime = timestamppb.New(ms.executionInfo.WorkflowExecutionExpirationTime.AsTime().Add(accum)) - } -} - // -- Propagation Methods of Time Skipping // propagateTimeSkippingToNextRun propagates both time skipping config and state to the next run in @@ -284,6 +261,15 @@ func (t *timeSkippingTransition) GateByFastForward(ff *persistencespb.FastForwar // Time Skipping Utility Functions // ============================================================================= +// AdjustNowWithTimeSkipping converts a real clock reading to the virtual clock frame inherited +// through time-skipping state propagation. +func AdjustNowWithTimeSkipping( + now time.Time, + statePropagation *commonpb.TimeSkippingStatePropagation, +) time.Time { + return now.Add(statePropagation.GetInitialSkippedDuration().AsDuration()) +} + func NewTimeSkippingInfoUtil(tsi *persistencespb.TimeSkippingInfo) *TimeSkippingInfoUtil { return &TimeSkippingInfoUtil{tsi: tsi} } diff --git a/service/history/workflow/timeskipping_test.go b/service/history/workflow/timeskipping_test.go index 8a404b3aed1..b83ce9a754e 100644 --- a/service/history/workflow/timeskipping_test.go +++ b/service/history/workflow/timeskipping_test.go @@ -653,38 +653,6 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { }) } -// TestWrapExecutionTimes covers the three invariants of wrapExecutionTimes: -// a nil/zero skipped duration is a no-op; a non-zero duration shifts every *set* -// execution timestamp forward by exactly that duration; unset timestamps stay unset. -func (s *mutableStateSuite) TestWrapExecutionTimes() { - base := time.Date(2026, 7, 1, 8, 0, 0, 0, time.UTC) - - s.Run("NilOrZeroDurationIsNoOp", func() { - s.mutableState.executionInfo.StartTime = timestamppb.New(base) - s.mutableState.wrapExecutionTimes(nil) - s.Equal(base, s.mutableState.executionInfo.StartTime.AsTime()) - s.mutableState.wrapExecutionTimes(durationpb.New(0)) - s.Equal(base, s.mutableState.executionInfo.StartTime.AsTime()) - }) - - s.Run("ShiftsSetTimestampsAndLeavesUnsetUntouched", func() { - accum := 2 * time.Hour - s.mutableState.executionState.StartTime = timestamppb.New(base) - s.mutableState.executionInfo.StartTime = timestamppb.New(base) - s.mutableState.executionInfo.ExecutionTime = timestamppb.New(base.Add(time.Minute)) - s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(base.Add(time.Hour)) - s.mutableState.executionInfo.WorkflowExecutionExpirationTime = nil // unset must stay unset - - s.mutableState.wrapExecutionTimes(durationpb.New(accum)) - - s.Equal(base.Add(accum), s.mutableState.executionState.StartTime.AsTime()) - s.Equal(base.Add(accum), s.mutableState.executionInfo.StartTime.AsTime()) - s.Equal(base.Add(time.Minute).Add(accum), s.mutableState.executionInfo.ExecutionTime.AsTime()) - s.Equal(base.Add(time.Hour).Add(accum), s.mutableState.executionInfo.WorkflowRunExpirationTime.AsTime()) - s.Nil(s.mutableState.executionInfo.WorkflowExecutionExpirationTime, "unset timestamp must remain unset") - }) -} - // TestApplyFastForward covers the full branch table of applyFastForward: // FastForward set / nil duration / nil config / Enabled=false. // The first-init virtual-time path is covered separately in TestInitTimeSkippingInfo. @@ -1681,3 +1649,43 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { }) } + +func TestAdjustNowWithTimeSkipping(t *testing.T) { + now := time.Date(2026, 8, 19, 9, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + statePropagation *commonpb.TimeSkippingStatePropagation + want time.Time + }{ + { + name: "nil propagation", + want: now, + }, + { + name: "nil initial skipped duration", + statePropagation: &commonpb.TimeSkippingStatePropagation{}, + want: now, + }, + { + name: "zero initial skipped duration", + statePropagation: &commonpb.TimeSkippingStatePropagation{ + InitialSkippedDuration: durationpb.New(0), + }, + want: now, + }, + { + name: "positive initial skipped duration", + statePropagation: &commonpb.TimeSkippingStatePropagation{ + InitialSkippedDuration: durationpb.New(2 * time.Hour), + }, + want: now.Add(2 * time.Hour), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, AdjustNowWithTimeSkipping(now, tc.statePropagation)) + }) + } +} diff --git a/tests/timeskipping_propagation_test.go b/tests/timeskipping_propagation_test.go index 781f046fe83..04ed207d813 100644 --- a/tests/timeskipping_propagation_test.go +++ b/tests/timeskipping_propagation_test.go @@ -625,27 +625,26 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInChildWf_ThreeGenerations() { "P's initiated event for C carries P's full accumulated at command time (3h = G's 1h + P's pt1 2h)") } -// TestTSPInChildWf_AdmissionTimestampsShifted verifies the time-shift block inside -// initTimeSkippingInfo (mutable_state_impl.go): when a child workflow boots with a -// non-zero PropagatedSkippedDuration, the boot-time admission timestamps on the -// child's MS are shifted forward by accum so they live in the virtual frame, while -// the WorkflowRunTimeoutTask's VisibilityTimestamp stays in the wall frame. +// TestTSPInChildWf_AdmissionTimestampsShifted verifies that a child workflow with a +// non-zero PropagatedSkippedDuration is admitted directly in the parent's virtual +// clock frame, while the WorkflowRunTimeoutTask's VisibilityTimestamp stays in the +// wall frame. // // Scenario: // - Parent TSC enabled. Parent runs StartTimer(t1, 1h) → idle → server skips 1h. // Parent's AccumulatedSkippedDuration ≈ 1h. -// - Parent issues StartChildWorkflow with WorkflowRunTimeout=2h. The child's -// initiated event carries InitialSkippedDuration=1h. On the child MS, -// applyTimeSkippingConfig seeds AccumulatedSkippedDuration=1h, and -// initTimeSkippingInfo shifts admission timestamps forward by accum=1h. +// - Parent issues StartChildWorkflow with WorkflowRunTimeout=2h and +// WorkflowExecutionTimeout=4h. The child's initiated event carries +// InitialSkippedDuration=1h. On the child MS, +// applyTimeSkippingConfig seeds AccumulatedSkippedDuration=1h, and the child +// start request uses wallChildBoot+1h as its admission time. // // Boot-time assertions (virtual frame): // - executionState.StartTime ≈ wallChildBoot + 1h // - executionInfo.StartTime ≈ wallChildBoot + 1h // - executionInfo.ExecutionTime ≈ wallChildBoot + 1h (no first-WT delay) // - executionInfo.WorkflowRunExpirationTime ≈ wallChildBoot + 1h + 2h -// - executionInfo.WorkflowExecutionExpirationTime: zero when ExecutionTimeout is -// unset on the child; otherwise shifted by the same formula. +// - executionInfo.WorkflowExecutionExpirationTime ≈ wallChildBoot + 1h + 4h // // Wall-clock assertion (the load-bearing one): the child's WorkflowRunTimeoutTask // must fire at wallChildBoot + 2h — i.e., virtual_expiration − accum — so a @@ -700,13 +699,14 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInChildWf_AdmissionTimestampsS CommandType: enumspb.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, Attributes: &commandpb.Command_StartChildWorkflowExecutionCommandAttributes{ StartChildWorkflowExecutionCommandAttributes: &commandpb.StartChildWorkflowExecutionCommandAttributes{ - Namespace: ns, - WorkflowId: childWFID, - WorkflowType: childWFType, - TaskQueue: tq, - WorkflowRunTimeout: durationpb.New(2 * time.Hour), - WorkflowTaskTimeout: durationpb.New(10 * time.Second), - ParentClosePolicy: enumspb.PARENT_CLOSE_POLICY_ABANDON, + Namespace: ns, + WorkflowId: childWFID, + WorkflowType: childWFType, + TaskQueue: tq, + WorkflowExecutionTimeout: durationpb.New(4 * time.Hour), + WorkflowRunTimeout: durationpb.New(2 * time.Hour), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + ParentClosePolicy: enumspb.PARENT_CLOSE_POLICY_ABANDON, }, }, } @@ -756,6 +756,7 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInChildWf_AdmissionTimestampsS const accum = time.Hour const runTimeout = 2 * time.Hour + const executionTimeout = 4 * time.Hour const tol = time.Minute expectedStart := wallChildBoot.Add(accum) @@ -772,15 +773,11 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInChildWf_AdmissionTimestampsS s.WithinDuration(expectedRunExpiration, childMS.State.ExecutionInfo.WorkflowRunExpirationTime.AsTime(), tol, "WorkflowRunExpirationTime = StartTime + RunTimeout in the virtual frame") - // WorkflowExecutionExpirationTime: the parent didn't set ExecutionTimeout, so the - // child's WorkflowExecutionTimeout is unset and the ExpirationTime is zero. The - // shift block in initTimeSkippingInfo only shifts non-nil values, so both branches - // are valid: zero is preserved as zero; non-zero is shifted by accum. weExpiration := childMS.State.ExecutionInfo.WorkflowExecutionExpirationTime - if weExpiration != nil && !weExpiration.AsTime().IsZero() { - s.WithinDuration(expectedRunExpiration, weExpiration.AsTime(), tol, - "WorkflowExecutionExpirationTime, when set, lives in the virtual frame and is shifted by accum") - } + s.NotNil(weExpiration) + expectedExecutionExpiration := wallChildBoot.Add(accum + executionTimeout) + s.WithinDuration(expectedExecutionExpiration, weExpiration.AsTime(), tol, + "transfer request must derive WorkflowExecutionExpirationTime from the virtual child start") recorder := env.GetTestCluster().GetHistoryTaskRecorder() s.NotNil(recorder) @@ -1185,6 +1182,9 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInReset() { s.Equal(1, optionsUpdatedCount, "reset run should contain exactly one reapplied OPTIONS_UPDATED event") s.NotNil(startedEvent) + s.WithinDuration(startedEvent.GetEventTime().AsTime(), resetMS.State.ExecutionState.GetStartTime().AsTime(), 10*time.Second, + "reset/rebuild must preserve the already-materialized history start time") + s.WithinDuration(startedEvent.GetEventTime().AsTime(), resetMS.State.ExecutionInfo.GetStartTime().AsTime(), 10*time.Second) startedTSC := startedEvent.GetWorkflowExecutionStartedEventAttributes().GetTimeSkippingConfig() s.NotNil(startedTSC, "WorkflowExecutionStarted should carry the original TimeSkippingConfig") s.False(startedTSC.GetEnabled(), @@ -1303,6 +1303,11 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInCaN() { }) s.NoError(err) s.NotEmpty(hist2.History.Events) + startedEventTime := hist2.History.Events[0].GetEventTime().AsTime() + s.WithinDuration(startedEventTime, run2MS.State.ExecutionState.GetStartTime().AsTime(), 10*time.Second, + "continue-as-new constructor time is already virtual and must not be shifted again") + s.WithinDuration(startedEventTime, run2MS.State.ExecutionInfo.GetStartTime().AsTime(), 10*time.Second) + s.WithinDuration(startedEventTime, run2MS.State.ExecutionInfo.GetExecutionTime().AsTime(), 10*time.Second) startedAttr := hist2.History.Events[0].GetWorkflowExecutionStartedEventAttributes() s.NotNil(startedAttr) s.Equal(run1ID, startedAttr.GetContinuedExecutionRunId(), @@ -1404,6 +1409,95 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInCaN_SkipSessionPropagated() "started event carries run 1's SessionSkipCount as InitialSkipCount") } +// TestTSPInCaN_ExecutionExpirationRemainsAbsolute verifies that time-skipping state +// propagation does not extend the workflow execution timeout across runs. The expiration +// timestamp is already expressed in virtual time and must remain unchanged when the next +// run inherits the accumulated skipped duration. +func (s *TimeSkippingPropagationTestSuite) TestTSPInCaN_ExecutionExpirationRemainsAbsolute() { + env := testcore.NewEnv(s.T()) + env.OverrideDynamicConfig(dynamicconfig.WorkflowTimeSkippingEnabled, true) + tv := testvars.New(s.T()) + ctx := s.Context() + + const executionTimeout = 3 * time.Hour + wfType := tv.WorkflowType() + wfID := tv.WorkflowID() + tq := tv.TaskQueue() + + start, err := env.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowId: wfID, + WorkflowType: wfType, + TaskQueue: tq, + WorkflowExecutionTimeout: durationpb.New(executionTimeout), + WorkflowRunTimeout: durationpb.New(24 * time.Hour), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + TimeSkippingConfig: &commonpb.TimeSkippingConfig{Enabled: true}, + }) + s.NoError(err) + run1ID := start.RunId + + state := 0 + handler := func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + fired := firedTimers(task) + switch { + case state == 0: + state = 1 + return cmdsResponse(timerCmd("t1", time.Hour)), nil + case state == 1 && fired["t1"]: + state = 2 + return cmdsResponse(continueAsNewCmd(wfType, tq)), nil + case state == 2: + state = 3 + return cmdsResponse(), nil + } + return cmdsResponse(), nil + } + + for i := range 3 { + _, pollErr := env.TaskPoller().PollAndHandleWorkflowTask(tv, handler) + if pollErr != nil { + s.T().Logf("iter %d: poll error: %v", i, pollErr) + } + } + + var status enumspb.WorkflowExecutionStatus + s.AwaitTrue(func() bool { + desc, describeErr := env.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: wfID}, + }) + if describeErr != nil { + return false + } + status = desc.WorkflowExecutionInfo.Status + return status != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + }, 15*time.Second, 100*time.Millisecond) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT, status) + + run1MS := s.getMutableState(env, wfID, run1ID) + originalExpiration := run1MS.State.ExecutionInfo.GetWorkflowExecutionExpirationTime().AsTime() + s.False(originalExpiration.IsZero()) + + run2MS := s.getMutableStateByID(ctx, env, wfID) + run2ID := run2MS.State.ExecutionState.RunId + s.NotEqual(run1ID, run2ID) + s.Equal(originalExpiration, run2MS.State.ExecutionInfo.GetWorkflowExecutionExpirationTime().AsTime(), + "continue-as-new must preserve the chain's absolute virtual execution-expiration deadline") + + hist2, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: wfID, RunId: run2ID}, + }) + s.NoError(err) + s.NotEmpty(hist2.History.Events) + terminalEvent := hist2.History.Events[len(hist2.History.Events)-1] + s.Equal(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TIMED_OUT, terminalEvent.GetEventType()) + s.WithinDuration(originalExpiration, terminalEvent.GetEventTime().AsTime(), 10*time.Second, + "workflow must time out at the original absolute virtual deadline") +} + // TestTSPInRetry verifies that a retried run is a same-lineage continuation: it inherits // the previous attempt's current TimeSkippingConfig AND its in-flight accumulated skip. // @@ -1529,6 +1623,10 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInRetry() { s.NotEmpty(hist2.History.Events) startedAttr := hist2.History.Events[0].GetWorkflowExecutionStartedEventAttributes() s.NotNil(startedAttr) + expectedAttempt2Start := hist2.History.Events[0].GetEventTime().AsTime() + s.WithinDuration(expectedAttempt2Start, attempt2MS.State.ExecutionState.GetStartTime().AsTime(), 10*time.Second, + "WFT-completion retry event and mutable state must be constructed in the same virtual frame") + s.WithinDuration(expectedAttempt2Start, attempt2MS.State.ExecutionInfo.GetStartTime().AsTime(), 10*time.Second) s.Equal(attempt1ID, startedAttr.GetContinuedExecutionRunId(), "attempt 2 references attempt 1 as its predecessor via ContinuedExecutionRunId") s.Equal(enumspb.CONTINUE_AS_NEW_INITIATOR_RETRY, startedAttr.GetInitiator(), @@ -1671,6 +1769,10 @@ func (s *TimeSkippingPropagationTestSuite) TestTSPInCron() { s.NotEmpty(hist2.History.Events) startedAttr := hist2.History.Events[0].GetWorkflowExecutionStartedEventAttributes() s.NotNil(startedAttr) + expectedRun2Start := hist2.History.Events[0].GetEventTime().AsTime() + s.WithinDuration(expectedRun2Start, run2MS.State.ExecutionState.GetStartTime().AsTime(), 90*time.Second, + "cron event and mutable state must be constructed in the same virtual frame") + s.WithinDuration(expectedRun2Start, run2MS.State.ExecutionInfo.GetStartTime().AsTime(), 90*time.Second) s.Equal(run1ID, startedAttr.GetContinuedExecutionRunId(), "run 2 references run 1 as its predecessor via ContinuedExecutionRunId") s.Equal(enumspb.CONTINUE_AS_NEW_INITIATOR_CRON_SCHEDULE, startedAttr.GetInitiator(), diff --git a/tests/timeskipping_test.go b/tests/timeskipping_test.go index 8da21c8b891..6008a87babd 100644 --- a/tests/timeskipping_test.go +++ b/tests/timeskipping_test.go @@ -74,6 +74,7 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_StartWorkflow_DCEnabled() { env := testcore.NewEnv(s.T()) env.OverrideDynamicConfig(dynamicconfig.WorkflowTimeSkippingEnabled, true) tv := testvars.New(s.T()) + beforeStart := time.Now() // Request leaves MaxSkipPerSession empty; the frontend populates it from dynamic config. inputConfig := &commonpb.TimeSkippingConfig{ @@ -92,11 +93,17 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_StartWorkflow_DCEnabled() { TimeSkippingConfig: inputConfig, }) s.NoError(err) + afterStart := time.Now() ms := s.getMutableState(env, tv.WorkflowID(), resp.RunId) s.True(ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig().GetEnabled()) inputConfig.MaxSessionSkipCount = defaultMaxSkipPerSession // frontend populated this from dynamic config s.True(proto.Equal(inputConfig, ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig())) + startTime := ms.State.ExecutionState.GetStartTime().AsTime() + s.False(startTime.Before(beforeStart), "zero-offset start must not move before the admission window") + s.False(startTime.After(afterStart), "zero-offset start must not move after the admission window") + s.Equal(startTime, ms.State.ExecutionInfo.GetStartTime().AsTime()) + s.Equal(startTime, ms.State.ExecutionInfo.GetExecutionTime().AsTime()) } // TestTimeSkipping_SignalWithStart_DCEnabled verifies that SignalWithStartWorkflowExecution @@ -237,6 +244,17 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_UpdateWorkflowOptions_DCEnabled // No time-skipping config before any update. ms := s.getMutableState(env, tv.WorkflowID(), runID) s.Nil(ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig()) + initialStateStart := ms.State.ExecutionState.GetStartTime().AsTime() + initialInfoStart := ms.State.ExecutionInfo.GetStartTime().AsTime() + initialExecutionTime := ms.State.ExecutionInfo.GetExecutionTime().AsTime() + initialRunExpiration := ms.State.ExecutionInfo.GetWorkflowRunExpirationTime().AsTime() + assertAdmissionTimesUnchanged := func() { + current := s.getMutableState(env, tv.WorkflowID(), runID) + s.Equal(initialStateStart, current.State.ExecutionState.GetStartTime().AsTime()) + s.Equal(initialInfoStart, current.State.ExecutionInfo.GetStartTime().AsTime()) + s.Equal(initialExecutionTime, current.State.ExecutionInfo.GetExecutionTime().AsTime()) + s.Equal(initialRunExpiration, current.State.ExecutionInfo.GetWorkflowRunExpirationTime().AsTime()) + } // First update: enable with a max_elapsed_duration. MaxSkipPerSession is left empty and // populated by the frontend from dynamic config; the persisted config and event carry it. @@ -249,6 +267,7 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_UpdateWorkflowOptions_DCEnabled ms = s.getMutableState(env, tv.WorkflowID(), runID) s.True(proto.Equal(config1, ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig())) + assertAdmissionTimesUnchanged() events := collectOptionsEvents() s.Len(events, 1) s.True(proto.Equal(config1, events[0].GetWorkflowExecutionOptionsUpdatedEventAttributes().GetTimeSkippingConfig())) @@ -263,6 +282,7 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_UpdateWorkflowOptions_DCEnabled ms = s.getMutableState(env, tv.WorkflowID(), runID) s.True(proto.Equal(config2, ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig())) + assertAdmissionTimesUnchanged() events = collectOptionsEvents() s.Len(events, 2) s.True(proto.Equal(config2, events[1].GetWorkflowExecutionOptionsUpdatedEventAttributes().GetTimeSkippingConfig())) @@ -275,6 +295,7 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_UpdateWorkflowOptions_DCEnabled ms = s.getMutableState(env, tv.WorkflowID(), runID) s.True(proto.Equal(config3, ms.State.ExecutionInfo.GetTimeSkippingInfo().GetConfig())) + assertAdmissionTimesUnchanged() events = collectOptionsEvents() s.Len(events, 3) s.True(proto.Equal(config3, events[2].GetWorkflowExecutionOptionsUpdatedEventAttributes().GetTimeSkippingConfig())) diff --git a/tests/xdc/timeskipping_replication_test.go b/tests/xdc/timeskipping_replication_test.go index cea975da397..7d2f93d9248 100644 --- a/tests/xdc/timeskipping_replication_test.go +++ b/tests/xdc/timeskipping_replication_test.go @@ -178,8 +178,10 @@ func (s *timeSkippingReplicationSuite) TestBasicSkipReplicates() { s.waitForTimeSkippingInfoSynced(ctx, nsID, wfID, runID) - active := s.getExecutionInfoFromCluster(ctx, 0, nsID, wfID, runID).GetTimeSkippingInfo() - standby := s.getExecutionInfoFromCluster(ctx, 1, nsID, wfID, runID).GetTimeSkippingInfo() + activeInfo := s.getExecutionInfoFromCluster(ctx, 0, nsID, wfID, runID) + standbyInfo := s.getExecutionInfoFromCluster(ctx, 1, nsID, wfID, runID) + active := activeInfo.GetTimeSkippingInfo() + standby := standbyInfo.GetTimeSkippingInfo() s.True(proto.Equal(active.GetConfig(), standby.GetConfig())) s.Equal( active.GetAccumulatedSkippedDuration().AsDuration(), @@ -187,6 +189,10 @@ func (s *timeSkippingReplicationSuite) TestBasicSkipReplicates() { ) s.InDelta(float64(startDelay), float64(standby.GetAccumulatedSkippedDuration().AsDuration()), float64(accumTol), "standby's accumulated skip should match the configured startDelay within tolerance") + s.Equal(activeInfo.GetStartTime().AsTime(), standbyInfo.GetStartTime().AsTime(), + "replication must preserve the already-virtual start timestamp without applying the offset again") + s.Equal(activeInfo.GetExecutionTime().AsTime(), standbyInfo.GetExecutionTime().AsTime()) + s.Equal(activeInfo.GetWorkflowRunExpirationTime().AsTime(), standbyInfo.GetWorkflowRunExpirationTime().AsTime()) } // TestFastForwardDisablePropagates verifies that completing a registered FastForward From 1cbaf6cf70d9e2cf664097b67d77a2df3870877b Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 19 Aug 2026 11:21:22 -0700 Subject: [PATCH 008/139] Trace inbound Nexus HTTP requests (#11560) ## What changed? Wrapped the frontend Nexus dispatch routes with the shared OpenTelemetry HTTP handler. ## Why? Nexus HTTP requests need an inbound server span to connect the caller trace. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- common/telemetry/http.go | 21 ++++ common/telemetry/http_test.go | 18 +++ .../frontend/nexus_operation_http_handler.go | 25 +++- temporal/fx.go | 2 + tests/nexus_api_test.go | 33 ++--- tests/nexus_api_validation_test.go | 14 +-- tests/nexus_otel_test.go | 114 ++++++++++++++---- tests/nexus_test_base.go | 12 ++ 8 files changed, 181 insertions(+), 58 deletions(-) diff --git a/common/telemetry/http.go b/common/telemetry/http.go index c825e49a0aa..979cabd141b 100644 --- a/common/telemetry/http.go +++ b/common/telemetry/http.go @@ -35,6 +35,27 @@ func (i HTTPClientTransportInstrumenter) Instrument(rt http.RoundTripper) http.R return i(rt) } +// HTTPServerHandlerInstrumenter instruments an HTTP server handler and uses spanName for its spans. +type HTTPServerHandlerInstrumenter func(handler http.Handler, spanName string) http.Handler + +// NewHTTPServerHandlerInstrumenter binds a tracer provider and propagator for reuse across HTTP server handlers. +func NewHTTPServerHandlerInstrumenter( + tracerProvider trace.TracerProvider, + propagator propagation.TextMapPropagator, +) HTTPServerHandlerInstrumenter { + return func(handler http.Handler, spanName string) http.Handler { + return NewHTTPHandler(handler, spanName, tracerProvider, propagator) + } +} + +// Instrument instruments handler, or returns it unchanged when the instrumenter is unset. +func (i HTTPServerHandlerInstrumenter) Instrument(handler http.Handler, spanName string) http.Handler { + if i == nil { + return handler + } + return i(handler, spanName) +} + // NewHTTPClientTransport instruments outbound HTTP requests with OpenTelemetry client spans // and injects trace context using propagator. If propagator is nil, it defaults to W3C Trace Context. func NewHTTPClientTransport( diff --git a/common/telemetry/http_test.go b/common/telemetry/http_test.go index 3aafffeea72..450205e0e60 100644 --- a/common/telemetry/http_test.go +++ b/common/telemetry/http_test.go @@ -419,6 +419,24 @@ func TestNewHTTPClientTransport(t *testing.T) { // Verifies server handler construction, standard tracing, and debug body handling. func TestNewHTTPHandler(t *testing.T) { + // The instrumenter carries HTTP tracing configuration without exposing it to handler owners. + t.Run("ServerHandlerInstrumenter", func(t *testing.T) { + t.Parallel() + + handler := http.NewServeMux() + var disabled HTTPServerHandlerInstrumenter + require.Same(t, handler, disabled.Instrument(handler, "test-handler")) + + recorder := tracetest.NewSpanRecorder() + tp := trace.NewTracerProvider(trace.WithSpanProcessor(recorder)) + instrumenter := NewHTTPServerHandlerInstrumenter(tp, nil) + instrumenter.Instrument(handler, "test-handler").ServeHTTP( + httptest.NewRecorder(), + httptest.NewRequest(http.MethodGet, "http://example.com", nil), + ) + require.Equal(t, "test-handler", recorder.Ended()[0].Name()) + }) + // Debug-only headers and payloads must remain absent unless debug mode is enabled. t.Run("SkipsHeadersAndPayloadsByDefault", func(t *testing.T) { t.Parallel() diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index e47ef3825ef..3f1226ccdcb 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -6,6 +6,7 @@ import ( "net/http" "net/url" "path" + "strings" "time" "github.com/gorilla/mux" @@ -25,6 +26,7 @@ import ( "go.temporal.io/server/common/routing" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/telemetry" "go.temporal.io/server/service/frontend/configs" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -44,6 +46,7 @@ type NexusOperationHTTPHandler struct { namespaceRateLimitInterceptor interceptor.NamespaceRateLimitInterceptor namespaceConcurrencyLimitInterceptor *interceptor.ConcurrentRequestLimitInterceptor rateLimitInterceptor *interceptor.RateLimitInterceptor + httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter } func NewNexusOperationHTTPHandler( @@ -64,6 +67,7 @@ func NewNexusOperationHTTPHandler( rateLimitInterceptor *interceptor.RateLimitInterceptor, logger log.Logger, httpTraceProvider commonnexus.HTTPClientTraceProvider, + httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { return &NexusOperationHTTPHandler{ base: nexusrpc.BaseHTTPHandler{ @@ -79,6 +83,7 @@ func NewNexusOperationHTTPHandler( namespaceConcurrencyLimitInterceptor: namespaceConcurrencyLimitInterceptor, rateLimitInterceptor: rateLimitInterceptor, preprocessErrorCounter: metricsHandler.Counter(metrics.NexusRequestPreProcessErrors.Name()).Record, + httpServerHandlerInstrumenter: httpServerHandlerInstrumenter, nexusHandler: nexusrpc.NewHTTPHandler(nexusrpc.HandlerOptions{ Handler: &nexusHandler{ logger: logger, @@ -106,10 +111,22 @@ func NewNexusOperationHTTPHandler( } func (h *NexusOperationHTTPHandler) RegisterRoutes(r *mux.Router) { - r.PathPrefix("/" + commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Representation() + "/"). - HandlerFunc(h.dispatchNexusTaskByNamespaceAndTaskQueue) - r.PathPrefix("/" + commonnexus.RouteDispatchNexusTaskByEndpoint.Representation() + "/"). - HandlerFunc(h.dispatchNexusTaskByEndpoint) + register := func(route, apiName string, handler http.HandlerFunc) { + spanName := strings.TrimPrefix(apiName, "/") + instrumentedHandler := h.httpServerHandlerInstrumenter.Instrument(handler, spanName) + r.PathPrefix("/" + route + "/").Handler(instrumentedHandler) + } + + register( + commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue.Representation(), + configs.DispatchNexusTaskByNamespaceAndTaskQueueAPIName, + h.dispatchNexusTaskByNamespaceAndTaskQueue, + ) + register( + commonnexus.RouteDispatchNexusTaskByEndpoint.Representation(), + configs.DispatchNexusTaskByEndpointAPIName, + h.dispatchNexusTaskByEndpoint, + ) } func (h *NexusOperationHTTPHandler) writeFailure(writer http.ResponseWriter, r *http.Request, err error) { diff --git a/temporal/fx.go b/temporal/fx.go index 44630f5e700..dd7c9331afd 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -1037,6 +1037,7 @@ var TraceExportModule = fx.Options( // - telemetry.ServerStatsHandler // - telemetry.ClientStatsHandler // - telemetry.HTTPClientTransportInstrumenter +// - telemetry.HTTPServerHandlerInstrumenter var ServiceTracingModule = fx.Options( fx.Supply([]otelsdktrace.BatchSpanProcessorOption{}), fx.Provide( @@ -1107,6 +1108,7 @@ var ServiceTracingModule = fx.Options( fx.Provide(telemetry.NewServerStatsHandler), fx.Provide(telemetry.NewClientStatsHandler), fx.Provide(telemetry.NewHTTPClientTransportInstrumenter), + fx.Provide(telemetry.NewHTTPServerHandlerInstrumenter), fx.Provide(metrics.NewServerStatsHandler), ) diff --git a/tests/nexus_api_test.go b/tests/nexus_api_test.go index 7544eec7a80..0559abd7c95 100644 --- a/tests/nexus_api_test.go +++ b/tests/nexus_api_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" "net/http" "testing" "time" @@ -273,9 +272,9 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Outcomes(useTemporalFailures endpoint := env.createNexusEndpoint(s.Context(), s.T(), tc.endpointName, testcore.RandomizeStr("task-queue")) var dispatchURL string if dispatchOnlyByEndpoint { - dispatchURL = getDispatchByEndpointURL(env.HttpAPIAddress(), endpoint.Id) + dispatchURL = env.dispatchByEndpointURL(endpoint.Id) } else { - dispatchURL = getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), endpoint.Spec.Target.GetWorker().TaskQueue) + dispatchURL = env.dispatchByTaskQueueURL(endpoint.Spec.Target.GetWorker().TaskQueue) } httpCaller, headerCapture := newHeaderCaptureCaller() @@ -420,9 +419,9 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Claims(useTemporalFailures b testEndpoint := env.createNexusEndpoint(s.Context(), s.T(), testcore.RandomizeStr("test-endpoint"), taskQueue) var dispatchURL string if dispatchOnlyByEndpoint { - dispatchURL = getDispatchByEndpointURL(env.HttpAPIAddress(), testEndpoint.Id) + dispatchURL = env.dispatchByEndpointURL(testEndpoint.Id) } else { - dispatchURL = getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), taskQueue) + dispatchURL = env.dispatchByTaskQueueURL(taskQueue) } client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: dispatchURL, Service: "test-service"}) @@ -534,9 +533,9 @@ func (s *NexusApiTestSuite) TestNexusCancelOperation_Outcomes(useTemporalFailure endpoint := env.createNexusEndpoint(s.Context(), s.T(), tc.endpointName, testcore.RandomizeStr("task-queue")) var dispatchURL string if dispatchOnlyByEndpoint { - dispatchURL = getDispatchByEndpointURL(env.HttpAPIAddress(), endpoint.Id) + dispatchURL = env.dispatchByEndpointURL(endpoint.Id) } else { - dispatchURL = getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), endpoint.Spec.Target.GetWorker().TaskQueue) + dispatchURL = env.dispatchByTaskQueueURL(endpoint.Spec.Target.GetWorker().TaskQueue) } httpCaller, headerCapture := newHeaderCaptureCaller() @@ -631,7 +630,7 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_WithNamespaceAndTaskQueue_Su }) s.NoError(err) - u := getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), taskQueue) + u := env.dispatchByTaskQueueURL(taskQueue) client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: u, Service: "test-service"}) s.NoError(err) // Versioned poller gets task @@ -687,7 +686,7 @@ func (s *NexusApiTestSuite) TestNexusClientNameMetricPropagation(useTemporalFail // Trigger a Nexus start operation via HTTP to unblock the poller. client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ - BaseURL: getDispatchByEndpointURL(env.HttpAPIAddress(), endpoint.Id), + BaseURL: env.dispatchByEndpointURL(endpoint.Id), Service: "test-service", }) s.NoError(err) @@ -712,19 +711,3 @@ func (s *NexusApiTestSuite) TestNexusClientNameMetricPropagation(useTemporalFail func nexusEchoHandler(_ *testing.T, res *workflowservice.PollNexusTaskQueueResponse) (*nexusTaskResponse, error) { return &nexusTaskResponse{StartResult: &nexus.HandlerStartOperationResultSync[*commonpb.Payload]{Value: res.Request.GetStartOperation().GetPayload()}}, nil } - -func getDispatchByNsAndTqURL(address string, namespace string, taskQueue string) string { - return fmt.Sprintf( - "http://%s/%s", - address, - commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue. - Path(commonnexus.NamespaceAndTaskQueue{ - Namespace: namespace, - TaskQueue: taskQueue, - }), - ) -} - -func getDispatchByEndpointURL(address string, endpoint string) string { - return fmt.Sprintf("http://%s/%s", address, commonnexus.RouteDispatchNexusTaskByEndpoint.Path(endpoint)) -} diff --git a/tests/nexus_api_validation_test.go b/tests/nexus_api_validation_test.go index 88db95b14b1..dde819faddf 100644 --- a/tests/nexus_api_validation_test.go +++ b/tests/nexus_api_validation_test.go @@ -34,7 +34,7 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_WithNamespaceAndTa // Also use this test to verify that namespaces are unescaped in the path. taskQueue := testcore.RandomizeStr("task-queue") namespace := "namespace not/found" - u := getDispatchByNsAndTqURL(env.HttpAPIAddress(), namespace, taskQueue) + u := env.dispatchByNamespaceAndTaskQueueURL(namespace, taskQueue) client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: u, Service: "test-service"}) s.NoError(err) capture := env.StartNamespaceMetricCaptureFor(namespace) @@ -65,7 +65,7 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_WithNamespaceAndTa namespace.WriteString("namespace-is-a-very-long-string") } - u := getDispatchByNsAndTqURL(env.HttpAPIAddress(), namespace.String(), taskQueue) + u := env.dispatchByNamespaceAndTaskQueueURL(namespace.String(), taskQueue) client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: u, Service: "test-service"}) s.NoError(err) capture := env.StartGlobalMetricCapture() @@ -193,9 +193,9 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_Forbidden() { var dispatchURL string if dispatchOnlyByEndpoint { - dispatchURL = getDispatchByEndpointURL(env.HttpAPIAddress(), testEndpoint.Id) + dispatchURL = env.dispatchByEndpointURL(testEndpoint.Id) } else { - dispatchURL = getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), taskQueue) + dispatchURL = env.dispatchByTaskQueueURL(taskQueue) } client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: dispatchURL, Service: "test-service"}) @@ -235,9 +235,9 @@ func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_PayloadSizeLimit() var dispatchURL string if dispatchOnlyByEndpoint { - dispatchURL = getDispatchByEndpointURL(env.HttpAPIAddress(), testEndpoint.Id) + dispatchURL = env.dispatchByEndpointURL(testEndpoint.Id) } else { - dispatchURL = getDispatchByNsAndTqURL(env.HttpAPIAddress(), env.Namespace().String(), taskQueue) + dispatchURL = env.dispatchByTaskQueueURL(taskQueue) } client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: dispatchURL, Service: "test-service"}) @@ -368,7 +368,7 @@ func (s *NexusAPIValidationTestSuite) TestNexus_RespondNexusTaskMethods_Validate func (s *NexusAPIValidationTestSuite) TestNexusStartOperation_ByEndpoint_EndpointNotFound() { env := newNexusTestEnv(s.T(), false, testcore.WithDedicatedCluster()) - u := getDispatchByEndpointURL(env.HttpAPIAddress(), uuid.NewString()) + u := env.dispatchByEndpointURL(uuid.NewString()) client, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{BaseURL: u, Service: "test-service"}) s.NoError(err) capture := env.StartGlobalMetricCapture() diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go index 0de5a4604e5..8b87d74fc43 100644 --- a/tests/nexus_otel_test.go +++ b/tests/nexus_otel_test.go @@ -26,18 +26,20 @@ import ( "go.temporal.io/server/chasm/lib/callback" "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" ) type nexusHTTPSpan struct { - TraceID int - SpanID int - Name string - ServiceName string - Kind oteltrace.SpanKind - URLPath string + TraceID int + SpanID int + ParentSpanID int + Name string + ServiceName string + Kind oteltrace.SpanKind + URLPath string } type NexusOTELSuite struct { @@ -126,7 +128,7 @@ func (s *NexusOTELSuite) TestCallback() { s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) } -// Verifies asynchronous start and cancellation use the instrumented external Nexus HTTP client. +// Verifies asynchronous start and cancellation connect real History client and Frontend server spans. func (s *NexusOTELSuite) TestOperation() { exporter := tracetest.NewInMemoryExporter() callerEnv := s.newTestEnv(exporter) @@ -197,6 +199,15 @@ func (s *NexusOTELSuite) TestOperation() { Kind: oteltrace.SpanKindClient, URLPath: operationURLPath, }, + { + TraceID: 1, + SpanID: 2, + ParentSpanID: 1, + Name: "temporal.api.nexusservice.v1.NexusService/DispatchByEndpoint", + ServiceName: "io.temporal.frontend", + Kind: oteltrace.SpanKindServer, + URLPath: operationURLPath, + }, { TraceID: 2, SpanID: 1, @@ -205,10 +216,19 @@ func (s *NexusOTELSuite) TestOperation() { Kind: oteltrace.SpanKindClient, URLPath: operationURLPath + "/cancel", }, + { + TraceID: 2, + SpanID: 2, + ParentSpanID: 1, + Name: "temporal.api.nexusservice.v1.NexusService/DispatchByEndpoint", + ServiceName: "io.temporal.frontend", + Kind: oteltrace.SpanKindServer, + URLPath: operationURLPath + "/cancel", + }, }) } -// Verifies worker-target Nexus operations trace requests routed through the local frontend client. +// Verifies worker-target Nexus operations connect local Frontend client and server spans. func (s *NexusOTELSuite) TestWorkerOperation() { exporter := tracetest.NewInMemoryExporter() env := s.newTestEnv(exporter) @@ -246,14 +266,25 @@ func (s *NexusOTELSuite) TestWorkerOperation() { s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) } operationURLPath := "/nexus/endpoints/" + endpoint.Id + "/services/" + service.Name + "/" + operation.Name() - httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ - TraceID: 1, - SpanID: 1, - Name: "HTTP POST", - ServiceName: "io.temporal.history", - Kind: oteltrace.SpanKindClient, - URLPath: operationURLPath, - }}) + httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{ + { + TraceID: 1, + SpanID: 1, + Name: "HTTP POST", + ServiceName: "io.temporal.history", + Kind: oteltrace.SpanKindClient, + URLPath: operationURLPath, + }, + { + TraceID: 1, + SpanID: 2, + ParentSpanID: 1, + Name: "temporal.api.nexusservice.v1.NexusService/DispatchByEndpoint", + ServiceName: "io.temporal.frontend", + Kind: oteltrace.SpanKindServer, + URLPath: operationURLPath, + }, + }) spanContext := oteltrace.SpanContextFromContext( propagation.TraceContext{}.Extract(s.Context(), propagation.MapCarrier(headers)), ) @@ -262,6 +293,44 @@ func (s *NexusOTELSuite) TestWorkerOperation() { s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) } +// Verifies the namespace and task queue dispatch route is instrumented independently of forwarding. +func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { + exporter := tracetest.NewInMemoryExporter() + env := s.newTestEnv(exporter) + taskQueue := env.Tv().TaskQueue().GetName() + pollerErrCh := env.nexusTaskPoller(s.Context(), s.T(), taskQueue, nexusEchoHandler) + dispatchURL, err := url.Parse(env.dispatchByTaskQueueURL(taskQueue)) + s.NoError(err) + nexusClient, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ + BaseURL: dispatchURL.String(), + Service: "test-service", + }) + s.NoError(err) + + // Inject a fixed traceparent so this test exercises the server independently of client instrumentation. + // Nexus API tests separately verify that frontend request headers reach workers. + const ( + traceID = "4bf92f3577b34da6a3ce929d0e0e4736" + parentSpanID = "00f067aa0ba902b7" + ) + _, err = nexusrpc.StartOperation(s.Context(), nexusClient, op, env.Tv().Any().String(), nexus.StartOperationOptions{ + Header: nexus.Header{"traceparent": "00-" + traceID + "-" + parentSpanID + "-01"}, + }) + s.NoError(err) + s.NoError(<-pollerErrCh) + + httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ + TraceID: 1, + SpanID: 1, + Name: "temporal.api.nexusservice.v1.NexusService/DispatchByNamespaceAndTaskQueue", + ServiceName: "io.temporal.frontend", + Kind: oteltrace.SpanKindServer, + URLPath: dispatchURL.Path + "/test-service/my-operation", + }}) + s.Require().Equal(traceID, httpSpans[0].SpanContext.TraceID().String()) + s.Require().Equal(parentSpanID, httpSpans[0].Parent.SpanID().String()) +} + // requireNexusHTTPSpans compares all exported HTTP spans after assigning stable local IDs // and returns the matching raw spans for context propagation assertions. func (s *NexusOTELSuite) requireNexusHTTPSpans( @@ -321,12 +390,13 @@ func (s *NexusOTELSuite) nexusHTTPSpans( } } result = append(result, nexusHTTPSpan{ - TraceID: traceIDs[traceID], - SpanID: spanIDs[traceID][span.SpanContext.SpanID()], - Name: span.Name, - ServiceName: serviceName, - Kind: span.SpanKind, - URLPath: urlPath, + TraceID: traceIDs[traceID], + SpanID: spanIDs[traceID][span.SpanContext.SpanID()], + ParentSpanID: spanIDs[traceID][span.Parent.SpanID()], + Name: span.Name, + ServiceName: serviceName, + Kind: span.SpanKind, + URLPath: urlPath, }) } return result, httpSpans diff --git a/tests/nexus_test_base.go b/tests/nexus_test_base.go index 0c3100fe61e..02cdf3676d5 100644 --- a/tests/nexus_test_base.go +++ b/tests/nexus_test_base.go @@ -103,6 +103,18 @@ func (env *NexusTestEnv) dispatchByEndpointURL(endpoint string) string { return "http://" + env.HttpAPIAddress() + "/" + cnexus.RouteDispatchNexusTaskByEndpoint.Path(endpoint) } +func (env *NexusTestEnv) dispatchByTaskQueueURL(taskQueue string) string { + return env.dispatchByNamespaceAndTaskQueueURL(env.Namespace().String(), taskQueue) +} + +func (env *NexusTestEnv) dispatchByNamespaceAndTaskQueueURL(namespace string, taskQueue string) string { + return "http://" + env.HttpAPIAddress() + "/" + cnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue. + Path(cnexus.NamespaceAndTaskQueue{ + Namespace: namespace, + TaskQueue: taskQueue, + }) +} + // nexusTaskResponse represents a successful response from a nexus task handler. // A nil response indicates no response should be sent (e.g., handler timed out). type nexusTaskResponse struct { From d0924bd7f56b0b19f7e396769a437659ea17ba98 Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 19 Aug 2026 11:22:30 -0700 Subject: [PATCH 009/139] improvements on time-skipping task regeneration (#11404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? 1. perf improvement: no time-skipping task regen on full refresh 2. new functional test: claim Nexus HSM timers are part of in-fight nexus operation, and won’t be skipped, add functional test 3. new functional test: add a functional test to verify time skipping won't change retention time 4. trivial bug fix: time-skipping: task regen didn’t read EnableWorkflowExecutionTimeoutTimer ## Why? - correctness related No. 4 though it is an edgy case - perf related No.1 - test coverage No.2,3 ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- service/history/workflow/task_generator.go | 54 +++-- .../history/workflow/task_generator_mock.go | 14 ++ .../history/workflow/task_generator_test.go | 198 +++++++++++++++--- service/history/workflow/task_refresher.go | 6 + .../history/workflow/task_refresher_test.go | 20 +- tests/timeskipping_test.go | 196 +++++++++++++++++ 6 files changed, 435 insertions(+), 53 deletions(-) diff --git a/service/history/workflow/task_generator.go b/service/history/workflow/task_generator.go index 70d11408ccd..07853dfd640 100644 --- a/service/history/workflow/task_generator.go +++ b/service/history/workflow/task_generator.go @@ -80,8 +80,8 @@ type ( GenerateActivityTimerTasks() error GenerateUserTimerTasks() error - // time skipping tasks RegenerateTimerTasksForTimeSkipping() error + GenerateTimeSkippingFastForwardTimerTask() error // replication tasks GenerateHistoryReplicationTasks( @@ -1066,18 +1066,20 @@ func (r *TaskGeneratorImpl) RegenerateTimerTasksForTimeSkipping() error { }) } - // (2) execution and run timeout timers - executionTimeoutTimer := r.mutableState.GetExecutionInfo().WorkflowExecutionExpirationTime - if !timeNotSet(executionTimeoutTimer) { + // (2) execution and run timeout timers. + executionInfo := r.mutableState.GetExecutionInfo() + executionTimeoutTimer := executionInfo.WorkflowExecutionExpirationTime + if !timeNotSet(executionTimeoutTimer) && + executionInfo.WorkflowExecutionTimerTaskStatus == TimerTaskStatusCreated { r.mutableState.AddTasks(&tasks.WorkflowExecutionTimeoutTask{ // TaskID is set by shard - NamespaceID: r.mutableState.GetExecutionInfo().NamespaceId, - WorkflowID: r.mutableState.GetExecutionInfo().WorkflowId, - FirstRunID: r.mutableState.GetExecutionInfo().FirstExecutionRunId, + NamespaceID: executionInfo.NamespaceId, + WorkflowID: executionInfo.WorkflowId, + FirstRunID: executionInfo.FirstExecutionRunId, VisibilityTimestamp: timestamp.TimeValue(executionTimeoutTimer), }) } - runTimeoutTimer := r.mutableState.GetExecutionInfo().WorkflowRunExpirationTime + runTimeoutTimer := executionInfo.WorkflowRunExpirationTime if !timeNotSet(runTimeoutTimer) { // Version must match the workflow's start version so the executor's // CheckTaskVersion passes for global namespaces (see timer_queue_active_task_executor.go). @@ -1093,20 +1095,9 @@ func (r *TaskGeneratorImpl) RegenerateTimerTasksForTimeSkipping() error { }) } - // (3) fast-forward timer — regenerate when configured so its real-time - // VisibilityTimestamp tracks the new accumulated skip. - tsi := r.mutableState.GetExecutionInfo().GetTimeSkippingInfo() - if tsi.GetConfig().GetEnabled() { - fastForward := tsi.GetFastForwardInfo() - if fastForward != nil && !fastForward.GetHasReached() { - r.mutableState.AddTasks(&tasks.TimeSkippingTimerTask{ - // TaskID is set by shard - WorkflowKey: r.mutableState.GetWorkflowKey(), - VisibilityTimestamp: fastForward.GetTargetTime().AsTime(), - VersionedTransition: tsi.GetFastForwardInfoLastUpdateVersionedTransition(), - ArchetypeID: r.mutableState.ChasmTree().ArchetypeID(), - }) - } + // (3) fast-forward timer + if err := r.GenerateTimeSkippingFastForwardTimerTask(); err != nil { + return err } // (4) start delays (start-with-delay, cron, retry in CAN, etc). @@ -1152,6 +1143,23 @@ func (r *TaskGeneratorImpl) RegenerateTimerTasksForTimeSkipping() error { } } - // todo@time-skipping: ChasmTaskPure is not supported yet. + // Generic StateMachineTimerTask (nexus HSM), WorkflowTaskTimeoutTask, and ActivityTimeoutTask + // are treated as a part of in-flight nexus operation and won't be regenerated. + // DeleteHistoryEventTask is not impacted by time skipping, and doesn't need regeneration. + return nil +} + +func (r *TaskGeneratorImpl) GenerateTimeSkippingFastForwardTimerTask() error { + tsi := r.mutableState.GetExecutionInfo().GetTimeSkippingInfo() + if !NewTimeSkippingInfoUtil(tsi).HasPendingFastForward() { + return nil + } + r.mutableState.AddTasks(&tasks.TimeSkippingTimerTask{ + // TaskID is set by shard + WorkflowKey: r.mutableState.GetWorkflowKey(), + VisibilityTimestamp: tsi.GetFastForwardInfo().GetTargetTime().AsTime(), + VersionedTransition: tsi.GetFastForwardInfoLastUpdateVersionedTransition(), + ArchetypeID: r.mutableState.ChasmTree().ArchetypeID(), + }) return nil } diff --git a/service/history/workflow/task_generator_mock.go b/service/history/workflow/task_generator_mock.go index 78986935aa5..3f6ec22170a 100644 --- a/service/history/workflow/task_generator_mock.go +++ b/service/history/workflow/task_generator_mock.go @@ -274,6 +274,20 @@ func (mr *MockTaskGeneratorMockRecorder) GenerateStartWorkflowTaskTasks(workflow return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateStartWorkflowTaskTasks", reflect.TypeOf((*MockTaskGenerator)(nil).GenerateStartWorkflowTaskTasks), workflowTaskScheduledEventID) } +// GenerateTimeSkippingFastForwardTimerTask mocks base method. +func (m *MockTaskGenerator) GenerateTimeSkippingFastForwardTimerTask() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateTimeSkippingFastForwardTimerTask") + ret0, _ := ret[0].(error) + return ret0 +} + +// GenerateTimeSkippingFastForwardTimerTask indicates an expected call of GenerateTimeSkippingFastForwardTimerTask. +func (mr *MockTaskGeneratorMockRecorder) GenerateTimeSkippingFastForwardTimerTask() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateTimeSkippingFastForwardTimerTask", reflect.TypeOf((*MockTaskGenerator)(nil).GenerateTimeSkippingFastForwardTimerTask)) +} + // GenerateUpsertVisibilityTask mocks base method. func (m *MockTaskGenerator) GenerateUpsertVisibilityTask() error { m.ctrl.T.Helper() diff --git a/service/history/workflow/task_generator_test.go b/service/history/workflow/task_generator_test.go index 934168631c1..a48d65bac98 100644 --- a/service/history/workflow/task_generator_test.go +++ b/service/history/workflow/task_generator_test.go @@ -1176,6 +1176,15 @@ func TestGenerateWorkerCommandsTasks(t *testing.T) { } } +// timeSkippingTestConfig builds the minimal config the time-skipping regen paths read. +// EnableWorkflowExecutionTimeoutTimer must be set explicitly: a bare &configs.Config{} leaves it +// nil and regen calls it. +func timeSkippingTestConfig(executionTimeoutTimerEnabled bool) *configs.Config { + return &configs.Config{ + EnableWorkflowExecutionTimeoutTimer: dynamicconfig.GetBoolPropertyFn(executionTimeoutTimerEnabled), + } +} + func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping(t *testing.T) { t.Parallel() @@ -1217,7 +1226,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping(t *testing.T) { capturedTasks = append(capturedTasks, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) // Both pending user timers must be regenerated. @@ -1273,7 +1282,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ForceRegenerates( emitCount += len(ts) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) // First call emits. require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) @@ -1335,7 +1344,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_EdgeCases(t *test tc.setupTimers(mutableState) } - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) }) } @@ -1359,18 +1368,21 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ExecutionTimers(t runExpiry := now.Add(3 * time.Hour) for _, tc := range []struct { - name string - execExpirationTime *timestamppb.Timestamp - runExpirationTime *timestamppb.Timestamp - wantExecTimeout bool - wantRunTimeout bool + name string + execExpirationTime *timestamppb.Timestamp + runExpirationTime *timestamppb.Timestamp + executionTimerTaskStatus int32 + executionTimeoutTimerOffDC bool + wantExecTimeout bool + wantRunTimeout bool }{ { - name: "both execution and run expirations set", - execExpirationTime: timestamppb.New(execExpiry), - runExpirationTime: timestamppb.New(runExpiry), - wantExecTimeout: true, - wantRunTimeout: true, + name: "both execution and run expirations set", + execExpirationTime: timestamppb.New(execExpiry), + runExpirationTime: timestamppb.New(runExpiry), + executionTimerTaskStatus: TimerTaskStatusCreated, + wantExecTimeout: true, + wantRunTimeout: true, }, { name: "only run expiration set", @@ -1379,6 +1391,23 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ExecutionTimers(t wantExecTimeout: false, wantRunTimeout: true, }, + { + name: "execution timeout timer not created", + execExpirationTime: timestamppb.New(execExpiry), + runExpirationTime: timestamppb.New(runExpiry), + executionTimeoutTimerOffDC: true, + wantExecTimeout: false, + wantRunTimeout: true, + }, + { + name: "created execution timeout timer with feature disabled", + execExpirationTime: timestamppb.New(execExpiry), + runExpirationTime: timestamppb.New(runExpiry), + executionTimerTaskStatus: TimerTaskStatusCreated, + executionTimeoutTimerOffDC: true, + wantExecTimeout: true, + wantRunTimeout: true, + }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() @@ -1388,11 +1417,12 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ExecutionTimers(t mutableState.EXPECT().GetPendingActivityInfos().Return(nil).AnyTimes() mutableState.EXPECT().Now().Return(time.Now().UTC()).AnyTimes() mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ - NamespaceId: namespaceID, - WorkflowId: workflowID, - FirstExecutionRunId: firstRunID, - WorkflowExecutionExpirationTime: tc.execExpirationTime, - WorkflowRunExpirationTime: tc.runExpirationTime, + NamespaceId: namespaceID, + WorkflowId: workflowID, + FirstExecutionRunId: firstRunID, + WorkflowExecutionExpirationTime: tc.execExpirationTime, + WorkflowRunExpirationTime: tc.runExpirationTime, + WorkflowExecutionTimerTaskStatus: tc.executionTimerTaskStatus, TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ AccumulatedSkippedDuration: durationpb.New(skippedDuration), }, @@ -1415,7 +1445,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ExecutionTimers(t captured = append(captured, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(!tc.executionTimeoutTimerOffDC), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) // Expected task count: always user timer, plus any timeout tasks whose expiration is set. @@ -1564,7 +1594,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_FastForwardTimer( captured = append(captured, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) var fastForwardTasks []*tasks.TimeSkippingTimerTask @@ -1591,6 +1621,115 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_FastForwardTimer( } } +func TestTaskGeneratorImpl_GenerateTimeSkippingFastForwardTimerTask(t *testing.T) { + t.Parallel() + + fastForwardTarget := time.Now().UTC().Add(2 * time.Hour) + fastForwardVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: 42, TransitionCount: 7} + const archetypeID = uint32(7) + + pendingFastForward := func(accumulated time.Duration) *persistencespb.TimeSkippingInfo { + return &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(2 * time.Hour)}, + }, + AccumulatedSkippedDuration: durationpb.New(accumulated), + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: timestamppb.New(fastForwardTarget), + }, + FastForwardInfoLastUpdateVersionedTransition: fastForwardVT, + } + } + + for _, tc := range []struct { + name string + tsi *persistencespb.TimeSkippingInfo + wantTask bool + }{ + { + name: "PendingFastForward/Emits", + tsi: pendingFastForward(time.Hour), + wantTask: true, + }, + { + // The refresher path must not lose the wake-up before the first skip, which is + // why this method deliberately omits the parent's accumulated-skip early return. + name: "ZeroAccumulatedSkip/StillEmits", + tsi: pendingFastForward(0), + wantTask: true, + }, + { + name: "NoTimeSkippingInfo/NoTask", + tsi: nil, + wantTask: false, + }, + { + name: "AlreadyReached/NoTask", + tsi: func() *persistencespb.TimeSkippingInfo { + tsi := pendingFastForward(time.Hour) + tsi.FastForwardInfo.HasReached = true + return tsi + }(), + wantTask: false, + }, + { + name: "Disabled/NoTask", + tsi: func() *persistencespb.TimeSkippingInfo { + tsi := pendingFastForward(time.Hour) + tsi.Config.Enabled = false + return tsi + }(), + wantTask: false, + }, + { + name: "NoFastForwardInfo/NoTask", + tsi: func() *persistencespb.TimeSkippingInfo { + tsi := pendingFastForward(time.Hour) + tsi.FastForwardInfo = nil + return tsi + }(), + wantTask: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + mutableState := historyi.NewMockMutableState(ctrl) + mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + TimeSkippingInfo: tc.tsi, + }).AnyTimes() + mutableState.EXPECT().GetWorkflowKey().Return(tests.WorkflowKey).AnyTimes() + mockChasmTree := historyi.NewMockChasmTree(ctrl) + mockChasmTree.EXPECT().ArchetypeID().Return(archetypeID).AnyTimes() + mutableState.EXPECT().ChasmTree().Return(mockChasmTree).AnyTimes() + + var captured []tasks.Task + mutableState.EXPECT().AddTasks(gomock.Any()).Do(func(ts ...tasks.Task) { + captured = append(captured, ts...) + }).AnyTimes() + + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) + require.NoError(t, taskGenerator.GenerateTimeSkippingFastForwardTimerTask()) + + if !tc.wantTask { + require.Empty(t, captured) + return + } + + require.Len(t, captured, 1) + task, ok := captured[0].(*tasks.TimeSkippingTimerTask) + require.True(t, ok, "expected *tasks.TimeSkippingTimerTask, got %T", captured[0]) + require.Equal(t, tests.WorkflowKey, task.WorkflowKey) + require.Equal(t, fastForwardTarget, task.VisibilityTimestamp) + protorequire.ProtoEqual(t, fastForwardVT, task.VersionedTransition) + require.Equal(t, archetypeID, task.ArchetypeID) + require.Equal(t, int64(0), task.TaskID, "TaskID must be zero (set by shard)") + }) + } +} + // TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_BackoffTimer covers step (4) // of RegenerateTimerTasksForTimeSkipping: the backoff-timer regeneration that supports // start-with-delay, cron, retry, and CaN-with-backoff. @@ -1696,7 +1835,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_BackoffTimer(t *t captured = append(captured, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) var backoffTasks []*tasks.WorkflowBackoffTimerTask @@ -1766,7 +1905,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_ActivityRetry(t * captured = append(captured, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) var retryTasks []*tasks.ActivityRetryTimerTask @@ -1814,7 +1953,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_BackoffTimer_Star wantErr := errors.New("boom") mutableState.EXPECT().GetStartVersion().Return(int64(0), wantErr) - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) err := taskGenerator.RegenerateTimerTasksForTimeSkipping() require.ErrorIs(t, err, wantErr) } @@ -1857,11 +1996,12 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_AllFieldsPopulate mutableState.EXPECT().GetPendingActivityInfos().Return(nil).AnyTimes() mutableState.EXPECT().Now().Return(now).AnyTimes() mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ - NamespaceId: tests.NamespaceID.String(), - WorkflowId: tests.WorkflowID, - FirstExecutionRunId: tests.RunID, - WorkflowExecutionExpirationTime: timestamppb.New(now.Add(24 * time.Hour)), - WorkflowRunExpirationTime: timestamppb.New(now.Add(3 * time.Hour)), + NamespaceId: tests.NamespaceID.String(), + WorkflowId: tests.WorkflowID, + FirstExecutionRunId: tests.RunID, + WorkflowExecutionExpirationTime: timestamppb.New(now.Add(24 * time.Hour)), + WorkflowRunExpirationTime: timestamppb.New(now.Add(3 * time.Hour)), + WorkflowExecutionTimerTaskStatus: TimerTaskStatusCreated, // StartTime < ExecutionTime with a cron schedule drives the backoff-timer block // (WORKFLOW_BACKOFF_TYPE_CRON, a non-zero enum value). StartTime: timestamppb.New(now), @@ -1900,7 +2040,7 @@ func TestTaskGeneratorImpl_RegenerateTimerTasksForTimeSkipping_AllFieldsPopulate captured = append(captured, ts...) }).AnyTimes() - taskGenerator := NewTaskGenerator(nil, mutableState, &configs.Config{}, nil, log.NewTestLogger()) + taskGenerator := NewTaskGenerator(nil, mutableState, timeSkippingTestConfig(true), nil, log.NewTestLogger()) require.NoError(t, taskGenerator.RegenerateTimerTasksForTimeSkipping()) // TaskID is set by the shard, never the generator, so it is the one field every task diff --git a/service/history/workflow/task_refresher.go b/service/history/workflow/task_refresher.go index 39caa383041..fc8920de62c 100644 --- a/service/history/workflow/task_refresher.go +++ b/service/history/workflow/task_refresher.go @@ -488,6 +488,12 @@ func (r *TaskRefresherImpl) refreshTasksForTimeSkipping( if tsi == nil { return nil } + + if transitionhistory.Compare(minVersionedTransition, EmptyVersionedTransition) == 0 { + // On a full refresh the per-component helpers regenerate timers with correct virtual + // time conversion. Full time-skipping task regeneration is not necessary. + return taskGenerator.GenerateTimeSkippingFastForwardTimerTask() + } if transitionhistory.Compare( tsi.GetLastUpdateVersionedTransition(), minVersionedTransition, diff --git a/service/history/workflow/task_refresher_test.go b/service/history/workflow/task_refresher_test.go index f3bab190f04..05cc3a7fa6d 100644 --- a/service/history/workflow/task_refresher_test.go +++ b/service/history/workflow/task_refresher_test.go @@ -1453,8 +1453,10 @@ func (s *taskRefresherSuite) TestRefreshTasksForTimeSkipping() { for _, tc := range []struct { name string tsi *persistencespb.TimeSkippingInfo + status enumspb.WorkflowExecutionStatus minVersionedTransition *persistencespb.VersionedTransition wantRegen bool + wantFastForwardRegen bool }{ { // Skip at transition 5, watermark 3 → new to this peer → re-stamp. @@ -1477,8 +1479,19 @@ func (s *taskRefresherSuite) TestRefreshTasksForTimeSkipping() { minVersionedTransition: &persistencespb.VersionedTransition{NamespaceFailoverVersion: common.EmptyVersion, TransitionCount: 4}, wantRegen: false, }, + { + name: "FullRefresh/RegeneratesFastForwardOnly", + tsi: tsiAt(5), + minVersionedTransition: EmptyVersionedTransition, + wantRegen: false, + wantFastForwardRegen: true, + }, } { s.Run(tc.name, func() { + status := tc.status + if status == enumspb.WORKFLOW_EXECUTION_STATUS_UNSPECIFIED { + status = enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + } mutableState, err := NewMutableStateFromDB( s.mockShard, s.mockShard.GetEventsCache(), @@ -1493,7 +1506,7 @@ func (s *taskRefresherSuite) TestRefreshTasksForTimeSkipping() { ExecutionState: &persistencespb.WorkflowExecutionState{ RunId: tests.RunID, State: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, - Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + Status: status, }, NextEventId: int64(20), }, @@ -1506,7 +1519,12 @@ func (s *taskRefresherSuite) TestRefreshTasksForTimeSkipping() { if tc.wantRegen { times = 1 } + fastForwardTimes := 0 + if tc.wantFastForwardRegen { + fastForwardTimes = 1 + } s.mockTaskGenerator.EXPECT().RegenerateTimerTasksForTimeSkipping().Return(nil).Times(times) + s.mockTaskGenerator.EXPECT().GenerateTimeSkippingFastForwardTimerTask().Return(nil).Times(fastForwardTimes) err = s.taskRefresher.refreshTasksForTimeSkipping(mutableState, s.mockTaskGenerator, tc.minVersionedTransition) s.NoError(err) diff --git a/tests/timeskipping_test.go b/tests/timeskipping_test.go index 6008a87babd..5b18eaeb09a 100644 --- a/tests/timeskipping_test.go +++ b/tests/timeskipping_test.go @@ -1,11 +1,14 @@ package tests import ( + "context" "errors" + "sync/atomic" "testing" "time" "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -23,6 +26,8 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/nexus/nexustest" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/taskpoller" @@ -482,6 +487,91 @@ func timeSkippingTransitions(events []*historypb.HistoryEvent) []*historypb.Hist return out } +// indexOfEventType returns the index of the first event with the given type, or -1. +func indexOfEventType(events []*historypb.HistoryEvent, t enumspb.EventType) int { + for i, e := range events { + if e.GetEventType() == t { + return i + } + } + return -1 +} + +// TestTimeSkipping_RetentionClearsSkippedWorkflowImmediately checks the retention deadline +// end-to-end by its observable effect rather than by inspecting a task timestamp. +// +// The namespace has retention 0 and archival disabled, so GenerateWorkflowCloseTasks emits a +// DeleteHistoryEventTask at closeTime + 0. That closeTime is *virtual*: the workflow skips a +// 1-day user timer before completing, so MutableState.AddTasks must subtract the accumulated +// skip for the deadline to land at real close time. If it did not, the deadline would sit a day +// out in wall clock and the execution would still be readable when this test gives up. +func (s *TimeSkippingTestSuite) TestTimeSkipping_RetentionClearsSkippedWorkflowImmediately() { + const skippedTimer = 24 * time.Hour + + env := testcore.NewEnv( + s.T(), + testcore.WithDynamicConfig(dynamicconfig.WorkflowTimeSkippingEnabled, true), + testcore.WithDynamicConfig(dynamicconfig.RetentionTimerJitterDuration, time.Duration(0)), + ) + + // NewEnv's namespace has 1 day retention, which cannot be lowered through the frontend. + ns := namespace.Name(testcore.RandomizeStr("ts-retention")) + _, err := env.RegisterNamespace(ns, 0, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + s.NoError(err) + + tv := testvars.New(s.T()) + startResp, err := env.FrontendClient().StartWorkflowExecution(s.Context(), &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: ns.String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + WorkflowRunTimeout: durationpb.New(48 * time.Hour), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + TimeSkippingConfig: &commonpb.TimeSkippingConfig{Enabled: true}, + }) + s.NoError(err) + execution := &commonpb.WorkflowExecution{WorkflowId: tv.WorkflowID(), RunId: startResp.GetRunId()} + + poller := taskpoller.New(s.T(), env.FrontendClient(), ns.String()) + _, err = poller.PollAndHandleWorkflowTask(tv, func(_ *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{startTimerCmd("timer-1", skippedTimer)}, + }, nil + }) + s.NoError(err) + + wallStart := time.Now() + _, err = poller.PollAndHandleWorkflowTask(tv, func(_ *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{completeWorkflowCmd()}, + }, nil + }) + s.NoError(err) + s.Less(time.Since(wallStart), 5*time.Minute, "the 1 day timer must be skipped, not waited out") + + // Both mutable state and history events go away: DeleteHistoryEventTask drives the full + // execution deletion, so Describe stops resolving and the history read fails too. + s.AwaitTruef(func() bool { + _, describeErr := env.FrontendClient().DescribeWorkflowExecution(s.Context(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: ns.String(), + Execution: execution, + }) + if describeErr == nil { + return false + } + var notFound *serviceerror.NotFound + return errors.As(describeErr, ¬Found) + }, 60*time.Second, 500*time.Millisecond, "mutable state was not deleted at real close time") + + _, err = env.FrontendClient().GetWorkflowExecutionHistory(s.Context(), &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: ns.String(), + Execution: execution, + }) + var notFound *serviceerror.NotFound + s.ErrorAs(err, ¬Found, "history events must be deleted along with mutable state") +} + // TestTimeSkipping_TimerAndActivity verifies that when a workflow has both a long user // timer and a pending activity, time-skipping is blocked until the activity completes. // Once the activity is done and the workflow task is drained, time-skipping fires and @@ -716,6 +806,112 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_PendingSignalExternalBlocksSkip wallElapsed) } +// TestTimeSkipping_PendingNexusOperationBlocksSkip verifies that a pending nexus operation — +// including one parked in retry backoff — blocks time skipping. This is what makes the +// StateMachineTimerTask (nexus HSM) exclusion in RegenerateTimerTasksForTimeSkipping safe: a skip +// can never happen while such a timer is live, so a nexus HSM timer can never be left stale by one. +// +// Sequence: +// +// WT1 → schedule nexus operation + start a 1h timer. Close tx: nexus op pending → no skip. +// Nexus attempt 1 → retryable handler error → operation parks in BACKING_OFF, still pending. +// Nexus attempt 2 → sync success → NexusOperationCompleted → WT2. +// WT2 → drain. Close tx: no pending nexus op, only the 1h timer → skip fires. +// WT3 → timer fired via skip → complete workflow. +// +// The load-bearing assertion is ordering: TIME_SKIPPING_TRANSITIONED must appear *after* +// NEXUS_OPERATION_COMPLETED. That is timing-independent — were the nexus gate missing, the skip +// would fire at WT1 close, well ahead of the operation completing. +func (s *TimeSkippingTestSuite) TestTimeSkipping_PendingNexusOperationBlocksSkip() { + env := newNexusTestEnv(s.T(), true, + testcore.WithDynamicConfig(dynamicconfig.WorkflowTimeSkippingEnabled, true), + ) + tv := testvars.New(s.T()) + ctx := s.Context() + + // Attempt 1 fails retryably so the operation parks in BACKING_OFF with a live nexus HSM + // timer; attempt 2 succeeds so the workflow can make progress and the test can observe the + // skip that follows. + var attempts atomic.Int32 + handler := nexustest.Handler{ + OnStartOperation: func( + _ context.Context, service, _ string, _ *nexus.LazyValue, _ nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + if service != "service" { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, `expected service to equal "service"`) + } + if attempts.Add(1) == 1 { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "intentional retryable error") + } + return &nexus.HandlerStartOperationResultSync[any]{Value: "ok"}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), handler) + + // Run timeout must exceed the 1h timer, or the skip shifts the run-timeout task into the + // past and the workflow times out before WT3. + wallStart := time.Now() + runID := s.startWorkflowWithTimeSkipping(env.TestEnv, tv, 4*time.Hour) + poller := taskpoller.New(s.T(), env.FrontendClient(), env.Namespace().String()) + + // WT1: nexus operation + 1h timer. The nexus operation is pending at close, so no skip. + _, err := poller.PollAndHandleWorkflowTask(tv, func(_ *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpointName, + Service: "service", + Operation: "operation", + Input: testcore.MustToPayload(s.T(), "input"), + }, + }, + }, + startTimerCmd("timer-1", time.Hour), + }, + }, nil + }) + s.NoError(err) + + // WT2 arrives once the nexus operation completes (after the retry). Draining it is the + // first close transaction where the workflow is idle, so the skip fires here. + _, err = poller.PollAndHandleWorkflowTask(tv, taskpoller.DrainWorkflowTask) + s.NoError(err) + + // WT3: the 1h timer fired because the skip re-stamped it to near-now wall. + _, err = poller.PollAndHandleWorkflowTask(tv, func(_ *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{completeWorkflowCmd()}, + }, nil + }) + s.NoError(err) + wallElapsed := time.Since(wallStart) + + history := env.GetHistory(env.Namespace().String(), &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: runID, + }) + + // The operation really did fail once and retry, so a nexus HSM timer was live while the + // workflow was otherwise idle. + s.GreaterOrEqual(int(attempts.Load()), 2, "nexus operation must have been retried at least once") + s.True(hasEventType(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED), "nexus operation must complete") + + nexusCompletedIdx := indexOfEventType(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED) + transitionIdx := indexOfEventType(history, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED) + s.NotEqual(-1, transitionIdx, "skip must fire once the nexus operation is no longer pending") + s.Greater(transitionIdx, nexusCompletedIdx, + "skip must not happen while the nexus operation is pending: TIME_SKIPPING_TRANSITIONED at index %d must follow NEXUS_OPERATION_COMPLETED at index %d", + transitionIdx, nexusCompletedIdx) + + s.True(hasEventType(history, enumspb.EVENT_TYPE_TIMER_FIRED), "timer must fire via the post-nexus skip") + s.True(hasEventType(history, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED), "workflow must complete") + s.Less(wallElapsed, 5*time.Minute, + "wall elapsed = %v; the 1h timer must be skipped once the nexus operation completes", wallElapsed) +} + // TestTimeSkipping_StartWithDelay_NoBound verifies that time-skipping with no // bound shifts a WorkflowStartDelay backoff into the near-now wall-clock window: // the first WT becomes available immediately instead of waiting wallStart + 1h. From 44ae6d566d7c7580574e7eb3fb6643594913a631 Mon Sep 17 00:00:00 2001 From: Kannan Date: Wed, 19 Aug 2026 11:57:58 -0700 Subject: [PATCH 010/139] Fix poller scaling decisions for backlog tasks and rate-limited queues (#11618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What - Don't send scale down signal when task is matched from backlog even if the poll wait time is high. - Do not send scale up signal when task queue is rate limited. ## Why - When a task comes from DB backlog, the poll wait time reflects DB read path latency, not excess pollers — the -1 is not appropriate. Instead we want to apply the normal scale up check. - Similarly, when dispatch is bottlenecked by a task queue rate limit, scaling up pollers won't help. ## How did you test it? Unit tests Co-authored-by: Claude Opus 4.6 --- common/metrics/tags.go | 9 +- .../matching/physical_task_queue_manager.go | 22 +++-- .../physical_task_queue_manager_interface.go | 3 +- .../physical_task_queue_manager_mock.go | 9 +- .../physical_task_queue_manager_test.go | 84 ++++++++++++++----- .../matching/task_queue_partition_manager.go | 2 +- 6 files changed, 95 insertions(+), 34 deletions(-) diff --git a/common/metrics/tags.go b/common/metrics/tags.go index e8ecd03ba7d..b0d60c018e2 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -341,10 +341,11 @@ const ( ) const ( - PollerScaleReasonIdle ReasonString = "idle" - PollerScaleReasonBacklog ReasonString = "backlog" - PollerScaleReasonTaskRate ReasonString = "task_rate" - PollerScaleReasonRateLimited ReasonString = "rate_limited" + PollerScaleReasonIdle ReasonString = "idle" + PollerScaleReasonBacklog ReasonString = "backlog" + PollerScaleReasonTaskRate ReasonString = "task_rate" + PollerScaleReasonRateLimited ReasonString = "rate_limited" + PollerScaleReasonTaskQueueRateLimited ReasonString = "task_queue_rate_limited" ) // PollerScaleDecisionTag records the direction of a poller scaling decision (scale up, scale diff --git a/service/matching/physical_task_queue_manager.go b/service/matching/physical_task_queue_manager.go index 87d6faa27b0..25fb8030f60 100644 --- a/service/matching/physical_task_queue_manager.go +++ b/service/matching/physical_task_queue_manager.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" @@ -879,20 +880,26 @@ func (c *physicalTaskQueueManagerImpl) GetFairnessWeightOverrides() fairnessWeig func (c *physicalTaskQueueManagerImpl) MakePollerScalingDecision( ctx context.Context, - pollStartTime time.Time) *taskqueuepb.PollerScalingDecision { - return c.makePollerScalingDecisionImpl(pollStartTime, func() *taskqueuepb.TaskQueueStats { + pollStartTime time.Time, + taskSource enumsspb.TaskSource, +) *taskqueuepb.PollerScalingDecision { + return c.makePollerScalingDecisionImpl(pollStartTime, taskSource, func() *taskqueuepb.TaskQueueStats { return c.partitionMgr.GetPhysicalQueueAdjustedStats(ctx, c) }) } func (c *physicalTaskQueueManagerImpl) makePollerScalingDecisionImpl( pollStartTime time.Time, + taskSource enumsspb.TaskSource, statsFn func() *taskqueuepb.TaskQueueStats, ) *taskqueuepb.PollerScalingDecision { pollWaitTime := c.partitionMgr.engine.timeSource.Since(pollStartTime) - // If a poller has waited around a while, we can always suggest a decrease. - if pollWaitTime >= c.partitionMgr.config.PollerScalingWaitTime() { - // Decrease if any poll matched after sitting idle for some configured period + // Only suggest scaling down for non-backlog tasks. Sync-matched tasks typically have + // tens of milliseconds latency, so a long wait means the poller was genuinely + // idle — there are more pollers than needed. For backlog tasks, the wait most likely + // comes from the DB read path (write → reader → matcher), not excess pollers. Skip + // the scale-down and fall through to the scale-up checks below. + if pollWaitTime >= c.partitionMgr.config.PollerScalingWaitTime() && taskSource != enumsspb.TASK_SOURCE_DB_BACKLOG { c.recordPollerScaleDecision(metrics.PollerScaleDecisionDown, metrics.PollerScaleReasonIdle) return &taskqueuepb.PollerScalingDecision{ PollRequestDeltaSuggestion: -1, @@ -913,6 +920,11 @@ func (c *physicalTaskQueueManagerImpl) makePollerScalingDecisionImpl( delta := int32(0) var reason metrics.ReasonString stats := statsFn() + // If dispatch is bottlenecked by a task queue rate limit, adding pollers won't help. + if stats.GetRateLimitingActive() { + c.recordPollerScaleDecision(metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonTaskQueueRateLimited) + return nil + } if stats.GetApproximateBacklogCount() > 0 && stats.GetApproximateBacklogAge().AsDuration() > c.partitionMgr.config.PollerScalingBacklogAgeScaleUp() { // Always increase when there is a backlog, even if we're a partition. It's also important to increase for diff --git a/service/matching/physical_task_queue_manager_interface.go b/service/matching/physical_task_queue_manager_interface.go index 72249d26a5a..3c2cd8e3f3e 100644 --- a/service/matching/physical_task_queue_manager_interface.go +++ b/service/matching/physical_task_queue_manager_interface.go @@ -8,6 +8,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" @@ -67,7 +68,7 @@ type ( QueueKey() *PhysicalTaskQueueKey // MakePollerScalingDecision makes a decision on whether to scale pollers up or down based on the current state // of the task queue and the task about to be returned. - MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time) *taskqueuepb.PollerScalingDecision + MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, taskSource enumsspb.TaskSource) *taskqueuepb.PollerScalingDecision // GetFairnessWeightOverrides returns current fairness weight overrides for this queue. GetFairnessWeightOverrides() fairnessWeightOverrides UpdateRemotePriorityBacklogs(remotePriorityBacklogSet) diff --git a/service/matching/physical_task_queue_manager_mock.go b/service/matching/physical_task_queue_manager_mock.go index f6904fc4a1a..cbf42a03414 100644 --- a/service/matching/physical_task_queue_manager_mock.go +++ b/service/matching/physical_task_queue_manager_mock.go @@ -16,6 +16,7 @@ import ( enums "go.temporal.io/api/enums/v1" taskqueue "go.temporal.io/api/taskqueue/v1" + enums0 "go.temporal.io/server/api/enums/v1" matchingservice "go.temporal.io/server/api/matchingservice/v1" persistence "go.temporal.io/server/api/persistence/v1" taskqueue0 "go.temporal.io/server/api/taskqueue/v1" @@ -215,17 +216,17 @@ func (mr *MockphysicalTaskQueueManagerMockRecorder) LegacyDescribeTaskQueue(incl } // MakePollerScalingDecision mocks base method. -func (m *MockphysicalTaskQueueManager) MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time) *taskqueue.PollerScalingDecision { +func (m *MockphysicalTaskQueueManager) MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, taskSource enums0.TaskSource) *taskqueue.PollerScalingDecision { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakePollerScalingDecision", ctx, pollStartTime) + ret := m.ctrl.Call(m, "MakePollerScalingDecision", ctx, pollStartTime, taskSource) ret0, _ := ret[0].(*taskqueue.PollerScalingDecision) return ret0 } // MakePollerScalingDecision indicates an expected call of MakePollerScalingDecision. -func (mr *MockphysicalTaskQueueManagerMockRecorder) MakePollerScalingDecision(ctx, pollStartTime any) *gomock.Call { +func (mr *MockphysicalTaskQueueManagerMockRecorder) MakePollerScalingDecision(ctx, pollStartTime, taskSource any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakePollerScalingDecision", reflect.TypeOf((*MockphysicalTaskQueueManager)(nil).MakePollerScalingDecision), ctx, pollStartTime) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakePollerScalingDecision", reflect.TypeOf((*MockphysicalTaskQueueManager)(nil).MakePollerScalingDecision), ctx, pollStartTime, taskSource) } // MarkAlive mocks base method. diff --git a/service/matching/physical_task_queue_manager_test.go b/service/matching/physical_task_queue_manager_test.go index 0dceca5356a..ef90aa30ffa 100644 --- a/service/matching/physical_task_queue_manager_test.go +++ b/service/matching/physical_task_queue_manager_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/suite" enumspb "go.temporal.io/api/enums/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservicemock/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common" @@ -435,7 +436,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpOnBacklog() { ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -450,7 +451,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpAddRateExceedsDispa TasksDispatchRate: 10, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -464,12 +465,12 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpUsesDefaultRatioThr // 110/100 = 1.1, below the 1.2 default. belowThreshold := &taskqueuepb.TaskQueueStats{TasksAddRate: 110, TasksDispatchRate: 100} - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return belowThreshold }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return belowThreshold }) s.Nil(decision) // 130/100 = 1.3, above the 1.2 default. aboveThreshold := &taskqueuepb.TaskQueueStats{TasksAddRate: 130, TasksDispatchRate: 100} - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return aboveThreshold }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return aboveThreshold }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -483,14 +484,14 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpRatioThresholdIsCon // above the actual ratio should suppress the scale-up. highRatio := &taskqueuepb.TaskQueueStats{TasksAddRate: 100, TasksDispatchRate: 10} s.tqMgr.partitionMgr.config.PollerScalingTaskAddToDispatchRatio = func() float64 { return 20 } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return highRatio }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return highRatio }) s.Nil(decision) // 110/100 = 1.1, which would not scale up under the default 1.2 threshold. Lowering the // threshold below the actual ratio should trigger a scale-up. lowRatio := &taskqueuepb.TaskQueueStats{TasksAddRate: 110, TasksDispatchRate: 100} s.tqMgr.partitionMgr.config.PollerScalingTaskAddToDispatchRatio = func() float64 { return 1.0 } - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return lowRatio }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return lowRatio }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -500,7 +501,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingNoChangeOnNoBacklogFa ApproximateBacklogCount: 0, ApproximateBacklogAge: durationpb.New(0), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -519,12 +520,12 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingNonRootPartition() { ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) fakeStats.ApproximateBacklogCount = 0 - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -547,7 +548,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingStickyQueue() { TasksAddRate: 100, TasksDispatchRate: 10, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -556,10 +557,55 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDownOnLongSyncMatch() fakeStats := &taskqueuepb.TaskQueueStats{ ApproximateBacklogCount: 0, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.LessOrEqual(decision.PollRequestDeltaSuggestion, int32(-1)) } +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingHoldOnLongBacklogMatch() { + // Backlog task with long poll wait and no sustained backlog: no scaling decision. + fakeStats := &taskqueuepb.TaskQueueStats{ + ApproximateBacklogCount: 0, + } + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.Nil(decision) +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingBacklogFallsThruToScaleUp() { + // Backlog task with long poll wait and sustained backlog: scale up (+1). + fakeStats := &taskqueuepb.TaskQueueStats{ + ApproximateBacklogCount: 100, + ApproximateBacklogAge: durationpb.New(1 * time.Minute), + } + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.NotNil(decision) + s.Equal(int32(1), decision.PollRequestDeltaSuggestion) +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingHoldOnTaskQueueRateLimited() { + // Long poll + backlog task + rate limit active: suppress -1 (backlog), then hold instead of +1 (rate limited). + fakeStats := &taskqueuepb.TaskQueueStats{ + ApproximateBacklogCount: 100, + ApproximateBacklogAge: durationpb.New(1 * time.Minute), + RateLimitingActive: true, + } + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.Nil(decision) +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricTaskQueueRateLimited() { + handler := s.enablePollerScaleDecisionMetrics() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + + fakeStats := &taskqueuepb.TaskQueueStats{ + ApproximateBacklogCount: 100, + ApproximateBacklogAge: durationpb.New(1 * time.Minute), + RateLimitingActive: true, + } + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonTaskQueueRateLimited) +} + func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionsAreRateLimited() { rl := quotas.NewMockRateLimiter(s.controller) rl.EXPECT().AllowN(gomock.Any(), gomock.Any()).Return(true).Times(1) @@ -570,10 +616,10 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionsAreRateLimit ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -602,7 +648,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleDo defer handler.StopCapture(capture) fakeStats := &taskqueuepb.TaskQueueStats{ApproximateBacklogCount: 0} - s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionDown, metrics.PollerScaleReasonIdle) } @@ -618,7 +664,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUp ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonBacklog) } @@ -634,7 +680,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUp TasksAddRate: 100, TasksDispatchRate: 10, } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonTaskRate) } @@ -650,7 +696,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricHoldRat ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonRateLimited) } @@ -669,7 +715,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricNoSigna TasksAddRate: 10, TasksDispatchRate: 100, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) s.Empty(capture.Snapshot()[metrics.PollerScaleDecisionCounter.Name()]) } @@ -682,7 +728,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricDisable defer handler.StopCapture(capture) fakeStats := &taskqueuepb.TaskQueueStats{ApproximateBacklogCount: 0} - s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Empty(capture.Snapshot()[metrics.PollerScaleDecisionCounter.Name()]) } diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index c876886cddd..be19db8a949 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -849,7 +849,7 @@ func (pm *taskQueuePartitionManagerImpl) PollTask( task, err := dbq.PollTask(ctx, pollMetadata) if task != nil { - task.pollerScalingDecision = dbq.MakePollerScalingDecision(ctx, pollMetadata.localPollStartTime) + task.pollerScalingDecision = dbq.MakePollerScalingDecision(ctx, pollMetadata.localPollStartTime, task.source) } // Update poller timestamp when poll ends, unless cancelled (e.g., shutdown/disconnect). From bfb8142d21b516f6b30539260c9e4840c7fd1a30 Mon Sep 17 00:00:00 2001 From: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Date: Wed, 19 Aug 2026 11:59:07 -0700 Subject: [PATCH 011/139] Fix batch operations targeting paused executions (#11642) ## What changed? Batch operations now target Paused executions in addition to Running ones. The filter auto-appended by adjustQueryBatchTypeEnum changed from ExecutionStatus='Running' to ExecutionStatus='Running' OR ExecutionStatus='Paused', affecting all workflow batch types (terminate/signal/cancel/update-options) and activity batch types (unpause/update-options/reset/terminate/cancel). ## Why? A paused execution is still non-terminal. Users issuing a batch terminate/signal/cancel reasonably expect it to apply to paused targets, but the old Running-only filter silently skipped them. ## How did you test it? - [X] built - [ ] run locally and tested manually - [X] covered by existing tests - [X] added new unit test(s) - [X] added new functional test(s) ## Potential risks - The filter now references ExecutionStatus='Paused' on every batch operation. This is ok as it's an additive clause to the visibilty query - This would be changing behavior for batch callers as paused activities are now affected. --- service/worker/batcher/activities.go | 15 +++-- service/worker/batcher/activities_test.go | 54 ++++++++++------ tests/activity_api_batch_cancel_test.go | 37 +++++++---- tests/activity_api_batch_terminate_test.go | 69 +++++++++++++------- tests/workflow_api_batch_signal_test.go | 74 ++++++++++++++++++++++ tests/workflow_api_batch_terminate_test.go | 1 + 6 files changed, 190 insertions(+), 60 deletions(-) diff --git a/service/worker/batcher/activities.go b/service/worker/batcher/activities.go index 3226051efb4..d9b1934e891 100644 --- a/service/worker/batcher/activities.go +++ b/service/worker/batcher/activities.go @@ -37,8 +37,8 @@ import ( ) const ( - pageSize = 1000 - statusRunningQueryFilter = "ExecutionStatus='Running'" + pageSize = 1000 + statusRunningOrPausedQueryFilter = "ExecutionStatus='Running' OR ExecutionStatus='Paused'" // defaultTaskTimeout bounds how long processing a single task may take so // that one hung operation cannot block the task processor forever. @@ -540,9 +540,10 @@ func (a *activities) adjustQueryBatchTypeEnum(query string, batchType enumspb.Ba enumspb.BATCH_OPERATION_TYPE_SIGNAL, enumspb.BATCH_OPERATION_TYPE_SIGNAL_WORKFLOW, enumspb.BATCH_OPERATION_TYPE_CANCEL, enumspb.BATCH_OPERATION_TYPE_CANCEL_WORKFLOW, enumspb.BATCH_OPERATION_TYPE_UPDATE_EXECUTION_OPTIONS, enumspb.BATCH_OPERATION_TYPE_UPDATE_WORKFLOW_EXECUTION_OPTIONS, - enumspb.BATCH_OPERATION_TYPE_UNPAUSE_ACTIVITY, enumspb.BATCH_OPERATION_TYPE_UPDATE_ACTIVITY_OPTIONS, enumspb.BATCH_OPERATION_TYPE_RESET_ACTIVITY, - enumspb.BATCH_OPERATION_TYPE_TERMINATE_ACTIVITY, enumspb.BATCH_OPERATION_TYPE_CANCEL_ACTIVITY: - return fmt.Sprintf("(%s) AND (%s)", query, statusRunningQueryFilter) + enumspb.BATCH_OPERATION_TYPE_UNPAUSE_ACTIVITY, enumspb.BATCH_OPERATION_TYPE_UPDATE_ACTIVITY_OPTIONS, + enumspb.BATCH_OPERATION_TYPE_RESET_ACTIVITY, enumspb.BATCH_OPERATION_TYPE_TERMINATE_ACTIVITY, + enumspb.BATCH_OPERATION_TYPE_CANCEL_ACTIVITY: + return fmt.Sprintf("(%s) AND (%s)", query, statusRunningOrPausedQueryFilter) default: return query } @@ -867,8 +868,8 @@ func isNonRetryableError(err error, batchType enumspb.BatchOperationType) bool { // spending every AttemptsOnRetryableError -- processTaskWithRetries // retries in place with no backoff. // - // Only these two are listed. UNPAUSE_ACTIVITY, UPDATE_ACTIVITY_OPTIONS - // and RESET_ACTIVITY get the same Running filter appended by + // Only these two are listed. UNPAUSE_ACTIVITY, UPDATE_ACTIVITY_OPTIONS, + // and RESET_ACTIVITY get the same open-status filter appended by // adjustQueryBatchTypeEnum and so reach this path too, but their // FailedPreconditions include states a retry does clear (pending reset, // pending cancellation, deferred Reset(RestoreOriginalOptions)). diff --git a/service/worker/batcher/activities_test.go b/service/worker/batcher/activities_test.go index 39b951b131d..0365f66f99d 100644 --- a/service/worker/batcher/activities_test.go +++ b/service/worker/batcher/activities_test.go @@ -371,31 +371,31 @@ func (s *activitiesSuite) TestAdjustQueryBatchTypeEnum() { { name: "Acceptance", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, { name: "Acceptance with parenthesis", query: "(A=B)", - expectedResult: fmt.Sprintf("((A=B)) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("((A=B)) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, { name: "Acceptance with multiple conditions", query: "(A=B) OR C=D", - expectedResult: fmt.Sprintf("((A=B) OR C=D) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("((A=B) OR C=D) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, { name: "Contains status - 1", query: "ExecutionStatus=Completed", - expectedResult: fmt.Sprintf("(ExecutionStatus=Completed) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(ExecutionStatus=Completed) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, { name: "Contains status - 2", query: "A=B OR ExecutionStatus='Completed'", - expectedResult: fmt.Sprintf("(A=B OR ExecutionStatus='Completed') AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B OR ExecutionStatus='Completed') AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, { @@ -407,53 +407,53 @@ func (s *activitiesSuite) TestAdjustQueryBatchTypeEnum() { { name: "Terminate workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_WORKFLOW, }, //nolint:staticcheck // SA1019: verifies batches started before the enum split { name: "Terminate legacy workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE, }, { name: "Cancel workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_CANCEL_WORKFLOW, }, //nolint:staticcheck // SA1019: verifies batches started before the enum split { name: "Cancel legacy workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_CANCEL, }, { name: "Signal workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_SIGNAL_WORKFLOW, }, //nolint:staticcheck // SA1019: verifies batches started before the enum split { name: "Signal legacy workflow variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_SIGNAL, }, { name: "Update workflow execution options variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_UPDATE_WORKFLOW_EXECUTION_OPTIONS, }, //nolint:staticcheck // SA1019: verifies batches started before the enum split { name: "Update legacy workflow execution options variant", query: "A=B", - expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningQueryFilter), + expectedResult: fmt.Sprintf("(A=B) AND (%s)", statusRunningOrPausedQueryFilter), batchType: enumspb.BATCH_OPERATION_TYPE_UPDATE_EXECUTION_OPTIONS, }, { @@ -487,19 +487,37 @@ func (s *activitiesSuite) TestAdjustQueryBatchTypeEnum() { batchType: enumspb.BATCH_OPERATION_TYPE_DELETE, }, { - // A caller must be able to terminate/cancel only running activities via + // A caller must be able to terminate/cancel non-terminal activities via // query; the server adds the filter so the caller doesn't have to. - name: "Terminate activity is filtered to running", + name: "Terminate activity is filtered to running and paused", query: "ActivityType='foo'", - expectedResult: fmt.Sprintf("(ActivityType='foo') AND (%s)", statusRunningQueryFilter), + expectedResult: "(ActivityType='foo') AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", batchType: enumspb.BATCH_OPERATION_TYPE_TERMINATE_ACTIVITY, }, { - name: "Cancel activity is filtered to running", + name: "Cancel activity is filtered to running and paused", query: "ActivityType='foo'", - expectedResult: fmt.Sprintf("(ActivityType='foo') AND (%s)", statusRunningQueryFilter), + expectedResult: "(ActivityType='foo') AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", batchType: enumspb.BATCH_OPERATION_TYPE_CANCEL_ACTIVITY, }, + { + name: "Unpause workflow activity is filtered to running and paused workflows", + query: "WorkflowType='foo'", + expectedResult: "(WorkflowType='foo') AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", + batchType: enumspb.BATCH_OPERATION_TYPE_UNPAUSE_ACTIVITY, + }, + { + name: "Update workflow activity options is filtered to running and paused workflows", + query: "WorkflowType='foo'", + expectedResult: "(WorkflowType='foo') AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", + batchType: enumspb.BATCH_OPERATION_TYPE_UPDATE_ACTIVITY_OPTIONS, + }, + { + name: "Reset workflow activity is filtered to running and paused workflows", + query: "WorkflowType='foo'", + expectedResult: "(WorkflowType='foo') AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", + batchType: enumspb.BATCH_OPERATION_TYPE_RESET_ACTIVITY, + }, { // Delete applies regardless of execution status, so no filter is added. name: "Delete activity is not filtered", diff --git a/tests/activity_api_batch_cancel_test.go b/tests/activity_api_batch_cancel_test.go index 8003b2214cb..d3b48307bf5 100644 --- a/tests/activity_api_batch_cancel_test.go +++ b/tests/activity_api_batch_cancel_test.go @@ -84,11 +84,9 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Success( } } -// TestActivityBatchCancel_ExcludesNonRunning verifies that a batch cancel query -// omitting `ExecutionStatus = 'Running'` still only targets the running -// activity: the server must add the filter automatically (adjustQueryBatchTypeEnum) -// so an already-completed activity of the same type is excluded, not (re-)canceled. -func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_ExcludesNonRunning() { +// TestActivityBatchCancel_TargetsRunningAndPaused verifies that a batch cancel +// query targets non-terminal activities while excluding completed activities. +func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_TargetsRunningAndPaused() { env := newStandaloneActivityBatchEnv(s.T()) t := s.T() ctx := s.Context() @@ -99,6 +97,16 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Excludes // task can't race and pick up the running activity's task instead. runningID := testcore.RandomizeStr(fmt.Sprintf("%s-running", t.Name())) runningResp := env.startAndValidateActivity(ctx, t, runningID, testcore.RandomizeStr(fmt.Sprintf("%s-running-tq", t.Name()))) + pausedID := testcore.RandomizeStr(fmt.Sprintf("%s-paused", t.Name())) + pausedResp := env.startAndValidateActivity(ctx, t, pausedID, testcore.RandomizeStr(fmt.Sprintf("%s-paused-tq", t.Name()))) + _, err := env.FrontendClient().PauseActivityExecution(ctx, &workflowservice.PauseActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: pausedID, + RunId: pausedResp.RunId, + Identity: "test-identity", + Reason: "test-pause", + }) + require.NoError(t, err) // This activity completes normally before the batch runs, so it must be // excluded from a batch operation scoped only by ActivityType. @@ -106,7 +114,7 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Excludes completedID := testcore.RandomizeStr(fmt.Sprintf("%s-completed", t.Name())) completedResp := env.startAndValidateActivity(ctx, t, completedID, completedTaskQueue) pollResp := env.pollActivityTaskAndValidate(ctx, t, completedID, completedTaskQueue, completedResp.RunId) - _, err := env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp.TaskToken, Result: defaultResult, @@ -116,9 +124,13 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Excludes // Query intentionally omits ExecutionStatus = 'Running'. query := fmt.Sprintf("ActivityType = '%s'", activityType) - // Wait for both activities to be indexed *and* for the completed one to no - // longer be indexed as Running, so the batch sees exactly one target. - waitForRunningFilterToSettle(ctx, t, env, query, 2, 1) + // Wait for all activities to be indexed and the batch filter to see the + // running and paused activities, but not the completed activity. + waitForActivityBatchFilterToSettle(ctx, t, env, query, map[string]enumspb.ActivityExecutionStatus{ + runningID: enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING, + pausedID: enumspb.ACTIVITY_EXECUTION_STATUS_PAUSED, + completedID: enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + }) jobID := uuid.NewString() _, err = env.SdkClient().WorkflowService().StartBatchOperation(ctx, &workflowservice.StartBatchOperationRequest{ @@ -135,8 +147,8 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Excludes }) s.NoError(err) - // The server-added ExecutionStatus = 'Running' filter must exclude the - // already-completed activity: only 1 execution is ever targeted, not 2. + // The server-added open-status filter must include the paused activity and + // exclude the completed activity. //nolint:forbidigo // tests with waits s.EventuallyWithT(func(c *assert.CollectT) { desc, err := env.FrontendClient().DescribeBatchOperation(ctx, &workflowservice.DescribeBatchOperationRequest{ @@ -144,10 +156,11 @@ func (s *ActivityAPIBatchCancelClientTestSuite) TestActivityBatchCancel_Excludes JobId: jobID, }) require.NoError(c, err) - require.EqualValues(c, 1, desc.GetTotalOperationCount()) + require.EqualValues(c, 2, desc.GetTotalOperationCount()) }, 10*time.Second, 200*time.Millisecond) env.eventuallyCanceled(ctx, t, runningID, runningResp.RunId) + env.eventuallyCanceled(ctx, t, pausedID, pausedResp.RunId) // The already-completed activity must remain untouched by the batch. descResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ diff --git a/tests/activity_api_batch_terminate_test.go b/tests/activity_api_batch_terminate_test.go index af730f65ada..56c05c6a823 100644 --- a/tests/activity_api_batch_terminate_test.go +++ b/tests/activity_api_batch_terminate_test.go @@ -59,6 +59,7 @@ func newStandaloneActivityBatchEnvWithBatchOperations(t *testing.T, enabled bool cluster.OverrideDynamicConfig(t, dynamicconfig.EnableChasm, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.Enabled, nsValues(true)) cluster.OverrideDynamicConfig(t, activity.EnableCallbacks, nsValues(true)) + cluster.OverrideDynamicConfig(t, activity.EnableStandaloneActivityOperatorCommands, nsValues(true)) cluster.OverrideDynamicConfig(t, dynamicconfig.FrontendEnableBatchOperationsForStandaloneActivities, nsValues(enabled)) return env } @@ -113,21 +114,24 @@ func assertBatchOperationType( }, 10*time.Second, 200*time.Millisecond) } -// waitForRunningFilterToSettle waits until visibility has settled on the target -// set that a terminate/cancel batch over `query` will actually see: `wantTotal` -// executions match `query`, and `wantRunning` of those also match the -// `ExecutionStatus='Running'` filter the batcher appends -// (adjustQueryBatchTypeEnum). -func waitForRunningFilterToSettle( +// waitForActivityBatchFilterToSettle waits until visibility has settled on the +// exact target set that a terminate/cancel batch over `query` will see. +func waitForActivityBatchFilterToSettle( ctx context.Context, t *testing.T, env *standaloneActivityEnv, query string, - wantTotal int, - wantRunning int, + expectedStatuses map[string]enumspb.ActivityExecutionStatus, ) { t.Helper() + wantOpen := 0 + for _, status := range expectedStatuses { + if status == enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING || status == enumspb.ACTIVITY_EXECUTION_STATUS_PAUSED { + wantOpen++ + } + } + //nolint:forbidigo // for tests with waits require.EventuallyWithT(t, func(c *assert.CollectT) { listResp, err := env.FrontendClient().ListActivityExecutions(ctx, &workflowservice.ListActivityExecutionsRequest{ @@ -136,16 +140,22 @@ func waitForRunningFilterToSettle( Query: query, }) require.NoError(c, err) - require.Len(c, listResp.GetExecutions(), wantTotal) + require.Len(c, listResp.GetExecutions(), len(expectedStatuses)) + + actualStatuses := make(map[string]enumspb.ActivityExecutionStatus, len(listResp.GetExecutions())) + for _, execution := range listResp.GetExecutions() { + actualStatuses[execution.GetActivityId()] = execution.GetStatus() + } + require.Equal(c, expectedStatuses, actualStatuses) // Mirrors the query the batcher itself counts with, so this waits on // exactly the value that drives TotalOperationCount. countResp, err := env.FrontendClient().CountActivityExecutions(ctx, &workflowservice.CountActivityExecutionsRequest{ Namespace: env.Namespace().String(), - Query: fmt.Sprintf("(%s) AND (ExecutionStatus='Running')", query), + Query: fmt.Sprintf("(%s) AND (ExecutionStatus='Running' OR ExecutionStatus='Paused')", query), }) require.NoError(c, err) - require.EqualValues(c, wantRunning, countResp.GetCount()) + require.EqualValues(c, wantOpen, countResp.GetCount()) }, 20*time.Second, 100*time.Millisecond) } @@ -386,11 +396,9 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_No }, 10*time.Second, 200*time.Millisecond) } -// TestActivityBatchTerminate_ExcludesNonRunning verifies that a batch terminate -// query omitting `ExecutionStatus = 'Running'` still only targets the running -// activity: the server must add the filter automatically (adjustQueryBatchTypeEnum) -// so an already-completed activity of the same type is excluded, not (re-)terminated. -func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_ExcludesNonRunning() { +// TestActivityBatchTerminate_TargetsRunningAndPaused verifies that a batch terminate +// query targets non-terminal activities while excluding completed activities. +func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_TargetsRunningAndPaused() { env := newStandaloneActivityBatchEnv(s.T()) t := s.T() ctx := s.Context() @@ -401,6 +409,16 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_Ex // task can't race and pick up the running activity's task instead. runningID := testcore.RandomizeStr(fmt.Sprintf("%s-running", t.Name())) runningResp := env.startAndValidateActivity(ctx, t, runningID, testcore.RandomizeStr(fmt.Sprintf("%s-running-tq", t.Name()))) + pausedID := testcore.RandomizeStr(fmt.Sprintf("%s-paused", t.Name())) + pausedResp := env.startAndValidateActivity(ctx, t, pausedID, testcore.RandomizeStr(fmt.Sprintf("%s-paused-tq", t.Name()))) + _, err := env.FrontendClient().PauseActivityExecution(ctx, &workflowservice.PauseActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: pausedID, + RunId: pausedResp.RunId, + Identity: "test-identity", + Reason: "test-pause", + }) + require.NoError(t, err) // This activity completes normally before the batch runs, so it must be // excluded from a batch operation scoped only by ActivityType. @@ -408,7 +426,7 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_Ex completedID := testcore.RandomizeStr(fmt.Sprintf("%s-completed", t.Name())) completedResp := env.startAndValidateActivity(ctx, t, completedID, completedTaskQueue) pollResp := env.pollActivityTaskAndValidate(ctx, t, completedID, completedTaskQueue, completedResp.RunId) - _, err := env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp.TaskToken, Result: defaultResult, @@ -418,9 +436,13 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_Ex // Query intentionally omits ExecutionStatus = 'Running'. query := fmt.Sprintf("ActivityType = '%s'", activityType) - // Wait for both activities to be indexed *and* for the completed one to no - // longer be indexed as Running, so the batch sees exactly one target. - waitForRunningFilterToSettle(ctx, t, env, query, 2, 1) + // Wait for all activities to be indexed and the batch filter to see the + // running and paused activities, but not the completed activity. + waitForActivityBatchFilterToSettle(ctx, t, env, query, map[string]enumspb.ActivityExecutionStatus{ + runningID: enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING, + pausedID: enumspb.ACTIVITY_EXECUTION_STATUS_PAUSED, + completedID: enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + }) jobID := uuid.NewString() _, err = env.SdkClient().WorkflowService().StartBatchOperation(ctx, &workflowservice.StartBatchOperationRequest{ @@ -437,8 +459,8 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_Ex }) s.NoError(err) - // The server-added ExecutionStatus = 'Running' filter must exclude the - // already-completed activity: only 1 execution is ever targeted, not 2. + // The server-added open-status filter must include the paused activity and + // exclude the completed activity. //nolint:forbidigo // for tests with waits s.EventuallyWithT(func(c *assert.CollectT) { desc, err := env.FrontendClient().DescribeBatchOperation(ctx, &workflowservice.DescribeBatchOperationRequest{ @@ -446,10 +468,11 @@ func (s *ActivityAPIBatchTerminateClientTestSuite) TestActivityBatchTerminate_Ex JobId: jobID, }) require.NoError(c, err) - require.EqualValues(c, 1, desc.GetTotalOperationCount()) + require.EqualValues(c, 2, desc.GetTotalOperationCount()) }, 10*time.Second, 200*time.Millisecond) env.eventuallyTerminated(ctx, t, runningID, runningResp.RunId) + env.eventuallyTerminated(ctx, t, pausedID, pausedResp.RunId) // The already-completed activity must remain untouched by the batch. descResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ diff --git a/tests/workflow_api_batch_signal_test.go b/tests/workflow_api_batch_signal_test.go index 6f20cf61346..d30e720869d 100644 --- a/tests/workflow_api_batch_signal_test.go +++ b/tests/workflow_api_batch_signal_test.go @@ -84,6 +84,80 @@ func (s *WorkflowAPIBatchSignalClientTestSuite) TestWorkflowBatchSignal_Separate s.Equal("first-signal-data,second-signal-data", result) } +func (s *WorkflowAPIBatchSignalClientTestSuite) TestWorkflowBatchSignal_PausedWorkflow() { + env := newWorkflowBatchEnv(s.T()) + t := s.T() + ctx := s.Context() + + workflowType := testcore.RandomizeStr(t.Name()) + env.SdkWorker().RegisterWorkflowWithOptions(signalReceivingWorkflow, workflow.RegisterOptions{Name: workflowType}) + + run, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: testcore.RandomizeStr(t.Name()), + TaskQueue: env.WorkerTaskQueue(), + }, workflowType) + require.NoError(t, err) + + _, err = env.FrontendClient().PauseWorkflowExecution(ctx, &workflowservice.PauseWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + Identity: "batch-signal-test", + Reason: "test", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + + query := fmt.Sprintf("WorkflowId = '%s'", run.GetID()) + //nolint:forbidigo // for tests with waits + require.EventuallyWithT(t, func(c *assert.CollectT) { + resp, err := env.FrontendClient().ListWorkflowExecutions(ctx, &workflowservice.ListWorkflowExecutionsRequest{ + Namespace: env.Namespace().String(), + PageSize: 1, + Query: fmt.Sprintf("%s AND ExecutionStatus = 'Paused'", query), + }) + require.NoError(c, err) + require.Len(c, resp.GetExecutions(), 1) + }, 10*time.Second, 100*time.Millisecond) + + inputPayloads, err := converter.GetDefaultDataConverter().ToPayloads("signal-while-paused") + require.NoError(t, err) + jobID := uuid.NewString() + _, err = env.SdkClient().WorkflowService().StartBatchOperation(ctx, &workflowservice.StartBatchOperationRequest{ + Namespace: env.Namespace().String(), + Operation: &workflowservice.StartBatchOperationRequest_SignalOperation{ + SignalOperation: &batchpb.BatchOperationSignal{ + Signal: batchSignalTestSignalName, + Input: inputPayloads, + Identity: "batch-signaler", + }, + }, + VisibilityQuery: query, + JobId: jobID, + Reason: "test", + }) + require.NoError(t, err) + assertWorkflowBatchOperationSucceeded(ctx, t, env, jobID) + + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID()) + require.NoError(t, err) + require.Equal(t, enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, desc.GetWorkflowExecutionInfo().GetStatus()) + + _, err = env.FrontendClient().UnpauseWorkflowExecution(ctx, &workflowservice.UnpauseWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowId: run.GetID(), + RunId: run.GetRunID(), + Identity: "batch-signal-test", + Reason: "test", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + + var result string + require.NoError(t, run.Get(ctx, &result)) + require.Equal(t, "signal-while-paused", result) +} + // signalWorkflowViaBatch starts a batch signal operation targeting a single // workflow execution and returns the batch job ID. func (s *WorkflowAPIBatchSignalClientTestSuite) signalWorkflowViaBatch( diff --git a/tests/workflow_api_batch_terminate_test.go b/tests/workflow_api_batch_terminate_test.go index 8883b0cb39b..ca30a36a859 100644 --- a/tests/workflow_api_batch_terminate_test.go +++ b/tests/workflow_api_batch_terminate_test.go @@ -39,6 +39,7 @@ func newWorkflowBatchEnv(t *testing.T) *testcore.TestEnv { t, testcore.WithWorkerService("batch operations"), testcore.WithDynamicConfig(dynamicconfig.FrontendMaxConcurrentBatchOperationPerNamespace, testcore.ClientSuiteLimit), + testcore.WithDynamicConfig(dynamicconfig.WorkflowPauseEnabled, true), ) } From 5e43259a3624a98048cb3bc277006fb4a73bb75f Mon Sep 17 00:00:00 2001 From: Lakshay <54310363+Lakshaymiddha@users.noreply.github.com> Date: Thu, 20 Aug 2026 00:29:53 +0530 Subject: [PATCH 012/139] Fix data race in UpdateWithStart: read ExecutionState.Status before releasing workflow lock (#11602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Captured `ExecutionState.Status` before calling `GetReleaseFn()(nil)` in `updateWithStart.Invoke` to avoid a data race with concurrent goroutines that may acquire the lock and modify `ExecutionState` after it is released. Added a regression test that confirms the race under `-race`. Fixes #11600 ## Why? `Invoke` in `service/history/api/multioperation/api.go` releases the workflow lock at line 196 via `workflowLease.GetReleaseFn()(nil)`, then reads `workflowLease.GetMutableState().GetExecutionState().Status` at line 201 — after the lock is released. Any concurrent goroutine waiting on `Lock()` for the same workflow (e.g. a signal, terminate, or another update) can acquire the lock and modify `ExecutionState` between lines 196 and 201, creating a data race. This matches the pattern noted in the `Updater` struct itself: > WARNING: any references to mutable state data *have to* be copied to avoid data races when used outside the workflow lease. ## How did you test it? - [x] added new unit test(s) The test spawns a concurrent writer that modifies `ExecutionState.Status` after the lock is released. **Before fix:** ``` go test -race -tags test_dep -count=1 \ -run TestUpdateWithStartSuite/TestInvoke_CompletedUpdate_StatusCapturedBeforeRelease \ ./service/history/api/multioperation/ WARNING: DATA RACE Read at ... api.go:201 --- FAIL ``` **After fix:** ``` ok go.temporal.io/server/service/history/api/multioperation ``` ## Potential risks Minimal - single line moved before the release call. No API or persistence behavior change. --- service/history/api/multioperation/api.go | 5 +- .../history/api/multioperation/api_test.go | 172 ++++++++++++++++++ 2 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 service/history/api/multioperation/api_test.go diff --git a/service/history/api/multioperation/api.go b/service/history/api/multioperation/api.go index 91150502b62..ac88ec15390 100644 --- a/service/history/api/multioperation/api.go +++ b/service/history/api/multioperation/api.go @@ -193,12 +193,15 @@ func (uws *updateWithStart) Invoke(ctx context.Context) (*historyservice.Execute // If Update is complete, return it. if outcome, err := workflowLease.GetMutableState().GetUpdateOutcome(ctx, updateID); err == nil { workflowKey := workflowLease.GetContext().GetWorkflowKey() + // Capture status before releasing the lock to avoid a data race with concurrent goroutines that may + // modify ExecutionState after acquiring the lock. + status := workflowLease.GetMutableState().GetExecutionState().Status workflowLease.GetReleaseFn()(nil) return makeResponse( &historyservice.StartWorkflowExecutionResponse{ RunId: workflowKey.RunID, Started: false, // set explicitly for emphasis - Status: workflowLease.GetMutableState().GetExecutionState().Status, + Status: status, }, uws.updater.CreateResponse(workflowKey, outcome, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED), ), nil diff --git a/service/history/api/multioperation/api_test.go b/service/history/api/multioperation/api_test.go new file mode 100644 index 00000000000..8b2cce60fad --- /dev/null +++ b/service/history/api/multioperation/api_test.go @@ -0,0 +1,172 @@ +package multioperation + +import ( + "context" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + enumspb "go.temporal.io/api/enums/v1" + updatepb "go.temporal.io/api/update/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/updateworkflow" + historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/tests" + "go.uber.org/mock/gomock" +) + +const ( + testUpdateID = "test-update-id" +) + +type ( + updateWithStartSuite struct { + suite.Suite + *require.Assertions + + controller *gomock.Controller + + currentContext *historyi.MockWorkflowContext + currentMutableState *historyi.MockMutableState + consistencyChecker *api.MockWorkflowConsistencyChecker + } +) + +func TestUpdateWithStartSuite(t *testing.T) { + s := new(updateWithStartSuite) + suite.Run(t, s) +} + +func (s *updateWithStartSuite) SetupSuite() { +} + +func (s *updateWithStartSuite) TearDownSuite() { +} + +func (s *updateWithStartSuite) SetupTest() { + s.Assertions = require.New(s.T()) + + s.controller = gomock.NewController(s.T()) + + s.currentContext = historyi.NewMockWorkflowContext(s.controller) + s.currentMutableState = historyi.NewMockMutableState(s.controller) + s.consistencyChecker = api.NewMockWorkflowConsistencyChecker(s.controller) +} + +func (s *updateWithStartSuite) TearDownTest() { + s.controller.Finish() +} + +// TestInvoke_CompletedUpdate_StatusCapturedBeforeRelease verifies that when +// Invoke finds a completed update outcome, it captures ExecutionState.Status +// before releasing the workflow lock. A concurrent writer that starts after +// the lock is released must not race with the Status read. +// +// Run with -race to verify: +// +// go test -race -tags test_dep -count=1 \ +// -run TestUpdateWithStartSuite/TestInvoke_CompletedUpdate_StatusCapturedBeforeRelease \ +// ./service/history/api/multioperation/ +func (s *updateWithStartSuite) TestInvoke_CompletedUpdate_StatusCapturedBeforeRelease() { + ctx := context.Background() + + // executionState is the race target: the concurrent writer modifies Status + // after the lock is released. This must be a mutable struct so the + // goroutine can write to it. + executionState := &persistencespb.WorkflowExecutionState{ + RunId: tests.RunID, + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + } + + // writerStarted is closed once the writer goroutine has begun writing, + // ensuring the concurrent write is in progress before Invoke reads Status. + lockReleased := make(chan struct{}) + writerStarted := make(chan struct{}) + var stop atomic.Bool + + // Track that release was called with nil error (clean release, not error path). + var releaseCalledWithNilErr atomic.Bool + + releaseFn := historyi.ReleaseWorkflowContextFunc(func(err error) { + if err == nil { + releaseCalledWithNilErr.Store(true) + } + close(lockReleased) + // Wait until the writer goroutine has started writing before returning, + // so the subsequent Status read in Invoke overlaps with the write. + <-writerStarted + }) + + lease := api.NewWorkflowLease(s.currentContext, releaseFn, s.currentMutableState) + + s.consistencyChecker.EXPECT(). + GetWorkflowLease(gomock.Any(), nil, gomock.Any(), gomock.Any()). + Return(lease, nil) + + s.currentMutableState.EXPECT(). + GetUpdateOutcome(gomock.Any(), gomock.Any()). + Return(&updatepb.Outcome{}, nil) + + s.currentMutableState.EXPECT(). + GetExecutionState(). + Return(executionState). + AnyTimes() + + s.currentContext.EXPECT(). + GetWorkflowKey(). + Return(tests.WorkflowKey). + AnyTimes() + + // Concurrent writer: simulates another goroutine that acquires the lock + // after release and modifies the execution state. Signals writerStarted + // after the first write so the release function knows it's safe to return. + go func() { + <-lockReleased + executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED + close(writerStarted) + for !stop.Load() { + executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED + } + }() + defer stop.Store(true) + + updateReq := createUpdateRequest(testUpdateID) + + uws := &updateWithStart{ + namespaceId: tests.NamespaceID, + consistencyChecker: s.consistencyChecker, + updateReq: updateReq, + startReq: &historyservice.StartWorkflowExecutionRequest{ + StartRequest: &workflowservice.StartWorkflowExecutionRequest{ + WorkflowId: tests.WorkflowID, + }, + }, + updater: updateworkflow.NewUpdater(nil, nil, nil, updateReq), + } + + resp, err := uws.Invoke(ctx) + s.NoError(err) + s.NotNil(resp) + + startResp := resp.Responses[0].GetStartWorkflow() + s.Equal(tests.RunID, startResp.RunId) + s.False(startResp.Started) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, startResp.Status) + s.True(releaseCalledWithNilErr.Load(), "release function should have been called with nil error") +} + +func createUpdateRequest(updateID string) *historyservice.UpdateWorkflowExecutionRequest { + return &historyservice.UpdateWorkflowExecutionRequest{ + Request: &workflowservice.UpdateWorkflowExecutionRequest{ + Request: &updatepb.Request{ + Meta: &updatepb.Meta{ + UpdateId: updateID, + }, + }, + }, + } +} From 6cea9e6eb4eee555c9115eaf8b6f10431c355f8c Mon Sep 17 00:00:00 2001 From: samm Date: Wed, 19 Aug 2026 13:05:11 -0700 Subject: [PATCH 013/139] Applies small fixes for chasm nexus operations (#11605) ## What changed? 1. Binds httpCaller after setting httpClient 2. Uses TransitionStarted.Possible for complete-before-start 3. Records request time for nexus operation cancel ## Why? 1. Binds httpCaller after setting httpClient When clusterID isn't set, a non-nil httpCaller with a nil receiver gets passed to `nexusrpc.NewHTTPClient`. Since the httpCaller is non-nil, it will skip the check that sets the nil httpCaller to the default caller. 2. Uses TransitionStarted.Possible for complete-before-start Matches HSM, for if a start response is lost, and a completion lands while while the operation is in BACKING_OFF. 3. Records request time for nexus operation cancel This just seems like it wasn't being recorded. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- chasm/lib/nexusoperation/fx.go | 32 ++--- chasm/lib/nexusoperation/fx_test.go | 137 +++++++++++++++++++++ chasm/lib/nexusoperation/handler.go | 8 +- chasm/lib/nexusoperation/operation.go | 2 +- chasm/lib/nexusoperation/operation_test.go | 35 ++++++ chasm/lib/workflow/nexus_events.go | 3 +- chasm/lib/workflow/nexus_events_test.go | 6 +- tests/nexus_standalone_test.go | 4 + 8 files changed, 205 insertions(+), 22 deletions(-) create mode 100644 chasm/lib/nexusoperation/fx_test.go diff --git a/chasm/lib/nexusoperation/fx.go b/chasm/lib/nexusoperation/fx.go index 364451fab8d..96db2bbcabf 100644 --- a/chasm/lib/nexusoperation/fx.go +++ b/chasm/lib/nexusoperation/fx.go @@ -144,7 +144,9 @@ func clientProviderFactory( return func(ctx context.Context, namespaceID string, entry *persistencespb.NexusEndpointEntry, service string) (*nexusrpc.HTTPClient, error) { var url string var httpClient *http.Client - httpCaller := httpClient.Do + // Populate source header for worker targets, and route internally. Callback assumes external target if unset. + needsCallbackSourceHeader := false + switch variant := entry.Endpoint.Spec.Target.Variant.(type) { case *persistencespb.NexusEndpointTarget_External_: url = variant.External.GetUrl() @@ -153,27 +155,27 @@ func clientProviderFactory( if err != nil { return nil, err } - if clusterID != "" { - httpCaller = func(r *http.Request) (*http.Response, error) { - resp, callErr := httpClient.Do(r) - commonnexus.SetFailureSourceOnContext(ctx, resp) - return resp, callErr - } - } case *persistencespb.NexusEndpointTarget_Worker_: url = cl.BaseURL() + "/" + commonnexus.RouteDispatchNexusTaskByEndpoint.Path(entry.Id) httpClient = &cl.Client - if clusterID != "" { - httpCaller = func(r *http.Request) (*http.Response, error) { + needsCallbackSourceHeader = true + default: + return nil, serviceerror.NewInternal("got unexpected endpoint target") + } + + httpCaller := httpClient.Do + if clusterID != "" { + httpCaller = func(r *http.Request) (*http.Response, error) { + if needsCallbackSourceHeader { r.Header.Set(nexusCallbackSourceHeader, clusterID) - resp, callErr := httpClient.Do(r) - commonnexus.SetFailureSourceOnContext(ctx, resp) - return resp, callErr } + resp, callErr := httpClient.Do(r) + // nexusrpc.HTTPClient does not return the raw HTTP response, so copy the failure-source header into the call context. + commonnexus.SetFailureSourceOnContext(ctx, resp) + return resp, callErr } - default: - return nil, serviceerror.NewInternal("got unexpected endpoint target") } + return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ BaseURL: url, Service: service, diff --git a/chasm/lib/nexusoperation/fx_test.go b/chasm/lib/nexusoperation/fx_test.go new file mode 100644 index 00000000000..630a1462870 --- /dev/null +++ b/chasm/lib/nexusoperation/fx_test.go @@ -0,0 +1,137 @@ +package nexusoperation + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/cluster" + "go.uber.org/mock/gomock" +) + +type testRoundTripper func(*http.Request) (*http.Response, error) + +func (f testRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +func TestClientProviderFactoryUsesSelectedHTTPClient(t *testing.T) { + errEndpointClientCalled := errors.New("endpoint client called") + errFrontendClientCalled := errors.New("frontend client called") + + tests := []struct { + name string + clusterID string + target *persistencespb.NexusEndpointTarget + expected error + }{ + { + name: "external without cluster ID", + target: &persistencespb.NexusEndpointTarget{ + Variant: &persistencespb.NexusEndpointTarget_External_{ + External: &persistencespb.NexusEndpointTarget_External{ + Url: "http://external.invalid", + }, + }, + }, + expected: errEndpointClientCalled, + }, + { + name: "external with cluster ID", + clusterID: "cluster-id", + target: &persistencespb.NexusEndpointTarget{ + Variant: &persistencespb.NexusEndpointTarget_External_{ + External: &persistencespb.NexusEndpointTarget_External{ + Url: "http://external.invalid", + }, + }, + }, + expected: errEndpointClientCalled, + }, + { + name: "worker without cluster ID", + target: &persistencespb.NexusEndpointTarget{ + Variant: &persistencespb.NexusEndpointTarget_Worker_{ + Worker: &persistencespb.NexusEndpointTarget_Worker{}, + }, + }, + expected: errFrontendClientCalled, + }, + { + name: "worker with cluster ID", + clusterID: "cluster-id", + target: &persistencespb.NexusEndpointTarget{ + Variant: &persistencespb.NexusEndpointTarget_Worker_{ + Worker: &persistencespb.NexusEndpointTarget_Worker{}, + }, + }, + expected: errFrontendClientCalled, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + endpointTransport := testRoundTripper(func(*http.Request) (*http.Response, error) { + return nil, errEndpointClientCalled + }) + frontendTransport := testRoundTripper(func(*http.Request) (*http.Response, error) { + return nil, errFrontendClientCalled + }) + + rpcFactory := common.NewMockRPCFactory(ctrl) + rpcFactory.EXPECT().CreateLocalFrontendHTTPClient().Return( + &common.FrontendHTTPClient{ + Client: http.Client{Transport: frontendTransport}, + Address: "frontend.invalid", + Scheme: "http", + }, + nil, + ) + + clusterMetadata := cluster.NewMockMetadata(ctrl) + clusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{ + "current": {ClusterID: test.clusterID}, + }) + clusterMetadata.EXPECT().GetCurrentClusterName().Return("current") + + provider, err := clientProviderFactory( + func(string, string) http.RoundTripper { + return endpointTransport + }, + clusterMetadata, + rpcFactory, + nil, + ) + require.NoError(t, err) + + client, err := provider( + context.Background(), + "namespace-id", + &persistencespb.NexusEndpointEntry{ + Id: "endpoint-id", + Endpoint: &persistencespb.NexusEndpoint{ + Spec: &persistencespb.NexusEndpointSpec{ + Target: test.target, + }, + }, + }, + "service", + ) + require.NoError(t, err) + + _, err = client.StartOperation( + context.Background(), + "operation", + nil, + nexus.StartOperationOptions{}, + ) + require.ErrorIs(t, err, test.expected) + }) + } +} diff --git a/chasm/lib/nexusoperation/handler.go b/chasm/lib/nexusoperation/handler.go index 1ce4b077c70..3e005654f99 100644 --- a/chasm/lib/nexusoperation/handler.go +++ b/chasm/lib/nexusoperation/handler.go @@ -11,6 +11,7 @@ import ( nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" "go.temporal.io/server/common/contextutil" "go.temporal.io/server/common/log" + "google.golang.org/protobuf/types/known/timestamppb" ) type handler struct { @@ -207,9 +208,10 @@ func (h *handler) RequestCancelNexusOperation( ref, func(o *Operation, ctx chasm.MutableContext, req *nexusoperationpb.RequestCancelNexusOperationRequest) (*nexusoperationpb.RequestCancelNexusOperationResponse, error) { if err := o.RequestCancel(ctx, &nexusoperationpb.CancellationState{ - RequestId: req.GetFrontendRequest().GetRequestId(), - Identity: req.GetFrontendRequest().GetIdentity(), - Reason: req.GetFrontendRequest().GetReason(), + RequestId: req.GetFrontendRequest().GetRequestId(), + Identity: req.GetFrontendRequest().GetIdentity(), + Reason: req.GetFrontendRequest().GetReason(), + RequestedTime: timestamppb.New(ctx.Now(o)), }); err != nil { return nil, err } diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index b4e3544709a..4ce985df2fb 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -263,7 +263,7 @@ func (o *Operation) HandleNexusCompletion( links := completion.GetLinks() // For completion-before-start, apply the started transition first. - if o.GetStatus() == nexusoperationpb.OPERATION_STATUS_SCHEDULED { + if TransitionStarted.Possible(o) { startTime := timestamp.TimeValuePtr(completion.GetStartTime()) if err := o.onStarted(ctx, completion.GetOperationToken(), startTime, links); err != nil { return err diff --git a/chasm/lib/nexusoperation/operation_test.go b/chasm/lib/nexusoperation/operation_test.go index 35aafed6f03..e2ac5ea4919 100644 --- a/chasm/lib/nexusoperation/operation_test.go +++ b/chasm/lib/nexusoperation/operation_test.go @@ -13,6 +13,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" + "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" @@ -158,6 +159,17 @@ func TestHandleNexusCompletion(t *testing.T) { require.NoError(t, TransitionStarted.Apply(op, ctx, EventStarted{OperationToken: "tok"})) return op } + newBackingOffOp := func(t *testing.T, ctx *chasm.MockMutableContext) *Operation { + t.Helper() + op := newScheduledTestOperation(t, ctx) + require.NoError(t, transitionAttemptFailed.Apply(op, ctx, EventAttemptFailed{ + Failure: &failurepb.Failure{ + Message: "retryable failure", + }, + RetryPolicy: backoff.NewConstantDelayRetryPolicy(time.Minute), + })) + return op + } ctrl := gomock.NewController(t) nsRegistry := namespace.NewMockRegistry(ctrl) nsRegistry.EXPECT().GetNamespaceName(namespace.ID("ns-id")).Return(namespace.Name("ns-name"), nil).AnyTimes() @@ -226,6 +238,29 @@ func TestHandleNexusCompletion(t *testing.T) { require.Equal(t, "tok", op.GetOperationToken()) require.Equal(t, defaultTime, op.GetStartedTime().AsTime()) }) + + t.Run("CompletionDuringRetryBackoff", func(t *testing.T) { + ctx := newCtx() + op := newBackingOffOp(t, ctx) + startTime := defaultTime.Add(-time.Second) + + err := op.HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + StartTime: timestamppb.New(startTime), + RequestId: op.GetRequestId(), + OperationToken: "tok", + Outcome: &persistencespb.ChasmNexusCompletion_Success{ + Success: mustToPayload(t, "result"), + }, + }) + + require.NoError(t, err) + require.Equal(t, nexusoperationpb.OPERATION_STATUS_SUCCEEDED, op.GetStatus()) + require.Equal(t, "tok", op.GetOperationToken()) + require.Equal(t, startTime, op.GetStartedTime().AsTime()) + require.Equal(t, startTime, op.GetLastAttemptCompleteTime().AsTime()) + require.Nil(t, op.GetLastAttemptFailure()) + require.Nil(t, op.GetNextAttemptScheduleTime()) + }) }) t.Run("Failure", func(t *testing.T) { diff --git a/chasm/lib/workflow/nexus_events.go b/chasm/lib/workflow/nexus_events.go index 8e6296fc71b..d2bf020713d 100644 --- a/chasm/lib/workflow/nexus_events.go +++ b/chasm/lib/workflow/nexus_events.go @@ -95,7 +95,8 @@ func (d CancelRequestedEventDefinition) Apply(ctx chasm.MutableContext, wf *Work } return op.RequestCancel(ctx, &nexusoperationpb.CancellationState{ - ParentData: cancelParentData, + RequestedTime: event.GetEventTime(), + ParentData: cancelParentData, }) } diff --git a/chasm/lib/workflow/nexus_events_test.go b/chasm/lib/workflow/nexus_events_test.go index b16e55c164b..144f931dde0 100644 --- a/chasm/lib/workflow/nexus_events_test.go +++ b/chasm/lib/workflow/nexus_events_test.go @@ -375,10 +375,11 @@ func TestCancelRequestedEventDefinitionApply(t *testing.T) { t.Run("creates cancellation child", func(t *testing.T) { tcx := newTestContext(t, defaultConfig) event, key := scheduleOperation(t, tcx) + requestedTime := time.Now().UTC() applyEventDefinition[CancelRequestedEventDefinition](t, tcx, &historypb.HistoryEvent{ EventId: int64(20), - EventTime: timestamppb.Now(), + EventTime: timestamppb.New(requestedTime), Attributes: &historypb.HistoryEvent_NexusOperationCancelRequestedEventAttributes{ NexusOperationCancelRequestedEventAttributes: &historypb.NexusOperationCancelRequestedEventAttributes{ ScheduledEventId: event.EventId, @@ -389,8 +390,9 @@ func TestCancelRequestedEventDefinitionApply(t *testing.T) { field, ok := tcx.wf.Operations[key] require.True(t, ok) op := field.Get(tcx.chasmCtx) - _, hasCancellation := op.Cancellation.TryGet(tcx.chasmCtx) + cancellation, hasCancellation := op.Cancellation.TryGet(tcx.chasmCtx) require.True(t, hasCancellation) + require.Equal(t, requestedTime, cancellation.GetRequestedTime().AsTime()) }) t.Run("tolerates missing operation", func(t *testing.T) { diff --git a/tests/nexus_standalone_test.go b/tests/nexus_standalone_test.go index ac849b65b57..4daea03d738 100644 --- a/tests/nexus_standalone_test.go +++ b/tests/nexus_standalone_test.go @@ -868,6 +868,7 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationCancel() { }) s.NoError(err) + beforeCancelRequest := time.Now() _, err = env.FrontendClient().RequestCancelNexusOperationExecution(s.Context(), &workflowservice.RequestCancelNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), OperationId: "test-op", @@ -875,6 +876,7 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationCancel() { Reason: "test cancellation", }) s.NoError(err) + afterCancelRequest := time.Now() // Verify state after cancel — operation is still running descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ @@ -900,6 +902,8 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationCancel() { }, descResp.GetInfo(), protorequire.IgnoreFields("operation_token", "last_attempt_complete_time", "request_id", "schedule_time", "expiration_time", "execution_duration", "cancellation_info")) cancellationInfo := descResp.GetInfo().GetCancellationInfo() s.NotNil(cancellationInfo) + s.NotNil(cancellationInfo.GetRequestedTime()) + s.WithinRange(cancellationInfo.GetRequestedTime().AsTime(), beforeCancelRequest, afterCancelRequest) s.NotEqual(enumspb.NEXUS_OPERATION_CANCELLATION_STATE_UNSPECIFIED, cancellationInfo.GetState()) protorequire.ProtoEqual(s.T(), &nexuspb.NexusOperationExecutionCancellationInfo{ Attempt: 1, From 7fc5c02b1cc79102f26545dc2454d80bb7c40de0 Mon Sep 17 00:00:00 2001 From: Quinn Klassen Date: Wed, 19 Aug 2026 14:09:21 -0700 Subject: [PATCH 014/139] Prevent malformed retry delays from poisoning Nexus completions (#11617) ## What changed? Validate the retry delay is a valid proto duration. ## Why? Prevent malformed retry delays from poisoning Nexus completions. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ## Potential risks Potentially users could have been sending an invalid proto duration, unclear how exactly, and now we would fail their request. --- service/frontend/workflow_handler.go | 18 +++++ service/frontend/workflow_handler_test.go | 84 +++++++++++++++++++++++ tests/activity_standalone_test.go | 63 +++++++++++++++++ 3 files changed, 165 insertions(+) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 048557e88ca..f14f55ae610 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -19,6 +19,7 @@ import ( commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" filterpb "go.temporal.io/api/filter/v1" historypb "go.temporal.io/api/history/v1" querypb "go.temporal.io/api/query/v1" @@ -1818,6 +1819,17 @@ func (wh *WorkflowHandler) RespondActivityTaskCompletedById(ctx context.Context, return &workflowservice.RespondActivityTaskCompletedByIdResponse{}, nil } +func validateActivityFailureNextRetryDelays(activityFailure *failurepb.Failure) error { + for current := activityFailure; current != nil; current = current.GetCause() { + if delay := current.GetApplicationFailureInfo().GetNextRetryDelay(); delay != nil { + if err := delay.CheckValid(); err != nil { + return serviceerror.NewInvalidArgumentf("NextRetryDelay is not a valid duration: %v", err) + } + } + } + return nil +} + // RespondActivityTaskFailed is called by application worker when it is done processing an ActivityTask. It will // result in a new 'ActivityTaskFailed' event being written to the workflow history and a new WorkflowTask // created for the workflow instance so new commands could be made. Use the 'taskToken' provided as response of @@ -1852,6 +1864,9 @@ func (wh *WorkflowHandler) RespondActivityTaskFailed( if request.GetFailure() != nil && request.GetFailure().GetApplicationFailureInfo() == nil { return nil, errFailureMustHaveApplicationFailureInfo } + if err := validateActivityFailureNextRetryDelays(request.GetFailure()); err != nil { + return nil, err + } if len(request.GetIdentity()) > wh.config.MaxIDLengthLimit() { return nil, errIdentityTooLong @@ -1987,6 +2002,9 @@ func (wh *WorkflowHandler) RespondActivityTaskFailedById(ctx context.Context, re sizeLimitWarn := wh.config.BlobSizeLimitWarn(namespaceEntry.Name().String()) response := workflowservice.RespondActivityTaskFailedByIdResponse{} + if err := validateActivityFailureNextRetryDelays(request.GetFailure()); err != nil { + return nil, err + } if request.GetLastHeartbeatDetails() != nil { if err := common.CheckEventBlobSizeLimit( diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index ed1c09168b1..ef0e8274a72 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -5331,6 +5331,90 @@ func (s *WorkflowHandlerSuite) TestPatchSchedule_ValidationAndErrors() { // carry (10000 years). Anything beyond it fails durationpb's CheckValid. const maxProtoDurationSeconds = int64(315576000000) +func TestValidateActivityFailureNextRetryDelays(t *testing.T) { + applicationFailure := func(delay *durationpb.Duration, cause *failurepb.Failure) *failurepb.Failure { + return &failurepb.Failure{ + Cause: cause, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{NextRetryDelay: delay}, + }, + } + } + + testCases := []struct { + name string + failure *failurepb.Failure + wantErr bool + }{ + {name: "nil failure"}, + {name: "nil retry delay", failure: applicationFailure(nil, nil)}, + {name: "zero retry delay", failure: applicationFailure(durationpb.New(0), nil)}, + {name: "negative retry delay", failure: applicationFailure(durationpb.New(-time.Second), nil)}, + { + name: "maximum duration", + failure: applicationFailure(&durationpb.Duration{Seconds: maxProtoDurationSeconds}, nil), + }, + { + name: "minimum duration", + failure: applicationFailure(&durationpb.Duration{Seconds: -maxProtoDurationSeconds}, nil), + }, + { + name: "mismatched positive seconds", + failure: applicationFailure(&durationpb.Duration{Seconds: 1, Nanos: -1}, nil), + wantErr: true, + }, + { + name: "mismatched negative seconds", + failure: applicationFailure(&durationpb.Duration{Seconds: -1, Nanos: 1}, nil), + wantErr: true, + }, + { + name: "nanos above range", + failure: applicationFailure(&durationpb.Duration{Nanos: 1000000000}, nil), + wantErr: true, + }, + { + name: "nanos below range", + failure: applicationFailure(&durationpb.Duration{Nanos: -1000000000}, nil), + wantErr: true, + }, + { + name: "seconds above range", + failure: applicationFailure(&durationpb.Duration{Seconds: maxProtoDurationSeconds + 1}, nil), + wantErr: true, + }, + { + name: "seconds below range", + failure: applicationFailure(&durationpb.Duration{Seconds: -maxProtoDurationSeconds - 1}, nil), + wantErr: true, + }, + { + name: "invalid nested retry delay", + failure: applicationFailure(nil, &failurepb.Failure{ + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{}, + }, + Cause: applicationFailure(&durationpb.Duration{Seconds: 1, Nanos: -1}, nil), + }), + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateActivityFailureNextRetryDelays(tc.failure) + if !tc.wantErr { + require.NoError(t, err) + return + } + + var invalidArgument *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, "NextRetryDelay is not a valid duration") + }) + } +} + // newScheduleSpecHandler builds the minimum WorkflowHandler that canonicalizeScheduleSpec // needs: a config, a spec builder, and a logger for the non-enforcing path. configure is // nil to exercise the shipped dynamic config defaults. diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index d5f1b23244e..906505ea4cf 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1492,6 +1492,69 @@ func (s *standaloneActivityTestSuite) TestFail() { env.validateFailure(s.Context(), t, activityID, runID, nil, env.Tv().WorkerIdentity()) }) + t.Run("InvalidNextRetryDelay", func(t *testing.T) { + for _, tc := range []struct { + name string + byID bool + }{ + {name: "ByToken"}, + {name: "ByID", byID: true}, + } { + t.Run(tc.name, func(t *testing.T) { + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + + startResp := env.startAndValidateActivity(s.Context(), t, activityID, taskQueue) + runID := startResp.RunId + pollResp := env.pollActivityTaskAndValidate(s.Context(), t, activityID, taskQueue, runID) + + invalidFailure := &failurepb.Failure{ + Message: "invalid retry delay", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + NonRetryable: true, + NextRetryDelay: &durationpb.Duration{Seconds: 1, Nanos: -1}, + }, + }, + } + + respondFailed := func(failure *failurepb.Failure) error { + if tc.byID { + _, err := env.FrontendClient().RespondActivityTaskFailedById(s.Context(), &workflowservice.RespondActivityTaskFailedByIdRequest{ + Namespace: env.Namespace().String(), + RunId: runID, + ActivityId: activityID, + Failure: failure, + }) + return err + } + _, err := env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + Namespace: env.Namespace().String(), + TaskToken: pollResp.TaskToken, + Failure: failure, + }) + return err + } + + err := respondFailed(invalidFailure) + var invalidArgument *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, "NextRetryDelay is not a valid duration") + + describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: runID, + }) + require.NoError(t, err) + require.Equal(t, enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING, describeResp.GetInfo().GetStatus()) + + require.NoError(t, respondFailed(defaultFailure)) + env.validateFailure(s.Context(), t, activityID, runID, nil, "") + }) + } + }) + t.Run("StaleToken", func(t *testing.T) { activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) From bb09631947f9d5189f7f1daaae51cb593cc37a6f Mon Sep 17 00:00:00 2001 From: Feiyang Xie Date: Wed, 19 Aug 2026 14:47:25 -0700 Subject: [PATCH 015/139] edge case fixes for time skipping propagation and fast-forward completion (#11373) ## What changed and why? 1. Edge case: If fast-forward completes during a workflow transaction but when time skipping checks at close transaction this fast-forward time is before ms.Now(), time skipping should still get disabled. Now we don't look at time points that are in the past and it is a bug. 2. Time-skipping propagation: Always propagate the time-skipping configuration and fast-forward state to the next run and other executions, regardless of whether time skipping is currently active. This ensures that read APIs (for example, Describe and PollFastForward) continue to return meaningful information instead of nil. - Otherwise, in an edge case where fast-forward completes in the first run and the user polls after the second run has become the current run of the workflow execution, the poll API would return a NotFound error instead of a completed poll result. - Similarly, the Describe API would return a nil configuration instead of the original configuration that should have been propagated. 3. Trivial changes: - simiplify parameter of `propagateTimeSkippingToNextRun` - unify UT names of timeskipping_test.go --- .../history/workflow/mutable_state_impl.go | 4 +- .../workflow/mutable_state_impl_test.go | 7 +- service/history/workflow/retry.go | 2 +- service/history/workflow/timeskipping.go | 193 ++-- service/history/workflow/timeskipping_test.go | 914 ++++++++++++------ 5 files changed, 761 insertions(+), 359 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index acaf25f57e1..c58e2fa6b7d 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -2840,7 +2840,7 @@ func (ms *MutableStateImpl) addWorkflowExecutionStartedEventForContinueAsNew( } declinedTargetVersionUpgrade := computeDeclinedTargetVersionUpgrade(previousExecutionInfo, inheritedPinnedVersion != nil) - tsc, stateProp := propagateTimeSkippingToNextRun(previousExecutionInfo) + tsc, stateProp := propagateTimeSkippingToNextRun(previousExecutionInfo.GetTimeSkippingInfo()) createRequest := &workflowservice.StartWorkflowExecutionRequest{ RequestId: uuid.NewString(), Namespace: ms.namespaceEntry.Name().String(), @@ -6463,7 +6463,7 @@ func (ms *MutableStateImpl) AddStartChildWorkflowExecutionInitiatedEvent( if err := ms.checkMutability(opTag); err != nil { return nil, nil, err } - childTSC, childTSStateProp := propagateTimeSkippingToChild(ms.executionInfo) + childTSC, childTSStateProp := propagateTimeSkippingToOtherExecution(ms.GetExecutionInfo().GetTimeSkippingInfo()) event, batchID := ms.hBuilder.AddStartChildWorkflowExecutionInitiatedEvent( workflowTaskCompletedEventID, command, diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 81b6989a006..46a4cc0d4de 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -4449,12 +4449,12 @@ func (s *mutableStateSuite) TestAddStartChildWorkflowExecutionInitiatedEvent_Tim expectInitialSkip: durationpb.New(time.Hour), }, { - name: "accumulated skip only", + name: "disabled parent config still propagates a disabled child config", parentTSI: &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: false}, AccumulatedSkippedDuration: durationpb.New(15 * time.Minute), }, - expectNilCfg: true, + expectCfg: &commonpb.TimeSkippingConfig{Enabled: false}, expectInitialSkip: durationpb.New(15 * time.Minute), }, { @@ -4510,8 +4510,9 @@ func (s *mutableStateSuite) TestAddStartChildWorkflowExecutionInitiatedEvent_Tim // The snapshot must be a clone — mutating it must not leak back into parent state. if tc.parentTSI.GetConfig() != nil { + origEnabled := tc.parentTSI.GetConfig().GetEnabled() gotTSC.Enabled = !gotTSC.GetEnabled() - s.True(tc.parentTSI.GetConfig().GetEnabled(), + s.Equal(origEnabled, tc.parentTSI.GetConfig().GetEnabled(), "mutation of snapshot leaked into parent's Config") } }) diff --git a/service/history/workflow/retry.go b/service/history/workflow/retry.go index 918818d262d..049a204f5bb 100644 --- a/service/history/workflow/retry.go +++ b/service/history/workflow/retry.go @@ -263,7 +263,7 @@ func SetupNewWorkflowForRetryOrCron( VersioningOverride: pinnedOverride, } - tsc, stateProp := propagateTimeSkippingToNextRun(previousExecutionInfo) + tsc, stateProp := propagateTimeSkippingToNextRun(previousExecutionInfo.GetTimeSkippingInfo()) createRequest.TimeSkippingConfig = tsc attempt := int32(1) diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 710c720450f..65d9c311c95 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -22,22 +22,24 @@ import ( // ============================================================================= // Time Skipping Configuration Management // ============================================================================= +// initTimeSkippingInfo can be called either when (1) a user starts an execution or +// (2) another run is started with in the same execution (the workflow chain-of-runs model) +// `propagatedState` is only set in the latter case. func (ms *MutableStateImpl) initTimeSkippingInfo( config *commonpb.TimeSkippingConfig, - timeSkippingStatePropagation *commonpb.TimeSkippingStatePropagation, + propagatedState *commonpb.TimeSkippingStatePropagation, ) { - initialSkip := timeSkippingStatePropagation.GetInitialSkippedDuration() + initialSkip := propagatedState.GetInitialSkippedDuration() if config == nil && initialSkip == nil { return } - ms.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: config, AccumulatedSkippedDuration: initialSkip, - SessionSkipCount: timeSkippingStatePropagation.GetInitialSkipCount(), + SessionSkipCount: propagatedState.GetInitialSkipCount(), } ms.wrapTimeSourceWithTimeSkipping() - ms.applyFastForward(timeSkippingStatePropagation.GetFastForwardTargetTime()) + ms.applyFastForward(propagatedState.GetFastForwardTargetTime()) ms.timeSkippingInfoUpdated = true } @@ -48,46 +50,68 @@ func (ms *MutableStateImpl) updateTimeSkippingInfo( if tsi == nil { return } + // we allow setting config to nil in updating tsc ms.executionInfo.TimeSkippingInfo.Config = config + tsi.SessionSkipCount = 0 ms.applyFastForward(nil) ms.timeSkippingInfoUpdated = true - tsi.SessionSkipCount = 0 } // applyFastForward (re)computes the FastForwardInfo using the new TimeSkippingConfig (TSC) and propagated time-skippingstates. // This method should be called whenever the TimeSkippingConfig is initialized or updated. -// An invariant of the FastForwardInfo is that after this method is called, if the current TSC has a FastForward value, -// the FastForwardInfo should never be nil. +// +// Invariant: FastForwardInfo is non-nil whenever the config still has a fast-forward config, and nil +// otherwise. Keeping it alive even when skipping is disabled lets a poller still observe a completed +// fast-forward carried over from a prior run. func (ms *MutableStateImpl) applyFastForward(propagatedTargetTime *timestamppb.Timestamp) { - tsc := ms.GetExecutionInfo().GetTimeSkippingInfo().GetConfig() - tsi := ms.executionInfo.TimeSkippingInfo + tsi := ms.executionInfo.GetTimeSkippingInfo() + if tsi == nil { + return + } + tsc := tsi.GetConfig() + ffConfig := tsc.GetFastForwardConfig() - if !tsc.GetEnabled() || tsc.GetFastForwardConfig().GetDuration().AsDuration() <= 0 { + // no ff + if ffConfig == nil { if tsi.FastForwardInfo != nil { ms.setAndStampFastForwardInfo(nil) } return } + // ff of different states: var targetTime time.Time + var hasReached bool if propagatedTargetTime != nil { targetTime = propagatedTargetTime.AsTime() + // A propagated target may already be due; a due target is a completed fast-forward. + hasReached = !ms.Now().Before(targetTime) } else { // if there is no propagated target time, // fast-forward refers to a new duration from now. - targetTime = ms.Now().Add(tsc.GetFastForwardConfig().GetDuration().AsDuration()) + targetTime = ms.Now().Add(ffConfig.GetDuration().AsDuration()) } ffVersionedTransition := ms.setAndStampFastForwardInfo(&persistencespb.FastForwardInfo{ TargetTime: timestamppb.New(targetTime), - HasReached: false, - }) - ms.AddTasks(&tasks.TimeSkippingTimerTask{ - WorkflowKey: ms.GetWorkflowKey(), - VisibilityTimestamp: targetTime, - VersionedTransition: ffVersionedTransition, - ArchetypeID: ms.ChasmTree().ArchetypeID(), + HasReached: hasReached, }) + + // actions based on ff states: 1) disable time skipping 2) add ff timer task + if hasReached { + tsc.Enabled = false + return + } + + // schedule the wake-up timer only while skipping is still enabled + if tsc.GetEnabled() { + ms.AddTasks(&tasks.TimeSkippingTimerTask{ + WorkflowKey: ms.GetWorkflowKey(), + VisibilityTimestamp: targetTime, + VersionedTransition: ffVersionedTransition, + ArchetypeID: ms.ChasmTree().ArchetypeID(), + }) + } } // setAndStampFastForwardInfo sets the fast-forward info and stamps it with the current transaction's versioned transition. @@ -106,62 +130,42 @@ func (ms *MutableStateImpl) setAndStampFastForwardInfo( // -- Propagation Methods of Time Skipping -// propagateTimeSkippingToNextRun propagates both time skipping config and state to the next run in -// the chain (CaN, retry, cron). The config is deep-cloned so the next run can mutate it without -// affecting the source. +// propagateTimeSkippingToNextRun propagates both the time-skipping config and state to the next run +// in the chain of runs(continue-as-new, retry, cron). The config is propagated regardless of whether time +// skipping is actively running, so reading APIs can always retrieve the latest effective configuration from the current run. func propagateTimeSkippingToNextRun( - source *persistencespb.WorkflowExecutionInfo, + tsi *persistencespb.TimeSkippingInfo, ) (*commonpb.TimeSkippingConfig, *commonpb.TimeSkippingStatePropagation) { - - tsi := source.GetTimeSkippingInfo() + if tsi == nil { + return nil, nil + } util := NewTimeSkippingInfoUtil(tsi) - - // if disabled, we just return nil for the new TSC var newTSC *commonpb.TimeSkippingConfig - if util.IsEnabled() { + if tsi.Config != nil { newTSC = common.CloneProto(tsi.GetConfig()) } - - // state propagation (virtual time, fast forward, skip count) - var stateProp *commonpb.TimeSkippingStatePropagation - if accum := util.GetAccumulatedSkippedDuration(); accum > 0 { - stateProp = &commonpb.TimeSkippingStatePropagation{ - InitialSkippedDuration: durationpb.New(accum), - } - } - if util.HasPendingFastForward() { - if stateProp == nil { - stateProp = &commonpb.TimeSkippingStatePropagation{} - } - stateProp.FastForwardTargetTime = tsi.GetFastForwardInfo().GetTargetTime() - } - // The skip count is scoped to a session; once time skipping is disabled the session has - // ended, so the count does not carry to the next run. - if skipCount := tsi.GetSessionSkipCount(); skipCount > 0 && util.IsEnabled() { - if stateProp == nil { - stateProp = &commonpb.TimeSkippingStatePropagation{} - } - stateProp.InitialSkipCount = skipCount + stateProp := &commonpb.TimeSkippingStatePropagation{ + InitialSkipCount: tsi.GetSessionSkipCount(), + InitialSkippedDuration: durationpb.New(util.GetAccumulatedSkippedDuration()), + FastForwardTargetTime: util.GetFastForwardTargetTime(), } return newTSC, stateProp } -// propagateTimeSkippingToChild snapshots the parent's time skipping into a child workflow. The -// child shares the parent's virtual clock (its start time is shifted forward by the parent's -// accumulated skipped duration) but is otherwise a fresh execution: -// - FastForward is per-execution and is never propagated to children. -// - The child starts a fresh skip session: SessionSkipCount is not propagated (starts at 0). -// - It does inherit the parent's per-session budget (MaxSkipPerSession). Children start -// internally, bypassing the frontend that populates the default budget, so inheriting the -// parent's already-resolved limit avoids a 0 that would disable skipping on the first skip. -func propagateTimeSkippingToChild( - source *persistencespb.WorkflowExecutionInfo, +// propagateTimeSkippingToOtherExecution snapshots the current execution's time skipping into another +// execution (e.g. a child workflow), which shares the current execution's virtual clock. Two rules: +// 1. State: nothing propagates except virtual time. +// 2. Config: everything propagates except the fast-forward config, and the whole config can be +// suppressed by DisablePropagation. +func propagateTimeSkippingToOtherExecution( + tsi *persistencespb.TimeSkippingInfo, ) (*commonpb.TimeSkippingConfig, *commonpb.TimeSkippingStatePropagation) { - tsi := source.GetTimeSkippingInfo() + if tsi == nil { + return nil, nil + } tsc := tsi.GetConfig() - util := NewTimeSkippingInfoUtil(tsi) + accum := NewTimeSkippingInfoUtil(tsi).GetAccumulatedSkippedDuration() - accum := util.GetAccumulatedSkippedDuration() var stateProp *commonpb.TimeSkippingStatePropagation if accum > 0 { stateProp = &commonpb.TimeSkippingStatePropagation{ @@ -170,16 +174,14 @@ func propagateTimeSkippingToChild( } } - // only propagates state - if !util.IsEnabled() || tsc.GetDisablePropagation() { + if tsc == nil || tsc.GetDisablePropagation() { return nil, stateProp } - // propagate both config and state - return &commonpb.TimeSkippingConfig{ - Enabled: true, - MaxSessionSkipCount: tsc.GetMaxSessionSkipCount(), - }, stateProp + // Propagate the whole config except the per-execution fast-forward. + newTSC := common.CloneProto(tsc) + newTSC.FastForwardConfig = nil + return newTSC, stateProp } // ============================================================================= @@ -223,11 +225,15 @@ func NewTimeSkippingTransition(currentTime time.Time) *timeSkippingTransition { // signal. Nil-safe. A transition without a current time is never valid — every meaningful field is // derived relative to the current time, so without it there is nothing to apply. func (t *timeSkippingTransition) IsValid() bool { - return t != nil && !t.CurrentTime.IsZero() && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) + return t.isInitialized() && (!t.TargetTime.IsZero() || t.DisabledAfterFastForward) +} + +func (t *timeSkippingTransition) isInitialized() bool { + return t != nil && !t.CurrentTime.IsZero() } func (t *timeSkippingTransition) TrackEarliestFutureTime(candidate time.Time) { - if t == nil || t.CurrentTime.IsZero() || candidate.IsZero() || candidate.Before(t.CurrentTime) { + if !t.isInitialized() || candidate.IsZero() || candidate.Before(t.CurrentTime) { return } if t.TargetTime.IsZero() || candidate.Before(t.TargetTime) { @@ -244,16 +250,16 @@ func (t *timeSkippingTransition) GateByFastForward(ff *persistencespb.FastForwar return } ffTargetTime := ff.GetTargetTime().AsTime() - // If a real candidate is scheduled strictly before the fast-forward target, we skip to - // that and the fast-forward budget is not yet exhausted — leave time skipping enabled. + if !ffTargetTime.After(t.CurrentTime) { + t.TargetTime = time.Time{} + t.DisabledAfterFastForward = true + return + } + if !t.TargetTime.IsZero() && t.TargetTime.Before(ffTargetTime) { return } - // Otherwise the fast-forward target is the earliest target: skip to it (clamped to the - // present by TrackEarliestFutureTime) and disable time skipping — the budget is reached. - // This is what lets the budget cap a chain of runs: a run with no earlier candidate - // consumes the remaining budget by skipping to the fast-forward and disabling. - t.TrackEarliestFutureTime(ffTargetTime) + t.TargetTime = ffTargetTime t.DisabledAfterFastForward = true } @@ -296,14 +302,32 @@ func (util *TimeSkippingInfoUtil) HasPendingFastForward() bool { return false } ff := util.tsi.GetFastForwardInfo() - return ff != nil && - !ff.GetHasReached() && - ff.GetTargetTime() != nil && - !ff.GetTargetTime().AsTime().IsZero() + ffTargetTime := util.GetFastForwardTargetTime() + if ff == nil || ffTargetTime == nil { + return false + } + return !ff.GetHasReached() +} + +func (util *TimeSkippingInfoUtil) GetFastForwardTargetTime() *timestamppb.Timestamp { + if util == nil || util.tsi == nil { + return nil + } + ff := util.tsi.GetFastForwardInfo() + if ff == nil || ff.GetTargetTime() == nil { + return nil + } + targetTime := ff.GetTargetTime().AsTime() + if targetTime.IsZero() { + return nil + } + return ff.TargetTime } -// IsEnabled reports whether time skipping is enabled. Nil-safe: a nil util, nil info, or nil config -// all yield false. +// IsEnabled reports whether time skipping is still running for this execution. Prefer this over +// reading the config's Enabled flag directly. +// The `enabled` field is disabled internally in three places: +// 1) initTimeSkippingInfo and updateTimeSkippingInfo, 2) runtime time-skipping transition, 3) fast-forward timer task. func (util *TimeSkippingInfoUtil) IsEnabled() bool { if util == nil || util.tsi == nil { return false @@ -318,6 +342,7 @@ func (util *TimeSkippingInfoUtil) ToDescribeInfo(currentTime time.Time) *commonp return &commonpb.TimeSkippingInfo{ CurrentTime: timestamppb.New(currentTime), EffectiveConfig: common.CloneProto(util.tsi.GetConfig()), + FastForwardInfo: util.ToFastForwardInfo(), } } diff --git a/service/history/workflow/timeskipping_test.go b/service/history/workflow/timeskipping_test.go index b83ce9a754e..c16db2d08bb 100644 --- a/service/history/workflow/timeskipping_test.go +++ b/service/history/workflow/timeskipping_test.go @@ -16,207 +16,219 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/components/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/service/history/tests" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) -func (s *mutableStateSuite) TestPropagateTimeSkippingToNextRun_FastForwardInfo() { +func (s *mutableStateSuite) TestPropagateTimeSkippingToNextRun() { fixed := time.Date(2027, 1, 1, 12, 0, 0, 0, time.UTC) fixedTS := timestamppb.New(fixed) - s.Run("FastForwardNotYetReached_TargetTimePropagated", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{ - Enabled: true, - FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, - }, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - FastForwardInfo: &persistencespb.FastForwardInfo{ - TargetTime: fixedTS, - HasReached: false, - }, + s.Run("PendingFastForward_PropagatesTargetTime", func() { + tsi := &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, + }, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: fixedTS, + HasReached: false, }, } - tsc, stateProp := propagateTimeSkippingToNextRun(src) + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) s.Require().NotNil(stateProp.GetFastForwardTargetTime()) s.Equal(fixed, stateProp.GetFastForwardTargetTime().AsTime()) s.Require().NotNil(tsc) - s.True(proto.Equal(tsc, src.GetTimeSkippingInfo().GetConfig())) - }) - - s.Run("FastForwardAlreadyReached_TargetTimeOmitted", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{ - Enabled: false, - FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, - }, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - FastForwardInfo: &persistencespb.FastForwardInfo{ - TargetTime: fixedTS, - HasReached: true, - }, - }, - } - tsc, stateProp := propagateTimeSkippingToNextRun(src) - s.Nil(tsc, "disabled config is not propagated") - s.NotNil(stateProp) - s.Nil(stateProp.GetFastForwardTargetTime(), "consumed fast-forward must not be re-propagated") - s.Equal(time.Hour, stateProp.GetInitialSkippedDuration().AsDuration()) + s.True(proto.Equal(tsc, tsi.GetConfig())) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("disabled config still propagates virtual time", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{ - Enabled: false, - FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, - }, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - FastForwardInfo: &persistencespb.FastForwardInfo{ - TargetTime: timestamppb.New(s.mutableState.timeSource.Now().Add(3 * time.Hour)), - HasReached: true, - }, + s.Run("ReachedFastForward_StillPropagatesTargetTime", func() { + tsi := &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: false, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, + }, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: fixedTS, + HasReached: true, }, } - tsc, stateProp := propagateTimeSkippingToNextRun(src) - s.Nil(tsc) - s.Nil(stateProp.GetFastForwardTargetTime()) + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) + s.Require().NotNil(tsc, "config is propagated regardless of enabled") + s.True(proto.Equal(tsc, tsi.GetConfig())) + s.NotNil(stateProp) + s.Require().NotNil(stateProp.GetFastForwardTargetTime(), + "a completed fast-forward is still propagated so the next run can report completion") + s.Equal(fixed, stateProp.GetFastForwardTargetTime().AsTime()) s.Equal(time.Hour, stateProp.GetInitialSkippedDuration().AsDuration()) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("nil config still propagates virtual time", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: nil, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - }, + s.Run("NilConfig_PropagatesVirtualTime", func() { + tsi := &persistencespb.TimeSkippingInfo{ + Config: nil, + AccumulatedSkippedDuration: durationpb.New(time.Hour), } - tsc, stateProp := propagateTimeSkippingToNextRun(src) + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) s.Nil(tsc) s.Nil(stateProp.GetFastForwardTargetTime()) s.Equal(time.Hour, stateProp.GetInitialSkippedDuration().AsDuration()) + s.requireInitNoPanic(tsc, stateProp) }) // Regression: with no accumulated duration the stateProp starts nil; an active // (not-yet-reached) fast-forward must still be propagated without a nil deref. - s.Run("FastForwardPropagatedWhenNoAccumulatedDuration", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{ - Enabled: true, - FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, - }, - // no AccumulatedSkippedDuration - FastForwardInfo: &persistencespb.FastForwardInfo{ - TargetTime: fixedTS, - HasReached: false, - }, + s.Run("NoAccumulatedSkip_StillPropagatesFastForwardTarget", func() { + tsi := &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, + }, + // no AccumulatedSkippedDuration + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: fixedTS, + HasReached: false, }, } - tsc, stateProp := propagateTimeSkippingToNextRun(src) + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) s.Require().NotNil(tsc) s.True(tsc.GetEnabled()) s.Require().NotNil(stateProp) - s.Nil(stateProp.GetInitialSkippedDuration(), "no virtual time accumulated yet") + s.Zero(stateProp.GetInitialSkippedDuration().AsDuration(), "no virtual time accumulated yet") s.Require().NotNil(stateProp.GetFastForwardTargetTime()) s.Equal(fixed, stateProp.GetFastForwardTargetTime().AsTime()) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("NothingPropagatedWhenNoAccumAndNoFastForward", func() { - src := &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: true}, - }, + s.Run("NoAccumulatedSkipNoFastForward_EmptyState", func() { + tsi := &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true}, } - tsc, stateProp := propagateTimeSkippingToNextRun(src) + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) s.Require().NotNil(tsc, "enabled config is still propagated") s.True(tsc.GetEnabled()) - s.Nil(stateProp, "no accumulated time and no active fast-forward means nothing to propagate") + // stateProp is always returned; it just carries nothing meaningful here. + s.Require().NotNil(stateProp) + s.Zero(stateProp.GetInitialSkippedDuration().AsDuration()) + s.Nil(stateProp.GetFastForwardTargetTime()) + s.Zero(stateProp.GetInitialSkipCount()) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("NoTimeSkippingInfoPropagatesNothing", func() { - tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.WorkflowExecutionInfo{}) + s.Run("NilTimeSkippingInfo_PropagatesNothing", func() { + tsc, stateProp := propagateTimeSkippingToNextRun(nil) s.Nil(tsc) s.Nil(stateProp) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("SkipCountAloneStillPropagates", func() { - tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 50}, - SessionSkipCount: 2, - }, + s.Run("SkipCountOnly_PropagatesSkipCount", func() { + tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 50}, + SessionSkipCount: 2, }) s.Require().NotNil(tsc) s.Require().NotNil(stateProp) - s.Nil(stateProp.GetInitialSkippedDuration()) + s.Zero(stateProp.GetInitialSkippedDuration().AsDuration()) s.Equal(int32(2), stateProp.GetInitialSkipCount()) s.Equal(int32(50), tsc.MaxSessionSkipCount) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("FullPropagationOfAllTimeSippingStates", func() { - tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 50}, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - SessionSkipCount: 3, - }, + s.Run("FullState_PropagatesEverything", func() { + tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 50}, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + SessionSkipCount: 3, }) s.Require().NotNil(tsc) s.Equal(int32(50), tsc.GetMaxSessionSkipCount()) s.Require().NotNil(stateProp) s.Equal(int32(3), stateProp.GetInitialSkipCount()) + s.requireInitNoPanic(tsc, stateProp) }) - s.Run("SessionSkipCountWontPropagateWhenTimeSkippingDisabled", func() { - tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: false, MaxSessionSkipCount: 50}, - AccumulatedSkippedDuration: durationpb.New(time.Hour), - SessionSkipCount: 3, - }, + s.Run("DisabledConfig_StillPropagatesState", func() { + tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: false, MaxSessionSkipCount: 50}, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + SessionSkipCount: 3, }) - s.Nil(tsc) + // Everything propagates regardless of enabled so the next run reflects the latest state. + s.Require().NotNil(tsc, "config is propagated regardless of enabled") + s.False(tsc.GetEnabled()) s.Require().NotNil(stateProp) - s.Equal(int32(0), stateProp.GetInitialSkipCount()) + s.Equal(int32(3), stateProp.GetInitialSkipCount()) + s.Equal(time.Hour, stateProp.GetInitialSkippedDuration().AsDuration()) + s.requireInitNoPanic(tsc, stateProp) + }) + + s.Run("EmptyInfo_NilConfigAndEmptyState", func() { + // Internal config nil (but TimeSkippingInfo present): must not panic; config is nil, + // stateProp is still returned and empty. + tsc, stateProp := propagateTimeSkippingToNextRun(&persistencespb.TimeSkippingInfo{}) + s.Nil(tsc, "nil config -> nil propagated config") + s.Require().NotNil(stateProp) + s.Zero(stateProp.GetInitialSkippedDuration().AsDuration()) + s.Nil(stateProp.GetFastForwardTargetTime()) + s.Zero(stateProp.GetInitialSkipCount()) + s.requireInitNoPanic(tsc, stateProp) + }) + + s.Run("DisablePropagationFlag_PreservedInChainOfRuns", func() { + // Chain-of-runs clones the full config, so DisablePropagation is preserved. Contrast with + // propagateTimeSkippingToOtherExecution, which never carries the flag onto a child config. + tsi := &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + DisablePropagation: true, + MaxSessionSkipCount: 5, + }, + } + tsc, stateProp := propagateTimeSkippingToNextRun(tsi) + s.Require().NotNil(tsc) + s.True(tsc.GetDisablePropagation()) + s.Equal(int32(5), tsc.GetMaxSessionSkipCount()) + s.requireInitNoPanic(tsc, stateProp) }) } -func (s *mutableStateSuite) TestSnapshotTimeSkippingInfo_ForChildWorkflows() { +func (s *mutableStateSuite) TestPropagateTimeSkippingToOtherExecution() { accumSkip := time.Hour sessionMaxSkipCount := int32(10) - newSource := func() *persistencespb.WorkflowExecutionInfo { + newTSI := func() *persistencespb.TimeSkippingInfo { s.mutableState.timeSource = clock.NewEventTimeSource() - return &persistencespb.WorkflowExecutionInfo{ - TimeSkippingInfo: &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{ - Enabled: true, - FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, - MaxSessionSkipCount: sessionMaxSkipCount, - }, - AccumulatedSkippedDuration: durationpb.New(accumSkip), - SessionSkipCount: 3, - FastForwardInfo: &persistencespb.FastForwardInfo{ - TargetTime: timestamppb.New(s.mutableState.timeSource.Now().Add(3 * time.Hour)), - HasReached: false, - }, + return &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, + MaxSessionSkipCount: sessionMaxSkipCount, + }, + AccumulatedSkippedDuration: durationpb.New(accumSkip), + SessionSkipCount: 3, + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: timestamppb.New(s.mutableState.timeSource.Now().Add(3 * time.Hour)), + HasReached: false, }, } } - s.Run("nil-safe check", func() { - tsc, propagatedState := propagateTimeSkippingToChild(&persistencespb.WorkflowExecutionInfo{}) + s.Run("NilTimeSkippingInfo_PropagatesNothing", func() { + tsc, propagatedState := propagateTimeSkippingToOtherExecution(nil) s.Nil(tsc) s.Nil(propagatedState) + s.requireInitNoPanic(tsc, propagatedState) }) - s.Run("a valid full propagation of config and virtual time", func() { - src := newSource() - tsc, propagatedState := propagateTimeSkippingToChild(src) + s.Run("FullState_PropagatesConfigAndVirtualTime", func() { + tsi := newTSI() + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) s.Require().NotNil(tsc) s.True(tsc.GetEnabled()) // no fast-forward @@ -227,66 +239,104 @@ func (s *mutableStateSuite) TestSnapshotTimeSkippingInfo_ForChildWorkflows() { s.Equal(sessionMaxSkipCount, tsc.GetMaxSessionSkipCount()) // no accumulated session skip s.Equal(int32(0), propagatedState.GetInitialSkipCount()) + s.requireInitNoPanic(tsc, propagatedState) + }) + + s.Run("FastForward_NeverPropagatedToChild", func() { + tsi := newTSI() // enabled parent carrying an active, unreached fast-forward + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) + s.Require().NotNil(tsc) + s.Nil(tsc.GetFastForwardConfig(), "child never inherits the fast-forward config") + s.Require().NotNil(propagatedState) + s.Nil(propagatedState.GetFastForwardTargetTime(), + "fast-forward is per-execution; a child never inherits the fast-forward target time") + s.requireInitNoPanic(tsc, propagatedState) + }) + + s.Run("DisablePropagationFlag_NeverCarriedToChild", func() { + // Config only propagates when !DisablePropagation, so the flag is structurally always + // false on a propagated child config -- there is nothing to carry down the tree. + tsi := newTSI() + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) + s.Require().NotNil(tsc) + s.False(tsc.GetDisablePropagation()) + s.requireInitNoPanic(tsc, propagatedState) }) - s.Run("child workflow propagation can be turned off", func() { - src := newSource() - src.TimeSkippingInfo.Config.DisablePropagation = true - tsc, propagatedState := propagateTimeSkippingToChild(src) + s.Run("DisablePropagationSet_NoConfigPropagated", func() { + tsi := newTSI() + tsi.Config.DisablePropagation = true + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) s.Nil(tsc) s.Require().NotNil(propagatedState) s.Equal(accumSkip, propagatedState.GetInitialSkippedDuration().AsDuration()) s.Equal(int32(0), propagatedState.GetInitialSkipCount()) s.Nil(propagatedState.GetFastForwardTargetTime()) + s.requireInitNoPanic(tsc, propagatedState) }) - s.Run("nil config still propagates virtual time", func() { - src := newSource() - src.TimeSkippingInfo.Config = nil - tsc, propagatedState := propagateTimeSkippingToChild(src) + s.Run("NilConfig_PropagatesVirtualTime", func() { + tsi := newTSI() + tsi.Config = nil + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) s.Nil(tsc) s.Require().NotNil(propagatedState) s.Equal(int32(0), propagatedState.GetInitialSkipCount()) s.Equal(time.Hour, propagatedState.GetInitialSkippedDuration().AsDuration(), "virtual time is always propagated, even when config propagation is disabled") s.Nil(propagatedState.GetFastForwardTargetTime()) + s.requireInitNoPanic(tsc, propagatedState) }) - s.Run("disabled propagation still propagates virtual time", func() { - src := newSource() - src.TimeSkippingInfo.Config.Enabled = false - tsc, propagatedState := propagateTimeSkippingToChild(src) - s.Nil(tsc) + s.Run("DisabledParentConfig_PropagatesDisabledConfig", func() { + tsi := newTSI() + tsi.Config.Enabled = false + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) + // enabled is copied through: a disabled-but-propagating parent hands the child a disabled config. + s.Require().NotNil(tsc) + s.False(tsc.GetEnabled()) + s.Equal(sessionMaxSkipCount, tsc.GetMaxSessionSkipCount()) + s.Nil(tsc.GetFastForwardConfig()) s.Require().NotNil(propagatedState) s.Equal(time.Hour, propagatedState.GetInitialSkippedDuration().AsDuration(), - "virtual time is always propagated, even when config propagation is disabled") + "virtual time is always propagated regardless of enabled") s.Nil(propagatedState.GetFastForwardTargetTime()) s.Equal(int32(0), propagatedState.GetInitialSkipCount()) + s.requireInitNoPanic(tsc, propagatedState) }) + s.Run("ZeroAccumulatedSkip_ConfigButNilState", func() { + tsi := newTSI() + tsi.AccumulatedSkippedDuration = nil + tsc, propagatedState := propagateTimeSkippingToOtherExecution(tsi) + s.Require().NotNil(tsc, "an enabled config still propagates with no accumulated skip") + s.True(tsc.GetEnabled()) + s.Nil(propagatedState, "no accumulated skip -> no state to propagate") + s.requireInitNoPanic(tsc, propagatedState) + }) } func (s *mutableStateSuite) TestIsWorkflowSkippable() { // base case: ensures the other tests modifications are the reason to make the workflow not skippable - s.Run("BaseCase", func() { + s.Run("EnabledAndIdle_Skippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}} s.True(s.mutableState.isWorkflowSkippable()) }) // config tests: nil safe, and false safe proof - s.Run("FalseWhenTimeSkippingInfoNil", func() { + s.Run("NilTimeSkippingInfo_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = nil s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenConfigNil", func() { + s.Run("NilConfig_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{Config: nil} s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenConfigDisabled", func() { + s.Run("DisabledConfig_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: false}, } @@ -294,7 +344,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { }) // state and status tests - s.Run("FalseWhenWorkflowNotRunning", func() { + s.Run("WorkflowNotRunning_NotSkippable", func() { s.mutableState.executionState.State = enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED s.mutableState.executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ @@ -303,7 +353,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPaused", func() { + s.Run("WorkflowPaused_NotSkippable", func() { s.mutableState.executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, @@ -313,7 +363,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { }) // inflight work tests - s.Run("FalseWhenPendingWorkflowTask", func() { + s.Run("PendingWorkflowTask_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -322,7 +372,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPendingActivity", func() { + s.Run("PendingActivity_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -330,7 +380,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPendingSignalExternal", func() { + s.Run("PendingSignalExternal_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -338,7 +388,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPendingRequestCancelExternal", func() { + s.Run("PendingRequestCancelExternal_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -346,7 +396,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPendingChildExecution", func() { + s.Run("PendingChildExecution_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -354,7 +404,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.False(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenPendingNexusOperation", func() { + s.Run("PendingNexusOperation_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -369,7 +419,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { }) // conditions that won't impact the workflow skippability - s.Run("TrueWhenPendingTimersExist", func() { + s.Run("PendingTimers_Skippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -377,7 +427,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.True(s.mutableState.isWorkflowSkippable()) }) - s.Run("TrueWhenOnlyActivityInRetryBackoff", func() { + s.Run("ActivityInRetryBackoff_Skippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -391,7 +441,7 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { s.True(s.mutableState.isWorkflowSkippable()) }) - s.Run("FalseWhenActivityRetryDueNowOrPast", func() { + s.Run("ActivityRetryDueNowOrPast_NotSkippable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -409,14 +459,18 @@ func (s *mutableStateSuite) TestIsWorkflowSkippable() { }) } -// TestInitTimeSkippingInfo covers the 3 basic scenarios in which the function is called: -// nil inputs (no-op), config-only init (user-started), and init with propagated state. -func (s *mutableStateSuite) TestInitTimeSkippingInfo() { +func (s *mutableStateSuite) requireInitNoPanic( + cfg *commonpb.TimeSkippingConfig, + sp *commonpb.TimeSkippingStatePropagation, +) { + s.mutableState.timeSource = clock.NewEventTimeSource() + s.mutableState.executionInfo.TimeSkippingInfo = nil + s.mutableState.PopTasks() + s.NotPanics(func() { s.mutableState.initTimeSkippingInfo(cfg, sp) }) +} - // if the inputs are nil, the caller doesn't need to call the TSI - // yet we still add this test to ensure the function is safe with an noop implementation - // to call with nil inputs - s.Run("SafeInitWithNil_ForExecutionsWithoutTS", func() { +func (s *mutableStateSuite) TestInitTimeSkippingInfo() { + s.Run("NilConfigAndNilPropagation_NoOp", func() { s.mutableState.timeSource = clock.NewEventTimeSource() baseTime := s.mutableState.timeSource.Now() s.NotPanics(func() { @@ -424,9 +478,10 @@ func (s *mutableStateSuite) TestInitTimeSkippingInfo() { }) s.Nil(s.mutableState.executionInfo.TimeSkippingInfo) s.Equal(baseTime, s.mutableState.Now()) + s.False(s.mutableState.timeSkippingInfoUpdated, "a no-op init must not mark updated") }) - s.Run("InitWithConfigOnly_ForExecutionsWithTSStartedByUser", func() { + s.Run("ConfigOnlyNoPropagation_SetsFastForwardInfo", func() { s.mutableState.timeSource = clock.NewEventTimeSource() baseTime := s.mutableState.timeSource.Now() currentVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: tests.Version, TransitionCount: 5} @@ -448,9 +503,10 @@ func (s *mutableStateSuite) TestInitTimeSkippingInfo() { tsi.GetFastForwardInfo().GetTargetTime().AsTime()) // the fast-forward records the current versioned transition s.Require().True(proto.Equal(nextVT, tsi.GetFastForwardInfoLastUpdateVersionedTransition())) + s.True(s.mutableState.timeSkippingInfoUpdated, "a real init marks updated") }) - s.Run("InitWithPropagation_ForExecutionsWithTSStartedByPropagation", func() { + s.Run("WithPropagatedState_ShiftsClockAndSetsFastForward", func() { s.mutableState.timeSource = clock.NewEventTimeSource() // all local time @@ -477,48 +533,84 @@ func (s *mutableStateSuite) TestInitTimeSkippingInfo() { s.Equal(hasSkipped, tsi.GetAccumulatedSkippedDuration().AsDuration()) s.Equal(targetTime.UTC(), tsi.GetFastForwardInfo().GetTargetTime().AsTime()) + s.True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("SessionSkipCountSeededFromPropagation", func() { + // Invariant 4: propagation may carry an enabled config whose fast-forward target is already due + // once the virtual clock (real + accumulated skip) is established at init. Init must disable time + // skipping, matching the runtime flip-enabled model. + s.Run("EnabledConfigWithDueFastForwardTarget_DisablesSkipping", func() { s.mutableState.timeSource = clock.NewEventTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = nil - cfg := &commonpb.TimeSkippingConfig{Enabled: true} - propagation := &commonpb.TimeSkippingStatePropagation{InitialSkipCount: 4} + baseTime := s.mutableState.timeSource.Now() + hasSkipped := 2 * time.Hour + dueTarget := baseTime.Add(time.Hour) // < virtual now (baseTime + 2h) + cfg := &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}, + } + propagation := &commonpb.TimeSkippingStatePropagation{ + InitialSkippedDuration: durationpb.New(hasSkipped), + FastForwardTargetTime: timestamppb.New(dueTarget), + } + s.mutableState.PopTasks() s.mutableState.initTimeSkippingInfo(cfg, propagation) tsi := s.mutableState.executionInfo.GetTimeSkippingInfo() s.Require().NotNil(tsi) - s.Equal(int32(4), tsi.GetSessionSkipCount()) + s.False(tsi.GetConfig().GetEnabled(), "a due fast-forward at init disables time skipping") + s.Require().NotNil(tsi.GetFastForwardInfo()) + s.True(tsi.GetFastForwardInfo().GetHasReached()) + s.Empty(s.mutableState.PopTasks()[tasks.CategoryTimer], "a reached target schedules no timer") + s.True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("SessionSkipCountDefaultsToZeroWithoutPropagation", func() { + s.Run("InheritedSkipOnlyNoConfig_ShiftsClock", func() { s.mutableState.timeSource = clock.NewEventTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = nil - cfg := &commonpb.TimeSkippingConfig{Enabled: true} + baseTime := s.mutableState.timeSource.Now() + hasSkipped := 2 * time.Hour + propagation := &commonpb.TimeSkippingStatePropagation{ + InitialSkippedDuration: durationpb.New(hasSkipped), + } - s.mutableState.initTimeSkippingInfo(cfg, nil) + s.mutableState.initTimeSkippingInfo(nil, propagation) tsi := s.mutableState.executionInfo.GetTimeSkippingInfo() s.Require().NotNil(tsi) - s.Equal(int32(0), tsi.GetSessionSkipCount()) + s.Nil(tsi.GetConfig()) + s.Equal(hasSkipped, tsi.GetAccumulatedSkippedDuration().AsDuration()) + s.Equal(baseTime.Add(hasSkipped), s.mutableState.Now(), "inherited skip still shifts the virtual clock") + s.True(s.mutableState.timeSkippingInfoUpdated) }) -} - -func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { - // update assumes the TS info was already initialized; updating a workflow that never - // started time skipping is a no-op and must not initialize it as a side effect. - s.Run("NoopWhenTimeSkippingInfoNil", func() { + s.Run("WithPropagatedSkipCount_SeedsSessionSkipCount", func() { + s.mutableState.timeSource = clock.NewEventTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = nil - s.mutableState.timeSkippingInfoUpdated = false + cfg := &commonpb.TimeSkippingConfig{Enabled: true} + propagation := &commonpb.TimeSkippingStatePropagation{InitialSkipCount: 4} + s.mutableState.initTimeSkippingInfo(cfg, propagation) + tsi := s.mutableState.executionInfo.GetTimeSkippingInfo() + s.Require().NotNil(tsi) + s.Equal(int32(4), tsi.GetSessionSkipCount()) + s.True(s.mutableState.timeSkippingInfoUpdated) + }) - s.mutableState.updateTimeSkippingInfo(&commonpb.TimeSkippingConfig{Enabled: true}) - s.Nil(s.mutableState.executionInfo.GetTimeSkippingInfo(), "must not initialize TS info on update") - s.False(s.mutableState.timeSkippingInfoUpdated, "must not mark updated when there is nothing to update") + s.Run("WithoutPropagation_SessionSkipCountIsZero", func() { + s.mutableState.timeSource = clock.NewEventTimeSource() + s.mutableState.executionInfo.TimeSkippingInfo = nil + cfg := &commonpb.TimeSkippingConfig{Enabled: true} + s.mutableState.initTimeSkippingInfo(cfg, nil) + tsi := s.mutableState.executionInfo.GetTimeSkippingInfo() + s.Require().NotNil(tsi) + s.Equal(int32(0), tsi.GetSessionSkipCount()) + s.True(s.mutableState.timeSkippingInfoUpdated) }) +} - s.Run("UpdateTimeSkippingInfo_UpdateWithNil", func() { +func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { + s.Run("UpdateWithNilConfig_ClearsConfigAndFastForward", func() { s.mutableState.timeSource = clock.NewEventTimeSource() baseTime := s.mutableState.timeSource.Now() currentTSI := &persistencespb.TimeSkippingInfo{ @@ -547,7 +639,7 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { s.True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("UpdateTimeSkippingInfo_EnableTS", func() { + s.Run("EnableWithFastForward_InstallsFastForwardInfo", func() { s.mutableState.timeSource = clock.NewEventTimeSource() baseTime := s.mutableState.timeSource.Now() currentVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: tests.Version, TransitionCount: 5} @@ -579,9 +671,10 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { s.Equal(baseTime.Add(2*time.Hour).UTC(), newTSI.GetFastForwardInfo().GetTargetTime().AsTime()) s.False(newTSI.GetFastForwardInfo().GetHasReached()) s.Equal(time.Hour, newTSI.GetAccumulatedSkippedDuration().AsDuration()) + s.True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("UpdateTimeSkippingInfo_OverrideFFThenTurnOff", func() { + s.Run("OverrideFastForwardThenDisable_ReinstallsThenClears", func() { s.mutableState.timeSource = clock.NewEventTimeSource() baseTime := s.mutableState.timeSource.Now() currentVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: tests.Version, TransitionCount: 8} @@ -630,12 +723,11 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { s.True(proto.Equal(nextVT, tsc3TSI.GetFastForwardInfoLastUpdateVersionedTransition()), "clearing the fast-forward re-stamps the versioned transition rather than clearing it") s.Equal(time.Hour, tsc3TSI.GetAccumulatedSkippedDuration().AsDuration()) - }) // Updating the config restarts the skip session: the per-session skip counter from the // previous session must be cleared. - s.Run("ClearsSessionSkipCount", func() { + s.Run("Update_ResetsSessionSkipCount", func() { s.mutableState.timeSource = clock.NewEventTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{ @@ -650,6 +742,7 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { tsi := s.mutableState.executionInfo.GetTimeSkippingInfo() s.Require().NotNil(tsi) s.Equal(int32(0), tsi.GetSessionSkipCount()) + s.True(s.mutableState.timeSkippingInfoUpdated) }) } @@ -657,7 +750,25 @@ func (s *mutableStateSuite) TestUpdateTimeSkippingInfo() { // FastForward set / nil duration / nil config / Enabled=false. // The first-init virtual-time path is covered separately in TestInitTimeSkippingInfo. func (s *mutableStateSuite) TestApplyFastForward() { - s.Run("FastForward_WithNoPropagatedTargetTime", func() { + s.Run("NilTimeSkippingInfo_NoOp", func() { + s.mutableState.executionInfo.TimeSkippingInfo = nil + s.NotPanics(func() { s.mutableState.applyFastForward(nil) }) + s.Nil(s.mutableState.executionInfo.GetTimeSkippingInfo()) + }) + + s.Run("NilConfig_ClearsFastForwardInfo", func() { + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: nil, + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: timestamppb.New(s.mutableState.Now().Add(time.Hour)), + }, + } + s.NotPanics(func() { s.mutableState.applyFastForward(nil) }) + s.Nil(s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo(), + "no ff config (nil config) -> ff info cleared") + }) + + s.Run("NoPropagatedTarget_SetsTargetFromDuration", func() { fastForwardDuration := 3 * time.Hour s.mutableState.timeSource = clock.NewEventTimeSource() @@ -673,15 +784,20 @@ func (s *mutableStateSuite) TestApplyFastForward() { FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(fastForwardDuration)}}, AccumulatedSkippedDuration: durationpb.New(time.Hour), } + s.mutableState.PopTasks() s.mutableState.applyFastForward(nil) fastForward := s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo() s.Require().NotNil(fastForward) s.False(fastForward.GetHasReached()) - s.WithinDuration(fastForward.GetTargetTime().AsTime(), baseTime.Add(fastForwardDuration), 1*time.Second) + s.WithinDuration(fastForward.GetTargetTime().AsTime(), baseTime.Add(fastForwardDuration), 0) s.True(proto.Equal(nextVT, s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfoLastUpdateVersionedTransition())) + + timers := s.mutableState.PopTasks()[tasks.CategoryTimer] + s.Require().Len(timers, 1, "an enabled config with a pending target schedules a fast-forward timer") + s.WithinDuration(s.mutableState.ToRealTime(fastForward.GetTargetTime().AsTime()), timers[0].GetVisibilityTime(), 0) }) - s.Run("FastForward_WithPropagatedTargetTime", func() { + s.Run("FuturePropagatedTarget_SchedulesTimer", func() { fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) propagatedTarget := fixed.Add(2 * time.Hour) s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) @@ -691,15 +807,100 @@ func (s *mutableStateSuite) TestApplyFastForward() { FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}}, AccumulatedSkippedDuration: durationpb.New(time.Hour), } + s.mutableState.PopTasks() s.mutableState.applyFastForward(timestamppb.New(propagatedTarget)) fastForward := s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo() s.Require().NotNil(fastForward) s.Equal(propagatedTarget, fastForward.GetTargetTime().AsTime(), "propagated target used directly, not recomputed from ff duration") s.NotNil(s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfoLastUpdateVersionedTransition()) + s.False(fastForward.GetHasReached(), "a future target has not been reached yet") + + timers := s.mutableState.PopTasks()[tasks.CategoryTimer] + s.Require().Len(timers, 1, "a future target schedules a fast-forward timer") + // AddTasks converts the virtual target to real wall-clock (minus accumulated skip). + s.Equal(s.mutableState.ToRealTime(propagatedTarget), timers[0].GetVisibilityTime()) + }) + + s.Run("FuturePropagatedTargetDisabledConfig_NoTimer", func() { + fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + propagatedTarget := fixed.Add(2 * time.Hour) + s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: false, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}}, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + } + s.mutableState.PopTasks() + s.mutableState.applyFastForward(timestamppb.New(propagatedTarget)) + fastForward := s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo() + s.Require().NotNil(fastForward) + s.Equal(propagatedTarget, fastForward.GetTargetTime().AsTime(), + "propagated target used directly, not recomputed from ff duration") + s.False(fastForward.GetHasReached(), "a future target has not been reached yet") + s.Empty(s.mutableState.PopTasks()[tasks.CategoryTimer], "a disabled config schedules no timer") + }) + + s.Run("StalePropagatedTarget_MarkedReachedAndDisablesSkipping", func() { + fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + staleTarget := fixed.Add(-2 * time.Hour) + s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}}, + AccumulatedSkippedDuration: durationpb.New(time.Hour), + } + s.mutableState.PopTasks() + s.mutableState.applyFastForward(timestamppb.New(staleTarget)) + tsi := s.mutableState.executionInfo.TimeSkippingInfo + fastForward := tsi.GetFastForwardInfo() + s.Require().NotNil(fastForward) + s.Equal(staleTarget, fastForward.GetTargetTime().AsTime()) + s.True(fastForward.GetHasReached(), "an already-due propagated target is treated as reached") + s.False(tsi.GetConfig().GetEnabled(), + "an already-due target completes the fast-forward, so time skipping is disabled") + s.Empty(s.mutableState.PopTasks()[tasks.CategoryTimer], "a reached target schedules no timer") }) - s.Run("FastForward_NilDuration_NoOp", func() { + s.Run("DisabledConfigWithReachedTarget_PreservesFastForwardInfo", func() { + fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + completedTarget := fixed.Add(-2 * time.Hour) + s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: false, // completing a fast-forward disables skipping + FastForwardConfig: &commonpb.FastForwardConfig{Id: "ff-1", Duration: durationpb.New(3 * time.Hour)}}, + } + s.mutableState.PopTasks() + s.mutableState.applyFastForward(timestamppb.New(completedTarget)) + fastForward := s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo() + s.Require().NotNil(fastForward, "a completed fast-forward is preserved even when disabled") + s.True(fastForward.GetHasReached()) + s.Empty(s.mutableState.PopTasks()[tasks.CategoryTimer], "disabled config schedules no timer") + }) + + s.Run("DisabledConfigNoPropagatedTarget_InertFastForwardInfo", func() { + fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: false, + FastForwardConfig: &commonpb.FastForwardConfig{Duration: durationpb.New(3 * time.Hour)}}, + } + s.mutableState.PopTasks() + s.mutableState.applyFastForward(nil) + // Invariant: an existing ff config keeps ff-info alive, but a disabled config makes it inert + // (fresh target, not reached, no timer). + ff := s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo() + s.Require().NotNil(ff) + s.Equal(fixed.Add(3*time.Hour), ff.GetTargetTime().AsTime()) + s.False(ff.GetHasReached()) + s.Empty(s.mutableState.PopTasks()[tasks.CategoryTimer], "disabled config schedules no timer") + }) + + s.Run("NoFastForwardConfig_ClearsFastForwardInfo", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{ Enabled: true}, @@ -708,7 +909,7 @@ func (s *mutableStateSuite) TestApplyFastForward() { s.Nil(s.mutableState.executionInfo.TimeSkippingInfo.GetFastForwardInfo()) }) - s.Run("FastForward_ClearsExistingInfo", func() { + s.Run("NoFastForwardConfigWithExistingInfo_ClearsIt", func() { staleVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: 1, TransitionCount: 7} s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: false}, @@ -738,7 +939,7 @@ func (s *mutableStateSuite) TestApplyFastForward() { s.True(proto.Equal(staleVT, tsi.GetFastForwardInfoLastUpdateVersionedTransition())) }) - s.Run("FastForward_OverridesExistingInfo", func() { + s.Run("StaleExistingFastForwardInfo_Overridden", func() { fixed := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) s.mutableState.timeSource = clock.NewEventTimeSource().Update(fixed) s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ @@ -759,12 +960,11 @@ func (s *mutableStateSuite) TestApplyFastForward() { }) } -// TestFindNextSkipTarget exercises the full candidate-selection branch table: pending timers, -// activity retry backoffs, the workflow backoff (ExecutionTime) branch, the run/execution -// timeouts (all valid targets), and the fast-forward gate that both contributes a candidate and -// disables time skipping once its target is at or before now. func (s *mutableStateSuite) TestFindNextSkipTarget() { baseTime := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC) + fixedTimeSource := clock.NewEventTimeSource() + fixedTimeSource.Update(baseTime) + s.mutableState.timeSource = fixedTimeSource addTimer := func(id string, expiry time.Time) { s.mutableState.pendingTimerInfoIDs[id] = &persistencespb.TimerInfo{ @@ -796,11 +996,35 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.mutableState.executionInfo.WorkflowExecutionExpirationTime = nil } - s.Run("NoCandidates_InvalidTransition", func() { + s.Run("NoCandidates_NoTarget", func() { resetMS() s.Nil(s.mutableState.findNextSkipTarget()) }) + s.Run("NilTimeSkippingInfo_NoPanic", func() { + resetMS() + s.mutableState.executionInfo.TimeSkippingInfo = nil + var tr *timeSkippingTransition + s.NotPanics(func() { tr = s.mutableState.findNextSkipTarget() }) + s.Nil(tr, "nil TimeSkippingInfo with no candidates yields no transition") + }) + + s.Run("NoWorkflowTaskYet_TimeoutsAreNotTargets", func() { + resetMS() + // !HadOrHasWorkflowTask: start event applied but the first workflow task is not scheduled yet. + s.mutableState.executionInfo.LastCompletedWorkflowTaskStartedEventId = common.EmptyEventID + s.mutableState.executionInfo.WorkflowTaskScheduledEventId = common.EmptyEventID + // ExecutionTime == StartTime, so the backoff branch contributes no candidate either. + s.mutableState.executionInfo.ExecutionTime = timestamppb.New(baseTime) + // Both timeouts are in the future but must not be skip targets until a task has run -- + // otherwise a fresh child would skip to its run timeout before ever setting its timers. + s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(baseTime.Add(2 * time.Hour)) + s.mutableState.executionInfo.WorkflowExecutionExpirationTime = timestamppb.New(baseTime.Add(3 * time.Hour)) + + s.Nil(s.mutableState.findNextSkipTarget(), + "run/execution timeouts must not be skip targets before the first workflow task") + }) + s.Run("OneUserTimer_TargetIsTimer", func() { resetMS() t1 := baseTime.Add(2 * time.Hour) @@ -822,7 +1046,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(baseTime.Add(2*time.Hour), tr.TargetTime) }) - s.Run("UserTimer_PlusEarlierFastForward_TargetIsFastForward", func() { + s.Run("UserTimerAndEarlierFastForward_TargetIsFastForward", func() { resetMS() addTimer("t1", baseTime.Add(3*time.Hour)) fastForwardTarget := baseTime.Add(time.Hour) @@ -836,7 +1060,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.True(tr.DisabledAfterFastForward) }) - s.Run("Backoff_NotChildAndExecutionTimeFuture_IsCandidate", func() { + s.Run("FutureWorkflowBackoff_IsCandidate", func() { resetMS() // !HadOrHasWorkflowTask: no completed and no pending workflow task. s.mutableState.executionInfo.LastCompletedWorkflowTaskStartedEventId = common.EmptyEventID @@ -851,7 +1075,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.False(tr.DisabledAfterFastForward) }) - s.Run("Backoff_ChildWFCase_NotCandidate", func() { + s.Run("ChildWithoutBackoff_NoCandidate", func() { resetMS() s.mutableState.executionInfo.LastCompletedWorkflowTaskStartedEventId = common.EmptyEventID s.mutableState.executionInfo.WorkflowTaskScheduledEventId = common.EmptyEventID @@ -862,7 +1086,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { "child WF without backoff and no other candidate must yield invalid transition") }) - s.Run("Backoff_CaNWithInheritedSkip_PastExecutionTime_NotCandidate", func() { + s.Run("PastExecutionTime_NoCandidate", func() { resetMS() s.mutableState.executionInfo.LastCompletedWorkflowTaskStartedEventId = common.EmptyEventID s.mutableState.executionInfo.WorkflowTaskScheduledEventId = common.EmptyEventID @@ -912,7 +1136,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(early, tr.TargetTime) }) - s.Run("ActivityBackoff_PlusEarlierTimer_TargetIsTimer", func() { + s.Run("ActivityBackoffAndEarlierTimer_TargetIsTimer", func() { resetMS() timerTime := baseTime.Add(time.Hour) addTimer("t1", timerTime) @@ -926,7 +1150,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(timerTime, tr.TargetTime) }) - s.Run("RunTimeoutIsATarget", func() { + s.Run("RunTimeout_IsTarget", func() { resetMS() runTimeout := baseTime.Add(2 * time.Hour) s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(runTimeout) @@ -936,7 +1160,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(runTimeout, tr.TargetTime) }) - s.Run("ExecutionTimeoutIsATarget", func() { + s.Run("ExecutionTimeout_IsTarget", func() { resetMS() execTimeout := baseTime.Add(3 * time.Hour) s.mutableState.executionInfo.WorkflowExecutionExpirationTime = timestamppb.New(execTimeout) @@ -946,7 +1170,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(execTimeout, tr.TargetTime) }) - s.Run("EarliestOfRunAndExecutionTimeoutWins", func() { + s.Run("RunAndExecutionTimeout_EarliestWins", func() { resetMS() runTimeout := baseTime.Add(2 * time.Hour) s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(runTimeout) @@ -957,7 +1181,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Equal(runTimeout, tr.TargetTime, "the earlier run timeout must win") }) - s.Run("ExpiredTimeoutsAreNotTargets", func() { + s.Run("ExpiredTimeouts_NotTargets", func() { resetMS() s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(baseTime.Add(-time.Hour)) s.mutableState.executionInfo.WorkflowExecutionExpirationTime = timestamppb.New(baseTime.Add(-2 * time.Hour)) @@ -965,7 +1189,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.Nil(s.mutableState.findNextSkipTarget(), "timeouts in the past cannot be skip targets") }) - s.Run("FastForwardLaterThanRunTimeout_TimeoutWins", func() { + s.Run("FastForwardAfterRunTimeout_TimeoutWins", func() { resetMS() runTimeout := baseTime.Add(30 * time.Minute) s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(runTimeout) @@ -977,7 +1201,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.False(tr.DisabledAfterFastForward, "fast-forward not reached: it was not the chosen target") }) - s.Run("FastForwardEarlierThanRunTimeout_FastForwardWins", func() { + s.Run("FastForwardBeforeRunTimeout_FastForwardWins", func() { resetMS() fastForwardTarget := baseTime.Add(30 * time.Minute) s.mutableState.executionInfo.WorkflowRunExpirationTime = timestamppb.New(baseTime.Add(2 * time.Hour)) @@ -989,7 +1213,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.True(tr.DisabledAfterFastForward, "fast-forward is the earliest target: skipping to it disables time skipping") }) - s.Run("FastForwardLaterThanExecutionTimeout_TimeoutWins", func() { + s.Run("FastForwardAfterExecutionTimeout_TimeoutWins", func() { resetMS() execTimeout := baseTime.Add(30 * time.Minute) s.mutableState.executionInfo.WorkflowExecutionExpirationTime = timestamppb.New(execTimeout) @@ -1001,7 +1225,7 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.False(tr.DisabledAfterFastForward) }) - s.Run("ZeroValuedTimeoutsAreIgnored_FastForwardWins", func() { + s.Run("ZeroValuedTimeouts_FastForwardWins", func() { resetMS() fastForwardTarget := baseTime.Add(time.Hour) setFastForward(fastForwardTarget) @@ -1017,15 +1241,15 @@ func (s *mutableStateSuite) TestFindNextSkipTarget() { s.True(tr.DisabledAfterFastForward) }) - s.Run("FastForwardAtNow_DisablesAfterFastForward", func() { + s.Run("FastForwardTargetAtNow_DisablesSkipping", func() { resetMS() - // Fast-forward target is at (virtual) now: it has been reached, so time skipping - // is disabled and the target is now itself. + // Fast-forward target is at (virtual) now: already due, so it is a bare disable + // signal with no skip target. setFastForward(baseTime) tr := s.mutableState.findNextSkipTarget() s.Require().NotNil(tr) - s.Equal(baseTime, tr.TargetTime) + s.True(tr.TargetTime.IsZero()) s.True(tr.DisabledAfterFastForward) }) } @@ -1038,7 +1262,7 @@ func (s *mutableStateSuite) TestCloseTransactionHandleWorkflowTimeSkipping() { } s.Require().True(s.mutableState.findNextSkipTarget().IsValid(), "the user timer is a valid skip target") - s.Run("NilConfig", func() { + s.Run("NilConfig_NoSkip", func() { // Time skipping is not enabled (nil config), so the workflow is not skippable. s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{Config: nil} s.mutableState.timeSkippingInfoUpdated = false @@ -1051,7 +1275,7 @@ func (s *mutableStateSuite) TestCloseTransactionHandleWorkflowTimeSkipping() { s.False(s.mutableState.timeSkippingInfoUpdated, "a gated-out transaction must not change state") }) - s.Run("NonWorkflow", func() { + s.Run("NonWorkflow_NoSkip", func() { // Enable time skipping so the only thing rejecting the skip is the non-workflow archetype. s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, @@ -1072,16 +1296,66 @@ func (s *mutableStateSuite) TestCloseTransactionHandleWorkflowTimeSkipping() { s.False(s.mutableState.timeSkippingInfoUpdated, "a non-workflow must not change state (policy %v)", policy) } }) + + // The primary use case: an active, skippable workflow with a valid target skips time, + // emits the transitioned event, and asks for task regeneration. + s.Run("ActiveWithValidTarget_SkipsAndRegeneratesTasks", func() { + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true}, + } + s.mutableState.pendingTimerInfoIDs["t1"] = &persistencespb.TimerInfo{ + TimerId: "t1", + ExpiryTime: timestamppb.New(s.mutableState.Now().Add(time.Hour)), + } + s.mutableState.timeSkippingInfoUpdated = false + s.Require().True(s.mutableState.IsWorkflow()) + s.Require().True(s.mutableState.isWorkflowSkippable()) + + needRegen := s.mutableState.closeTransactionHandleWorkflowTimeSkipping( + context.Background(), historyi.TransactionPolicyActive) + + s.True(needRegen, "a skippable workflow with a valid target must skip time") + s.True(s.mutableState.timeSkippingInfoUpdated, "a successful skip mutates state") + s.Positive(s.mutableState.executionInfo.TimeSkippingInfo.GetAccumulatedSkippedDuration().AsDuration(), + "the skip advances virtual time toward the timer target") + }) + + s.Run("ActiveWithoutValidTarget_NoSkip", func() { + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true}, + } + s.mutableState.timeSkippingInfoUpdated = false + s.Require().True(s.mutableState.isWorkflowSkippable()) + s.Require().False(s.mutableState.findNextSkipTarget().IsValid(), "no candidate -> invalid transition") + + needRegen := s.mutableState.closeTransactionHandleWorkflowTimeSkipping( + context.Background(), historyi.TransactionPolicyActive) + s.False(needRegen, "no valid skip target -> no skip") + s.False(s.mutableState.timeSkippingInfoUpdated, "no skip -> no state change") + }) + + s.Run("PassivePolicy_NoSkip", func() { + s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true}, + } + s.mutableState.pendingTimerInfoIDs["t1"] = &persistencespb.TimerInfo{ + TimerId: "t1", + ExpiryTime: timestamppb.New(s.mutableState.Now().Add(time.Hour)), + } + s.mutableState.timeSkippingInfoUpdated = false + + needRegen := s.mutableState.closeTransactionHandleWorkflowTimeSkipping( + context.Background(), historyi.TransactionPolicyPassive) + s.False(needRegen, "passive policy never skips") + s.False(s.mutableState.timeSkippingInfoUpdated, "passive policy makes no state change") + }) } -// TestTimeSkippingTransition covers the pure timeSkippingTransition data structure: -// its constructor, validity check, earliest-future-time tracking, and fast-forward -// gating. It needs no mutable state, so it is a plain test rather than a suite method. func TestTimeSkippingTransition(t *testing.T) { t.Parallel() base := time.Date(2027, 1, 1, 12, 0, 0, 0, time.UTC) - t.Run("New sets only the current time", func(t *testing.T) { + t.Run("New_SetsOnlyCurrentTime", func(t *testing.T) { tr := NewTimeSkippingTransition(base) require.Equal(t, base, tr.CurrentTime) require.True(t, tr.TargetTime.IsZero()) @@ -1091,7 +1365,7 @@ func TestTimeSkippingTransition(t *testing.T) { // Invariant 1: every method is nil-safe — on a nil receiver, and (for GateByFastForward) // on a nil/absent fast-forward argument. - t.Run("nil safe", func(t *testing.T) { + t.Run("NilReceiver_NoPanic", func(t *testing.T) { var nilTr *timeSkippingTransition require.False(t, nilTr.IsValid(), "nil transition is never valid") require.NotPanics(t, func() { nilTr.TrackEarliestFutureTime(base.Add(time.Hour)) }) @@ -1116,15 +1390,15 @@ func TestTimeSkippingTransition(t *testing.T) { // Invariant 2: TrackEarliestFutureTime keeps the earliest strictly-trackable future time // and ignores anything that is not a usable future skip target. - t.Run("earliest future time", func(t *testing.T) { - t.Run("ignores zero and past candidates", func(t *testing.T) { + t.Run("TrackEarliestFutureTime", func(t *testing.T) { + t.Run("ZeroOrPastCandidate_Ignored", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.TrackEarliestFutureTime(time.Time{}) // zero candidate tr.TrackEarliestFutureTime(base.Add(-time.Hour)) // past candidate require.True(t, tr.TargetTime.IsZero()) }) - t.Run("keeps the earliest of several future candidates", func(t *testing.T) { + t.Run("MultipleFutureCandidates_KeepsEarliest", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.TrackEarliestFutureTime(base.Add(3 * time.Hour)) require.Equal(t, base.Add(3*time.Hour), tr.TargetTime) @@ -1136,7 +1410,7 @@ func TestTimeSkippingTransition(t *testing.T) { require.Equal(t, base.Add(time.Hour), tr.TargetTime) }) - t.Run("accepts a candidate equal to the current time", func(t *testing.T) { + t.Run("CandidateEqualToCurrentTime_Accepted", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.TrackEarliestFutureTime(base) require.Equal(t, base, tr.TargetTime) @@ -1147,8 +1421,8 @@ func TestTimeSkippingTransition(t *testing.T) { // it is the earliest target (nothing earlier tracked). When a real candidate is earlier the // fast-forward is not reached and skipping stays enabled. An absent/reached/zero fast-forward // is a no-op. - t.Run("fast-forward fallback and gating", func(t *testing.T) { - t.Run("taken as the target and disables when nothing earlier exists", func(t *testing.T) { + t.Run("GateByFastForward", func(t *testing.T) { + t.Run("NoEarlierTarget_TakesFastForwardTargetAndDisables", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(time.Hour))}) require.True(t, base.Add(time.Hour).Equal(tr.TargetTime)) @@ -1156,7 +1430,7 @@ func TestTimeSkippingTransition(t *testing.T) { require.True(t, tr.IsValid()) }) - t.Run("an earlier tracked target wins over a later fast-forward", func(t *testing.T) { + t.Run("EarlierTrackedTarget_WinsOverFastForward", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.TrackEarliestFutureTime(base.Add(time.Hour)) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(3 * time.Hour))}) @@ -1164,7 +1438,7 @@ func TestTimeSkippingTransition(t *testing.T) { require.False(t, tr.DisabledAfterFastForward) }) - t.Run("an earlier fast-forward wins over a later tracked target and disables", func(t *testing.T) { + t.Run("EarlierFastForward_WinsOverTrackedAndDisables", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.TrackEarliestFutureTime(base.Add(3 * time.Hour)) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(time.Hour))}) @@ -1172,7 +1446,7 @@ func TestTimeSkippingTransition(t *testing.T) { require.True(t, tr.DisabledAfterFastForward) }) - t.Run("ignores an already-reached fast-forward", func(t *testing.T) { + t.Run("ReachedFastForward_Ignored", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{ HasReached: true, @@ -1182,44 +1456,53 @@ func TestTimeSkippingTransition(t *testing.T) { require.False(t, tr.DisabledAfterFastForward) }) - t.Run("ignores a zero-valued target time", func(t *testing.T) { + t.Run("ZeroFastForwardTarget_Ignored", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(time.Time{})}) require.True(t, tr.TargetTime.IsZero()) require.False(t, tr.DisabledAfterFastForward) }) - t.Run("target equal to current disables and sets the target", func(t *testing.T) { + t.Run("TargetEqualToCurrentTime_BareDisableSignal", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base)}) - require.True(t, base.Equal(tr.TargetTime)) + require.True(t, tr.TargetTime.IsZero(), "a target at now is already due, not a future skip target") require.True(t, tr.DisabledAfterFastForward) require.True(t, tr.IsValid()) }) - t.Run("past target disables as a bare signal", func(t *testing.T) { + t.Run("PastFastForwardTarget_BareDisableSignal", func(t *testing.T) { tr := NewTimeSkippingTransition(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(-time.Hour))}) require.True(t, tr.TargetTime.IsZero(), "a past target is not a future skip target") require.True(t, tr.DisabledAfterFastForward) require.True(t, tr.IsValid()) }) + + t.Run("StaleFastForward_ClearsTrackedTargetAndDisables", func(t *testing.T) { + tr := NewTimeSkippingTransition(base) + tr.TrackEarliestFutureTime(base.Add(time.Hour)) + tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(-time.Hour))}) + require.True(t, tr.TargetTime.IsZero(), "a stale fast-forward clears the tracked target") + require.True(t, tr.DisabledAfterFastForward) + require.True(t, tr.IsValid(), "a bare disable signal is still a valid transition") + }) }) // Invariant 4: without a current time the transition is always invalid and no setter can // make it valid — every field is relative to the current time. - t.Run("no current time always invalid", func(t *testing.T) { - t.Run("a directly-set target is still invalid", func(t *testing.T) { + t.Run("NoCurrentTime", func(t *testing.T) { + t.Run("DirectlySetTarget_StillInvalid", func(t *testing.T) { tr := &timeSkippingTransition{TargetTime: base.Add(time.Hour)} require.False(t, tr.IsValid()) }) - t.Run("a directly-set disable signal is still invalid", func(t *testing.T) { + t.Run("DirectlySetDisableSignal_StillInvalid", func(t *testing.T) { tr := &timeSkippingTransition{DisabledAfterFastForward: true} require.False(t, tr.IsValid()) }) - t.Run("setters are no-ops without a current time", func(t *testing.T) { + t.Run("SettersWithoutCurrentTime_NoOp", func(t *testing.T) { tr := &timeSkippingTransition{} tr.TrackEarliestFutureTime(base) tr.GateByFastForward(&persistencespb.FastForwardInfo{TargetTime: timestamppb.New(base.Add(time.Hour))}) @@ -1250,7 +1533,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv } } - s.Run("ErrorWhenTimeSkippingInfoNil", func() { + s.Run("NilTimeSkippingInfo_ReturnsError", func() { s.mutableState.executionInfo.TimeSkippingInfo = nil targetTime := baseTime.Add(2 * time.Hour) err := s.mutableState.ApplyWorkflowExecutionTimeSkippingTransitionedEvent( @@ -1260,7 +1543,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv s.Require().Error(err) }) - s.Run("AccumulatesDuration", func() { + s.Run("WithTargetTime_AccumulatesDuration", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -1278,7 +1561,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv s.Require().True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("AccumulatesDurationAdditively", func() { + s.Run("WithExistingSkip_AccumulatesAdditively", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, AccumulatedSkippedDuration: durationpb.New(time.Hour), @@ -1295,7 +1578,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv s.Require().Equal(3*time.Hour, accumulated.AsDuration()) // 1h pre-existing + 2h new }) - s.Run("ErrorWhenNilTargetTimeAndNotDisabledAfterBound", func() { + s.Run("NilTargetTimeAndNotDisabledAfterFastForward_ReturnsError", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -1307,9 +1590,12 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv s.Require().Error(err) }) - s.Run("NilTargetTimeWithDisabledAfterBoundDoesNotAccumulateDuration", func() { + // disabledAfterFastForward only disables when a FastForwardInfo is present (the nil-guard in + // applyEvent). A large MaxSessionSkipCount keeps the skip-count cap from being the thing that + // disables, so these two isolate the ff==nil guard: Enabled must stay true. + s.Run("NilTargetTimeWithDisabledFlag_NoAccumulation", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: true}, + Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 100}, AccumulatedSkippedDuration: durationpb.New(time.Hour), } s.mutableState.timeSkippingInfoUpdated = false @@ -1321,13 +1607,14 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv accumulated := s.mutableState.GetExecutionInfo().TimeSkippingInfo.AccumulatedSkippedDuration s.Require().Equal(time.Hour, accumulated.AsDuration()) - s.Require().False(s.mutableState.GetExecutionInfo().TimeSkippingInfo.Config.Enabled) + s.Require().True(s.mutableState.GetExecutionInfo().TimeSkippingInfo.Config.Enabled, + "disabledAfterFastForward with nil FastForwardInfo must not disable") s.Require().True(s.mutableState.timeSkippingInfoUpdated) }) - s.Run("DisabledAfterBoundDisablesConfigAndAccumulatesDuration", func() { + s.Run("DisabledFlagWithNilFastForwardInfo_DoesNotDisable", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ - Config: &commonpb.TimeSkippingConfig{Enabled: true}, + Config: &commonpb.TimeSkippingConfig{Enabled: true, MaxSessionSkipCount: 100}, } targetTime := baseTime.Add(2 * time.Hour) @@ -1337,14 +1624,15 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv ) s.Require().NoError(err) - s.Require().False(s.mutableState.GetExecutionInfo().TimeSkippingInfo.Config.Enabled) + s.Require().True(s.mutableState.GetExecutionInfo().TimeSkippingInfo.Config.Enabled, + "disabledAfterFastForward with nil FastForwardInfo must not disable") accumulated := s.mutableState.GetExecutionInfo().TimeSkippingInfo.AccumulatedSkippedDuration s.Require().Equal(2*time.Hour, accumulated.AsDuration()) }) // Regression: a fast-forward-completion transition must both disable the config and record // the fast-forward as reached. A previous bug left Enabled unchanged on this path. - s.Run("FastForwardCompletionDisables", func() { + s.Run("FastForwardCompletion_DisablesSkipping", func() { targetTime := baseTime.Add(2 * time.Hour) staleVT := &persistencespb.VersionedTransition{NamespaceFailoverVersion: 1, TransitionCount: 3} s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ @@ -1377,7 +1665,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv // Every applied transition increments SessionSkipCount by one, and once the count reaches // the per-session cap the config is disabled. - s.Run("SessionSkipCountIncrementsUntilCapDisables", func() { + s.Run("SkipCountReachesCap_DisablesSkipping", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{ Enabled: true, @@ -1409,7 +1697,7 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionTimeSkippingTransitionedEv // A MaxSkipPerSession of 0 caps on the very first skip (1 >= 0), disabling immediately. // The frontend guarantees a populated config carries at least 1, so a 0 reaching the // runtime (e.g. a config that never had the cap populated) kills skipping right away. - s.Run("SessionSkipCountCapsImmediatelyWhenMaxUnset", func() { + s.Run("MaxSessionSkipUnset_DisablesOnFirstSkip", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -1439,7 +1727,7 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { return ts } - s.Run("ZeroOffsetWhenTimeSkippingInfoNil", func() { + s.Run("NilTimeSkippingInfo_ZeroOffset", func() { s.mutableState.timeSource = fixedTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = nil @@ -1451,7 +1739,7 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { s.Equal(fixedBase, s.mutableState.timeSource.Now()) }) - s.Run("OffsetTracksAccumulatedDuration", func() { + s.Run("AccumulatedSkip_OffsetsClock", func() { s.mutableState.timeSource = fixedTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ AccumulatedSkippedDuration: durationpb.New(skipped), @@ -1464,7 +1752,7 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { s.Equal(fixedBase.Add(skipped), s.mutableState.timeSource.Now()) }) - s.Run("OffsetFollowsLateTimeSkippingInfoAssignment", func() { + s.Run("LateTimeSkippingInfoAssignment_OffsetFollows", func() { // Wrap first with nil TimeSkippingInfo, then assign it — the closure must // pick up the new accumulated duration without a re-wrap. s.mutableState.timeSource = fixedTimeSource() @@ -1478,7 +1766,7 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { s.Equal(fixedBase.Add(skipped), s.mutableState.timeSource.Now()) }) - s.Run("IdempotentWhenAlreadyWrapped", func() { + s.Run("AlreadyWrapped_Idempotent", func() { s.mutableState.timeSource = fixedTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ AccumulatedSkippedDuration: durationpb.New(skipped), @@ -1491,7 +1779,7 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { s.Equal(wrappedOnce, s.mutableState.timeSource, "second call must not double-wrap") }) - s.Run("HBuilderUsesVirtualTime", func() { + s.Run("HistoryBuilder_UsesVirtualTime", func() { s.mutableState.timeSource = fixedTimeSource() s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ AccumulatedSkippedDuration: durationpb.New(skipped), @@ -1512,21 +1800,86 @@ func (s *mutableStateSuite) TestWrapTimeSourceWithTimeSkipping() { } func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { - s.Run("NilTSISafe", func() { + s.Run("NilTimeSkippingInfo_NilSafe", func() { util := NewTimeSkippingInfoUtil(nil) s.False(util.HasPendingFastForward()) s.False(util.IsEnabled()) + s.Zero(util.GetAccumulatedSkippedDuration()) + s.Nil(util.GetFastForwardTargetTime()) + s.Nil(util.ToDescribeInfo(time.Now())) + s.Nil(util.ToFastForwardInfo()) }) - s.Run("NilReceiverSafe", func() { + s.Run("NilReceiver_NilSafe", func() { var util *TimeSkippingInfoUtil s.False(util.IsEnabled()) s.False(util.HasPendingFastForward()) s.Zero(util.GetAccumulatedSkippedDuration()) + s.Nil(util.GetFastForwardTargetTime()) s.Nil(util.ToDescribeInfo(time.Now())) + s.Nil(util.ToFastForwardInfo()) + }) + + s.Run("GetFastForwardTarget_NilFastForwardInfo_ReturnsNil", func() { + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: true}}) + s.Nil(util.GetFastForwardTargetTime()) + }) + + s.Run("GetFastForwardTarget_NilTargetTime_ReturnsNil", func() { + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + FastForwardInfo: &persistencespb.FastForwardInfo{TargetTime: nil}}) + s.Nil(util.GetFastForwardTargetTime()) }) - s.Run("NilFFISafe", func() { + s.Run("GetFastForwardTarget_ZeroTargetTime_ReturnsNil", func() { + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + FastForwardInfo: &persistencespb.FastForwardInfo{TargetTime: timestamppb.New(time.Time{})}}) + s.Nil(util.GetFastForwardTargetTime()) + }) + + s.Run("GetFastForwardTarget_ValidTarget_ReturnsTarget", func() { + target := time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC) + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + FastForwardInfo: &persistencespb.FastForwardInfo{TargetTime: timestamppb.New(target)}}) + got := util.GetFastForwardTargetTime() + s.Require().NotNil(got) + s.Equal(target, got.AsTime()) + }) + + s.Run("ToFastForwardInfo_Populated_IncludesIdAndDuration", func() { + target := time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC) + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + FastForwardConfig: &commonpb.FastForwardConfig{Id: "ff-9", Duration: durationpb.New(3 * time.Hour)}}, + FastForwardInfo: &persistencespb.FastForwardInfo{TargetTime: timestamppb.New(target), HasReached: true}, + }) + got := util.ToFastForwardInfo() + s.Require().NotNil(got) + s.Equal(target, got.GetTargetTime().AsTime()) + s.True(got.GetHasCompleted()) + s.Equal("ff-9", got.GetFastForwardId()) + s.Equal(3*time.Hour, got.GetFastForwardDuration().AsDuration()) + }) + + s.Run("ToFastForwardInfo_NilConfig_EmptyIdAndDuration", func() { + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + Config: nil, + FastForwardInfo: &persistencespb.FastForwardInfo{TargetTime: timestamppb.New(time.Date(2027, 1, 1, 0, 0, 0, 0, time.UTC))}, + }) + got := util.ToFastForwardInfo() + s.Require().NotNil(got, "ff present -> non-nil even with nil config") + s.Empty(got.GetFastForwardId()) + s.Nil(got.GetFastForwardDuration()) + }) + + s.Run("IsEnabled_DisabledConfig_False", func() { + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{Enabled: false}}) + s.False(util.IsEnabled()) + }) + + s.Run("HasPendingFastForward_NilFastForwardInfo_False", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, FastForwardInfo: nil, @@ -1535,7 +1888,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.False(util.HasPendingFastForward()) }) - s.Run("NoPendingFastForwardWhenTargetTimeNil", func() { + s.Run("HasPendingFastForward_NilTargetTime_False", func() { util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, FastForwardInfo: &persistencespb.FastForwardInfo{HasReached: false, TargetTime: nil}, @@ -1543,7 +1896,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.False(util.HasPendingFastForward()) }) - s.Run("NoPendingFastForwardWhenTargetTimeZero", func() { + s.Run("HasPendingFastForward_ZeroTargetTime_False", func() { util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, FastForwardInfo: &persistencespb.FastForwardInfo{HasReached: false, TargetTime: timestamppb.New(time.Time{})}, @@ -1551,7 +1904,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.False(util.HasPendingFastForward()) }) - s.Run("NoPendingFastForwardWhenHasReached", func() { + s.Run("HasPendingFastForward_AlreadyReached_False", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, FastForwardInfo: &persistencespb.FastForwardInfo{ @@ -1567,7 +1920,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.False(util.HasPendingFastForward()) }) - s.Run("NoPendingFastForwardWhenDisabled", func() { + s.Run("HasPendingFastForward_DisabledConfig_False", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: false}, FastForwardInfo: &persistencespb.FastForwardInfo{ @@ -1583,7 +1936,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.False(util.HasPendingFastForward()) }) - s.Run("HasPendingFastForward", func() { + s.Run("HasPendingFastForward_ActiveTarget_True", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, FastForwardInfo: &persistencespb.FastForwardInfo{ @@ -1599,7 +1952,7 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.True(util.HasPendingFastForward()) }) - s.Run("IsEnabled", func() { + s.Run("IsEnabled_EnabledConfig_True", func() { s.mutableState.executionInfo.TimeSkippingInfo = &persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, } @@ -1607,12 +1960,12 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.True(util.IsEnabled()) }) - s.Run("IsEnabledNilConfig", func() { + s.Run("IsEnabled_NilConfig_False", func() { util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{Config: nil}) s.False(util.IsEnabled()) }) - s.Run("GetAccumulatedSkippedDuration", func() { + s.Run("GetAccumulatedSkippedDuration_ReturnsValue", func() { s.Zero(NewTimeSkippingInfoUtil(nil).GetAccumulatedSkippedDuration(), "nil info") s.Zero(NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{}).GetAccumulatedSkippedDuration(), "unset duration") util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ @@ -1621,11 +1974,11 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.Equal(90*time.Minute, util.GetAccumulatedSkippedDuration()) }) - s.Run("ToDescribeInfoNilTSI", func() { + s.Run("ToDescribeInfo_NilTimeSkippingInfo_ReturnsNil", func() { s.Nil(NewTimeSkippingInfoUtil(nil).ToDescribeInfo(time.Now())) }) - s.Run("ToDescribeInfoEnabled", func() { + s.Run("ToDescribeInfo_EnabledNoFastForward_NilFastForwardInfo", func() { now := time.Date(2027, 1, 1, 12, 0, 0, 0, time.UTC) util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: true}, @@ -1634,9 +1987,32 @@ func (s *mutableStateSuite) TestTimeSkippingInfoUtil() { s.NotNil(info) s.True(info.GetEffectiveConfig().GetEnabled()) s.Equal(now, info.GetCurrentTime().AsTime()) + s.Nil(info.GetFastForwardInfo(), "no fast-forward set") + }) + + s.Run("ToDescribeInfo_WithFastForward_IncludesFastForwardInfo", func() { + now := time.Date(2027, 1, 1, 12, 0, 0, 0, time.UTC) + targetTime := now.Add(time.Hour) + util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ + Config: &commonpb.TimeSkippingConfig{ + Enabled: true, + FastForwardConfig: &commonpb.FastForwardConfig{Id: "ff-1"}, + }, + FastForwardInfo: &persistencespb.FastForwardInfo{ + TargetTime: timestamppb.New(targetTime), + HasReached: true, + }, + }) + info := util.ToDescribeInfo(now) + s.NotNil(info) + ff := info.GetFastForwardInfo() + s.NotNil(ff) + s.Equal(targetTime, ff.GetTargetTime().AsTime()) + s.True(ff.GetHasCompleted()) + s.Equal("ff-1", ff.GetFastForwardId()) }) - s.Run("ToDescribeInfoDisabled", func() { + s.Run("ToDescribeInfo_DisabledConfig_ReflectsConfig", func() { util := NewTimeSkippingInfoUtil(&persistencespb.TimeSkippingInfo{ Config: &commonpb.TimeSkippingConfig{Enabled: false}, AccumulatedSkippedDuration: durationpb.New(2 * time.Hour), From 0ac4acad35fbc275028d8db833b3f3b3778c0285 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Wed, 19 Aug 2026 16:28:18 -0700 Subject: [PATCH 016/139] Pass the local sub state machine to sanitizeFn on apply (#11646) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## The bug `applyUpdatesToSubStateMachine` shadowed the variable it meant to populate: ```go var existing V // outer: never assigned if existing, ok := pendingInfos[key]; ok { // := declares a NEW existing, scoped to the if ... ms.approximateSize -= existing.Size() + getSizeOfKey(key) // inner one — correct here } val := updated if sanitizeFn != nil { val = common.CloneProto(updated) sanitizeFn(existing, val) // outer one — always nil } ``` `:=` only requires *one* new variable on the left (`ok`), and the `if` init statement is its own scope, so `existing` is redeclared there rather than assigned. It compiles because both variables are used, and `go vet` does not check shadowing by default. `sanitizeFn` has therefore always received `current == nil`. ## What this activates Two of the five `sanitizeFn` implementations depend on `current`; the other three (timers, request cancels, signals) ignore it or are `nil`, so they are unaffected. **Activities.** `getActivityTimerTaskStatus` short-circuits to `TimerTaskStatusNone` when `current` is nil, so every applied activity update cleared the *entire* timer task mask rather than only the bits whose deadline moved. The deadline comparison added in #11565 has never taken effect — that PR is currently a no-op. **Child executions.** `if current != nil { incoming.Clock = current.Clock }` never ran. `Clock` is a local shard vector clock that `sanitizeChildExecutionInfo` strips before replicating, so the incoming copy always arrives nil — the guard exists precisely to restore the local value. Without it, every replicated update to an existing child execution erased it. ## Risk Both are restorations of intended behavior, not new behavior, and both previously failed in the safe direction: - An over-cleared activity mask regenerates a timer task that is already pending. Duplicates are dropped at execution (`processSingleActivityTimeoutTask` re-derives the sequence and fires what expired), so the symptom was extra timer-queue writes, not missed timeouts. - A nil child clock is tolerated by callers — see the comment in `recordchildworkflowcompleted/api.go`: *"it should be fine e.g. that ci.Clock is nil"*. The child execution change is a **no-op unless the local cluster recorded a clock by starting the child itself**, which for a passive cluster means after a failover. On a standby that never started the child, both sides are nil. For activities, preserved bits suppress recreation of a timer task, so it is worth being explicit about why that is safe on the passive side: an expired-but-unresolved standby timer returns `ErrTaskRetry` and stays in the queue rather than being consumed, so there is nothing to recreate. The one path that does drop a task, past `StandbyTaskMissingEventsDiscardDelay`, logs a warning and increments `task_errors_discarded`, so it cannot happen silently. ## Tests Two new tests at the `applyUpdatesToSubStateMachines` level, one per activated `sanitizeFn`. **Both fail with the shadowing restored** (verified: `expected: 13, actual: 0` for the mask; `local child execution clock was not carried over` for the clock). The gap they close: the existing `getActivityTimerTaskStatus` tests call the decision function directly with a non-nil `current`, so no unit test could observe the *caller* passing nil. Nine passing tests, zero coverage of the wiring. Two pre-existing tests needed updating, both informative: - `TestApplyMutation` / `TestApplySnapshot` failed with *"Unexpected call to IsVersionFromSameCluster"* — that mock was never needed because the nil short-circuit made the cluster check unreachable. The gap is itself evidence the path was dead. - `verifyActivityInfos` asserted `s.Equal(int32(TimerTaskStatusNone), actual.TimerTaskStatus)`, encoding the bug and blocking any correct fix. Replaced with the invariant that actually matters: ```go s.Zero(actual.TimerTaskStatus&^originStatus, "TimerTaskStatus gained a bit that was not set locally") ``` Applying an update may carry over or drop locally set bits, but must never introduce one that was not already set — claiming a timer task exists when none does is the direction that loses timers. That holds regardless of which bits a given case preserves, so it will not need rewriting as the mask logic evolves. ## Related, tracked separately While auditing whether the mask can be trusted, one path was found that moves a deadline without invalidating the mask: unpausing an activity. Paused activities are excluded from the timer sequence, so a timeout task firing during a long pause is dropped as an invalid task, yet `unpauseActivityInfo` leaves the bit set. That is pre-existing and active-side, independent of this PR, and is being tracked as its own change. It is worth noting here because this PR removes the passive side's accidental repair for that class of stale bit: today's blanket wipe clears any stale bit on the next replicated activity update. --- `go build ./service/...`, `go vet`, and golangci-lint (repo config, `--new-from-rev=origin/main`) clean; `./service/history/workflow/... ./service/history/ndc/...` pass across 3 consecutive runs. Two pre-existing flakes are skipped in those runs and unrelated to this change (both reproduce on unmodified `main`): `TestTaskRefresherSuite/TestRefreshSubStateMachineTasks` (nanosecond-differing HSM deadlines, map iteration order — fails 7/12 on main) and `TestMutableStateSuite/*/TestApplyWorkflowExecutionOptionsUpdatedEvent_TimeSkippingConfig` (wall-clock resolution, ~1/30). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5 --- .../history/workflow/mutable_state_impl.go | 4 +- .../workflow/mutable_state_impl_test.go | 132 +++++- .../activity_timer_task_replication_test.go | 429 ++++++++++++++++++ 3 files changed, 560 insertions(+), 5 deletions(-) create mode 100644 tests/xdc/activity_timer_task_replication_test.go diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index c58e2fa6b7d..af190b9d827 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -9511,8 +9511,8 @@ func applyUpdatesToSubStateMachine[K comparable, V lastUpdatedStateTransitionGet } for key, updated := range updatedSubStateMachine { - var existing V - if existing, ok := pendingInfos[key]; ok { + existing, ok := pendingInfos[key] + if ok { if transitionhistory.Compare(existing.GetLastUpdateVersionedTransition(), updated.GetLastUpdateVersionedTransition()) == 0 { continue } diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 46a4cc0d4de..f005f34a354 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -5491,7 +5491,7 @@ func (s *mutableStateSuite) verifyChildExecutionInfos(expectedMap, actualMap, or } } -func (s *mutableStateSuite) verifyActivityInfos(expectedMap, actualMap map[int64]*persistencespb.ActivityInfo) { +func (s *mutableStateSuite) verifyActivityInfos(expectedMap, actualMap, originMap map[int64]*persistencespb.ActivityInfo) { s.Equal(len(expectedMap), len(actualMap)) for k, expected := range expectedMap { actual, ok := actualMap[k] @@ -5532,7 +5532,15 @@ func (s *mutableStateSuite) verifyActivityInfos(expectedMap, actualMap map[int64 s.True(proto.Equal(expected.LastUpdateVersionedTransition, actual.LastUpdateVersionedTransition), "LastUpdateVersionedTransition mismatch") // special handled fields - s.Equal(int32(TimerTaskStatusNone), actual.TimerTaskStatus, "TimerTaskStatus mismatch") + // TimerTaskStatus is not replicated state: it records which timer tasks this + // cluster created locally. Applying an update may carry over or drop those local + // bits (see getActivityTimerTaskStatus), but must never introduce a bit that was + // not already set locally, which would claim a task exists when none does. + originStatus := int32(TimerTaskStatusNone) + if origin, ok := originMap[k]; ok { + originStatus = origin.TimerTaskStatus + } + s.Zero(actual.TimerTaskStatus&^originStatus, "TimerTaskStatus gained a bit that was not set locally") } } @@ -5636,7 +5644,7 @@ func (s *mutableStateSuite) verifyMutableState(current, target, origin *MutableS s.True(proto.Equal(target.executionState, current.executionState), "executionState mismatch") s.Equal(target.pendingActivityTimerHeartbeats, current.pendingActivityTimerHeartbeats, "pendingActivityTimerHeartbeats mismatch") - s.verifyActivityInfos(target.pendingActivityInfoIDs, current.pendingActivityInfoIDs) + s.verifyActivityInfos(target.pendingActivityInfoIDs, current.pendingActivityInfoIDs, origin.pendingActivityInfoIDs) s.Equal(target.pendingActivityIDToEventID, current.pendingActivityIDToEventID, "pendingActivityIDToEventID mismatch") compareMapOfProto(s, current.pendingActivityInfoIDs, current.updateActivityInfos) s.Equal(map[int64]struct{}{89: {}}, current.deleteActivityInfos, "deleteActivityInfos mismatch") @@ -5896,6 +5904,10 @@ func (s *mutableStateSuite) TestApplySnapshot() { snapshot := s.buildSnapshot(targetMS) s.Nil(snapshot.ExecutionInfo.SubStateMachinesByType) + // Applying an activity update consults the cluster of the local activity info. + s.mockShard.Resource.ClusterMetadata.EXPECT(). + IsVersionFromSameCluster(gomock.Any(), gomock.Any()).Return(true).AnyTimes() + err = currentMS.ApplySnapshot(snapshot) s.NoError(err) s.NotNil(currentMS.GetExecutionInfo().SubStateMachinesByType) @@ -6121,6 +6133,10 @@ func (s *mutableStateSuite) TestApplyMutation() { UpdatedNodes: updateChasmNodes, }).Return(nil).Times(1) + // Applying an activity update consults the cluster of the local activity info. + s.mockShard.Resource.ClusterMetadata.EXPECT(). + IsVersionFromSameCluster(gomock.Any(), gomock.Any()).Return(true).AnyTimes() + err = currentMS.ApplyMutation(mutation) s.NoError(err) s.verifyMutableState(currentMS, targetMS, originMS) @@ -6341,6 +6357,116 @@ func (s *mutableStateSuite) TestNextActivityTimerTaskMask_NewActivity_NoMask() { ) } +// TestApplyUpdatesToSubStateMachines_ActivityTimerTaskStatusCarriedOver covers the wiring +// between applyUpdatesToSubStateMachines and getActivityTimerTaskStatus, rather than the +// decision function on its own. +// +// The getActivityTimerTaskStatus tests above call it directly with a non-nil `current`, so +// they cannot catch a caller that fails to hand over the local activity info. If `current` +// arrives nil the function short-circuits to TimerTaskStatusNone and the entire deadline +// comparison silently degrades to "always reset every bit", which is invisible at the unit +// level but is the whole behavior in production. +func (s *mutableStateSuite) TestApplyUpdatesToSubStateMachines_ActivityTimerTaskStatusCarriedOver() { + scheduledEventID := int64(90) + failoverVersion := s.namespaceEntry.FailoverVersion(tests.WorkflowID) + scheduledTime := time.Date(2026, 8, 11, 0, 0, 0, 0, time.UTC) + + // A started activity with a task outstanding for each timer that applies to it. + expectedStatus := int32(TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat) + local := &persistencespb.ActivityInfo{ + Version: failoverVersion, + ScheduledEventId: scheduledEventID, + ActivityId: "activityID_timer_task_status", + ScheduledTime: timestamppb.New(scheduledTime), + FirstScheduledTime: timestamppb.New(scheduledTime), + StartedEventId: int64(91), + StartedTime: timestamppb.New(scheduledTime.Add(time.Second)), + ScheduleToCloseTimeout: durationpb.New(time.Hour), + StartToCloseTimeout: durationpb.New(10 * time.Minute), + HeartbeatTimeout: durationpb.New(time.Minute), + TimerTaskStatus: expectedStatus, + LastUpdateVersionedTransition: &persistencespb.VersionedTransition{NamespaceFailoverVersion: failoverVersion, TransitionCount: 1}, + } + + dbState := s.buildWorkflowMutableState() + dbState.ActivityInfos[scheduledEventID] = local + mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) + s.NoError(err) + + // The replicated copy advances the versioned transition (otherwise the update is + // skipped as already applied) but leaves every deadline input untouched. + incoming := common.CloneProto(local) + incoming.LastUpdateVersionedTransition = &persistencespb.VersionedTransition{ + NamespaceFailoverVersion: failoverVersion, + TransitionCount: 2, + } + incoming.TimerTaskStatus = TimerTaskStatusNone // replicated state carries the peer's bits + + s.mockShard.Resource.ClusterMetadata.EXPECT(). + IsVersionFromSameCluster(failoverVersion, failoverVersion).Return(true).AnyTimes() + + err = mutableState.applyUpdatesToSubStateMachines( + map[int64]*persistencespb.ActivityInfo{scheduledEventID: incoming}, + nil, nil, nil, nil, + false, + ) + s.NoError(err) + + // No deadline moved, so every pending timer task is still valid and its bit must be + // carried over from the local activity info. + applied, ok := mutableState.pendingActivityInfoIDs[scheduledEventID] + s.True(ok) + s.Equal(expectedStatus, applied.TimerTaskStatus) +} + +// TestApplyUpdatesToSubStateMachines_ChildExecutionClockCarriedOver covers the other +// sanitizeFn that depends on the local sub state machine, alongside the activity one. +// +// ChildExecutionInfo.Clock is a local shard vector clock, and sanitizeChildExecutionInfo +// strips it before replicating, so the incoming copy always arrives nil. The local value +// therefore has to be restored on apply, otherwise every replicated update to an existing +// child execution erases it. +func (s *mutableStateSuite) TestApplyUpdatesToSubStateMachines_ChildExecutionClockCarriedOver() { + initiatedEventID := int64(80) + localClock := &clockspb.VectorClock{ClusterId: 1, ShardId: 2, Clock: 3} + + dbState := s.buildWorkflowMutableState() + local := dbState.ChildExecutionInfos[initiatedEventID] + s.NotNil(local, "expected the fixture to have a pending child execution") + local.Clock = localClock + local.LastUpdateVersionedTransition = &persistencespb.VersionedTransition{ + NamespaceFailoverVersion: local.Version, + TransitionCount: 1, + } + + mutableState, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, s.namespaceEntry, dbState, 123) + s.NoError(err) + + // The replicated copy advances the versioned transition and, as the sender sanitized + // it, carries no clock. + incoming := common.CloneProto(local) + incoming.Clock = nil + incoming.LastUpdateVersionedTransition = &persistencespb.VersionedTransition{ + NamespaceFailoverVersion: local.Version, + TransitionCount: 2, + } + + err = mutableState.applyUpdatesToSubStateMachines( + nil, + nil, + map[int64]*persistencespb.ChildExecutionInfo{initiatedEventID: incoming}, + nil, nil, + false, + ) + s.NoError(err) + + applied, ok := mutableState.pendingChildExecutionInfoIDs[initiatedEventID] + s.True(ok) + s.True(proto.Equal(localClock, applied.Clock), "local child execution clock was not carried over") +} + func (s *mutableStateSuite) TestUpdateActivityTaskStatusWithTimerHeartbeat() { dbState := s.buildWorkflowMutableState() scheduleEventId := int64(781) diff --git a/tests/xdc/activity_timer_task_replication_test.go b/tests/xdc/activity_timer_task_replication_test.go new file mode 100644 index 00000000000..2ea8d5673dc --- /dev/null +++ b/tests/xdc/activity_timer_task_replication_test.go @@ -0,0 +1,429 @@ +package xdc + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/sdk/activity" + sdkclient "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" + sdkworker "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/api/adminservice/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + historyspb "go.temporal.io/server/api/history/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/service/history/tasks" + historyworkflow "go.temporal.io/server/service/history/workflow" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// ActivityTimerTaskReplicationSuite covers how a standby cluster maintains an activity's +// timer task mask as updates replicate to it. +// +// The mask records which activity timeout tasks this cluster has already created. Getting +// it wrong is costly in both directions: keeping a bit whose deadline moved loses the +// timer, since CreateNextActivityTimer only inspects the earliest timer and skips it when +// the bit is set; clearing a bit whose deadline did not move regenerates a task that is +// already pending. These tests cover one direction each. +type ActivityTimerTaskReplicationSuite struct { + xdcBaseSuite +} + +func TestActivityTimerTaskReplicationSuite(t *testing.T) { + s := new(ActivityTimerTaskReplicationSuite) + suite.Run(t, s) +} + +func (s *ActivityTimerTaskReplicationSuite) SetupSuite() { + if s.dynamicConfigOverrides == nil { + s.dynamicConfigOverrides = make(map[dynamicconfig.Key]any) + } + // The timer task mask is carried across a replicated update by + // applyUpdatesToSubStateMachines, which only runs on the state-based replication path. + // xdcBaseSuite overrides the EnableTransitionHistory default with this field, so + // without setting it these tests would exercise the event-based SyncActivity path and + // never reach the logic under test. + s.enableTransitionHistory = true + s.setupSuite() +} + +func (s *ActivityTimerTaskReplicationSuite) SetupTest() { + s.setupTest() +} + +func (s *ActivityTimerTaskReplicationSuite) TearDownSuite() { + s.tearDownSuite() +} + +// TestRetryingActivityDoesNotAccumulateStandbyTimerTasks covers the "deadline unchanged, +// keep the bit" direction: a retrying activity must not make the standby regenerate its +// schedule-to-close timeout task on every replicated update. +// +// Each attempt produces two applies on the standby, one when the activity starts and one +// when it fails and the next attempt is scheduled. Before the mask was carried across an +// apply, every one of them cleared it, and the following task refresh recreated whichever +// timer was earliest. Between attempts that is schedule-to-close: clearing the started +// state removes start-to-close and heartbeat from the sequence, and schedule-to-start is +// normalized to the schedule-to-close duration when the caller does not set it, so it is +// anchored on the advancing ScheduledTime while schedule-to-close stays on the fixed +// FirstScheduledTime. Since that deadline never moves, the regenerated tasks all carried +// an identical fire time and piled onto a single instant in the timer queue. +// +// The assertion is that the count of far-future tasks does not grow between two attempt +// checkpoints. That is the property that matters, and it is robust to however many tasks +// initial replication creates; an exact count would not be, because the baseline is two: +// +// - On attempt 1, ScheduledTime still equals FirstScheduledTime, so the +// schedule-to-start and schedule-to-close deadlines coincide. The tie resolves toward +// schedule-to-start, so that task is created and then left behind as an orphan once +// the activity starts and its timer leaves the sequence. +// - On attempt 2, ScheduledTime has advanced, so schedule-to-close is strictly earlier +// and is created. +// - From attempt 3 on, the schedule-to-close bit is set and stays the earliest timer +// between attempts, so nothing further is created. +// +// Under the old behavior the count tracked the attempt count exactly (measured: 5 at +// attempt 5, 12 at attempt 12). +func (s *ActivityTimerTaskReplicationSuite) TestRetryingActivityDoesNotAccumulateStandbyTimerTasks() { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + const ( + // Long enough that the schedule-to-close deadline stays far in the future for the + // whole test, so it never fires and is never consumed. + scheduleToCloseTimeout = time.Hour + startToCloseTimeout = 2 * time.Second + retryInterval = time.Second + // Two checkpoints far enough apart that one-task-per-attempt growth is + // unmistakable. + firstCheckpoint = int32(5) + secondCheckpoint = int32(12) + // Separates the schedule-to-close band from the per-attempt timers, which fire + // within seconds. adminservice.Task does not carry the timeout type, so fire time + // is the only discriminator available. + farFutureCutoff = scheduleToCloseTimeout / 2 + ) + + var startedActivityCount atomic.Int32 + activityFunction := func() (string, error) { + startedActivityCount.Add(1) + return "", errors.New("bad-luck-please-retry") + } + + workflowFn := func(ctx workflow.Context) error { + var ret string + return workflow.ExecuteActivity(workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + ActivityID: "activity-id", + DisableEagerExecution: true, + StartToCloseTimeout: startToCloseTimeout, + ScheduleToCloseTimeout: scheduleToCloseTimeout, + RetryPolicy: &temporal.RetryPolicy{ + InitialInterval: retryInterval, + BackoffCoefficient: 1, + }, + }), activityFunction).Get(ctx, &ret) + } + + ns, workflowRun := s.startWorkflow(ctx, workflowFn, activityFunction) + + countBands := func(checkpoint int32) (farFuture, perAttempt int, ai *persistencespb.ActivityInfo) { + ai = s.awaitStandbyAttempt(ctx, ns, workflowRun.GetID(), workflowRun.GetRunID(), checkpoint) + cutoff := time.Now().UTC().Add(farFutureCutoff) + for _, task := range s.activityTimeoutTasks(ctx, s.clusters[1], workflowRun.GetID()) { + if task.GetFireTime().AsTime().After(cutoff) { + farFuture++ + } else { + perAttempt++ + } + } + return farFuture, perAttempt, ai + } + + firstFar, firstPerAttempt, standbyActivity := countBands(firstCheckpoint) + s.T().Logf("at attempt>=%d: farFuture=%d perAttempt=%d mask=%d", + firstCheckpoint, firstFar, firstPerAttempt, standbyActivity.GetTimerTaskStatus()) + + secondFar, secondPerAttempt, standbyActivity := countBands(secondCheckpoint) + s.T().Logf("at attempt>=%d: farFuture=%d perAttempt=%d mask=%d (activity attempts started: %d)", + secondCheckpoint, secondFar, secondPerAttempt, standbyActivity.GetTimerTaskStatus(), + startedActivityCount.Load()) + + s.Equal(firstFar, secondFar, + "standby accumulated far-future activity timeout tasks across retries: %d at attempt %d, %d at attempt %d; "+ + "the timer task mask is being cleared on every replicated apply", + firstFar, firstCheckpoint, secondFar, secondCheckpoint) + + // The opposite failure: over-preserving the mask would starve the per-attempt timers. + // Each attempt gets a fresh start-to-close deadline, so those tasks must keep coming. + s.Greater(secondPerAttempt, firstPerAttempt, + "standby stopped creating per-attempt activity timeout tasks; a preserved mask bit is "+ + "suppressing timers whose deadline did move") + + // The standby must also record that a schedule-to-close task exists, and the tasks + // that do exist must fire at the deadline the standby itself derives. + s.NotZero( + standbyActivity.GetTimerTaskStatus()&historyworkflow.TimerTaskStatusCreatedScheduleToClose, + "standby mask does not record a schedule-to-close task", + ) + expectedFireTime := standbyActivity.GetFirstScheduledTime().AsTime(). + Add(standbyActivity.GetScheduleToCloseTimeout().AsDuration()) + cutoff := time.Now().UTC().Add(farFutureCutoff) + for _, task := range s.activityTimeoutTasks(ctx, s.clusters[1], workflowRun.GetID()) { + if !task.GetFireTime().AsTime().After(cutoff) { + continue + } + s.Equal(workflowRun.GetRunID(), task.GetRunId()) + s.WithinDuration(expectedFireTime, task.GetFireTime().AsTime(), time.Second, + "far-future task does not fire at FirstScheduledTime+ScheduleToCloseTimeout") + } + + // A bit must never be set for a timer that does not currently apply, since + // CreateNextActivityTimer would then skip the earliest timer and stall the sequence. + if standbyActivity.GetStartedEventId() == common.EmptyEventID { + s.Zero(standbyActivity.GetTimerTaskStatus()&historyworkflow.TimerTaskStatusCreatedStartToClose, + "start-to-close bit is set while the standby sees the activity as not started") + } +} + +// TestHeartbeatingActivityRecreatesStandbyHeartbeatTimer covers the moving-deadline case: +// a heartbeat pushes the heartbeat deadline forward, so the pending heartbeat timeout task +// becomes wrong and the standby must replace it rather than rest on the bit it already has +// set. Heartbeat is the earliest timer while the activity runs, so a standby that stopped +// replacing it would also stall the rest of the sequence. +// +// Note this does not discriminate on how the timer task mask is carried across a +// replicated update: heartbeat is handled separately, because its deadline moves on every +// heartbeat and no event drives recreation. The standby timer executor clears the +// heartbeat bit itself once the task fires and its recorded visibility time has passed +// (the only bit it ever clears), and CreateNextActivityTimer persists the deadline each +// heartbeat task was created for. So this passes whether the mask carry logic preserves or +// clears bits. It is here as an invariant guard on that separate mechanism; the mask carry +// logic is covered by TestRetryingActivityDoesNotAccumulateStandbyTimerTasks, which fails +// in both directions. +func (s *ActivityTimerTaskReplicationSuite) TestHeartbeatingActivityRecreatesStandbyHeartbeatTimer() { + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + const ( + heartbeatTimeout = 4 * time.Second + heartbeatInterval = 500 * time.Millisecond + startToCloseTimeout = 60 * time.Second + scheduleToCloseTimeout = time.Hour + activityRunTime = 30 * time.Second + // Heartbeat deadlines land a few seconds out; start-to-close is a minute out and + // schedule-to-close an hour, so this cleanly isolates the heartbeat band. + heartbeatBandCutoff = 30 * time.Second + ) + + activityFunction := func(ctx context.Context) (string, error) { + deadline := time.Now().Add(activityRunTime) + for time.Now().Before(deadline) { + activity.RecordHeartbeat(ctx) + select { + case <-ctx.Done(): + return "", ctx.Err() + case <-time.After(heartbeatInterval): + } + } + return "done", nil + } + + workflowFn := func(ctx workflow.Context) error { + var ret string + return workflow.ExecuteActivity(workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + ActivityID: "activity-id", + DisableEagerExecution: true, + HeartbeatTimeout: heartbeatTimeout, + StartToCloseTimeout: startToCloseTimeout, + ScheduleToCloseTimeout: scheduleToCloseTimeout, + }), activityFunction).Get(ctx, &ret) + } + + ns, workflowRun := s.startWorkflow(ctx, workflowFn, activityFunction) + + // Latest heartbeat timeout task the standby holds, or zero if it holds none. + latestHeartbeatTask := func() time.Time { + var latest time.Time + cutoff := time.Now().UTC().Add(heartbeatBandCutoff) + for _, task := range s.activityTimeoutTasks(ctx, s.clusters[1], workflowRun.GetID()) { + fireTime := task.GetFireTime().AsTime() + if fireTime.Before(cutoff) && fireTime.After(latest) { + latest = fireTime + } + } + return latest + } + + // Wait until the standby has seen the activity start and recorded a heartbeat timer. + var firstFireTime time.Time + await.Require(ctx, s.T(), func(t *await.T) { + ai := s.standbyActivity(ctx, ns, workflowRun.GetID(), workflowRun.GetRunID()) + require.NotNil(t, ai) + require.NotEqual(t, common.EmptyEventID, ai.GetStartedEventId(), "activity not started on standby yet") + require.NotZero(t, ai.GetTimerTaskStatus()&historyworkflow.TimerTaskStatusCreatedHeartbeat, + "standby mask does not record a heartbeat task") + firstFireTime = latestHeartbeatTask() + require.False(t, firstFireTime.IsZero(), "no heartbeat timeout task on the standby") + }, 60*time.Second, 500*time.Millisecond) + s.T().Logf("first heartbeat task fires at %s", firstFireTime.Format(time.RFC3339Nano)) + + // As heartbeats keep arriving the deadline advances, so the standby must keep + // recreating the task rather than resting on the bit it already has set. + var secondFireTime time.Time + await.Require(ctx, s.T(), func(t *await.T) { + secondFireTime = latestHeartbeatTask() + require.False(t, secondFireTime.IsZero(), "standby has no heartbeat timeout task") + require.True(t, secondFireTime.After(firstFireTime), + "heartbeat timeout task did not advance past %s; the standby is not recreating it as the deadline moves", + firstFireTime) + }, 60*time.Second, 500*time.Millisecond) + s.T().Logf("heartbeat task advanced to %s (+%s)", + secondFireTime.Format(time.RFC3339Nano), secondFireTime.Sub(firstFireTime)) + + // The task the standby holds must match the deadline it derives from its own state: + // max(StartedTime, LastHeartbeatUpdateTime) + HeartbeatTimeout. + ai := s.standbyActivity(ctx, ns, workflowRun.GetID(), workflowRun.GetRunID()) + s.NotNil(ai) + lastHeartbeat := ai.GetStartedTime().AsTime() + if hb := ai.GetLastHeartbeatUpdateTime().AsTime(); hb.After(lastHeartbeat) { + lastHeartbeat = hb + } + s.WithinDuration(lastHeartbeat.Add(ai.GetHeartbeatTimeout().AsDuration()), latestHeartbeatTask(), 2*time.Second, + "heartbeat task does not fire at lastHeartbeat+HeartbeatTimeout") +} + +// startWorkflow registers the activity on the active cluster and starts the workflow, +// returning the namespace and the run. +func (s *ActivityTimerTaskReplicationSuite) startWorkflow( + ctx context.Context, + workflowFn any, + activityFn any, +) (string, sdkclient.WorkflowRun) { + ns := s.createGlobalNamespace() + activeSDKClient, err := sdkclient.Dial(sdkclient.Options{ + HostPort: s.clusters[0].Host().FrontendGRPCAddress(), + Namespace: ns, + Logger: log.NewSdkLogger(s.logger), + }) + s.NoError(err) + s.T().Cleanup(activeSDKClient.Close) + + taskQueue := testcore.RandomizeStr("tq") + worker := sdkworker.New(activeSDKClient, taskQueue, sdkworker.Options{}) + worker.RegisterWorkflow(workflowFn) + worker.RegisterActivity(activityFn) + s.NoError(worker.Start()) + s.T().Cleanup(worker.Stop) + + workflowRun, err := activeSDKClient.ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: testcore.RandomizeStr("wfid-" + s.T().Name()), + TaskQueue: taskQueue, + }, workflowFn) + s.NoError(err) + return ns, workflowRun +} + +// standbyActivity returns the standby's view of the workflow's single pending activity, or +// nil if it does not have exactly one. +func (s *ActivityTimerTaskReplicationSuite) standbyActivity( + ctx context.Context, + ns, workflowID, runID string, +) *persistencespb.ActivityInfo { + resp, err := s.clusters[1].AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + }) + if err != nil { + return nil + } + activityInfos := resp.GetDatabaseMutableState().GetActivityInfos() + if len(activityInfos) != 1 { + return nil + } + for _, ai := range activityInfos { + return ai + } + return nil +} + +// awaitStandbyAttempt waits for the active cluster to reach minAttempt and then for the +// standby to catch up, so the applies under test have actually run there. +func (s *ActivityTimerTaskReplicationSuite) awaitStandbyAttempt( + ctx context.Context, + ns, workflowID, runID string, + minAttempt int32, +) *persistencespb.ActivityInfo { + execution := &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID} + await.Require(ctx, s.T(), func(t *await.T) { + resp, err := s.clusters[0].AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: execution, + }) + require.NoError(t, err) + activityInfos := resp.GetDatabaseMutableState().GetActivityInfos() + require.Len(t, activityInfos, 1) + for _, ai := range activityInfos { + require.GreaterOrEqual(t, ai.GetAttempt(), minAttempt) + } + }, 60*time.Second, 500*time.Millisecond) + + var standbyActivity *persistencespb.ActivityInfo + await.Require(ctx, s.T(), func(t *await.T) { + ai := s.standbyActivity(ctx, ns, workflowID, runID) + require.NotNil(t, ai) + require.GreaterOrEqual(t, ai.GetAttempt(), minAttempt) + standbyActivity = ai + }, 60*time.Second, 500*time.Millisecond) + return standbyActivity +} + +// activityTimeoutTasks returns the cluster's activity timeout tasks for the given +// workflow. The xdc suites run with a single history shard, so shard 1 holds every task. +func (s *ActivityTimerTaskReplicationSuite) activityTimeoutTasks( + ctx context.Context, + cluster *testcore.TestCluster, + workflowID string, +) []*adminservice.Task { + var result []*adminservice.Task + var nextPageToken []byte + for { + resp, err := cluster.AdminClient().ListHistoryTasks(ctx, &adminservice.ListHistoryTasksRequest{ + ShardId: 1, + Category: int32(tasks.CategoryIDTimer), + TaskRange: &historyspb.TaskRange{ + InclusiveMinTaskKey: &historyspb.TaskKey{FireTime: timestamppb.New(time.Unix(0, 0).UTC())}, + ExclusiveMaxTaskKey: &historyspb.TaskKey{FireTime: timestamppb.New(time.Now().UTC().Add(10 * 365 * 24 * time.Hour))}, + }, + BatchSize: 1000, + NextPageToken: nextPageToken, + }) + s.NoError(err) + + for _, task := range resp.GetTasks() { + if task.GetWorkflowId() != workflowID { + continue + } + if task.GetTaskType() != enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT { + continue + } + result = append(result, task) + } + + nextPageToken = resp.GetNextPageToken() + if len(nextPageToken) == 0 { + return result + } + } +} From 69637784d8d83863501d6afee5a6ef51735a606f Mon Sep 17 00:00:00 2001 From: Alex Stanfield <13949480+chaptersix@users.noreply.github.com> Date: Wed, 19 Aug 2026 19:26:46 -0500 Subject: [PATCH 017/139] fix: floor future action times at generator watermark (#11622) ## Summary - Generator execution clamps its processed time range to `LastProcessedTime`, but `computeFutureActionTimes` (used by both `Describe` and `UpdateFutureActionTimes`) only floored its starting time at `max(now, UpdateTime)`, ignoring the watermark. - This let Describe/`FutureActionTimes` advertise occurrences at or before `LastProcessedTime` that the generator had already processed or silently skipped. - Floors the starting time at `LastProcessedTime` as well, so every advertised future action is guaranteed to be strictly after the generator's high water mark. - Reuses `common/util.MaxTime` for this floor and for the existing `UpdateTime`-vs-`LastProcessedTime` clamp in `GeneratorTaskHandler.Execute`, instead of hand-rolled `if X.After(Y) { Y = X }` checks. ## Test plan - Added `TestGeneratorTask_FutureActionTimesRespectLastProcessedTimeWatermark` in `chasm/lib/scheduler/generator_tasks_test.go`, which pushes `LastProcessedTime` ahead of "now" and asserts all advertised future times are strictly after it. Verified it fails without the fix and passes with it. - Built on a new scheduler-specific `chasmtest` helper (`newSchedulerTestEngine` + `updateScheduler`/`readScheduler` in `helper_test.go`), adapted from PR 0 of the `sch-readable` stack, rather than the `newTestEnv` rapid harness, so the watermark mutation and task execution cross the same transaction/read boundaries as production. Left out the parts of that PR not applicable here (frontend-client plumbing, the generic side-effect-task firing helper), since `GeneratorTaskHandler` is a pure task handler that doesn't touch the frontend client. - `go test -tags test_dep ./chasm/lib/scheduler/...` passes. --- chasm/lib/scheduler/generator.go | 10 +- chasm/lib/scheduler/generator_tasks.go | 6 +- chasm/lib/scheduler/generator_tasks_test.go | 47 +++++++++ chasm/lib/scheduler/helper_test.go | 108 +++++++++++++++++++- 4 files changed, 159 insertions(+), 12 deletions(-) diff --git a/chasm/lib/scheduler/generator.go b/chasm/lib/scheduler/generator.go index 3eaad9a4d72..77cb5fd0dce 100644 --- a/chasm/lib/scheduler/generator.go +++ b/chasm/lib/scheduler/generator.go @@ -7,6 +7,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/util" "go.temporal.io/server/service/worker/scheduler" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -96,11 +97,10 @@ func (g *Generator) computeFutureActionTimes( } futureTimes := make([]*timestamppb.Timestamp, 0, count) - // Start from max(now, updateTime) to ensure we skip times before the last update. - t := ctx.Now(g) - if updateTime := sched.Info.GetUpdateTime().AsTime(); updateTime.After(t) { - t = updateTime - } + // Start from max(now, updateTime, lastProcessedTime) to ensure we skip times + // before the last update and times already processed (or skipped) by the + // generator's high water mark. + t := util.MaxTime(ctx.Now(g), sched.Info.GetUpdateTime().AsTime(), g.GetLastProcessedTime().AsTime()) for len(futureTimes) < count { res, err := spec.GetNextTime(sched.jitterSeed(), t) if err != nil || res.Next.IsZero() { diff --git a/chasm/lib/scheduler/generator_tasks.go b/chasm/lib/scheduler/generator_tasks.go index 5a442155ee4..153bb039950 100644 --- a/chasm/lib/scheduler/generator_tasks.go +++ b/chasm/lib/scheduler/generator_tasks.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/util" queueerrors "go.temporal.io/server/service/history/queues/errors" "go.temporal.io/server/service/worker/scheduler" "go.uber.org/fx" @@ -73,9 +74,8 @@ func (g *GeneratorTaskHandler) Execute( // If the high water mark is earlier than when a schedule was updated, we must skip any actions that hadn't // yet been processed. - if scheduler.Info.GetUpdateTime().AsTime().After(generator.LastProcessedTime.AsTime()) { - generator.LastProcessedTime = scheduler.Info.GetUpdateTime() - } + generator.LastProcessedTime = timestamppb.New( + util.MaxTime(generator.LastProcessedTime.AsTime(), scheduler.Info.GetUpdateTime().AsTime())) // Process time range between last high water mark and system time. t1 := generator.LastProcessedTime.AsTime() diff --git a/chasm/lib/scheduler/generator_tasks_test.go b/chasm/lib/scheduler/generator_tasks_test.go index cfec1e91708..8ef619ff481 100644 --- a/chasm/lib/scheduler/generator_tasks_test.go +++ b/chasm/lib/scheduler/generator_tasks_test.go @@ -1,6 +1,7 @@ package scheduler_test import ( + "context" "errors" "testing" "time" @@ -9,6 +10,7 @@ import ( schedulepb "go.temporal.io/api/schedule/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common/metrics" @@ -307,6 +309,51 @@ func TestGeneratorTask_NonIdle_ClearsIdleCloseTime(t *testing.T) { require.Nil(t, sched.IdleCloseTime, "expected IdleCloseTime to be cleared when the schedule has work") } +// TestGeneratorTask_FutureActionTimesRespectLastProcessedTimeWatermark pins the +// fix for advertised future actions preceding the generator's own high water +// mark. Generator execution clamps its processed range to LastProcessedTime, +// so any occurrence at or before it has already been processed (or skipped) - +// UpdateFutureActionTimes must not advertise it as still upcoming. +// +// Uses the scheduler-specific chasm test engine helper, rather than the +// newTestEnv rapid harness, so the watermark mutation and the task execution +// that reads it cross the same transaction boundary as production. +func TestGeneratorTask_FutureActionTimesRespectLastProcessedTimeWatermark(t *testing.T) { + env := newSchedulerTestEngine(t, defaultSchedule()) + + // Push the generator's high water mark ten intervals ahead of "now", + // simulating a watermark that execution has already advanced past. + watermark := env.timeSource.Now().Add(10 * defaultInterval) + var generator *scheduler.Generator + require.NoError(t, env.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + generator = s.Generator.Get(ctx) + generator.LastProcessedTime = timestamppb.New(watermark) + return nil + })) + + handler := scheduler.NewGeneratorTaskHandler(scheduler.GeneratorTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + SpecProcessor: scheduler.NewSpecProcessor(defaultConfig(), metrics.NoopMetricsHandler, env.logger, newLegacySpecBuilder(0, 0)), + SpecBuilder: newLegacySpecBuilder(0, 0), + }) + dropped, err := chasmtest.ExecutePureTask( + context.Background(), env.engine, generator, handler, chasm.TaskAttributes{}, &schedulerpb.GeneratorTask{}) + require.NoError(t, err) + require.False(t, dropped) + + require.NoError(t, env.readScheduler(func(s *scheduler.Scheduler, ctx chasm.Context) error { + generator := s.Generator.Get(ctx) + require.NotEmpty(t, generator.GetFutureActionTimes()) + for _, future := range generator.GetFutureActionTimes() { + require.True(t, future.AsTime().After(watermark), + "advertised future action %v is not after watermark %v", future.AsTime(), watermark) + } + return nil + })) +} + func TestGeneratorTask_UpdateFutureActionTimes_SkipsBeforeUpdateTime(t *testing.T) { env := newTestEnv(t) handler := newGeneratorHandler(env) diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 73a368e52b2..2345aa02ec2 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -10,10 +10,12 @@ import ( commonpb "go.temporal.io/api/common/v1" schedulepb "go.temporal.io/api/schedule/v1" workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" @@ -190,6 +192,7 @@ func newRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler. // engineTestConfig holds configuration options for newTestEngineContext. type engineTestConfig struct { specProcessor scheduler.SpecProcessor + timeSource *clock.EventTimeSource engineOpts []chasmtest.EngineOption } @@ -208,20 +211,32 @@ func withEngineSpecProcessor(sp scheduler.SpecProcessor) engineTestOption { // time source, for tests that need to advance time explicitly. func withEngineTimeSource(ts *clock.EventTimeSource) engineTestOption { return func(c *engineTestConfig) { + c.timeSource = ts c.engineOpts = append(c.engineOpts, chasmtest.WithTimeSource(ts)) } } +func newEngineTestConfig(opts ...engineTestOption) *engineTestConfig { + config := &engineTestConfig{} + for _, opt := range opts { + opt(config) + } + return config +} + // newTestEngineContext builds a CHASM registry with the core and scheduler // libraries registered, wraps it in a chasmtest.Engine, and returns the // engine along with an engine-bound context ready for chasm.StartExecution / // ReadComponent / etc. func newTestEngineContext(t *testing.T, logger log.Logger, opts ...engineTestOption) (*chasmtest.Engine, context.Context) { - config := &engineTestConfig{} - for _, opt := range opts { - opt(config) - } + return newTestEngineContextFromConfig(t, logger, newEngineTestConfig(opts...)) +} +func newTestEngineContextFromConfig( + t *testing.T, + logger log.Logger, + config *engineTestConfig, +) (*chasmtest.Engine, context.Context) { specProcessor := config.specProcessor if specProcessor == nil { specProcessor = newRealSpecProcessor(gomock.NewController(t), logger) @@ -235,6 +250,91 @@ func newTestEngineContext(t *testing.T, logger log.Logger, opts ...engineTestOpt return engine, chasm.NewEngineContext(context.Background(), engine) } +// schedulerTestEngine bundles a CHASM test engine, a created Scheduler root +// component, and convenience accessors, for tests that drive a Scheduler +// through its actual task handlers rather than the newTestEnv rapid harness. +type schedulerTestEngine struct { + engine *chasmtest.Engine + engineCtx context.Context + rootRef chasm.ComponentRef + logger log.Logger + timeSource *clock.EventTimeSource +} + +// newSchedulerTestEngine builds a schedulerTestEngine and creates a schedule +// on it via the real CreateSchedule handler path. If no time source is +// supplied via withEngineTimeSource, a controllable one is created and +// wired in automatically. +func newSchedulerTestEngine( + t *testing.T, + schedule *schedulepb.Schedule, + opts ...engineTestOption, +) *schedulerTestEngine { + t.Helper() + + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + config := newEngineTestConfig(opts...) + if config.timeSource == nil { + config.timeSource = clock.NewEventTimeSource() + config.timeSource.Update(time.Now()) + config.engineOpts = append(config.engineOpts, chasmtest.WithTimeSource(config.timeSource)) + } + engine, engineCtx := newTestEngineContextFromConfig(t, logger, config) + rootRef := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: scheduleID, + }) + _, err := scheduler.NewTestHandler(logger).CreateSchedule(engineCtx, &schedulerpb.CreateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: namespace, + ScheduleId: scheduleID, + Schedule: schedule, + RequestId: "create-request", + }, + }) + require.NoError(t, err) + return &schedulerTestEngine{ + engine: engine, + engineCtx: engineCtx, + rootRef: rootRef, + logger: logger, + timeSource: config.timeSource, + } +} + +// updateScheduler runs update against the Scheduler root component through +// the engine's UpdateComponent path. +func (e *schedulerTestEngine) updateScheduler( + update func(*scheduler.Scheduler, chasm.MutableContext) error, +) error { + _, _, err := chasm.UpdateComponent( + e.engineCtx, + e.rootRef, + func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, update(s, ctx) + }, + struct{}{}, + ) + return err +} + +// readScheduler runs read against the Scheduler root component through the +// engine's read-only ReadComponent path. +func (e *schedulerTestEngine) readScheduler( + read func(*scheduler.Scheduler, chasm.Context) error, +) error { + _, err := chasm.ReadComponent( + e.engineCtx, + e.rootRef, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + return struct{}{}, read(s, ctx) + }, + struct{}{}, + ) + return err +} + // newTestEnv creates a new test environment with the given options. func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { config := &testEnvConfig{} From f1c8590f68b61d2318d9c4daf473850be24d2207 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 19 Aug 2026 17:27:32 -0700 Subject: [PATCH 018/139] Add Nexus operation context to handler-side logs (#11663) ## What changed Adds a few Nexus-specific log tags to the handler-side frontend logger. ## Why Mainly for the request ID to debug Nexus calls across namespaces better. --------- Co-authored-by: Claude Opus 5 (1M context) --- common/log/tag/tags.go | 4 ++++ service/frontend/nexus_handler.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index 73d8ef9134d..f9b9f659bfc 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -1013,6 +1013,10 @@ func NexusOperation(operation string) ZapTag { return NewStringTag("nexus-operation", operation) } +func NexusService(service string) ZapTag { + return NewStringTag("nexus-service", service) +} + // NexusTaskQueueName returns tag for NexusTaskQueueName func NexusTaskQueueName(taskQueueName string) ZapTag { return NewStringTag("nexus-task-queue-name", taskQueueName) diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 89ff09fe175..31e900883f6 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -308,6 +308,19 @@ func (c *operationContext) enrichNexusOperationMetrics(service, operation string } } +// enrichNexusOperationLogs adds Nexus operation context to the handler-side logger. +func (c *operationContext) enrichNexusOperationLogs(service, operation, requestID string) { + tags := []tag.Tag{ + tag.NexusService(service), + tag.NexusOperation(operation), + tag.Endpoint(c.endpointName), + } + if requestID != "" { + tags = append(tags, tag.RequestID(requestID)) + } + c.logger = log.With(c.logger, tags...) +} + // Key to extract a nexusContext object from a context.Context. type nexusContextKey struct{} @@ -397,6 +410,7 @@ func (h *nexusHandler) StartOperation( } ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) + oc.enrichNexusOperationLogs(service, operation, options.RequestID) defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) var links []*nexuspb.Link @@ -637,6 +651,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, } ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) + oc.enrichNexusOperationLogs(service, operation, "") defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) request := oc.matchingRequest(&nexuspb.Request{ From b1d6efc0a7d2cbf218034ed29042c5f5d92b7862 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 19 Aug 2026 17:27:43 -0700 Subject: [PATCH 019/139] Tag Nexus pre-dispatch failure logs (#11664) ## What changed Adds logs tags for failures on the Nexus frontend path. ## Why Have more details to correlate issues with requests. --------- Co-authored-by: Claude Opus 5 (1M context) --- common/log/tag/tags.go | 5 +++ .../frontend/nexus_operation_http_handler.go | 43 +++++++++++++------ 2 files changed, 36 insertions(+), 12 deletions(-) diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index f9b9f659bfc..4109a1ceec8 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -1027,6 +1027,11 @@ func NexusEndpointTargetNamespaceID(namespaceID string) ZapTag { return NewStringTag("nexus-endpoint-target-namespace-id", namespaceID) } +// NexusEndpointID returns a tag for a Nexus endpoint's ID, all that is known if its lookup failed. +func NexusEndpointID(endpointID string) ZapTag { + return NewStringTag("nexus-endpoint-id", endpointID) +} + // WorkflowRuleID returns tag for WorkflowRuleID func WorkflowRuleID(ruleID string) ZapTag { return NewStringTag("wf-rule-id", ruleID) diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index 3f1226ccdcb..fa5fd022364 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -3,6 +3,7 @@ package frontend import ( "context" "errors" + "fmt" "net/http" "net/url" "path" @@ -139,26 +140,31 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByNamespaceAndTaskQueue(w h var err error nc := h.baseNexusContext(configs.DispatchNexusTaskByNamespaceAndTaskQueueAPIName, r.Header) params := prepareRequest(commonnexus.RouteDispatchNexusTaskByNamespaceAndTaskQueue, w, r) + logger := log.With( + h.logger, + tag.WorkflowNamespace(params.Namespace), + tag.NexusTaskQueueName(params.TaskQueue), + ) if nc.taskQueue, err = url.PathUnescape(params.TaskQueue); err != nil { - h.logger.Error("invalid URL", tag.Error(err)) + logger.Error("invalid URL", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL")) return } if nc.namespaceName, err = url.PathUnescape(params.Namespace); err != nil { - h.logger.Error("invalid URL", tag.Error(err)) + logger.Error("invalid URL", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL")) return } if err = h.namespaceValidationInterceptor.ValidateName(nc.namespaceName); err != nil { - h.logger.Error("invalid namespace name", tag.Error(err)) + logger.Error("invalid namespace name", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "%v", err.Error())) return } rWithAuthCtx, err := h.parseTLSAndAuthInfo(r, nc) if err != nil { - h.logger.Error("failed to get claims", tag.Error(err)) + logger.Error("failed to get claims", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthenticated, "unauthorized")) return } @@ -166,7 +172,7 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByNamespaceAndTaskQueue(w h u, err := mux.CurrentRoute(r).URL("namespace", params.Namespace, "task_queue", params.TaskQueue) if err != nil { - h.logger.Error("invalid URL", tag.Error(err)) + logger.Error("invalid URL", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error")) return } @@ -177,16 +183,17 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByNamespaceAndTaskQueue(w h // Handler for [nexushttp.RouteSet.DispatchNexusTaskByEndpoint]. func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseWriter, r *http.Request) { endpointIDEscaped := prepareRequest(commonnexus.RouteDispatchNexusTaskByEndpoint, w, r) + logger := log.With(h.logger, tag.NexusEndpointID(endpointIDEscaped)) endpointID, err := url.PathUnescape(endpointIDEscaped) if err != nil { - h.logger.Error("invalid URL", tag.Error(err)) + logger.Error("invalid URL", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL")) return } endpointEntry, err := h.enpointRegistry.GetByID(r.Context(), endpointID) if err != nil { - h.logger.Error("invalid Nexus endpoint ID", tag.Error(err)) + logger.Error("invalid Nexus endpoint ID", tag.Error(err)) s, ok := status.FromError(err) if !ok { s = serviceerror.ToStatus(err) @@ -210,7 +217,9 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseW return } - nc, ok := h.nexusContextFromEndpoint(endpointEntry, w, r) + logger = log.With(logger, tag.Endpoint(endpointEntry.Endpoint.GetSpec().GetName())) + + nc, ok := h.nexusContextFromEndpoint(endpointEntry, w, r, logger) if !ok { // nexusContextFromEndpoint already writes the failure response. return @@ -218,7 +227,7 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseW rWithAuthCtx, err := h.parseTLSAndAuthInfo(r, nc) if err != nil { - h.logger.Error("failed to get claims", tag.Error(err)) + logger.Error("failed to get claims", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnauthenticated, "unauthorized")) return } @@ -226,7 +235,7 @@ func (h *NexusOperationHTTPHandler) dispatchNexusTaskByEndpoint(w http.ResponseW u, err := mux.CurrentRoute(r).URL("endpoint", endpointIDEscaped) if err != nil { - h.logger.Error("invalid URL", tag.Error(err)) + logger.Error("invalid URL", tag.Error(err)) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error")) return } @@ -251,12 +260,21 @@ func (h *NexusOperationHTTPHandler) baseNexusContext(apiName string, header http // endpoint is valid for dispatching. // For security reasons, at the moment only worker target endpoints are considered valid, in the future external // endpoints may also be supported. -func (h *NexusOperationHTTPHandler) nexusContextFromEndpoint(entry *persistencespb.NexusEndpointEntry, w http.ResponseWriter, r *http.Request) (*nexusContext, bool) { +func (h *NexusOperationHTTPHandler) nexusContextFromEndpoint( + entry *persistencespb.NexusEndpointEntry, + w http.ResponseWriter, + r *http.Request, + logger log.Logger, +) (*nexusContext, bool) { switch v := entry.Endpoint.Spec.GetTarget().GetVariant().(type) { case *persistencespb.NexusEndpointTarget_Worker_: nsName, err := h.namespaceRegistry.GetNamespaceName(namespace.ID(v.Worker.GetNamespaceId())) if err != nil { - h.logger.Error("failed to get namespace name by ID", tag.Error(err)) + logger.Error( + "failed to get namespace name by ID", + tag.Error(err), + tag.NexusEndpointTargetNamespaceID(v.Worker.GetNamespaceId()), + ) var notFoundErr *serviceerror.NamespaceNotFound if errors.As(err, ¬FoundErr) { h.writeFailure(w, r, &nexus.HandlerError{ @@ -276,6 +294,7 @@ func (h *NexusOperationHTTPHandler) nexusContextFromEndpoint(entry *persistences nc.endpointID = entry.Id return nc, true default: + logger.Error("unsupported Nexus endpoint target type", tag.NewStringTag("target-type", fmt.Sprintf("%T", v))) h.writeFailure(w, r, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid endpoint target")) return nil, false } From ba919854a1b1837d39b6386dab499e6914174bf6 Mon Sep 17 00:00:00 2001 From: Qian Chen Date: Wed, 19 Aug 2026 18:05:00 -0700 Subject: [PATCH 020/139] Emit namespace replication lifecycle wide events (#11632) ## What changed? - Added a separate `namespace_replication_lifecycle` wide event with `created`, `received`, `processed`, and `dlqed` phases. - Included namespace/task identity, source and target clusters, source task ID, retry attempt count, deterministic task fingerprint, and the serialized namespace replication task. - Included the successful `CreateNamespaceRequest` or resolved `UpdateNamespaceRequest` as `persistence_request` on `processed`; duplicate, stale, and skipped tasks omit it. - Passed receiver-side diagnostic metadata through a typed context so the existing `TaskExecutor.Execute` and create/update handler signatures remain unchanged. - Added the dedicated, default-off `system.emitNamespaceReplicationLifecycleEvents` dynamic-config gate, checked explicitly at both the processor and processed-event emitter. - Preserved the namespace replication queue message ID as `source_task_id` when reading tasks. ## Why? Namespace CRUD events describe user-visible namespace mutations, but do not show whether the resulting namespace replication task was queued, received, applied, retried, or sent to the DLQ. These events provide that transport and processing audit trail without additional persistence reads. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) Commands: ```text go test -tags test_dep ./common/wideevents ./common/namespace/nsreplication ./service/worker/replicator ./service/frontend make GOLANGCI_LINT_FIX=false GOLANGCI_LINT_BASE_REV=origin/main lint-code ``` Local two-cluster testing covered create, update, and failover after rebasing onto `origin/main`. Each task produced linked `created -> received -> processed` events, and `processed` contained the expected persistence request. With the dynamic-config flag off, namespace replication still completed and neither cluster emitted a matching lifecycle event. --- common/dynamicconfig/constants.go | 2 +- .../replication_task_executor.go | 141 ++++++++- .../replication_task_executor_test.go | 201 ++++++++++++- .../transmission_task_handler.go | 45 ++- .../transmission_task_handler_test.go | 60 ++++ .../namespace_replication_queue.go | 1 + .../persistence-tests/queue_persistence.go | 3 + common/wideevents/context.go | 26 ++ common/wideevents/namespace_events.go | 5 +- .../namespace_replication_events.go | 241 +++++++++++++++ .../namespace_replication_events_test.go | 274 ++++++++++++++++++ service/frontend/namespace_handler_test.go | 8 +- service/frontend/workflow_handler.go | 8 +- service/frontend/workflow_handler_test.go | 1 + service/worker/fx.go | 9 + .../replication_message_processor.go | 143 ++++++--- .../replication_message_processor_test.go | 236 ++++++++++++++- service/worker/replicator/replicator.go | 15 + service/worker/service.go | 13 + 19 files changed, 1359 insertions(+), 73 deletions(-) create mode 100644 common/wideevents/namespace_replication_events.go create mode 100644 common/wideevents/namespace_replication_events_test.go diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 1c417249ff4..91ab4903c51 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -239,7 +239,7 @@ response to a StartWorkflowExecution request and skipping the trip through match EmitNamespaceLifecycleEvents = NewGlobalBoolSetting( "system.emitNamespaceLifecycleEvents", false, - `EmitNamespaceLifecycleEvents controls whether services emit NamespaceLifecycle wide events for namespace CRUD and handover phases. Cluster-level; default off.`, + `EmitNamespaceLifecycleEvents controls whether services emit NamespaceLifecycle and NamespaceReplicationLifecycle wide events. Cluster-level; default off.`, ) PersistenceHealthSignalMetricsEnabled = NewGlobalBoolSetting( "system.persistenceHealthSignalMetricsEnabled", diff --git a/common/namespace/nsreplication/replication_task_executor.go b/common/namespace/nsreplication/replication_task_executor.go index 3d2a2ed4c51..5f33450ab3b 100644 --- a/common/namespace/nsreplication/replication_task_executor.go +++ b/common/namespace/nsreplication/replication_task_executor.go @@ -5,17 +5,21 @@ package nsreplication import ( "context" + otellog "go.opentelemetry.io/otel/log" enumspb "go.temporal.io/api/enums/v1" replicationpb "go.temporal.io/api/replication/v1" "go.temporal.io/api/serviceerror" enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/common/wideevents" + "google.golang.org/protobuf/proto" ) var ( @@ -49,13 +53,18 @@ type ( Execute(ctx context.Context, task *replicationspb.NamespaceTaskAttributes) error } + // TaskExecutorOption configures a namespace replication task executor. + TaskExecutorOption func(*taskExecutorImpl) + taskExecutorImpl struct { - currentCluster string - metadataManager persistence.MetadataManager - dataMerger NamespaceDataMerger - admitter NamespaceReplicationAdmitter - logger log.Logger - testHooks testhooks.TestHooks + currentCluster string + metadataManager persistence.MetadataManager + dataMerger NamespaceDataMerger + admitter NamespaceReplicationAdmitter + logger log.Logger + eventLogger otellog.Logger + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn + testHooks testhooks.TestHooks } ) @@ -67,8 +76,9 @@ func NewTaskExecutor( admitter NamespaceReplicationAdmitter, logger log.Logger, testHooks testhooks.TestHooks, + options ...TaskExecutorOption, ) TaskExecutor { - return &taskExecutorImpl{ + executor := &taskExecutorImpl{ currentCluster: currentCluster, metadataManager: metadataManagerV2, dataMerger: dataMerger, @@ -76,6 +86,21 @@ func NewTaskExecutor( logger: logger, testHooks: testHooks, } + for _, option := range options { + option(executor) + } + return executor +} + +// WithNamespaceReplicationLifecycleEvents configures processed lifecycle event emission. +func WithNamespaceReplicationLifecycleEvents( + eventLogger otellog.Logger, + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn, +) TaskExecutorOption { + return func(executor *taskExecutorImpl) { + executor.eventLogger = eventLogger + executor.emitNamespaceLifecycleEvents = emitNamespaceLifecycleEvents + } } // Execute handles receiving of the namespace replication task @@ -103,6 +128,15 @@ func (h *taskExecutorImpl) executeValidatedTask( task *replicationspb.NamespaceTaskAttributes, ) error { if shouldProcess, err := h.shouldProcessTask(ctx, task); !shouldProcess || err != nil { + if !shouldProcess && err == nil { + h.emitNamespaceReplicationProcessed( + ctx, + wideevents.NamespaceReplicationOutcomeNotAdmitted, + nil, + nil, + nil, + ) + } return err } @@ -237,12 +271,26 @@ func (h *taskExecutorImpl) handleNamespaceCreationReplicationTask( if recordExists { // name -> id & id -> name check pass, this is duplication request + h.emitNamespaceReplicationProcessed( + ctx, + wideevents.NamespaceReplicationOutcomeDuplicate, + nil, + nil, + nil, + ) return nil } return err } - return err + h.emitNamespaceReplicationProcessed( + ctx, + wideevents.NamespaceReplicationOutcomeCreated, + nil, + request, + nil, + ) + return nil } // handleNamespaceUpdateReplicationTask handles the namespace update replication task @@ -276,6 +324,7 @@ func (h *taskExecutorImpl) handleNamespaceUpdateReplicationTask( } return err } + localNamespacePreMutation := h.cloneLocalNamespaceForReplicationEvent(ctx, resp.Namespace) recordUpdated := false request := &persistence.UpdateNamespaceRequest{ @@ -329,10 +378,84 @@ func (h *taskExecutorImpl) handleNamespaceUpdateReplicationTask( } if !recordUpdated { + if localNamespacePreMutation != nil { + h.emitNamespaceReplicationProcessed( + ctx, + wideevents.NamespaceReplicationOutcomeNoChange, + localNamespacePreMutation, + nil, + nil, + ) + } return nil } - return h.metadataManager.UpdateNamespace(ctx, request) + if err := h.metadataManager.UpdateNamespace(ctx, request); err != nil { + return err + } + if localNamespacePreMutation != nil { + h.emitNamespaceReplicationProcessed( + ctx, + wideevents.NamespaceReplicationOutcomeUpdated, + localNamespacePreMutation, + nil, + request, + ) + } + return nil +} + +func (h *taskExecutorImpl) cloneLocalNamespaceForReplicationEvent( + ctx context.Context, + namespaceDetail *persistencespb.NamespaceDetail, +) *persistencespb.NamespaceDetail { + if h.eventLogger == nil || + h.emitNamespaceLifecycleEvents == nil || + !h.emitNamespaceLifecycleEvents() { + return nil + } + if _, ok := wideevents.NamespaceReplicationTaskContextFromContext(ctx); !ok { + return nil + } + clonedNamespace, ok := proto.Clone(namespaceDetail).(*persistencespb.NamespaceDetail) + if !ok { + h.logger.Error("Failed to clone local namespace for namespace replication lifecycle event") + return nil + } + return clonedNamespace +} + +func (h *taskExecutorImpl) emitNamespaceReplicationProcessed( + ctx context.Context, + outcome wideevents.NamespaceReplicationOutcome, + localNamespacePreMutation *persistencespb.NamespaceDetail, + createRequest *persistence.CreateNamespaceRequest, + updateRequest *persistence.UpdateNamespaceRequest, +) { + if h.eventLogger == nil || + h.emitNamespaceLifecycleEvents == nil || + !h.emitNamespaceLifecycleEvents() { + return + } + + metadata, ok := wideevents.NamespaceReplicationTaskContextFromContext(ctx) + if !ok { + return + } + + sourceTaskID := metadata.SourceTaskID + wideevents.EmitNamespaceReplicationLifecycle(h.eventLogger, wideevents.NamespaceReplicationLifecycleInput{ + Phase: wideevents.NamespaceReplicationProcessed, + Outcome: outcome, + EventData: metadata.EventData, + SourceCluster: metadata.SourceCluster, + TargetCluster: metadata.TargetCluster, + SourceTaskID: &sourceTaskID, + AttemptCount: metadata.AttemptCount, + LocalNamespacePreMutation: localNamespacePreMutation, + CreateNamespaceRequest: createRequest, + UpdateNamespaceRequest: updateRequest, + }) } func (h *taskExecutorImpl) validateNamespaceReplicationTask(task *replicationspb.NamespaceTaskAttributes) error { diff --git a/common/namespace/nsreplication/replication_task_executor_test.go b/common/namespace/nsreplication/replication_task_executor_test.go index 8b8cde153be..03ec06c7972 100644 --- a/common/namespace/nsreplication/replication_task_executor_test.go +++ b/common/namespace/nsreplication/replication_task_executor_test.go @@ -2,12 +2,15 @@ package nsreplication import ( "context" + "encoding/json" "errors" "testing" "time" "github.com/google/uuid" "github.com/stretchr/testify/suite" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" enumspb "go.temporal.io/api/enums/v1" namespacepb "go.temporal.io/api/namespace/v1" replicationpb "go.temporal.io/api/replication/v1" @@ -15,24 +18,50 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/common/wideevents" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) type ( + replicationEventCaptureLogger struct { + embedded.Logger + records []otellog.Record + } + namespaceReplicationTaskExecutorSuite struct { suite.Suite controller *gomock.Controller mockMetadataMgr *persistence.MockMetadataManager namespaceReplicator *taskExecutorImpl + eventLogger *replicationEventCaptureLogger } ) +func namespaceUpdateRequestMatcher(expected *persistence.UpdateNamespaceRequest) gomock.Matcher { + return gomock.Cond(func(actual *persistence.UpdateNamespaceRequest) bool { + return actual != nil && + actual.IsGlobalNamespace == expected.IsGlobalNamespace && + actual.NotificationVersion == expected.NotificationVersion && + proto.Equal(actual.Namespace, expected.Namespace) + }) +} + +func (l *replicationEventCaptureLogger) Emit(_ context.Context, record otellog.Record) { + l.records = append(l.records, record) +} + +func (l *replicationEventCaptureLogger) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + func TestNamespaceReplicationTaskExecutorSuite(t *testing.T) { s := new(namespaceReplicationTaskExecutorSuite) suite.Run(t, s) @@ -49,6 +78,7 @@ func (s *namespaceReplicationTaskExecutorSuite) SetupTest() { s.controller = gomock.NewController(s.T()) s.mockMetadataMgr = persistence.NewMockMetadataManager(s.controller) logger := log.NewTestLogger() + s.eventLogger = &replicationEventCaptureLogger{} s.namespaceReplicator = NewTaskExecutor( "some random standby cluster name", s.mockMetadataMgr, @@ -56,6 +86,10 @@ func (s *namespaceReplicationTaskExecutorSuite) SetupTest() { NewDefaultAdmitter(), logger, testhooks.TestHooks{}, + WithNamespaceReplicationLifecycleEvents( + s.eventLogger, + dynamicconfig.GetBoolPropertyFn(true), + ), ).(*taskExecutorImpl) } @@ -63,6 +97,102 @@ func (s *namespaceReplicationTaskExecutorSuite) TearDownTest() { s.controller.Finish() } +func (s *namespaceReplicationTaskExecutorSuite) replicationEventContext( + task *replicationspb.NamespaceTaskAttributes, +) context.Context { + eventData, ok := wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider().Extract( + &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK, + Attributes: &replicationspb.ReplicationTask_NamespaceTaskAttributes{ + NamespaceTaskAttributes: task, + }, + }, + ) + s.Require().True(ok) + return wideevents.SetNamespaceReplicationTaskContext(context.Background(), wideevents.NamespaceReplicationTaskContext{ + SourceCluster: "source-cluster", + TargetCluster: "target-cluster", + SourceTaskID: 42, + AttemptCount: 2, + EventData: eventData, + }) +} + +func (s *namespaceReplicationTaskExecutorSuite) processedPersistenceRequest( + outcome wideevents.NamespaceReplicationOutcome, +) map[string]any { + s.Require().Len(s.eventLogger.records, 1) + values := make(map[string]otellog.Value) + s.eventLogger.records[0].WalkAttributes(func(kv otellog.KeyValue) bool { + values[kv.Key] = kv.Value + return true + }) + s.Equal("processed", values["phase"].AsString()) + var details map[string]any + s.Require().NoError(json.Unmarshal([]byte(values["details"].AsString()), &details)) + s.Equal(string(outcome), details["outcome"]) + s.Equal("source-cluster", details["source_cluster"]) + s.Equal("target-cluster", details["target_cluster"]) + s.InDelta(float64(42), details["source_task_id"], 0) + s.InDelta(float64(2), details["attempt_count"], 0) + + request, ok := details["persistence_request"] + if !ok { + return nil + } + return request.(map[string]any) +} + +func (s *namespaceReplicationTaskExecutorSuite) processedLocalNamespacePreMutation() map[string]any { + s.Require().Len(s.eventLogger.records, 1) + values := make(map[string]otellog.Value) + s.eventLogger.records[0].WalkAttributes(func(kv otellog.KeyValue) bool { + values[kv.Key] = kv.Value + return true + }) + var details map[string]any + s.Require().NoError(json.Unmarshal([]byte(values["details"].AsString()), &details)) + value, ok := details["local_namespace_pre_mutation"] + s.Require().True(ok) + return value.(map[string]any) +} + +func (s *namespaceReplicationTaskExecutorSuite) TestEmitProcessedRequiresExplicitConfigAndContext() { + task := &replicationspb.NamespaceTaskAttributes{ + Id: "namespace-id", + Info: &namespacepb.NamespaceInfo{Name: "namespace-name"}, + } + + s.namespaceReplicator.emitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(false) + s.namespaceReplicator.emitNamespaceReplicationProcessed( + s.replicationEventContext(task), + wideevents.NamespaceReplicationOutcomeNoChange, + nil, + nil, + nil, + ) + s.Empty(s.eventLogger.records) + + s.namespaceReplicator.emitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.namespaceReplicator.emitNamespaceReplicationProcessed( + context.Background(), + wideevents.NamespaceReplicationOutcomeNoChange, + nil, + nil, + nil, + ) + s.Empty(s.eventLogger.records) + + s.namespaceReplicator.emitNamespaceReplicationProcessed( + s.replicationEventContext(task), + wideevents.NamespaceReplicationOutcomeNoChange, + nil, + nil, + nil, + ) + s.Len(s.eventLogger.records, 1) +} + func (s *namespaceReplicationTaskExecutorSuite) TestExecute_RegisterNamespaceTask_NameUUIDCollision() { operation := enumsspb.NAMESPACE_OPERATION_CREATE id := uuid.NewString() @@ -253,8 +383,12 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_RegisterNamespaceTas }, IsGlobalNamespace: true, }) - err := s.namespaceReplicator.Execute(context.Background(), task) + err := s.namespaceReplicator.Execute(s.replicationEventContext(task), task) s.Nil(err) + request := s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeCreated) + s.Equal("CreateNamespaceRequest", request["request_type"]) + s.Equal(true, request["is_global_namespace"]) + s.Equal(id, request["namespace"].(map[string]any)["info"].(map[string]any)["id"]) } func (s *namespaceReplicationTaskExecutorSuite) TestExecute_RegisterNamespaceTask_Duplicate() { @@ -298,8 +432,33 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_RegisterNamespaceTas }, }}, nil).Times(1) s.mockMetadataMgr.EXPECT().CreateNamespace(gomock.Any(), gomock.Any()).Return(nil, errors.New("test")) - err := s.namespaceReplicator.Execute(context.Background(), task) + err := s.namespaceReplicator.Execute(s.replicationEventContext(task), task) s.Nil(err) + s.Nil(s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeDuplicate)) +} + +func (s *namespaceReplicationTaskExecutorSuite) TestExecute_RegisterNamespaceTask_NotAdmitted() { + task := &replicationspb.NamespaceTaskAttributes{ + Id: uuid.NewString(), + NamespaceOperation: enumsspb.NAMESPACE_OPERATION_CREATE, + Info: &namespacepb.NamespaceInfo{ + Name: uuid.NewString(), + State: enumspb.NAMESPACE_STATE_REGISTERED, + }, + Config: &namespacepb.NamespaceConfig{}, + ReplicationConfig: &replicationpb.NamespaceReplicationConfig{ + Clusters: []*replicationpb.ClusterReplicationConfig{ + {ClusterName: "another-cluster"}, + }, + }, + } + s.mockMetadataMgr.EXPECT().GetNamespace(gomock.Any(), &persistence.GetNamespaceRequest{ + Name: task.GetInfo().GetName(), + }).Return(nil, &serviceerror.NamespaceNotFound{}) + + err := s.namespaceReplicator.Execute(s.replicationEventContext(task), task) + s.Require().NoError(err) + s.Nil(s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeNotAdmitted)) } func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_NamespaceNotExist() { @@ -383,8 +542,11 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, IsGlobalNamespace: true, }) - err := s.namespaceReplicator.Execute(context.Background(), updateTask) + err := s.namespaceReplicator.Execute(s.replicationEventContext(updateTask), updateTask) s.Nil(err) + request := s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeCreated) + s.Equal("CreateNamespaceRequest", request["request_type"]) + s.Equal("12", request["namespace"].(map[string]any)["config_version"]) } func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_UpdateConfig_UpdateActiveCluster() { @@ -457,7 +619,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ s.mockMetadataMgr.EXPECT().GetMetadata(gomock.Any()).Return(&persistence.GetMetadataResponse{ NotificationVersion: updateFailoverVersion, }, nil).Times(1) - s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), &persistence.UpdateNamespaceRequest{ + s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), namespaceUpdateRequestMatcher(&persistence.UpdateNamespaceRequest{ Namespace: &persistencespb.NamespaceDetail{ Info: &persistencespb.NamespaceInfo{ Id: id, @@ -485,9 +647,17 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, IsGlobalNamespace: false, NotificationVersion: updateFailoverVersion, - }) - err := s.namespaceReplicator.Execute(context.Background(), updateTask) + })) + err := s.namespaceReplicator.Execute(s.replicationEventContext(updateTask), updateTask) s.Nil(err) + request := s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeUpdated) + s.Equal("UpdateNamespaceRequest", request["request_type"]) + s.Equal("1", request["namespace"].(map[string]any)["config_version"]) + s.InDelta(float64(updateFailoverVersion), request["notification_version"].(float64), 0) + localPreMutation := s.processedLocalNamespacePreMutation() + s.Equal("0", localPreMutation["config_version"]) + s.Equal(id, localPreMutation["info"].(map[string]any)["id"]) + s.Empty(localPreMutation["info"].(map[string]any)["name"]) } func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_UpdateConfig_NoUpdateActiveCluster() { @@ -553,7 +723,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ s.mockMetadataMgr.EXPECT().GetMetadata(gomock.Any()).Return(&persistence.GetMetadataResponse{ NotificationVersion: updateFailoverVersion, }, nil).Times(1) - s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), &persistence.UpdateNamespaceRequest{ + s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), namespaceUpdateRequestMatcher(&persistence.UpdateNamespaceRequest{ Namespace: &persistencespb.NamespaceDetail{ Info: &persistencespb.NamespaceInfo{ Id: id, @@ -579,7 +749,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, IsGlobalNamespace: false, NotificationVersion: updateFailoverVersion, - }) + })) err := s.namespaceReplicator.Execute(context.Background(), updateTask) s.Nil(err) } @@ -647,7 +817,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ s.mockMetadataMgr.EXPECT().GetMetadata(gomock.Any()).Return(&persistence.GetMetadataResponse{ NotificationVersion: updateFailoverVersion, }, nil).Times(1) - s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), &persistence.UpdateNamespaceRequest{ + s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), namespaceUpdateRequestMatcher(&persistence.UpdateNamespaceRequest{ Namespace: &persistencespb.NamespaceDetail{ Info: &persistencespb.NamespaceInfo{ Id: id, @@ -661,7 +831,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, IsGlobalNamespace: false, NotificationVersion: updateFailoverVersion, - }) + })) err := s.namespaceReplicator.Execute(context.Background(), updateTask) s.Nil(err) } @@ -732,8 +902,13 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, nil).Times(1) s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), gomock.Any()).Times(0) - err := s.namespaceReplicator.Execute(context.Background(), updateTask) + err := s.namespaceReplicator.Execute(s.replicationEventContext(updateTask), updateTask) s.Nil(err) + s.Nil(s.processedPersistenceRequest(wideevents.NamespaceReplicationOutcomeNoChange)) + localPreMutation := s.processedLocalNamespacePreMutation() + s.Equal("2", localPreMutation["config_version"]) + s.Equal("60", localPreMutation["failover_version"]) + s.Equal(id, localPreMutation["info"].(map[string]any)["id"]) } // TestExecute_UpdateNamespaceTask_FailoverPropagatesNormalState verifies the @@ -784,7 +959,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ s.mockMetadataMgr.EXPECT().GetMetadata(gomock.Any()).Return(&persistence.GetMetadataResponse{ NotificationVersion: updateFailoverVersion, }, nil).Times(1) - s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), &persistence.UpdateNamespaceRequest{ + s.mockMetadataMgr.EXPECT().UpdateNamespace(gomock.Any(), namespaceUpdateRequestMatcher(&persistence.UpdateNamespaceRequest{ Namespace: &persistencespb.NamespaceDetail{ Info: &persistencespb.NamespaceInfo{Id: id}, ReplicationConfig: &persistencespb.NamespaceReplicationConfig{ @@ -797,7 +972,7 @@ func (s *namespaceReplicationTaskExecutorSuite) TestExecute_UpdateNamespaceTask_ }, IsGlobalNamespace: false, NotificationVersion: updateFailoverVersion, - }) + })) err := s.namespaceReplicator.Execute(context.Background(), updateTask) s.NoError(err) } diff --git a/common/namespace/nsreplication/transmission_task_handler.go b/common/namespace/nsreplication/transmission_task_handler.go index 09c47a9fc19..fb8820e41eb 100644 --- a/common/namespace/nsreplication/transmission_task_handler.go +++ b/common/namespace/nsreplication/transmission_task_handler.go @@ -3,14 +3,17 @@ package nsreplication import ( "context" + otellog "go.opentelemetry.io/otel/log" enumspb "go.temporal.io/api/enums/v1" namespacepb "go.temporal.io/api/namespace/v1" replicationpb "go.temporal.io/api/replication/v1" enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/wideevents" ) // NOTE: the counterpart of namespace replication receiving logic is in service/worker package @@ -34,8 +37,11 @@ type ( } replicator struct { - namespaceReplicationQueue persistence.NamespaceReplicationQueue - logger log.Logger + namespaceReplicationQueue persistence.NamespaceReplicationQueue + logger log.Logger + eventLogger otellog.Logger + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn + currentCluster string } ) @@ -43,10 +49,16 @@ type ( func NewReplicator( namespaceReplicationQueue persistence.NamespaceReplicationQueue, logger log.Logger, + eventLogger otellog.Logger, + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn, + currentCluster string, ) Replicator { return &replicator{ - namespaceReplicationQueue: namespaceReplicationQueue, - logger: logger, + namespaceReplicationQueue: namespaceReplicationQueue, + logger: logger, + eventLogger: eventLogger, + emitNamespaceLifecycleEvents: emitNamespaceLifecycleEvents, + currentCluster: currentCluster, } } @@ -113,12 +125,27 @@ func (r *replicator) HandleTransmissionTask( task.NamespaceTaskAttributes.ReplicationConfig.State = replicationConfig.State } - return r.namespaceReplicationQueue.Publish( - ctx, - &replicationspb.ReplicationTask{ - TaskType: taskType, - Attributes: task, + replicationTask := &replicationspb.ReplicationTask{ + TaskType: taskType, + Attributes: task, + } + err := r.namespaceReplicationQueue.Publish(ctx, replicationTask) + if err != nil { + return err + } + + if r.emitNamespaceLifecycleEvents() { + eventData, ok := wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider().Extract(replicationTask) + if !ok { + return nil + } + wideevents.EmitNamespaceReplicationLifecycle(r.eventLogger, wideevents.NamespaceReplicationLifecycleInput{ + Phase: wideevents.NamespaceReplicationCreated, + EventData: eventData, + SourceCluster: r.currentCluster, }) + } + return nil } func convertClusterReplicationConfigToProto( diff --git a/common/namespace/nsreplication/transmission_task_handler_test.go b/common/namespace/nsreplication/transmission_task_handler_test.go index bb36f1498c5..b464e636385 100644 --- a/common/namespace/nsreplication/transmission_task_handler_test.go +++ b/common/namespace/nsreplication/transmission_task_handler_test.go @@ -2,16 +2,20 @@ package nsreplication import ( "context" + "encoding/json" "testing" "time" "github.com/stretchr/testify/suite" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" enumspb "go.temporal.io/api/enums/v1" namespacepb "go.temporal.io/api/namespace/v1" replicationpb "go.temporal.io/api/replication/v1" enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" @@ -20,6 +24,11 @@ import ( ) type ( + transmissionCaptureLogger struct { + embedded.Logger + records []otellog.Record + } + transmissionTaskSuite struct { suite.Suite @@ -30,6 +39,14 @@ type ( } ) +func (l *transmissionCaptureLogger) Emit(_ context.Context, record otellog.Record) { + l.records = append(l.records, record) +} + +func (l *transmissionCaptureLogger) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + func TestTransmissionTaskSuite(t *testing.T) { s := new(transmissionTaskSuite) suite.Run(t, s) @@ -48,6 +65,9 @@ func (s *transmissionTaskSuite) SetupTest() { s.namespaceReplicator = NewReplicator( s.namespaceReplicationQueue, log.NewTestLogger(), + nil, + dynamicconfig.GetBoolPropertyFn(false), + "cluster-a", ).(*replicator) } @@ -56,6 +76,10 @@ func (s *transmissionTaskSuite) TearDownTest() { } func (s *transmissionTaskSuite) TestHandleTransmissionTask_RegisterNamespaceTask_IsGlobalNamespace() { + eventLogger := &transmissionCaptureLogger{} + s.namespaceReplicator.eventLogger = eventLogger + s.namespaceReplicator.emitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + taskType := enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK id := primitives.NewUUID().String() name := "some random namespace test name" @@ -145,6 +169,42 @@ func (s *transmissionTaskSuite) TestHandleTransmissionTask_RegisterNamespaceTask false, // forceReplicate ) s.Require().NoError(err) + s.Require().Len(eventLogger.records, 1) + s.Equal("namespace_replication_lifecycle", eventLogger.records[0].EventName()) + attrs := make(map[string]otellog.Value) + eventLogger.records[0].WalkAttributes(func(kv otellog.KeyValue) bool { + attrs[kv.Key] = kv.Value + return true + }) + s.Equal("created", attrs["phase"].AsString()) + s.Equal(name, attrs["namespace"].AsString()) + s.Equal(id, attrs["namespace_id"].AsString()) + var details map[string]any + s.Require().NoError(json.Unmarshal([]byte(attrs["details"].AsString()), &details)) + s.Equal("cluster-a", details["source_cluster"]) + s.NotEmpty(details["task_fingerprint"]) + s.Equal( + "some random test owner", + details["task"].(map[string]any)["info"].(map[string]any)["owner_email"], + ) + + s.namespaceReplicationQueue.EXPECT().Publish(gomock.Any(), gomock.Any()).Return(nil) + s.namespaceReplicator.emitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(false) + err = s.namespaceReplicator.HandleTransmissionTask( + context.Background(), + namespaceOperation, + info, + config, + replicationConfig, + true, + configVersion, + failoverVersion, + isGlobalNamespace, + nil, + false, + ) + s.Require().NoError(err) + s.Len(eventLogger.records, 1) } func (s *transmissionTaskSuite) TestHandleTransmissionTask_RegisterNamespaceTask_NotGlobalNamespace() { diff --git a/common/persistence/namespace_replication_queue.go b/common/persistence/namespace_replication_queue.go index fbcb2739d09..cfb46aecd4a 100644 --- a/common/persistence/namespace_replication_queue.go +++ b/common/persistence/namespace_replication_queue.go @@ -133,6 +133,7 @@ func (q *namespaceReplicationQueueImpl) GetReplicationMessages( return nil, lastMessageID, fmt.Errorf("failed to decode task: %v", err) } + replicationTask.SourceTaskId = message.ID lastMessageID = message.ID replicationTasks = append(replicationTasks, replicationTask) } diff --git a/common/persistence/persistence-tests/queue_persistence.go b/common/persistence/persistence-tests/queue_persistence.go index 64895ccdbde..0960538bfb6 100644 --- a/common/persistence/persistence-tests/queue_persistence.go +++ b/common/persistence/persistence-tests/queue_persistence.go @@ -88,6 +88,9 @@ func (s *QueuePersistenceSuite) TestNamespaceReplicationQueue() { s.Nil(err, "GetReplicationMessages failed.") s.Len(result, numMessages) s.Equal(int64(numMessages-1), lastRetrievedMessageID) + for i, task := range result { + s.Equal(int64(i), task.SourceTaskId) + } } // TestQueueMetadataOperations tests queue metadata operations diff --git a/common/wideevents/context.go b/common/wideevents/context.go index 8eba7187af3..af047a7238f 100644 --- a/common/wideevents/context.go +++ b/common/wideevents/context.go @@ -3,6 +3,7 @@ package wideevents import "context" type replicationTaskOriginCtxKey struct{} +type namespaceReplicationTaskContextKey struct{} // ReplicationTaskOrigin identifies the source task whose processing produced an event. type ReplicationTaskOrigin struct { @@ -24,3 +25,28 @@ func ReplicationTaskOriginFromContext(ctx context.Context) ReplicationTaskOrigin } return ReplicationTaskOrigin{} } + +// NamespaceReplicationTaskContext contains receiver-side lifecycle metadata for a namespace task. +type NamespaceReplicationTaskContext struct { + SourceCluster string + TargetCluster string + SourceTaskID int64 + AttemptCount int + EventData NamespaceReplicationTaskEventData +} + +// SetNamespaceReplicationTaskContext stamps namespace replication metadata onto ctx. +func SetNamespaceReplicationTaskContext( + ctx context.Context, + metadata NamespaceReplicationTaskContext, +) context.Context { + return context.WithValue(ctx, namespaceReplicationTaskContextKey{}, metadata) +} + +// NamespaceReplicationTaskContextFromContext returns the stamped metadata when present. +func NamespaceReplicationTaskContextFromContext( + ctx context.Context, +) (NamespaceReplicationTaskContext, bool) { + metadata, ok := ctx.Value(namespaceReplicationTaskContextKey{}).(NamespaceReplicationTaskContext) + return metadata, ok +} diff --git a/common/wideevents/namespace_events.go b/common/wideevents/namespace_events.go index 69e2508ce37..c841833a373 100644 --- a/common/wideevents/namespace_events.go +++ b/common/wideevents/namespace_events.go @@ -3,8 +3,9 @@ package wideevents import "go.opentelemetry.io/otel/log" // NamespaceLifecycleEventName is the stable event name for the generic, phase-discriminated wide -// event describing namespace-level activity (replication, failover, configuration, admission, -// handover, etc.). This package owns only the stable envelope: the set of phase values and the +// event describing namespace lifecycle activity (failover, configuration, admission, handover, +// etc.). Namespace replication tasks use NamespaceReplicationLifecyclePayload instead. This package +// owns only the stable envelope: the set of phase values and the // contents of Details are supplied by the emitter. const NamespaceLifecycleEventName = "namespace_lifecycle" diff --git a/common/wideevents/namespace_replication_events.go b/common/wideevents/namespace_replication_events.go new file mode 100644 index 00000000000..2009c2536db --- /dev/null +++ b/common/wideevents/namespace_replication_events.go @@ -0,0 +1,241 @@ +package wideevents + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "maps" + + "go.opentelemetry.io/otel/log" + enumsspb "go.temporal.io/server/api/enums/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/persistence" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +const NamespaceReplicationLifecycleEventName = "namespace_replication_lifecycle" + +type NamespaceReplicationPhase string + +type NamespaceReplicationOutcome string + +const ( + NamespaceReplicationCreated NamespaceReplicationPhase = "created" + NamespaceReplicationReceived NamespaceReplicationPhase = "received" + NamespaceReplicationProcessed NamespaceReplicationPhase = "processed" + NamespaceReplicationDLQed NamespaceReplicationPhase = "dlqed" + + NamespaceReplicationOutcomeCreated NamespaceReplicationOutcome = "created" + NamespaceReplicationOutcomeUpdated NamespaceReplicationOutcome = "updated" + NamespaceReplicationOutcomeNotAdmitted NamespaceReplicationOutcome = "not_admitted" + NamespaceReplicationOutcomeDuplicate NamespaceReplicationOutcome = "duplicate" + NamespaceReplicationOutcomeNoChange NamespaceReplicationOutcome = "no_change" +) + +// NamespaceReplicationTaskEventData contains the task-specific fields shared by every lifecycle phase. +// TaskFingerprintData may be set when the bytes carried by the queue are more stable than +// deterministically marshaling TaskPayload. +type NamespaceReplicationTaskEventData struct { + TaskType int32 + TaskKind string + Namespace string + NamespaceID string + Operation string + ConfigVersion *int64 + FailoverVersion *int64 + TaskPayload proto.Message + TaskFingerprintData []byte + Details map[string]any +} + +// NamespaceReplicationTaskEventDataProvider identifies namespace replication tasks and extracts the +// task-specific data needed by the lifecycle event. +type NamespaceReplicationTaskEventDataProvider interface { + Extract(task *replicationspb.ReplicationTask) (NamespaceReplicationTaskEventData, bool) +} + +type defaultNamespaceReplicationTaskEventDataProvider struct{} + +// NewDefaultNamespaceReplicationTaskEventDataProvider returns the OSS namespace task provider. +func NewDefaultNamespaceReplicationTaskEventDataProvider() NamespaceReplicationTaskEventDataProvider { + return defaultNamespaceReplicationTaskEventDataProvider{} +} + +func (defaultNamespaceReplicationTaskEventDataProvider) Extract( + task *replicationspb.ReplicationTask, +) (NamespaceReplicationTaskEventData, bool) { + if task.GetTaskType() != enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK { + return NamespaceReplicationTaskEventData{}, false + } + + attributes := task.GetNamespaceTaskAttributes() + if attributes == nil { + return NamespaceReplicationTaskEventData{}, false + } + + configVersion := attributes.GetConfigVersion() + failoverVersion := attributes.GetFailoverVersion() + return NamespaceReplicationTaskEventData{ + TaskType: int32(task.GetTaskType()), + TaskKind: "namespace", + Namespace: attributes.GetInfo().GetName(), + NamespaceID: attributes.GetId(), + Operation: attributes.GetNamespaceOperation().String(), + ConfigVersion: &configVersion, + FailoverVersion: &failoverVersion, + TaskPayload: attributes, + }, true +} + +// NamespaceReplicationLifecyclePayload uses the namespace lifecycle envelope while retaining a +// distinct event name. Replication-specific data is carried in Details. +type NamespaceReplicationLifecyclePayload NamespaceLifecyclePayload + +func (p NamespaceReplicationLifecyclePayload) EventName() string { + return NamespaceReplicationLifecycleEventName +} + +func (p NamespaceReplicationLifecyclePayload) Attributes() []log.KeyValue { + return NamespaceLifecyclePayload(p).Attributes() +} + +type NamespaceReplicationLifecycleInput struct { + Phase NamespaceReplicationPhase + Outcome NamespaceReplicationOutcome + EventData NamespaceReplicationTaskEventData + SourceCluster string + TargetCluster string + SourceTaskID *int64 + AttemptCount int + Error error + LocalNamespacePreMutation *persistencespb.NamespaceDetail + CreateNamespaceRequest *persistence.CreateNamespaceRequest + UpdateNamespaceRequest *persistence.UpdateNamespaceRequest +} + +func EmitNamespaceReplicationLifecycle(logger log.Logger, in NamespaceReplicationLifecycleInput) { + if in.EventData.TaskPayload == nil { + return + } + + taskJSON, err := protojson.MarshalOptions{UseProtoNames: true}.Marshal(in.EventData.TaskPayload) + if err != nil { + return + } + taskBytes := in.EventData.TaskFingerprintData + if len(taskBytes) == 0 { + taskBytes, err = proto.MarshalOptions{Deterministic: true}.Marshal(in.EventData.TaskPayload) + if err != nil { + return + } + } + fingerprint := sha256.Sum256(taskBytes) + + details := make(map[string]any, len(in.EventData.Details)+12) + maps.Copy(details, in.EventData.Details) + details["task_type"] = in.EventData.TaskType + details["task_kind"] = in.EventData.TaskKind + details["operation"] = in.EventData.Operation + details["task_fingerprint"] = hex.EncodeToString(fingerprint[:]) + details["task"] = json.RawMessage(taskJSON) + if in.Outcome != "" { + details["outcome"] = in.Outcome + } + if in.EventData.ConfigVersion != nil { + details["config_version"] = *in.EventData.ConfigVersion + } + if in.EventData.FailoverVersion != nil { + details["failover_version"] = *in.EventData.FailoverVersion + } + if in.SourceCluster != "" { + details["source_cluster"] = in.SourceCluster + } + if in.TargetCluster != "" { + details["target_cluster"] = in.TargetCluster + } + if in.SourceTaskID != nil { + details["source_task_id"] = *in.SourceTaskID + } + if in.AttemptCount > 0 { + details["attempt_count"] = in.AttemptCount + } + if in.Error != nil { + details["error"] = in.Error.Error() + } + if localNamespacePreMutation := marshalNamespaceDetail(in.LocalNamespacePreMutation); len(localNamespacePreMutation) > 0 { + details["local_namespace_pre_mutation"] = localNamespacePreMutation + } + if persistenceRequest := marshalNamespacePersistenceRequest( + in.CreateNamespaceRequest, + in.UpdateNamespaceRequest, + ); len(persistenceRequest) > 0 { + details["persistence_request"] = persistenceRequest + } + Emit(logger, NamespaceReplicationLifecyclePayload{ + Phase: string(in.Phase), + Namespace: in.EventData.Namespace, + NamespaceID: in.EventData.NamespaceID, + Details: details, + }) +} + +func marshalNamespaceDetail(namespace *persistencespb.NamespaceDetail) json.RawMessage { + if namespace == nil { + return nil + } + namespaceJSON, err := protojson.MarshalOptions{ + UseProtoNames: true, + EmitUnpopulated: true, + }.Marshal(namespace) + if err != nil { + return nil + } + return namespaceJSON +} + +func marshalNamespacePersistenceRequest( + createRequest *persistence.CreateNamespaceRequest, + updateRequest *persistence.UpdateNamespaceRequest, +) json.RawMessage { + type persistenceRequest struct { + RequestType string `json:"request_type"` + Namespace json.RawMessage `json:"namespace"` + IsGlobalNamespace bool `json:"is_global_namespace"` + NotificationVersion *int64 `json:"notification_version,omitempty"` + } + + var request persistenceRequest + switch { + case createRequest != nil: + namespaceJSON := marshalNamespaceDetail(createRequest.Namespace) + if len(namespaceJSON) == 0 { + return nil + } + request = persistenceRequest{ + RequestType: "CreateNamespaceRequest", + Namespace: json.RawMessage(namespaceJSON), + IsGlobalNamespace: createRequest.IsGlobalNamespace, + } + case updateRequest != nil: + namespaceJSON := marshalNamespaceDetail(updateRequest.Namespace) + if len(namespaceJSON) == 0 { + return nil + } + request = persistenceRequest{ + RequestType: "UpdateNamespaceRequest", + Namespace: json.RawMessage(namespaceJSON), + IsGlobalNamespace: updateRequest.IsGlobalNamespace, + NotificationVersion: &updateRequest.NotificationVersion, + } + default: + return nil + } + + requestJSON, err := json.Marshal(request) + if err != nil { + return nil + } + return requestJSON +} diff --git a/common/wideevents/namespace_replication_events_test.go b/common/wideevents/namespace_replication_events_test.go new file mode 100644 index 00000000000..a8b7e18c135 --- /dev/null +++ b/common/wideevents/namespace_replication_events_test.go @@ -0,0 +1,274 @@ +package wideevents + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/log" + enumspb "go.temporal.io/api/enums/v1" + namespacepb "go.temporal.io/api/namespace/v1" + replicationpb "go.temporal.io/api/replication/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/persistence" + "google.golang.org/protobuf/types/known/durationpb" +) + +func TestNamespaceReplicationLifecycleEventName(t *testing.T) { + require.Equal(t, "namespace_replication_lifecycle", NamespaceReplicationLifecyclePayload{}.EventName()) +} + +func TestNamespaceReplicationTaskContext(t *testing.T) { + task := namespaceReplicationTaskForTest() + want := NamespaceReplicationTaskContext{ + SourceCluster: "cluster-a", + TargetCluster: "cluster-b", + SourceTaskID: 17, + AttemptCount: 2, + EventData: namespaceReplicationEventDataForTest(t, task), + } + + _, ok := NamespaceReplicationTaskContextFromContext(context.Background()) + require.False(t, ok) + got, ok := NamespaceReplicationTaskContextFromContext( + SetNamespaceReplicationTaskContext(context.Background(), want), + ) + require.True(t, ok) + require.Equal(t, want, got) +} + +func TestEmitNamespaceReplicationLifecycle(t *testing.T) { + logger := &captureLogger{} + sourceTaskID := int64(0) + task := namespaceReplicationTaskForTest() + eventData := namespaceReplicationEventDataForTest(t, task) + eventData.Details = map[string]any{"custom_detail": "custom-value"} + + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationDLQed, + EventData: eventData, + SourceCluster: "cluster-a", + TargetCluster: "cluster-b", + SourceTaskID: &sourceTaskID, + AttemptCount: 3, + Error: errors.New("persistence unavailable"), + }) + + require.Len(t, logger.records, 1) + require.Equal(t, NamespaceReplicationLifecycleEventName, logger.records[0].EventName()) + got := namespaceReplicationRecordValues(logger.records[0]) + require.Equal(t, map[string]any{ + "phase": "dlqed", + "namespace": "payments", + "namespace_id": "namespace-id", + "details": got["details"], + }, got) + details := namespaceReplicationRecordDetails(t, logger.records[0]) + require.InDelta(t, float64(enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK), details["task_type"], 0) + require.Equal(t, "namespace", details["task_kind"]) + require.Equal(t, "Create", details["operation"]) + require.InDelta(t, float64(7), details["config_version"], 0) + require.InDelta(t, float64(11), details["failover_version"], 0) + require.Equal(t, "cluster-a", details["source_cluster"]) + require.Equal(t, "cluster-b", details["target_cluster"]) + require.InDelta(t, float64(0), details["source_task_id"], 0) + require.InDelta(t, float64(3), details["attempt_count"], 0) + require.Equal(t, "persistence unavailable", details["error"]) + require.Equal(t, "custom-value", details["custom_detail"]) + require.Len(t, details["task_fingerprint"], 64) + + taskJSON := details["task"].(map[string]any) + require.Equal(t, "NAMESPACE_OPERATION_CREATE", taskJSON["namespace_operation"]) + require.Equal(t, "namespace-id", taskJSON["id"]) + require.Equal(t, "7", taskJSON["config_version"]) + require.Equal(t, "11", taskJSON["failover_version"]) + require.Equal(t, "payments", taskJSON["info"].(map[string]any)["name"]) + require.Equal(t, "cluster-a", taskJSON["replication_config"].(map[string]any)["active_cluster_name"]) +} + +func TestNamespaceReplicationTaskFingerprintLinksPhases(t *testing.T) { + logger := &captureLogger{} + task := namespaceReplicationTaskForTest() + + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationCreated, + EventData: namespaceReplicationEventDataForTest(t, task), + }) + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationProcessed, + Outcome: NamespaceReplicationOutcomeCreated, + EventData: namespaceReplicationEventDataForTest(t, task), + }) + task.ConfigVersion++ + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationCreated, + EventData: namespaceReplicationEventDataForTest(t, task), + }) + + require.Len(t, logger.records, 3) + first := namespaceReplicationRecordDetails(t, logger.records[0])["task_fingerprint"] + require.Equal(t, first, namespaceReplicationRecordDetails(t, logger.records[1])["task_fingerprint"]) + require.NotEqual(t, first, namespaceReplicationRecordDetails(t, logger.records[2])["task_fingerprint"]) +} + +func TestNamespaceReplicationProcessedIncludesCreatePersistenceRequest(t *testing.T) { + logger := &captureLogger{} + request := &persistence.CreateNamespaceRequest{ + Namespace: &persistencespb.NamespaceDetail{ + Info: &persistencespb.NamespaceInfo{Id: "namespace-id", Name: "payments"}, + }, + IsGlobalNamespace: true, + } + + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationProcessed, + Outcome: NamespaceReplicationOutcomeCreated, + EventData: namespaceReplicationEventDataForTest(t, namespaceReplicationTaskForTest()), + CreateNamespaceRequest: request, + }) + + details := namespaceReplicationRecordDetails(t, logger.records[0]) + got := details["persistence_request"].(map[string]any) + require.Equal(t, "CreateNamespaceRequest", got["request_type"]) + require.Equal(t, true, got["is_global_namespace"]) + require.NotContains(t, got, "notification_version") + namespace := got["namespace"].(map[string]any) + require.Equal(t, "namespace-id", namespace["info"].(map[string]any)["id"]) + require.Equal(t, "0", namespace["config_version"]) +} + +func TestNamespaceReplicationProcessedIncludesUpdatePersistenceRequest(t *testing.T) { + logger := &captureLogger{} + localNamespacePreMutation := &persistencespb.NamespaceDetail{ + Info: &persistencespb.NamespaceInfo{Id: "namespace-id", Name: "payments"}, + ConfigVersion: 7, + } + request := &persistence.UpdateNamespaceRequest{ + Namespace: &persistencespb.NamespaceDetail{ + Info: &persistencespb.NamespaceInfo{Id: "namespace-id", Name: "payments"}, + ConfigVersion: 8, + }, + IsGlobalNamespace: true, + NotificationVersion: 19, + } + + EmitNamespaceReplicationLifecycle(logger, NamespaceReplicationLifecycleInput{ + Phase: NamespaceReplicationProcessed, + Outcome: NamespaceReplicationOutcomeUpdated, + EventData: namespaceReplicationEventDataForTest(t, namespaceReplicationTaskForTest()), + LocalNamespacePreMutation: localNamespacePreMutation, + UpdateNamespaceRequest: request, + }) + + details := namespaceReplicationRecordDetails(t, logger.records[0]) + got := details["persistence_request"].(map[string]any) + require.Equal(t, "UpdateNamespaceRequest", got["request_type"]) + require.InDelta(t, float64(19), got["notification_version"].(float64), 0) + require.Equal(t, "8", got["namespace"].(map[string]any)["config_version"]) + + before := details["local_namespace_pre_mutation"].(map[string]any) + require.Equal(t, "7", before["config_version"]) + require.Equal(t, "payments", before["info"].(map[string]any)["name"]) +} + +func TestDefaultNamespaceReplicationTaskEventDataProvider(t *testing.T) { + provider := NewDefaultNamespaceReplicationTaskEventDataProvider() + attributes := namespaceReplicationTaskForTest() + task := &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK, + Attributes: &replicationspb.ReplicationTask_NamespaceTaskAttributes{ + NamespaceTaskAttributes: attributes, + }, + } + + eventData, ok := provider.Extract(task) + require.True(t, ok) + require.Equal(t, int32(enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK), eventData.TaskType) + require.Equal(t, "namespace", eventData.TaskKind) + require.Equal(t, "payments", eventData.Namespace) + require.Equal(t, "namespace-id", eventData.NamespaceID) + require.Equal(t, "Create", eventData.Operation) + require.Equal(t, int64(7), *eventData.ConfigVersion) + require.Equal(t, int64(11), *eventData.FailoverVersion) + require.Same(t, attributes, eventData.TaskPayload) + + _, ok = provider.Extract(&replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_TASK_QUEUE_USER_DATA, + }) + require.False(t, ok) +} + +func namespaceReplicationEventDataForTest( + t *testing.T, + attributes *replicationspb.NamespaceTaskAttributes, +) NamespaceReplicationTaskEventData { + t.Helper() + eventData, ok := NewDefaultNamespaceReplicationTaskEventDataProvider().Extract(&replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK, + Attributes: &replicationspb.ReplicationTask_NamespaceTaskAttributes{ + NamespaceTaskAttributes: attributes, + }, + }) + require.True(t, ok) + return eventData +} + +func namespaceReplicationTaskForTest() *replicationspb.NamespaceTaskAttributes { + return &replicationspb.NamespaceTaskAttributes{ + NamespaceOperation: enumsspb.NAMESPACE_OPERATION_CREATE, + Id: "namespace-id", + Info: &namespacepb.NamespaceInfo{ + Name: "payments", + State: enumspb.NAMESPACE_STATE_REGISTERED, + Description: "payment workflows", + OwnerEmail: "payments@example.com", + Data: map[string]string{"region": "west"}, + }, + Config: &namespacepb.NamespaceConfig{ + WorkflowExecutionRetentionTtl: durationpb.New(72 * time.Hour), + HistoryArchivalState: enumspb.ARCHIVAL_STATE_DISABLED, + VisibilityArchivalState: enumspb.ARCHIVAL_STATE_DISABLED, + CustomSearchAttributeAliases: map[string]string{"OrderId": "order_id"}, + }, + ReplicationConfig: &replicationpb.NamespaceReplicationConfig{ + ActiveClusterName: "cluster-a", + Clusters: []*replicationpb.ClusterReplicationConfig{ + {ClusterName: "cluster-a"}, + {ClusterName: "cluster-b"}, + }, + State: enumspb.REPLICATION_STATE_NORMAL, + }, + ConfigVersion: 7, + FailoverVersion: 11, + FailoverHistory: []*replicationpb.FailoverStatus{{FailoverVersion: 11}}, + } +} + +func namespaceReplicationRecordValues(record log.Record) map[string]any { + values := make(map[string]any) + record.WalkAttributes(func(kv log.KeyValue) bool { + switch kv.Value.Kind() { + case log.KindString: + values[kv.Key] = kv.Value.AsString() + case log.KindInt64: + values[kv.Key] = kv.Value.AsInt64() + default: + } + return true + }) + return values +} + +func namespaceReplicationRecordDetails(t *testing.T, record log.Record) map[string]any { + t.Helper() + detailsJSON, ok := namespaceReplicationRecordValues(record)["details"].(string) + require.True(t, ok) + var details map[string]any + require.NoError(t, json.Unmarshal([]byte(detailsJSON), &details)) + return details +} diff --git a/service/frontend/namespace_handler_test.go b/service/frontend/namespace_handler_test.go index 598c550a1bb..aff3283af26 100644 --- a/service/frontend/namespace_handler_test.go +++ b/service/frontend/namespace_handler_test.go @@ -89,7 +89,13 @@ func (s *namespaceHandlerCommonSuite) SetupTest() { s.mockMetadataMgr = persistence.NewMockMetadataManager(s.controller) s.mockClusterMetadata = cluster.NewMockMetadata(s.controller) s.mockProducer = persistence.NewMockNamespaceReplicationQueue(s.controller) - s.mockNamespaceReplicator = nsreplication.NewReplicator(s.mockProducer, logger) + s.mockNamespaceReplicator = nsreplication.NewReplicator( + s.mockProducer, + logger, + nil, + dc.GetBoolPropertyFn(false), + "cluster-a", + ) s.archivalMetadata = archiver.NewArchivalMetadata( dcCollection, "", diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index f14f55ae610..433272e5998 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -356,7 +356,13 @@ func NewWorkflowHandler( persistenceMetadataManager, namespaceRegistry, clusterMetadata, - nsreplication.NewReplicator(namespaceReplicationQueue, logger), + nsreplication.NewReplicator( + namespaceReplicationQueue, + logger, + eventLogger, + config.EmitNamespaceLifecycleEvents, + clusterMetadata.GetCurrentClusterName(), + ), archivalMetadata, archiverProvider, timeSource, diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index ef0e8274a72..a0eea679ae3 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -160,6 +160,7 @@ func (s *WorkflowHandlerSuite) SetupTest() { s.mockProducer = persistence.NewMockNamespaceReplicationQueue(s.controller) s.mockHistoryArchiver = archiver.NewMockHistoryArchiver(s.controller) s.mockVisibilityArchiver = archiver.NewMockVisibilityArchiver(s.controller) + s.mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() s.tokenSerializer = tasktoken.NewSerializer() diff --git a/service/worker/fx.go b/service/worker/fx.go index 16cf0a77d77..18cfb5d363a 100644 --- a/service/worker/fx.go +++ b/service/worker/fx.go @@ -4,6 +4,7 @@ import ( "context" "os" + otellog "go.opentelemetry.io/otel/log" wcicomponent "go.temporal.io/auto-scaled-workers/wci/workercomponent" "go.temporal.io/server/api/adminservice/v1" "go.temporal.io/server/chasm" @@ -31,6 +32,7 @@ import ( "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service" "go.temporal.io/server/service/worker/batcher" workercommon "go.temporal.io/server/service/worker/common" @@ -92,6 +94,8 @@ var Module = fx.Options( dataMerger nsreplication.NamespaceDataMerger, admitter nsreplication.NamespaceReplicationAdmitter, logger log.Logger, + eventLogger otellog.Logger, + serviceConfig *Config, testHooks testhooks.TestHooks, ) nsreplication.TaskExecutor { return nsreplication.NewTaskExecutor( @@ -101,10 +105,15 @@ var Module = fx.Options( admitter, logger, testHooks, + nsreplication.WithNamespaceReplicationLifecycleEvents( + eventLogger, + serviceConfig.EmitNamespaceLifecycleEvents, + ), ) }), fx.Provide(nsreplication.NewNoopDataMerger), fx.Provide(nsreplication.NewDefaultAdmitter), + fx.Provide(wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider), fx.Provide(ServerProvider), fx.Provide(NewService), fx.Provide(fx.Annotate(NewWorkerManager, fx.ParamTags(workercommon.WorkerComponentTag))), diff --git a/service/worker/replicator/replication_message_processor.go b/service/worker/replicator/replication_message_processor.go index cdea941bced..f521909d9f7 100644 --- a/service/worker/replicator/replication_message_processor.go +++ b/service/worker/replicator/replication_message_processor.go @@ -6,6 +6,7 @@ import ( "sync/atomic" "time" + otellog "go.opentelemetry.io/otel/log" "go.temporal.io/api/serviceerror" "go.temporal.io/server/api/adminservice/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -13,6 +14,7 @@ import ( replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -22,6 +24,7 @@ import ( "go.temporal.io/server/common/namespace/nsreplication" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/rpc" + "go.temporal.io/server/common/wideevents" ) const ( @@ -43,6 +46,9 @@ func newReplicationMessageProcessor( currentCluster string, sourceCluster string, logger log.Logger, + eventLogger otellog.Logger, + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn, + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider, remotePeer adminservice.AdminServiceClient, metricsHandler metrics.Handler, namespaceTaskExecutor nsreplication.TaskExecutor, @@ -74,45 +80,51 @@ func newReplicationMessageProcessor( } return &replicationMessageProcessor{ - hostInfo: hostInfo, - serviceResolver: serviceResolver, - status: common.DaemonStatusInitialized, - currentCluster: currentCluster, - sourceCluster: sourceCluster, - logger: logger, - remotePeer: remotePeer, - namespaceTaskExecutor: namespaceTaskExecutor, - customTaskHandler: customTaskHandler, - metricsHandler: metricsHandler.WithTags(metrics.OperationTag(metrics.NamespaceReplicationTaskScope)), - retryPolicyForTask: retryPolicyForTask, - lastProcessedMessageID: -1, - lastRetrievedMessageID: -1, - done: make(chan struct{}), - namespaceReplicationQueue: namespaceReplicationQueue, - matchingClient: matchingClient, - namespaceRegistry: namespaceRegistry, + hostInfo: hostInfo, + serviceResolver: serviceResolver, + status: common.DaemonStatusInitialized, + currentCluster: currentCluster, + sourceCluster: sourceCluster, + logger: logger, + eventLogger: eventLogger, + emitNamespaceLifecycleEvents: emitNamespaceLifecycleEvents, + eventDataProvider: eventDataProvider, + remotePeer: remotePeer, + namespaceTaskExecutor: namespaceTaskExecutor, + customTaskHandler: customTaskHandler, + metricsHandler: metricsHandler.WithTags(metrics.OperationTag(metrics.NamespaceReplicationTaskScope)), + retryPolicyForTask: retryPolicyForTask, + lastProcessedMessageID: -1, + lastRetrievedMessageID: -1, + done: make(chan struct{}), + namespaceReplicationQueue: namespaceReplicationQueue, + matchingClient: matchingClient, + namespaceRegistry: namespaceRegistry, } } type ( replicationMessageProcessor struct { - hostInfo membership.HostInfo - serviceResolver membership.ServiceResolver - status int32 - currentCluster string - sourceCluster string - logger log.Logger - remotePeer adminservice.AdminServiceClient - namespaceTaskExecutor nsreplication.TaskExecutor - customTaskHandler func(ctx context.Context, task *replicationspb.ReplicationTask) error - metricsHandler metrics.Handler - retryPolicyForTask func(*replicationspb.ReplicationTask) backoff.RetryPolicy - lastProcessedMessageID int64 - lastRetrievedMessageID int64 - done chan struct{} - namespaceReplicationQueue persistence.NamespaceReplicationQueue - matchingClient matchingservice.MatchingServiceClient - namespaceRegistry namespace.Registry + hostInfo membership.HostInfo + serviceResolver membership.ServiceResolver + status int32 + currentCluster string + sourceCluster string + logger log.Logger + eventLogger otellog.Logger + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider + remotePeer adminservice.AdminServiceClient + namespaceTaskExecutor nsreplication.TaskExecutor + customTaskHandler func(ctx context.Context, task *replicationspb.ReplicationTask) error + metricsHandler metrics.Handler + retryPolicyForTask func(*replicationspb.ReplicationTask) backoff.RetryPolicy + lastProcessedMessageID int64 + lastRetrievedMessageID int64 + done chan struct{} + namespaceReplicationQueue persistence.NamespaceReplicationQueue + matchingClient matchingservice.MatchingServiceClient + namespaceRegistry namespace.Registry } ) @@ -177,9 +189,32 @@ func (p *replicationMessageProcessor) handleReplicationTasks() { taskCtx := headers.SetCallerInfo(context.TODO(), headers.SystemPreemptableCallerInfo) for taskIndex := range response.Messages.ReplicationTasks { task := response.Messages.ReplicationTasks[taskIndex] + eventData, emitEvents := p.namespaceReplicationEventData(task) + if emitEvents { + p.emitNamespaceReplicationEvent( + eventData, + task.GetSourceTaskId(), + wideevents.NamespaceReplicationReceived, + 0, + nil, + ) + } + + attemptCount := 0 policy := p.retryPolicyForTask(task) err := backoff.ThrottleRetry(func() error { - return p.handleReplicationTask(taskCtx, task) + attemptCount++ + attemptCtx := taskCtx + if emitEvents { + attemptCtx = wideevents.SetNamespaceReplicationTaskContext(taskCtx, wideevents.NamespaceReplicationTaskContext{ + SourceCluster: p.sourceCluster, + TargetCluster: p.currentCluster, + SourceTaskID: task.GetSourceTaskId(), + AttemptCount: attemptCount, + EventData: eventData, + }) + } + return p.handleReplicationTask(attemptCtx, task) }, policy, isTransientRetryableError) if err != nil { @@ -187,7 +222,6 @@ func (p *replicationMessageProcessor) handleReplicationTasks() { p.logger.Error("Failed to apply replication tasks", tag.Error(err)) dlqErr := backoff.ThrottleRetry(func() error { - return p.putNamespaceReplicationTaskToDLQ(taskCtx, task) }, policy, isTransientRetryableError) if dlqErr != nil { @@ -195,6 +229,16 @@ func (p *replicationMessageProcessor) handleReplicationTasks() { metrics.ReplicatorDLQFailures.With(p.metricsHandler).Record(1) return } + + if emitEvents { + p.emitNamespaceReplicationEvent( + eventData, + task.GetSourceTaskId(), + wideevents.NamespaceReplicationDLQed, + attemptCount, + err, + ) + } } } @@ -202,6 +246,33 @@ func (p *replicationMessageProcessor) handleReplicationTasks() { p.lastRetrievedMessageID = response.Messages.GetLastRetrievedMessageId() } +func (p *replicationMessageProcessor) emitNamespaceReplicationEvent( + eventData wideevents.NamespaceReplicationTaskEventData, + sourceTaskID int64, + phase wideevents.NamespaceReplicationPhase, + attemptCount int, + err error, +) { + wideevents.EmitNamespaceReplicationLifecycle(p.eventLogger, wideevents.NamespaceReplicationLifecycleInput{ + Phase: phase, + EventData: eventData, + SourceCluster: p.sourceCluster, + TargetCluster: p.currentCluster, + SourceTaskID: &sourceTaskID, + AttemptCount: attemptCount, + Error: err, + }) +} + +func (p *replicationMessageProcessor) namespaceReplicationEventData( + task *replicationspb.ReplicationTask, +) (wideevents.NamespaceReplicationTaskEventData, bool) { + if !p.emitNamespaceLifecycleEvents() { + return wideevents.NamespaceReplicationTaskEventData{}, false + } + return p.eventDataProvider.Extract(task) +} + func (p *replicationMessageProcessor) putNamespaceReplicationTaskToDLQ( ctx context.Context, task *replicationspb.ReplicationTask, diff --git a/service/worker/replicator/replication_message_processor_test.go b/service/worker/replicator/replication_message_processor_test.go index 634f43a4f12..55bdaf68696 100644 --- a/service/worker/replicator/replication_message_processor_test.go +++ b/service/worker/replicator/replication_message_processor_test.go @@ -1,14 +1,64 @@ package replicator import ( + "context" + "encoding/json" "testing" + "time" "github.com/stretchr/testify/require" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" + commonpb "go.temporal.io/api/common/v1" + namespacepb "go.temporal.io/api/namespace/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/api/adminservicemock/v1" enumsspb "go.temporal.io/server/api/enums/v1" replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/membership" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace/nsreplication" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/wideevents" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/wrapperspb" ) +type replicationEventCaptureLogger struct { + embedded.Logger + records []otellog.Record +} + +type customNamespaceReplicationTaskEventDataProvider struct{} + +func (customNamespaceReplicationTaskEventDataProvider) Extract( + task *replicationspb.ReplicationTask, +) (wideevents.NamespaceReplicationTaskEventData, bool) { + if int32(task.GetTaskType()) != 1002 { + return wideevents.NamespaceReplicationTaskEventData{}, false + } + return wideevents.NamespaceReplicationTaskEventData{ + TaskType: int32(task.GetTaskType()), + TaskKind: "custom_namespace_config", + NamespaceID: "custom-namespace-id", + Operation: "update_config", + TaskPayload: wrapperspb.Bytes(task.GetData().GetData()), + TaskFingerprintData: task.GetData().GetData(), + }, true +} + +func (l *replicationEventCaptureLogger) Emit(_ context.Context, record otellog.Record) { + l.records = append(l.records, record) +} + +func (l *replicationEventCaptureLogger) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + // TestRetryPolicyForTask verifies that the processor's per-task retry-policy // selector hands out a distinct policy for namespace tasks (which need the // wider CAS-tolerant budget) and a shared default policy for all other task @@ -17,7 +67,10 @@ func TestRetryPolicyForTask(t *testing.T) { p := newReplicationMessageProcessor( "currentCluster", "sourceCluster", - nil, // logger + nil, // logger + nil, // eventLogger + dynamicconfig.GetBoolPropertyFn(false), + wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider(), nil, // remotePeer metrics.NoopMetricsHandler, // metricsHandler — actually used by constructor nil, // namespaceTaskExecutor @@ -47,3 +100,184 @@ func TestRetryPolicyForTask(t *testing.T) { // Selector is stable across calls. require.Same(t, nsPolicy, p.retryPolicyForTask(nsTask)) } + +func TestHandleNamespaceReplicationTaskEmitsReceivedAndPassesProcessingContext(t *testing.T) { + p, task, executor, _, eventLogger := newReplicationEventTestProcessor(t, true, 2) + var processingContext wideevents.NamespaceReplicationTaskContext + var processingContextSet bool + executor.EXPECT().Execute(gomock.Any(), task.GetNamespaceTaskAttributes()).DoAndReturn( + func(ctx context.Context, _ *replicationspb.NamespaceTaskAttributes) error { + processingContext, processingContextSet = wideevents.NamespaceReplicationTaskContextFromContext(ctx) + return nil + }, + ) + + p.handleReplicationTasks() + require.Equal(t, []string{"received"}, replicationEventPhases(eventLogger.records)) + + received := replicationEventDetails(t, eventLogger.records[0]) + require.InDelta(t, float64(42), received["source_task_id"], 0) + require.Equal(t, "cluster-a", received["source_cluster"]) + require.Equal(t, "cluster-b", received["target_cluster"]) + require.True(t, processingContextSet) + eventData, ok := wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider().Extract(task) + require.True(t, ok) + require.Equal(t, wideevents.NamespaceReplicationTaskContext{ + SourceCluster: "cluster-a", + TargetCluster: "cluster-b", + SourceTaskID: 42, + AttemptCount: 1, + EventData: eventData, + }, processingContext) +} + +func TestHandleNamespaceReplicationTaskCountsRetries(t *testing.T) { + p, _, executor, _, eventLogger := newReplicationEventTestProcessor(t, true, 2) + attempt := 0 + var processingContext wideevents.NamespaceReplicationTaskContext + executor.EXPECT().Execute(gomock.Any(), gomock.Any()).Times(2).DoAndReturn( + func(ctx context.Context, _ *replicationspb.NamespaceTaskAttributes) error { + attempt++ + processingContext, _ = wideevents.NamespaceReplicationTaskContextFromContext(ctx) + if attempt == 1 { + return serviceerror.NewUnavailable("retry") + } + return nil + }, + ) + + p.handleReplicationTasks() + require.Equal(t, []string{"received"}, replicationEventPhases(eventLogger.records)) + require.Equal(t, 2, processingContext.AttemptCount) +} + +func TestHandleNamespaceReplicationTaskEmitsDLQed(t *testing.T) { + p, task, executor, queue, eventLogger := newReplicationEventTestProcessor(t, true, 1) + executor.EXPECT().Execute(gomock.Any(), task.GetNamespaceTaskAttributes()).Return(serviceerror.NewInvalidArgument("bad task")) + queue.EXPECT().PublishToDLQ(gomock.Any(), task).Return(nil) + + p.handleReplicationTasks() + require.Equal(t, []string{"received", "dlqed"}, replicationEventPhases(eventLogger.records)) + dlqed := replicationEventDetails(t, eventLogger.records[1]) + require.InDelta(t, float64(1), dlqed["attempt_count"], 0) + require.Equal(t, "bad task", dlqed["error"]) + require.NotContains(t, dlqed, "persistence_request") +} + +func TestHandleNamespaceReplicationTaskEventsDisabled(t *testing.T) { + p, task, executor, _, eventLogger := newReplicationEventTestProcessor(t, false, 1) + processingContextSet := false + executor.EXPECT().Execute(gomock.Any(), task.GetNamespaceTaskAttributes()).DoAndReturn( + func(ctx context.Context, _ *replicationspb.NamespaceTaskAttributes) error { + _, processingContextSet = wideevents.NamespaceReplicationTaskContextFromContext(ctx) + return nil + }, + ) + + p.handleReplicationTasks() + require.Empty(t, eventLogger.records) + require.False(t, processingContextSet) +} + +func TestCustomNamespaceReplicationTaskUsesEventDataProvider(t *testing.T) { + task := &replicationspb.ReplicationTask{ + TaskType: enumsspb.ReplicationTaskType(1002), + Data: &commonpb.DataBlob{Data: []byte("custom-task")}, + } + p := &replicationMessageProcessor{ + emitNamespaceLifecycleEvents: dynamicconfig.GetBoolPropertyFn(true), + eventDataProvider: customNamespaceReplicationTaskEventDataProvider{}, + } + + eventData, ok := p.namespaceReplicationEventData(task) + require.True(t, ok) + require.Equal(t, int32(1002), eventData.TaskType) + require.Equal(t, "custom_namespace_config", eventData.TaskKind) + require.Equal(t, "custom-namespace-id", eventData.NamespaceID) + require.Equal(t, "update_config", eventData.Operation) + require.Equal(t, []byte("custom-task"), eventData.TaskFingerprintData) +} + +func newReplicationEventTestProcessor( + t *testing.T, + enabled bool, + maximumAttempts int, +) (*replicationMessageProcessor, *replicationspb.ReplicationTask, *nsreplication.MockTaskExecutor, *persistence.MockNamespaceReplicationQueue, *replicationEventCaptureLogger) { + t.Helper() + controller := gomock.NewController(t) + executor := nsreplication.NewMockTaskExecutor(controller) + queue := persistence.NewMockNamespaceReplicationQueue(controller) + remotePeer := adminservicemock.NewMockAdminServiceClient(controller) + serviceResolver := membership.NewMockServiceResolver(controller) + hostInfo := membership.NewHostInfoFromAddress("worker") + eventLogger := &replicationEventCaptureLogger{} + task := namespaceReplicationTaskForProcessorTest() + policy := backoff.NewExponentialRetryPolicy(time.Millisecond).WithMaximumAttempts(maximumAttempts) + serviceResolver.EXPECT().Lookup("cluster-a").Return(hostInfo, nil) + remotePeer.EXPECT().GetNamespaceReplicationMessages(gomock.Any(), gomock.Any()).Return( + &adminservice.GetNamespaceReplicationMessagesResponse{ + Messages: &replicationspb.ReplicationMessages{ + ReplicationTasks: []*replicationspb.ReplicationTask{task}, + LastRetrievedMessageId: task.GetSourceTaskId(), + }, + }, + nil, + ) + p := &replicationMessageProcessor{ + hostInfo: hostInfo, + serviceResolver: serviceResolver, + currentCluster: "cluster-b", + sourceCluster: "cluster-a", + logger: log.NewNoopLogger(), + eventLogger: eventLogger, + emitNamespaceLifecycleEvents: dynamicconfig.GetBoolPropertyFn(enabled), + eventDataProvider: wideevents.NewDefaultNamespaceReplicationTaskEventDataProvider(), + remotePeer: remotePeer, + namespaceTaskExecutor: executor, + metricsHandler: metrics.NoopMetricsHandler, + retryPolicyForTask: func(*replicationspb.ReplicationTask) backoff.RetryPolicy { return policy }, + lastProcessedMessageID: -1, + lastRetrievedMessageID: -1, + namespaceReplicationQueue: queue, + } + return p, task, executor, queue, eventLogger +} + +func namespaceReplicationTaskForProcessorTest() *replicationspb.ReplicationTask { + return &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_NAMESPACE_TASK, + SourceTaskId: 42, + Attributes: &replicationspb.ReplicationTask_NamespaceTaskAttributes{ + NamespaceTaskAttributes: &replicationspb.NamespaceTaskAttributes{ + NamespaceOperation: enumsspb.NAMESPACE_OPERATION_UPDATE, + Id: "namespace-id", + Info: &namespacepb.NamespaceInfo{Name: "payments"}, + Config: &namespacepb.NamespaceConfig{}, + }, + }, + } +} + +func replicationEventPhases(records []otellog.Record) []string { + phases := make([]string, 0, len(records)) + for _, record := range records { + phases = append(phases, replicationEventValues(record)["phase"].AsString()) + } + return phases +} + +func replicationEventValues(record otellog.Record) map[string]otellog.Value { + values := make(map[string]otellog.Value) + record.WalkAttributes(func(kv otellog.KeyValue) bool { + values[kv.Key] = kv.Value + return true + }) + return values +} + +func replicationEventDetails(t *testing.T, record otellog.Record) map[string]any { + t.Helper() + var details map[string]any + require.NoError(t, json.Unmarshal([]byte(replicationEventValues(record)["details"].AsString()), &details)) + return details +} diff --git a/service/worker/replicator/replicator.go b/service/worker/replicator/replicator.go index 407de2fad6d..c5f38c83414 100644 --- a/service/worker/replicator/replicator.go +++ b/service/worker/replicator/replicator.go @@ -8,11 +8,13 @@ import ( "sync/atomic" "time" + otellog "go.opentelemetry.io/otel/log" "go.temporal.io/server/api/matchingservice/v1" replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/client" "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/goro" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" @@ -22,6 +24,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/namespace/nsreplication" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/wideevents" ) const replicationQueueCleanupInterval = 5 * time.Minute @@ -35,6 +38,9 @@ type ( customTaskHandler func(ctx context.Context, task *replicationspb.ReplicationTask) error clientBean client.Bean logger log.Logger + eventLogger otellog.Logger + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider metricsHandler metrics.Handler hostInfo membership.HostInfo serviceResolver membership.ServiceResolver @@ -57,6 +63,9 @@ func NewReplicator( clusterMetadata cluster.Metadata, clientBean client.Bean, logger log.Logger, + eventLogger otellog.Logger, + emitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn, + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider, metricsHandler metrics.Handler, hostInfo membership.HostInfo, serviceResolver membership.ServiceResolver, @@ -74,6 +83,9 @@ func NewReplicator( namespaceProcessors: make(map[string]*replicationMessageProcessor), clientBean: clientBean, logger: log.With(logger, tag.ComponentReplicator), + eventLogger: eventLogger, + emitNamespaceLifecycleEvents: emitNamespaceLifecycleEvents, + eventDataProvider: eventDataProvider, metricsHandler: metricsHandler, namespaceReplicationQueue: namespaceReplicationQueue, matchingClient: matchingClient, @@ -151,6 +163,9 @@ func (r *Replicator) listenToClusterMetadataChange() { currentClusterName, clusterName, log.With(r.logger, tag.ComponentReplicationTaskProcessor, tag.SourceCluster(clusterName)), + r.eventLogger, + r.emitNamespaceLifecycleEvents, + r.eventDataProvider, remoteAdminClient, r.metricsHandler, r.namespaceReplicationTaskExecutor, diff --git a/service/worker/service.go b/service/worker/service.go index 6cfef3f8cdd..fa9516c09fb 100644 --- a/service/worker/service.go +++ b/service/worker/service.go @@ -4,6 +4,7 @@ import ( "context" "net" + otellog "go.opentelemetry.io/otel/log" "go.temporal.io/api/serviceerror" sdkworker "go.temporal.io/sdk/worker" "go.temporal.io/server/api/matchingservice/v1" @@ -24,6 +25,7 @@ import ( "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/resource" "go.temporal.io/server/common/sdk" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/worker/parentclosepolicy" "go.temporal.io/server/service/worker/replicator" "go.temporal.io/server/service/worker/scanner" @@ -44,6 +46,8 @@ type ( // Archiver: Handles archival of workflow histories. Service struct { logger log.Logger + eventLogger otellog.Logger + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider clusterMetadata cluster.Metadata clientBean client.Bean clusterMetadataManager persistence.ClusterMetadataManager @@ -94,6 +98,7 @@ type ( BatcherRPS dynamicconfig.IntPropertyFnWithNamespaceFilter BatcherConcurrency dynamicconfig.IntPropertyFnWithNamespaceFilter EnableParentClosePolicyWorker dynamicconfig.BoolPropertyFn + EmitNamespaceLifecycleEvents dynamicconfig.BoolPropertyFn PerNamespaceWorkerCount dynamicconfig.TypedSubscribableWithNamespaceFilter[int] PerNamespaceWorkerOptions dynamicconfig.TypedSubscribableWithNamespaceFilter[sdkworker.Options] PerNamespaceWorkerStartRate dynamicconfig.FloatPropertyFn @@ -111,6 +116,8 @@ type ( func NewService( logger log.SnTaggedLogger, + eventLogger otellog.Logger, + eventDataProvider wideevents.NamespaceReplicationTaskEventDataProvider, serviceConfig *Config, sdkClientFactory sdk.ClientFactory, clusterMetadata cluster.Metadata, @@ -144,6 +151,8 @@ func NewService( config: serviceConfig, sdkClientFactory: sdkClientFactory, logger: logger, + eventLogger: eventLogger, + eventDataProvider: eventDataProvider, clusterMetadata: clusterMetadata, clientBean: clientBean, clusterMetadataManager: clusterMetadataManager, @@ -219,6 +228,7 @@ func NewConfig( BatcherRPS: dynamicconfig.BatcherRPS.Get(dc), BatcherConcurrency: dynamicconfig.BatcherConcurrency.Get(dc), EnableParentClosePolicyWorker: dynamicconfig.EnableParentClosePolicyWorker.Get(dc), + EmitNamespaceLifecycleEvents: dynamicconfig.EmitNamespaceLifecycleEvents.Get(dc), PerNamespaceWorkerCount: dynamicconfig.WorkerPerNamespaceWorkerCount.Subscribe(dc), PerNamespaceWorkerOptions: dynamicconfig.WorkerPerNamespaceWorkerOptions.Subscribe(dc), PerNamespaceWorkerStartRate: dynamicconfig.WorkerPerNamespaceWorkerStartRate.Get(dc), @@ -372,6 +382,9 @@ func (s *Service) startReplicator() { s.clusterMetadata, s.clientBean, s.logger, + s.eventLogger, + s.config.EmitNamespaceLifecycleEvents, + s.eventDataProvider, s.metricsHandler, s.hostInfo, s.workerServiceResolver, From 612823d3ead9eb614af73b68d4dd67de98c53b1f Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 19 Aug 2026 20:16:03 -0700 Subject: [PATCH 021/139] Log outbound queue circuit breaker state changes (#11661) ### What changed Sets gobreaker's OnStateChange hook on the outbound queue circuit breaker pool, logging every transition. ### Why Obtain more details for debugging curcuit breaker in production. --------- Co-authored-by: Claude Opus 5 (1M context) --- chasm/lib/callback/invocable_outbound.go | 2 +- common/log/tag/tags.go | 5 +++++ components/callbacks/nexus_invocation.go | 2 +- service/history/circuitbreakerpool/fx.go | 24 ++++++++++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 6ed0a65d6dc..41a02a55ece 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -46,7 +46,7 @@ func (n invocableOutbound) Invoke( traceLogger := log.With(h.logger, tag.WorkflowNamespace(ns.Name().String()), tag.Operation("CompleteNexusOperation"), - tag.String("destination", taskAttr.Destination), + tag.Destination(taskAttr.Destination), tag.WorkflowID(n.workflowID), tag.WorkflowRunID(n.runID), tag.AttemptStart(time.Now().UTC()), diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index 4109a1ceec8..fa9c16346fa 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -989,6 +989,11 @@ func Endpoint(endpoint string) ZapTag { return NewStringTag("endpoint", endpoint) } +// Destination returns a tag for an outbound task's destination. +func Destination(destination string) ZapTag { + return NewStringTag("destination", destination) +} + func BuildId(buildId string) ZapTag { return NewStringTag("build-id", buildId) } diff --git a/components/callbacks/nexus_invocation.go b/components/callbacks/nexus_invocation.go index e24f070f455..69feed61164 100644 --- a/components/callbacks/nexus_invocation.go +++ b/components/callbacks/nexus_invocation.go @@ -47,7 +47,7 @@ func (n nexusInvocation) Invoke(ctx context.Context, ns *namespace.Namespace, e traceLogger := log.With(e.Logger, tag.WorkflowNamespace(ns.Name().String()), tag.Operation("CompleteNexusOperation"), - tag.String("destination", task.destination), + tag.Destination(task.destination), tag.WorkflowID(n.workflowID), tag.WorkflowRunID(n.runID), tag.AttemptStart(time.Now().UTC()), diff --git a/service/history/circuitbreakerpool/fx.go b/service/history/circuitbreakerpool/fx.go index af21eb664b0..8cc393f2210 100644 --- a/service/history/circuitbreakerpool/fx.go +++ b/service/history/circuitbreakerpool/fx.go @@ -3,7 +3,10 @@ package circuitbreakerpool import ( "fmt" + "github.com/sony/gobreaker" "go.temporal.io/server/common/circuitbreaker" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/tasks" @@ -21,6 +24,7 @@ type OutboundQueueCircuitBreakerPool struct { func OutboundQueueCircuitBreakerPoolProvider( namespaceRegistry namespace.Registry, config *configs.Config, + logger log.SnTaggedLogger, ) *OutboundQueueCircuitBreakerPool { return &OutboundQueueCircuitBreakerPool{ CircuitBreakerPool: NewCircuitBreakerPool( @@ -37,6 +41,7 @@ func OutboundQueueCircuitBreakerPoolProvider( key.NamespaceID, key.Destination, ), + OnStateChange: onStateChange(key, nsName.String(), logger), }) initial, cancel := config.OutboundQueueCircuitBreakerSettings( nsName.String(), @@ -50,3 +55,22 @@ func OutboundQueueCircuitBreakerPoolProvider( ), } } + +// onStateChange logs breaker transitions. +func onStateChange( + key tasks.TaskGroupNamespaceIDAndDestination, + nsName string, + logger log.SnTaggedLogger, +) func(name string, from gobreaker.State, to gobreaker.State) { + return func(_ string, from gobreaker.State, to gobreaker.State) { + logger.Warn( + "outbound queue circuit breaker state change", + tag.WorkflowNamespace(nsName), + tag.WorkflowNamespaceID(key.NamespaceID), + tag.Destination(key.Destination), + tag.NewStringTag("task-group", key.TaskGroup), + tag.NewStringTag("from-state", from.String()), + tag.NewStringTag("to-state", to.String()), + ) + } +} From 71d375e764e2931e2e9b959bc07ad8447b02ca08 Mon Sep 17 00:00:00 2001 From: Prathyush PV Date: Thu, 20 Aug 2026 09:13:51 -0700 Subject: [PATCH 022/139] Return early from CHASM PollComponent when shard moves off host (#10878) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? A blocked `ChasmEngine.pollComponent` now returns a `ShardOwnershipLostError` as soon as its shard moves off this host — by adding a `select` case on the shard's lifecycle context — instead of blocking until the request context deadline. ## Why? Follow-up to #10860 (requested in review): `pollComponent` had the same gap as the `GetWorkflowExecutionHistory` long poll — nothing in its `select` was tied to shard lifecycle, so a poll in flight when its shard moved stalled until timeout. This lets the caller redirect to the new owner immediately. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- service/history/chasm_engine.go | 10 +++++ service/history/chasm_engine_test.go | 65 ++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/service/history/chasm_engine.go b/service/history/chasm_engine.go index c58de35d73f..2b79f838fc2 100644 --- a/service/history/chasm_engine.go +++ b/service/history/chasm_engine.go @@ -728,6 +728,11 @@ func (e *ChasmEngine) pollComponent( monotonicPredicate func(chasm.Context, chasm.Component) (bool, error), ) (retRef []byte, retError error) { + shardContext, err := e.getShardContext(ctx, requestRef) + if err != nil { + return nil, err + } + var ch <-chan struct{} var unsubscribe func() defer func() { @@ -772,6 +777,11 @@ func (e *ChasmEngine) pollComponent( if err != nil || ref != nil { return ref, err } + case <-shardContext.GetLifecycleContext().Done(): + return nil, &persistence.ShardOwnershipLostError{ + ShardID: shardContext.GetShardID(), + Msg: "shard closed", + } case <-ctx.Done(): return nil, ctx.Err() } diff --git a/service/history/chasm_engine_test.go b/service/history/chasm_engine_test.go index 9a5e5b515fb..29b051ee804 100644 --- a/service/history/chasm_engine_test.go +++ b/service/history/chasm_engine_test.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/contextutil" + "go.temporal.io/server/common/convert" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/membership" "go.temporal.io/server/common/metrics" @@ -1537,6 +1538,70 @@ func (s *chasmEngineSuite) testPollComponentWait(useEmptyRunID bool) { s.Equal(activityID, <-newActivityID) } +// TestPollComponent_ShardClosed verifies that a poll blocked waiting for notifications returns a +// ShardOwnershipLost service error as soon as the shard moves off this host, rather than blocking +// until the context deadline. +func (s *chasmEngineSuite) TestPollComponent_ShardClosed() { + tv := testvars.New(s.T()) + tv = tv.WithRunID(tv.Any().RunID()) + + resolvedKey := chasm.ExecutionKey{ + NamespaceID: string(tests.NamespaceID), + BusinessID: tv.WorkflowID(), + RunID: tv.RunID(), + } + pollRef := chasm.NewComponentRef[*testComponent](resolvedKey) + + s.mockExecutionManager.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()). + Return(&persistence.GetWorkflowExecutionResponse{ + State: s.buildPersistenceMutableState( + resolvedKey, + &persistencespb.ActivityInfo{}, + enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + nil), + }, nil). + AnyTimes() + + s.mockShard.Resource.HistoryServiceResolver.EXPECT(). + Lookup(convert.Int32ToString(s.mockShard.GetShardID())). + Return(membership.NewHostInfoFromAddress("owner-host:1234"), nil). + Times(1) + s.mockShard.Resource.HostInfoProvider.EXPECT(). + HostInfo(). + Return(membership.NewHostInfoFromAddress("current-host:5678")). + Times(1) + + pollErr := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // Predicate is never satisfied, so the poll subscribes and blocks. + _, err := s.engine.PollComponent( + ctx, + pollRef, + func(chasm.Context, chasm.Component) (bool, error) { + return false, nil + }, + ) + pollErr <- err + }() + + // Let the poll park in its select, then move the shard off this host. + time.Sleep(100 * time.Millisecond) //nolint:forbidigo + s.mockShard.UnloadForOwnershipLost() + + select { + case err := <-pollErr: + var solErr *serviceerrors.ShardOwnershipLost + s.ErrorAs(err, &solErr) + s.Equal("owner-host:1234", solErr.OwnerHost) + s.Equal("current-host:5678", solErr.CurrentHost) + case <-time.After(5 * time.Second): + s.FailNow("poll did not return after shard close") + } +} + // TestPollComponent_StaleState tests that PollComponent returns a user-friendly Unavailable error // when the submitted component reference is ahead of persisted state (e.g. due to namespace // failover). From e327c061b8997a84de47a74a858b467a128de1fc Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 09:47:39 -0700 Subject: [PATCH 023/139] Update Go lint tooling (#11673) Go 1.27 prerequisite that upgrades golangci-lint, goimports, and stringer to versions compatible with the new toolchain. --- Makefile | 7 +++---- common/testing/testlogger/testlogger.gen.go | 5 +++-- service/matching/loadcause_string_gen.go | 5 +++-- service/matching/unloadcause_string_gen.go | 5 +++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index fb4cd5cac91..23f4697b5ab 100644 --- a/Makefile +++ b/Makefile @@ -172,11 +172,10 @@ $(STAMPDIR): $(LOCALBIN): @mkdir -p $(LOCALBIN) -# When updating the version, update the golangci-lint GHA workflow as well. .PHONY: golangci-lint GOLANGCI_LINT_BASE_REV ?= $(MAIN_BRANCH) GOLANGCI_LINT_FIX ?= true -GOLANGCI_LINT_VERSION := v2.9.0 +GOLANGCI_LINT_VERSION := v2.13.0 GOLANGCI_LINT := $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION) $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/v2/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) @@ -229,7 +228,7 @@ YAMLFMT := $(LOCALBIN)/yamlfmt-$(YAMLFMT_VER) $(YAMLFMT): | $(LOCALBIN) $(call go-install-tool,$(YAMLFMT),github.com/google/yamlfmt/cmd/yamlfmt,$(YAMLFMT_VER)) -GOIMPORTS_VER := v0.36.0 +GOIMPORTS_VER := v0.49.0 GOIMPORTS := $(LOCALBIN)/goimports-$(GOIMPORTS_VER) $(STAMPDIR)/goimports-$(GOIMPORTS_VER): | $(STAMPDIR) $(LOCALBIN) $(call go-install-tool,$(GOIMPORTS),golang.org/x/tools/cmd/goimports,$(GOIMPORTS_VER)) @@ -263,7 +262,7 @@ $(STAMPDIR)/mockgen-$(MOCKGEN_VER): | $(STAMPDIR) $(LOCALBIN) @touch $@ $(MOCKGEN): $(STAMPDIR)/mockgen-$(MOCKGEN_VER) -STRINGER_VER := v0.36.0 +STRINGER_VER := v0.49.0 STRINGER := $(LOCALBIN)/stringer $(STAMPDIR)/stringer-$(STRINGER_VER): | $(STAMPDIR) $(LOCALBIN) $(call go-install-tool,$(STRINGER),golang.org/x/tools/cmd/stringer,$(STRINGER_VER)) diff --git a/common/testing/testlogger/testlogger.gen.go b/common/testing/testlogger/testlogger.gen.go index ac4796c2b6d..9f2cd93b676 100644 --- a/common/testing/testlogger/testlogger.gen.go +++ b/common/testing/testlogger/testlogger.gen.go @@ -22,8 +22,9 @@ const _Level_name = "DebugInfoWarnErrorFatalDPanicPanic" var _Level_index = [...]uint8{0, 5, 9, 13, 18, 23, 29, 34} func (i Level) String() string { - if i < 0 || i >= Level(len(_Level_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_Level_index)-1 { return "Level(" + strconv.FormatInt(int64(i), 10) + ")" } - return _Level_name[_Level_index[i]:_Level_index[i+1]] + return _Level_name[_Level_index[idx]:_Level_index[idx+1]] } diff --git a/service/matching/loadcause_string_gen.go b/service/matching/loadcause_string_gen.go index 1133a162467..4b38e423578 100644 --- a/service/matching/loadcause_string_gen.go +++ b/service/matching/loadcause_string_gen.go @@ -25,8 +25,9 @@ const _loadCause_name = "UnspecifiedTaskQueryDescribeUserDataNexusTaskPollOtherR var _loadCause_index = [...]uint8{0, 11, 15, 20, 28, 36, 45, 49, 58, 68, 73} func (i loadCause) String() string { - if i < 0 || i >= loadCause(len(_loadCause_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_loadCause_index)-1 { return "loadCause(" + strconv.FormatInt(int64(i), 10) + ")" } - return _loadCause_name[_loadCause_index[i]:_loadCause_index[i+1]] + return _loadCause_name[_loadCause_index[idx]:_loadCause_index[idx+1]] } diff --git a/service/matching/unloadcause_string_gen.go b/service/matching/unloadcause_string_gen.go index c7116943df5..c07ba98ec08 100644 --- a/service/matching/unloadcause_string_gen.go +++ b/service/matching/unloadcause_string_gen.go @@ -24,8 +24,9 @@ const _unloadCause_name = "UnspecifiedInitErrorIdleMembershipConflictShuttingDow var _unloadCause_index = [...]uint8{0, 11, 20, 24, 34, 42, 54, 59, 71, 81} func (i unloadCause) String() string { - if i < 0 || i >= unloadCause(len(_unloadCause_index)-1) { + idx := int(i) - 0 + if i < 0 || idx >= len(_unloadCause_index)-1 { return "unloadCause(" + strconv.FormatInt(int64(i), 10) + ")" } - return _unloadCause_name[_unloadCause_index[i]:_unloadCause_index[i+1]] + return _unloadCause_name[_unloadCause_index[idx]:_unloadCause_index[idx+1]] } From 05167abf6701a4d385b2c75948f32b02b4492ad1 Mon Sep 17 00:00:00 2001 From: Quinn Klassen Date: Thu, 20 Aug 2026 10:21:57 -0700 Subject: [PATCH 024/139] Make standalone activity completion callback attachment idempotent after closure (#11628) ## What changed? Makes Standalone Activity conflict updates idempotent by recording the `requestID` when attaching callbacks or links and recognizing duplicate request IDs. I needed to add a dedicated CHASM error for ## Why? This prevents a successful attachment whose response was lost from failing on retry or duplicating/replacing callbacks and links. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- chasm/chasmtest/test_engine.go | 3 +- chasm/chasmtest/test_engine_test.go | 1 + chasm/errors.go | 6 + chasm/lib/activity/handler.go | 3 +- service/history/chasm_engine.go | 3 +- service/history/chasm_engine_test.go | 1 + tests/activity_standalone_test.go | 167 +++++++++++++++++++++++---- 7 files changed, 159 insertions(+), 25 deletions(-) diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index e4a10732fc1..7e36c5bc3e7 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -240,8 +240,7 @@ func (e *Engine) UpdateComponent( options := constructTransitionOptions(opts...) if options.RequestID != "" { if _, ok := execution.requestIDs[options.RequestID]; ok { - return nil, serviceerror.NewFailedPreconditionf( - "request ID %s has already been used for this execution", options.RequestID) + return nil, chasm.ErrRequestIDAlreadyUsed } } diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index ec40d7f1715..4c0d9c3cd98 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -60,6 +60,7 @@ func TestUpdateComponentDeduplicatesRequestID(t *testing.T) { updatedRef, err = update() var failedPrecondition *serviceerror.FailedPrecondition require.ErrorAs(t, err, &failedPrecondition) + require.ErrorIs(t, err, chasm.ErrRequestIDAlreadyUsed) require.Nil(t, updatedRef) require.Equal(t, 1, updateCount) } diff --git a/chasm/errors.go b/chasm/errors.go index f3eace657ba..01647271356 100644 --- a/chasm/errors.go +++ b/chasm/errors.go @@ -1,5 +1,11 @@ package chasm +import "go.temporal.io/api/serviceerror" + +// ErrRequestIDAlreadyUsed is returned by UpdateComponent when the request ID supplied via +// WithRequestID has already been recorded on the execution. +var ErrRequestIDAlreadyUsed = serviceerror.NewFailedPrecondition("request ID has already been used for this execution") + type ExecutionAlreadyStartedError struct { Message string CurrentRequestID string diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index fa06092a173..50c675da471 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -157,8 +157,9 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St return nil, nil }, nil, + chasm.WithRequestID(requestID), ) - if err != nil { + if err != nil && !errors.Is(err, chasm.ErrRequestIDAlreadyUsed) { return nil, err } } diff --git a/service/history/chasm_engine.go b/service/history/chasm_engine.go index 2b79f838fc2..28e3c59d6dc 100644 --- a/service/history/chasm_engine.go +++ b/service/history/chasm_engine.go @@ -566,8 +566,7 @@ func (e *ChasmEngine) updateComponent( // reject it as non-retryable (FailedPrecondition) without running updateFn or persisting. if requestID != "" && executionLease.GetMutableState().HasRequestID(requestID) { executionLease.GetReleaseFn()(nil) - return nil, serviceerror.NewFailedPreconditionf( - "request ID %s has already been used for this execution", requestID) + return nil, chasm.ErrRequestIDAlreadyUsed } defer func() { diff --git a/service/history/chasm_engine_test.go b/service/history/chasm_engine_test.go index 29b051ee804..8213aa5d362 100644 --- a/service/history/chasm_engine_test.go +++ b/service/history/chasm_engine_test.go @@ -1059,6 +1059,7 @@ func (s *chasmEngineSuite) TestUpdateComponent_RequestIDIdempotency() { dupRef, err := update() var failedPrecondition *serviceerror.FailedPrecondition s.ErrorAs(err, &failedPrecondition) + s.ErrorIs(err, chasm.ErrRequestIDAlreadyUsed) s.Nil(dupRef) s.Equal(1, updateCount, "updateFn must not run again for a duplicate request ID") } diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 906505ea4cf..de4663bc684 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -282,6 +282,23 @@ func (s *standaloneActivityTestSuite) TestIDConflictPolicy() { AttachCompletionCallbacks: true, AttachLinks: true, } + workflowEventLink := func(workflowID string) *commonpb.Link { + return &commonpb.Link{ + Variant: &commonpb.Link_WorkflowEvent_{ + WorkflowEvent: &commonpb.Link_WorkflowEvent{ + Namespace: env.Namespace().String(), + WorkflowId: workflowID, + RunId: "run-id", + Reference: &commonpb.Link_WorkflowEvent_EventRef{ + EventRef: &commonpb.Link_WorkflowEvent_EventReference{ + EventId: 1, + EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + }, + }, + }, + }, + } + } t.Run("AttachesToNewActivity", func(t *testing.T) { newActivityID := testcore.RandomizeStr(t.Name()) @@ -354,23 +371,7 @@ func (s *standaloneActivityTestSuite) TestIDConflictPolicy() { bothTaskQueue := testcore.RandomizeStr(t.Name()) bothStartResp := env.startAndValidateActivity(ctx, t, bothActivityID, bothTaskQueue) - attachedLinks := []*commonpb.Link{ - { - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: env.Namespace().String(), - WorkflowId: "both-wf", - RunId: "both-run", - Reference: &commonpb.Link_WorkflowEvent_EventRef{ - EventRef: &commonpb.Link_WorkflowEvent_EventReference{ - EventId: 1, - EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, - }, - }, - }, - }, - }, - } + attachedLinks := []*commonpb.Link{workflowEventLink("both-wf")} resp, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), @@ -411,6 +412,7 @@ func (s *standaloneActivityTestSuite) TestIDConflictPolicy() { idempotentStartResp := env.startAndValidateActivity(s.Context(), t, idempotentActivityID, idempotentTaskQueue) requestID := env.Tv().Any().String() + originalLinks := []*commonpb.Link{workflowEventLink("idempotent-original")} startReq := &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: idempotentActivityID, @@ -426,15 +428,20 @@ func (s *standaloneActivityTestSuite) TestIDConflictPolicy() { CompletionCallbacks: []*commonpb.Callback{ {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/idempotent-cb"}}}, }, + Links: originalLinks, OnConflictOptions: onConflictOpts, } - // First call attaches the callback. + // First call attaches both values. resp1, err := env.FrontendClient().StartActivityExecution(s.Context(), startReq) require.NoError(t, err) require.False(t, resp1.GetStarted()) - // Second call with the same request ID should not duplicate the callback. + // A retry with the same request ID is ignored even if its payload changed. + startReq.CompletionCallbacks = []*commonpb.Callback{ + {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/retry-cb"}}}, + } + startReq.Links = []*commonpb.Link{workflowEventLink("idempotent-retry")} resp2, err := env.FrontendClient().StartActivityExecution(s.Context(), startReq) require.NoError(t, err) require.False(t, resp2.GetStarted()) @@ -445,9 +452,129 @@ func (s *standaloneActivityTestSuite) TestIDConflictPolicy() { RunId: idempotentStartResp.RunId, }) require.NoError(t, err) - // Only 1 callback: the second call with the same request ID should not add another. require.Len(t, descResp.Callbacks, 1) require.Equal(t, "http://localhost/idempotent-cb", descResp.Callbacks[0].GetInfo().GetCallback().GetNexus().GetUrl()) + protorequire.ProtoSliceEqual(t, originalLinks, descResp.GetInfo().GetLinks()) + }) + + t.Run("IdempotentWithSameRequestIdAfterClose", func(t *testing.T) { + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + requestID := env.Tv().Any().String() + ch, callbackAddress := newNexusCompletionHandler(t) + startReq := &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: env.Tv().WorkerIdentity(), + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + StartToCloseTimeout: durationpb.New(time.Minute), + IdConflictPolicy: enumspb.ACTIVITY_ID_CONFLICT_POLICY_USE_EXISTING, + RequestId: requestID, + CompletionCallbacks: []*commonpb.Callback{{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: callbackAddress}}, + }}, + OnConflictOptions: onConflictOpts, + } + + startResp, err := env.FrontendClient().StartActivityExecution(ctx, startReq) + require.NoError(t, err) + require.True(t, startResp.GetStarted()) + + pollResp := env.pollActivityTaskAndValidate(ctx, t, activityID, taskQueue, startResp.GetRunId()) + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: pollResp.GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + select { + case <-ch.requestCh: + case <-ctx.Done(): + require.Fail(t, "timed out waiting for completion callback") + } + + retryResp, err := env.FrontendClient().StartActivityExecution(ctx, startReq) + require.NoError(t, err) + require.False(t, retryResp.GetStarted()) + require.Equal(t, startResp.GetRunId(), retryResp.GetRunId()) + ch.requestCompleteCh <- nil + + descResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: startResp.GetRunId(), + }) + require.NoError(t, err) + require.Len(t, descResp.GetCallbacks(), 1) + }) + + t.Run("ConflictUpdateIdempotentAfterClose", func(t *testing.T) { + activityID := testcore.RandomizeStr(t.Name()) + taskQueue := testcore.RandomizeStr(t.Name()) + startResp := env.startAndValidateActivity(ctx, t, activityID, taskQueue) + requestID := env.Tv().Any().String() + ch, callbackAddress := newNexusCompletionHandler(t) + originalLinks := []*commonpb.Link{workflowEventLink("after-close-original")} + conflictReq := &workflowservice.StartActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + ActivityType: env.Tv().ActivityType(), + Identity: env.Tv().WorkerIdentity(), + Input: defaultInput, + TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue}, + StartToCloseTimeout: durationpb.New(time.Minute), + IdConflictPolicy: enumspb.ACTIVITY_ID_CONFLICT_POLICY_USE_EXISTING, + RequestId: requestID, + CompletionCallbacks: []*commonpb.Callback{{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: callbackAddress}}, + }}, + Links: originalLinks, + OnConflictOptions: onConflictOpts, + } + + conflictResp, err := env.FrontendClient().StartActivityExecution(ctx, conflictReq) + require.NoError(t, err) + require.False(t, conflictResp.GetStarted()) + require.Equal(t, startResp.GetRunId(), conflictResp.GetRunId()) + + pollResp := env.pollActivityTaskAndValidate(ctx, t, activityID, taskQueue, startResp.GetRunId()) + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: pollResp.GetTaskToken(), + Result: defaultResult, + Identity: defaultIdentity, + }) + require.NoError(t, err) + + select { + case <-ch.requestCh: + case <-ctx.Done(): + require.Fail(t, "timed out waiting for completion callback") + } + + conflictReq.CompletionCallbacks = []*commonpb.Callback{{ + Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/retry-cb"}}, + }} + conflictReq.Links = []*commonpb.Link{workflowEventLink("after-close-retry")} + retryResp, err := env.FrontendClient().StartActivityExecution(ctx, conflictReq) + require.NoError(t, err) + require.False(t, retryResp.GetStarted()) + require.Equal(t, startResp.GetRunId(), retryResp.GetRunId()) + ch.requestCompleteCh <- nil + + descResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ + Namespace: env.Namespace().String(), + ActivityId: activityID, + RunId: startResp.GetRunId(), + }) + require.NoError(t, err) + require.Len(t, descResp.GetCallbacks(), 1) + require.Equal(t, callbackAddress, descResp.GetCallbacks()[0].GetInfo().GetCallback().GetNexus().GetUrl()) + protorequire.ProtoSliceEqual(t, originalLinks, descResp.GetInfo().GetLinks()) }) }) From 430968b08a60bccaf9cbcece472e182226c81260 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 10:47:13 -0700 Subject: [PATCH 025/139] Use typed atomic values (#11675) Go 1.27 prerequisite that applies the `atomictypes` Go fixer to use typed atomic values. --- common/collection/concurrent_tx_map.go | 14 ++--- .../persistence_test_base.go | 4 +- .../tasks/execution_queue_scheduler_test.go | 14 ++--- service/history/shard/controller_impl.go | 18 ++----- service/matching/task_gc.go | 6 +-- service/worker/batcher/activities_test.go | 18 +++---- service/worker/pernamespaceworker.go | 16 ++---- .../worker/scanner/executions/scavenger.go | 16 ++---- service/worker/scanner/executor/executor.go | 20 +++---- service/worker/scanner/taskqueue/scavenger.go | 8 +-- service/worker/worker.go | 14 ++--- tests/ndc/replication_migration_back_test.go | 4 +- tests/ndc/replication_task_batching_test.go | 4 +- tests/query_workflow_test.go | 6 +-- tests/schedule_test.go | 52 +++++++++---------- tests/xdc/stream_based_replication_test.go | 4 +- 16 files changed, 93 insertions(+), 125 deletions(-) diff --git a/common/collection/concurrent_tx_map.go b/common/collection/concurrent_tx_map.go index 2cc7324f8a2..25e31ad864a 100644 --- a/common/collection/concurrent_tx_map.go +++ b/common/collection/concurrent_tx_map.go @@ -21,7 +21,7 @@ type ( ShardedConcurrentTxMap struct { shards [nShards]mapShard hashfn HashFunc - size int32 + size atomic.Int32 initialCap int } @@ -87,7 +87,7 @@ func (cmap *ShardedConcurrentTxMap) Put(key any, value any) { cmap.lazyInitShard(shard) _, ok := shard.items[key] if !ok { - atomic.AddInt32(&cmap.size, 1) + cmap.size.Add(1) } shard.items[key] = value shard.Unlock() @@ -103,7 +103,7 @@ func (cmap *ShardedConcurrentTxMap) PutIfNotExist(key any, value any) bool { _, ok = shard.items[key] if !ok { shard.items[key] = value - atomic.AddInt32(&cmap.size, 1) + cmap.size.Add(1) } shard.Unlock() return !ok @@ -117,7 +117,7 @@ func (cmap *ShardedConcurrentTxMap) Remove(key any) { _, ok := shard.items[key] if ok { delete(shard.items, key) - atomic.AddInt32(&cmap.size, -1) + cmap.size.Add(-1) } shard.Unlock() } @@ -151,7 +151,7 @@ func (cmap *ShardedConcurrentTxMap) PutOrDo(key any, value any, fn ActionFunc) ( if !ok { shard.items[key] = value v = value - atomic.AddInt32(&cmap.size, 1) + cmap.size.Add(1) } else { err = fn(key, v) } @@ -169,7 +169,7 @@ func (cmap *ShardedConcurrentTxMap) RemoveIf(key any, fn PredicateFunc) bool { if ok && fn(key, value) { removed = true delete(shard.items, key) - atomic.AddInt32(&cmap.size, -1) + cmap.size.Add(-1) } } shard.Unlock() @@ -218,7 +218,7 @@ func (cmap *ShardedConcurrentTxMap) Iter() MapIterator { // Len returns the number of items in the map func (cmap *ShardedConcurrentTxMap) Len() int { - return int(atomic.LoadInt32(&cmap.size)) + return int(cmap.size.Load()) } func (cmap *ShardedConcurrentTxMap) getShard(key any) *mapShard { diff --git a/common/persistence/persistence-tests/persistence_test_base.go b/common/persistence/persistence-tests/persistence_test_base.go index db3eb947686..d43786cc3c7 100644 --- a/common/persistence/persistence-tests/persistence_test_base.go +++ b/common/persistence/persistence-tests/persistence_test_base.go @@ -116,7 +116,7 @@ type ( // TestTransferTaskIDGenerator helper TestTransferTaskIDGenerator struct { - seqNum int64 + seqNum atomic.Int64 } ) @@ -365,7 +365,7 @@ func (s *TestBase) EqualTimes(t1, t2 time.Time) { // GenerateTransferTaskID helper func (g *TestTransferTaskIDGenerator) GenerateTransferTaskID() (int64, error) { - return atomic.AddInt64(&g.seqNum, 1), nil + return g.seqNum.Add(1), nil } // Publish is a utility method to add messages to the queue diff --git a/common/tasks/execution_queue_scheduler_test.go b/common/tasks/execution_queue_scheduler_test.go index dd9606b8b8e..2a5d1e15f0b 100644 --- a/common/tasks/execution_queue_scheduler_test.go +++ b/common/tasks/execution_queue_scheduler_test.go @@ -380,7 +380,7 @@ func (s *executionQueueSchedulerSuite) TestTTLExpiryRace_NoTaskOrphaning() { defer scheduler.Stop() const iterations = 50 - var processedCount int32 + var processedCount atomic.Int32 testWG := sync.WaitGroup{} for range iterations { @@ -390,7 +390,7 @@ func (s *executionQueueSchedulerSuite) TestTTLExpiryRace_NoTaskOrphaning() { mockTask.EXPECT().RetryPolicy().Return(s.retryPolicy).AnyTimes() mockTask.EXPECT().Execute().Return(nil).Times(1) mockTask.EXPECT().Ack().Do(func() { - atomic.AddInt32(&processedCount, 1) + processedCount.Add(1) testWG.Done() }).Times(1) @@ -416,10 +416,10 @@ func (s *executionQueueSchedulerSuite) TestTTLExpiryRace_NoTaskOrphaning() { // Success case <-time.After(30 * time.Second): s.Fail("Timed out waiting for tasks — possible task orphaning", - "processed %d/%d tasks", atomic.LoadInt32(&processedCount), iterations) + "processed %d/%d tasks", processedCount.Load(), iterations) } - s.Equal(int32(iterations), atomic.LoadInt32(&processedCount), + s.Equal(int32(iterations), processedCount.Load(), "All tasks must be processed, none orphaned") } @@ -523,7 +523,7 @@ func (s *executionQueueSchedulerSuite) TestParallelTrySubmit_SameWorkflow() { numSubmitters := 50 tasksPerSubmitter := 20 - var executionCount int32 + var executionCount atomic.Int32 testWG := sync.WaitGroup{} testWG.Add(numSubmitters * tasksPerSubmitter) @@ -539,7 +539,7 @@ func (s *executionQueueSchedulerSuite) TestParallelTrySubmit_SameWorkflow() { mockTask := NewMockTask(s.controller) mockTask.EXPECT().RetryPolicy().Return(s.retryPolicy).AnyTimes() mockTask.EXPECT().Execute().DoAndReturn(func() error { - atomic.AddInt32(&executionCount, 1) + executionCount.Add(1) return nil }).Times(1) mockTask.EXPECT().Ack().Do(func() { testWG.Done() }).Times(1) @@ -563,7 +563,7 @@ func (s *executionQueueSchedulerSuite) TestParallelTrySubmit_SameWorkflow() { s.Fail("Timed out waiting for parallel tasks to same workflow") } - s.Equal(int32(numSubmitters*tasksPerSubmitter), atomic.LoadInt32(&executionCount)) + s.Equal(int32(numSubmitters*tasksPerSubmitter), executionCount.Load()) } // ============================================================================= diff --git a/service/history/shard/controller_impl.go b/service/history/shard/controller_impl.go index 3314a3529f2..e21f67b7eb9 100644 --- a/service/history/shard/controller_impl.go +++ b/service/history/shard/controller_impl.go @@ -50,7 +50,7 @@ type ( contextTaggedLogger log.Logger hostInfoProvider membership.HostInfoProvider ownership *ownership - status int32 + status atomic.Int32 taggedMetricsHandler metrics.Handler // shardCountSubscriptions is a set of subscriptions that receive shard count updates whenever the set of // shards that this controller owns changes. @@ -103,11 +103,7 @@ func ControllerProvider( } func (c *ControllerImpl) Start() { - if !atomic.CompareAndSwapInt32( - &c.status, - common.DaemonStatusInitialized, - common.DaemonStatusStarted, - ) { + if !c.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } @@ -117,11 +113,7 @@ func (c *ControllerImpl) Start() { } func (c *ControllerImpl) Stop() { - if !atomic.CompareAndSwapInt32( - &c.status, - common.DaemonStatusStarted, - common.DaemonStatusStopped, - ) { + if !c.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } @@ -153,7 +145,7 @@ func (c *ControllerImpl) GetPingChecks() []pingable.Check { } func (c *ControllerImpl) Status() int32 { - return atomic.LoadInt32(&c.status) + return c.status.Load() } func (c *ControllerImpl) InitialShardsAcquired(ctx context.Context) error { @@ -259,7 +251,7 @@ func (c *ControllerImpl) getOrCreateShardContext(shardID int32) (historyi.Contro return nil, err } - if atomic.LoadInt32(&c.status) == common.DaemonStatusStopped { + if c.status.Load() == common.DaemonStatusStopped { hostInfo := c.hostInfoProvider.HostInfo() return nil, fmt.Errorf("ControllerImpl for host '%v' shutting down", hostInfo.Identity()) } diff --git a/service/matching/task_gc.go b/service/matching/task_gc.go index 386af257757..7c6e05366d9 100644 --- a/service/matching/task_gc.go +++ b/service/matching/task_gc.go @@ -13,7 +13,7 @@ type taskGC struct { db *taskQueueDB config *taskQueueConfig - lock int64 + lock atomic.Bool ackLevel int64 lastDeleteTime time.Time } @@ -93,9 +93,9 @@ func (tgc *taskGC) checkPrecond(ackLevel int64, batchSize int, ignoreTimeCond bo } func (tgc *taskGC) tryLock() bool { - return atomic.CompareAndSwapInt64(&tgc.lock, 0, 1) + return tgc.lock.CompareAndSwap(false, true) } func (tgc *taskGC) unlock() { - atomic.StoreInt64(&tgc.lock, 0) + tgc.lock.Store(false) } diff --git a/service/worker/batcher/activities_test.go b/service/worker/batcher/activities_test.go index 0365f66f99d..ddfb60311e5 100644 --- a/service/worker/batcher/activities_test.go +++ b/service/worker/batcher/activities_test.go @@ -1071,7 +1071,7 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll } // Fake worker pool: drain taskCh and report success for every real task. - var processed int64 + var processed atomic.Int64 fakeWorker := func( ctx context.Context, taskCh chan task, @@ -1090,7 +1090,7 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll if t.executionInfo == nil { continue } - atomic.AddInt64(&processed, 1) + processed.Add(1) select { case respCh <- taskResponse{err: nil, page: t.page}: case <-ctx.Done(): @@ -1123,7 +1123,7 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll s.NoError(encoded.Get(&hbd)) s.Equal(total, hbd.SuccessCount) s.Equal(0, hbd.ErrorCount) - s.Equal(int64(total), atomic.LoadInt64(&processed)) + s.Equal(int64(total), processed.Load()) mockSdk.AssertExpectations(s.T()) } @@ -1162,8 +1162,8 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll mockSdk := &mocks.Client{} mockSdk.On("WorkflowService").Return(s.mockFrontendClient) - var processed int64 - var invalidTargets int64 + var processed atomic.Int64 + var invalidTargets atomic.Int64 fakeWorker := func( ctx context.Context, taskCh chan task, @@ -1182,9 +1182,9 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll if task.targetExecution == nil || task.targetExecution.GetType() != enumspb.EXECUTION_TYPE_ACTIVITY || task.targetExecution.GetBusinessId() == "" || task.targetExecution.GetRunId() == "" { - atomic.AddInt64(&invalidTargets, 1) + invalidTargets.Add(1) } - atomic.AddInt64(&processed, 1) + processed.Add(1) select { case respCh <- taskResponse{page: task.page}: case <-ctx.Done(): @@ -1218,8 +1218,8 @@ func (s *activitiesSuite) TestProcessWorkflowsWithProactiveFetching_ProcessesAll s.Require().NoError(encoded.Get(&hbd)) s.Equal(total, hbd.SuccessCount) s.Equal(0, hbd.ErrorCount) - s.Equal(int64(total), atomic.LoadInt64(&processed)) - s.Zero(atomic.LoadInt64(&invalidTargets)) + s.Equal(int64(total), processed.Load()) + s.Zero(invalidTargets.Load()) mockSdk.AssertExpectations(s.T()) } diff --git a/service/worker/pernamespaceworker.go b/service/worker/pernamespaceworker.go index d40d5f49092..ad9990cda60 100644 --- a/service/worker/pernamespaceworker.go +++ b/service/worker/pernamespaceworker.go @@ -42,7 +42,7 @@ const ( type ( PerNamespaceWorkerManager struct { - status int32 + status atomic.Int32 // from init params or Start logger log.Logger @@ -129,18 +129,14 @@ func NewPerNamespaceWorkerManager( } func (wm *PerNamespaceWorkerManager) Running() bool { - return atomic.LoadInt32(&wm.status) == common.DaemonStatusStarted + return wm.status.Load() == common.DaemonStatusStarted } func (wm *PerNamespaceWorkerManager) Start( self membership.HostInfo, serviceResolver membership.ServiceResolver, ) { - if !atomic.CompareAndSwapInt32( - &wm.status, - common.DaemonStatusInitialized, - common.DaemonStatusStarted, - ) { + if !wm.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } @@ -163,11 +159,7 @@ func (wm *PerNamespaceWorkerManager) Start( } func (wm *PerNamespaceWorkerManager) Stop() { - if !atomic.CompareAndSwapInt32( - &wm.status, - common.DaemonStatusStarted, - common.DaemonStatusStopped, - ) { + if !wm.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } diff --git a/service/worker/scanner/executions/scavenger.go b/service/worker/scanner/executions/scavenger.go index 73c0103a522..b542b68898b 100644 --- a/service/worker/scanner/executions/scavenger.go +++ b/service/worker/scanner/executions/scavenger.go @@ -27,7 +27,7 @@ const ( type ( // Scavenger is the type that holds the state for executions scavenger daemon Scavenger struct { - status int32 + status atomic.Int32 numHistoryShards int32 activityContext context.Context @@ -101,11 +101,7 @@ func NewScavenger( // Start starts the scavenger func (s *Scavenger) Start() { - if !atomic.CompareAndSwapInt32( - &s.status, - common.DaemonStatusInitialized, - common.DaemonStatusStarted, - ) { + if !s.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } s.logger.Info("Executions scavenger starting") @@ -118,11 +114,7 @@ func (s *Scavenger) Start() { // Stop stops the scavenger func (s *Scavenger) Stop() { - if !atomic.CompareAndSwapInt32( - &s.status, - common.DaemonStatusStarted, - common.DaemonStatusStopped, - ) { + if !s.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } metrics.StoppedCount.With(s.metricsHandler).Record(1) @@ -135,7 +127,7 @@ func (s *Scavenger) Stop() { // Alive returns true if the scavenger is still running func (s *Scavenger) Alive() bool { - return atomic.LoadInt32(&s.status) == common.DaemonStatusStarted + return s.status.Load() == common.DaemonStatusStarted } // run does a single run over all executions and validates them diff --git a/service/worker/scanner/executor/executor.go b/service/worker/scanner/executor/executor.go index 2c5e5204ae9..6e80c34af3a 100644 --- a/service/worker/scanner/executor/executor.go +++ b/service/worker/scanner/executor/executor.go @@ -36,8 +36,8 @@ type ( size int maxDeferred int runQ *runQueue - outstanding int64 - status int32 + outstanding atomic.Int64 + status atomic.Int32 metricsHandler metrics.Handler stopC chan struct{} stopWG sync.WaitGroup @@ -74,7 +74,7 @@ func NewFixedSizePoolExecutor(size int, maxDeferred int, metricsHandler metrics. // Start starts the executor func (e *fixedPoolExecutor) Start() { - if !atomic.CompareAndSwapInt32(&e.status, common.DaemonStatusInitialized, common.DaemonStatusStarted) { + if !e.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } for i := 0; i < e.size; i++ { @@ -85,7 +85,7 @@ func (e *fixedPoolExecutor) Start() { // Stop stops the executor func (e *fixedPoolExecutor) Stop() { - if !atomic.CompareAndSwapInt32(&e.status, common.DaemonStatusStarted, common.DaemonStatusStopped) { + if !e.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } close(e.stopC) @@ -99,14 +99,14 @@ func (e *fixedPoolExecutor) Submit(task Task) bool { } added := e.runQ.add(task) if added { - atomic.AddInt64(&e.outstanding, 1) + e.outstanding.Add(1) } return added } // TaskCount returns the total of number of tasks currently outstanding func (e *fixedPoolExecutor) TaskCount() int64 { - return atomic.LoadInt64(&e.outstanding) + return e.outstanding.Load() } func (e *fixedPoolExecutor) worker() { @@ -120,18 +120,18 @@ func (e *fixedPoolExecutor) worker() { status := task.Run() switch status { case TaskStatusDone: - atomic.AddInt64(&e.outstanding, -1) + e.outstanding.Add(-1) metrics.ExecutorTasksDoneCount.With(e.metricsHandler).Record(1) case TaskStatusDefer: if e.runQ.deferredCount() < e.maxDeferred { e.runQ.addAndDefer(task) metrics.ExecutorTasksDeferredCount.With(e.metricsHandler).Record(1) } else { - atomic.AddInt64(&e.outstanding, -1) + e.outstanding.Add(-1) metrics.ExecutorTasksDroppedCount.With(e.metricsHandler).Record(1) } case TaskStatusErr: - atomic.AddInt64(&e.outstanding, -1) + e.outstanding.Add(-1) metrics.ExecutorTasksErrCount.With(e.metricsHandler).Record(1) default: panic(fmt.Sprintf("unknown task status: %v", status)) @@ -140,5 +140,5 @@ func (e *fixedPoolExecutor) worker() { } func (e *fixedPoolExecutor) alive() bool { - return atomic.LoadInt32(&e.status) == common.DaemonStatusStarted + return e.status.Load() == common.DaemonStatusStarted } diff --git a/service/worker/scanner/taskqueue/scavenger.go b/service/worker/scanner/taskqueue/scavenger.go index 324122624ed..1857976a984 100644 --- a/service/worker/scanner/taskqueue/scavenger.go +++ b/service/worker/scanner/taskqueue/scavenger.go @@ -23,7 +23,7 @@ type ( metricsHandler metrics.Handler logger log.Logger stats stats - status int32 + status atomic.Int32 stopC chan struct{} stopWG sync.WaitGroup @@ -100,7 +100,7 @@ func NewScavenger(db p.TaskManager, metricsHandler metrics.Handler, logger log.L // Start starts the scavenger func (s *Scavenger) Start() { - if !atomic.CompareAndSwapInt32(&s.status, common.DaemonStatusInitialized, common.DaemonStatusStarted) { + if !s.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } s.logger.Info("Taskqueue scavenger starting") @@ -113,7 +113,7 @@ func (s *Scavenger) Start() { // Stop stops the scavenger func (s *Scavenger) Stop() { - if !atomic.CompareAndSwapInt32(&s.status, common.DaemonStatusStarted, common.DaemonStatusStopped) { + if !s.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } metrics.StoppedCount.With(s.metricsHandler).Record(1) @@ -127,7 +127,7 @@ func (s *Scavenger) Stop() { // Alive returns true if the scavenger is still running func (s *Scavenger) Alive() bool { - return atomic.LoadInt32(&s.status) == common.DaemonStatusStarted + return s.status.Load() == common.DaemonStatusStarted } // run does a single run over all executorTask queues diff --git a/service/worker/worker.go b/service/worker/worker.go index 9a29fb15680..524b161113f 100644 --- a/service/worker/worker.go +++ b/service/worker/worker.go @@ -18,7 +18,7 @@ import ( type ( // workerManager maintains list of SDK workers. workerManager struct { - status int32 + status atomic.Int32 hostInfo membership.HostInfo logger log.Logger sdkClientFactory sdk.ClientFactory @@ -44,11 +44,7 @@ func NewWorkerManager( } func (wm *workerManager) Start() { - if !atomic.CompareAndSwapInt32( - &wm.status, - common.DaemonStatusInitialized, - common.DaemonStatusStarted, - ) { + if !wm.status.CompareAndSwap(common.DaemonStatusInitialized, common.DaemonStatusStarted) { return } @@ -101,11 +97,7 @@ func (wm *workerManager) Start() { } func (wm *workerManager) Stop() { - if !atomic.CompareAndSwapInt32( - &wm.status, - common.DaemonStatusStarted, - common.DaemonStatusStopped, - ) { + if !wm.status.CompareAndSwap(common.DaemonStatusStarted, common.DaemonStatusStopped) { return } diff --git a/tests/ndc/replication_migration_back_test.go b/tests/ndc/replication_migration_back_test.go index 47f9ecf01cc..9030f15f333 100644 --- a/tests/ndc/replication_migration_back_test.go +++ b/tests/ndc/replication_migration_back_test.go @@ -49,7 +49,7 @@ type ( mockAdminClient map[string]adminservice.AdminServiceClient namespace namespace.Name namespaceID namespace.ID - standByTaskID int64 + standByTaskID atomic.Int64 autoIncrementTaskID int64 passiveClusterName string @@ -559,7 +559,7 @@ func (s *ReplicationMigrationBackTestSuite) registerNamespace() { func (s *ReplicationMigrationBackTestSuite) GetReplicationMessagesMock() (*adminservice.StreamWorkflowReplicationMessagesResponse, error) { task := <-s.standByReplicationTasksChan - taskID := atomic.AddInt64(&s.standByTaskID, 1) + taskID := s.standByTaskID.Add(1) task.SourceTaskId = taskID tasks := []*replicationspb.ReplicationTask{task} diff --git a/tests/ndc/replication_task_batching_test.go b/tests/ndc/replication_task_batching_test.go index a4d83a34927..6b0fb4ed7e6 100644 --- a/tests/ndc/replication_task_batching_test.go +++ b/tests/ndc/replication_task_batching_test.go @@ -45,7 +45,7 @@ type ( mockAdminClient map[string]adminservice.AdminServiceClient namespace namespace.Name namespaceID namespace.ID - standByTaskID int64 + standByTaskID atomic.Int64 autoIncrementTaskID int64 passiveClusterName string @@ -222,7 +222,7 @@ func (s *NDCReplicationTaskBatchingTestSuite) registerNamespace() { func (s *NDCReplicationTaskBatchingTestSuite) GetReplicationMessagesMock() (*adminservice.StreamWorkflowReplicationMessagesResponse, error) { task := <-s.standByReplicationTasksChan - taskID := atomic.AddInt64(&s.standByTaskID, 1) + taskID := s.standByTaskID.Add(1) task.SourceTaskId = taskID tasks := []*replicationspb.ReplicationTask{task} diff --git a/tests/query_workflow_test.go b/tests/query_workflow_test.go index 5658029902c..e9c30cb257a 100644 --- a/tests/query_workflow_test.go +++ b/tests/query_workflow_test.go @@ -272,7 +272,7 @@ func (s *QueryWorkflowSuite) TestQueryWorkflow_QueryBeforeStart() { func (s *QueryWorkflowSuite) TestQueryWorkflow_QueryFailedWorkflowTask() { env := testcore.NewEnv(s.T()) testname := s.T().Name() - var failures int32 + var failures atomic.Int32 workflowFn := func(ctx workflow.Context) (string, error) { err := workflow.SetQueryHandler(ctx, testname, func() (string, error) { return "", nil @@ -281,7 +281,7 @@ func (s *QueryWorkflowSuite) TestQueryWorkflow_QueryFailedWorkflowTask() { if err != nil { s.T().Fatalf("SetQueryHandler failed: %s", err.Error()) } - atomic.AddInt32(&failures, 1) + failures.Add(1) // force workflow task to fail panic("Workflow failed") } @@ -305,7 +305,7 @@ func (s *QueryWorkflowSuite) TestQueryWorkflow_QueryFailedWorkflowTask() { s.AwaitTrue(func() bool { // wait for workflow task to fail 3 times - return atomic.LoadInt32(&failures) >= 3 + return failures.Load() >= 3 }, 10*time.Second, 50*time.Millisecond) _, err = env.SdkClient().QueryWorkflow(ctx, id, "", testname) diff --git a/tests/schedule_test.go b/tests/schedule_test.go index ea254106ef4..0018ca1cce9 100644 --- a/tests/schedule_test.go +++ b/tests/schedule_test.go @@ -1379,12 +1379,12 @@ func testInput(t *testing.T, newContext contextFactory) { RequestId: uuid.NewString(), } - var runs int32 + var runs atomic.Int32 workflowFn := func(ctx workflow.Context, arg1 *myData, arg2 map[int]float64) error { workflow.SideEffect(ctx, func(ctx workflow.Context) any { s.Equal(*input1, *arg1) s.Equal(input2, arg2) - atomic.AddInt32(&runs, 1) + runs.Add(1) return 0 }) return nil @@ -1395,7 +1395,7 @@ func testInput(t *testing.T, newContext contextFactory) { _, err = s.FrontendClient().CreateSchedule(ctx, req) s.NoError(err) - s.Eventually(func() bool { return atomic.LoadInt32(&runs) == 1 }, 8*time.Second, 200*time.Millisecond) + s.Eventually(func() bool { return runs.Load() == 1 }, 8*time.Second, 200*time.Millisecond) } func testLastCompletionAndError(t *testing.T, newContext contextFactory) { @@ -1430,7 +1430,7 @@ func testLastCompletionAndError(t *testing.T, newContext contextFactory) { } runs := make(map[string]struct{}) - var testComplete int32 + var testComplete atomic.Int32 workflowFn := func(ctx workflow.Context) (string, error) { var num int @@ -1458,7 +1458,7 @@ func testLastCompletionAndError(t *testing.T, newContext contextFactory) { case 3: s.Equal("this one succeeds", lcr) s.ErrorContains(lastErr, "this one fails") - atomic.StoreInt32(&testComplete, 1) + testComplete.Store(1) return "done", nil default: panic("shouldn't be running anymore") @@ -1470,7 +1470,7 @@ func testLastCompletionAndError(t *testing.T, newContext contextFactory) { _, err := s.FrontendClient().CreateSchedule(ctx, req) s.NoError(err) - s.Eventually(func() bool { return atomic.LoadInt32(&testComplete) == 1 }, 20*time.Second, 200*time.Millisecond) + s.Eventually(func() bool { return testComplete.Load() == 1 }, 20*time.Second, 200*time.Millisecond) } // testScheduleContinuesAfterWorkflowRetryFailure verifies a schedule keeps firing actions @@ -1487,10 +1487,10 @@ func testScheduleContinuesAfterWorkflowRetryFailure(t *testing.T, newContext con wid := testcore.RandomizeStr("sched-retry-fail-wf") wt := testcore.RandomizeStr("sched-retry-fail-wt") - var sawRetry int32 + var sawRetry atomic.Int32 workflowFn := func(ctx workflow.Context) error { if workflow.GetInfo(ctx).Attempt > 1 { - atomic.StoreInt32(&sawRetry, 1) + sawRetry.Store(1) } return errors.New("intentional failure to force a retry") } @@ -1546,11 +1546,11 @@ func testScheduleContinuesAfterWorkflowRetryFailure(t *testing.T, newContext con failedActions++ } } - return atomic.LoadInt32(&sawRetry) == 1 && failedActions >= 2 + return sawRetry.Load() == 1 && failedActions >= 2 }, 30*time.Second, 500*time.Millisecond, "schedule should keep recording FAILED actions after the workflow retry-fails") - s.Equal(int32(1), atomic.LoadInt32(&sawRetry), "scheduled workflow should have retried (attempt > 1)") + s.Equal(int32(1), sawRetry.Load(), "scheduled workflow should have retried (attempt > 1)") s.GreaterOrEqual(failedActions, 2, "schedule should record multiple retry-failed actions") s.GreaterOrEqual(lastDescribe.GetInfo().GetActionCount(), int64(2)) s.False(lastDescribe.GetSchedule().GetState().GetPaused(), "a retry-failed workflow must not pause the schedule") @@ -1830,10 +1830,10 @@ func testUpdateIntervalTakesEffect(t *testing.T, newContext contextFactory) { wid := "sched-test-update-interval-wf" wt := "sched-test-update-interval-wt" - var runs int32 + var runs atomic.Int32 workflowFn := func(ctx workflow.Context) error { workflow.SideEffect(ctx, func(ctx workflow.Context) any { - atomic.AddInt32(&runs, 1) + runs.Add(1) return 0 }) return nil @@ -1881,7 +1881,7 @@ func testUpdateIntervalTakesEffect(t *testing.T, newContext contextFactory) { // After updating to 1s interval, we should see runs start within a few seconds. s.Eventually( - func() bool { return atomic.LoadInt32(&runs) >= 2 }, + func() bool { return runs.Load() >= 2 }, 10*time.Second, 500*time.Millisecond, "expected at least 2 runs within 10s after updating interval to 1s", @@ -3496,10 +3496,10 @@ func testRefresh(t *testing.T, newContext contextFactory) { RequestId: uuid.NewString(), } - var runs int32 + var runs atomic.Int32 workflowFn := func(ctx workflow.Context) error { workflow.SideEffect(ctx, func(ctx workflow.Context) any { - atomic.AddInt32(&runs, 1) + runs.Add(1) return 0 }) s.NoError(workflow.Sleep(ctx, 10*time.Second)) // longer than execution timeout @@ -3510,7 +3510,7 @@ func testRefresh(t *testing.T, newContext contextFactory) { _, err := s.FrontendClient().CreateSchedule(newContext(s.Context()), req) s.NoError(err) - s.Eventually(func() bool { return atomic.LoadInt32(&runs) == 1 }, 20*time.Second, 200*time.Millisecond) + s.Eventually(func() bool { return runs.Load() == 1 }, 20*time.Second, 200*time.Millisecond) // workflow has started but is now sleeping. it will timeout in 2 seconds. @@ -3623,10 +3623,10 @@ func testRateLimit(t *testing.T, newContext contextFactory) { wid := "sched-test-rate-limit-wf-%d" wt := "sched-test-rate-limit-wt" - var runs int32 + var runs atomic.Int32 workflowFn := func(ctx workflow.Context) error { workflow.SideEffect(ctx, func(ctx workflow.Context) any { - atomic.AddInt32(&runs, 1) + runs.Add(1) return 0 }) return nil @@ -3665,7 +3665,7 @@ func testRateLimit(t *testing.T, newContext contextFactory) { // With no rate limit, we'd see 10/second == 50 workflows run. With a limit of 1/sec, we // expect to see around 5. - s.Less(atomic.LoadInt32(&runs), int32(10)) + s.Less(runs.Load(), int32(10)) } // testNextTimeCache only applies to V1. @@ -4090,11 +4090,11 @@ func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) wid := "sched-test-unpause-resumes-wf" wt := "sched-test-unpause-resumes-wt" - var runs int32 + var runs atomic.Int32 s.SdkWorker().RegisterWorkflowWithOptions( func(ctx workflow.Context) error { workflow.SideEffect(ctx, func(ctx workflow.Context) any { - atomic.AddInt32(&runs, 1) + runs.Add(1) return 0 }) return nil @@ -4125,7 +4125,7 @@ func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) s.NoError(err) // Wait for the schedule to fire at least once, confirming it's running. - s.Eventually(func() bool { return atomic.LoadInt32(&runs) >= 1 }, 15*time.Second, 500*time.Millisecond) + s.Eventually(func() bool { return runs.Load() >= 1 }, 15*time.Second, 500*time.Millisecond) // Pause. _, err = s.FrontendClient().PatchSchedule(newContext(s.Context()), &workflowservice.PatchScheduleRequest{ @@ -4141,9 +4141,9 @@ func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) // observes paused state, performs no-op scheduling, and then the schedule // becomes quiescent (no new runs over a stability window). stableSamples := 0 - lastRuns := atomic.LoadInt32(&runs) + lastRuns := runs.Load() s.Eventually(func() bool { - currentRuns := atomic.LoadInt32(&runs) + currentRuns := runs.Load() if currentRuns == lastRuns { stableSamples++ } else { @@ -4152,7 +4152,7 @@ func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) } return stableSamples >= 6 }, 15*time.Second, 500*time.Millisecond) - runsBeforeUnpause := atomic.LoadInt32(&runs) + runsBeforeUnpause := runs.Load() // Unpause. _, err = s.FrontendClient().PatchSchedule(newContext(s.Context()), &workflowservice.PatchScheduleRequest{ @@ -4166,7 +4166,7 @@ func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) // The generator should be kicked immediately on unpause and new runs should follow. s.Eventually( - func() bool { return atomic.LoadInt32(&runs) > runsBeforeUnpause }, + func() bool { return runs.Load() > runsBeforeUnpause }, 15*time.Second, 500*time.Millisecond, "schedule should resume processing after unpause", diff --git a/tests/xdc/stream_based_replication_test.go b/tests/xdc/stream_based_replication_test.go index 8b5fa7d0c56..cb9f2646640 100644 --- a/tests/xdc/stream_based_replication_test.go +++ b/tests/xdc/stream_based_replication_test.go @@ -1071,7 +1071,7 @@ func (s *streamBasedReplicationTestSuite) TestPassiveActivityRetryTimerReplicati s.NoError(err) defer sdkClient.Close() - var activityAttempts int32 + var activityAttempts atomic.Int32 // Workflow with activity that retries with 3 second intervals simpleWorkflow := func(ctx workflow.Context) (string, error) { @@ -1096,7 +1096,7 @@ func (s *streamBasedReplicationTestSuite) TestPassiveActivityRetryTimerReplicati // Activity that sleeps 3 seconds and fails twice, succeeds on 3rd attempt simpleActivity := func(ctx context.Context) (string, error) { - attempt := atomic.AddInt32(&activityAttempts, 1) + attempt := activityAttempts.Add(1) if attempt < 3 { return "", fmt.Errorf("failed attempt %d", attempt) } From 8cadb770111168c9b0a49b7183465cf749ef6dee Mon Sep 17 00:00:00 2001 From: michaely520 Date: Thu, 20 Aug 2026 11:32:16 -0700 Subject: [PATCH 026/139] Emit namespace migration workflow lifecycle events (#11658) ## What changed? Adds `namespace_lifecycle` start and finish events for the namespace handover, force replication, and catchup system workflows. The events carry workflow identity and the core operation inputs. Finished events classify the result as succeeded, canceled, or failed. Force replication reports its cumulative verified workflow count and emits only one start and one finish across a continue-as-new chain. Emission uses one shared activity, the existing `system.emitNamespaceLifecycleEvents` gate, disconnected cleanup for cancellation, and workflow versioning for replay compatibility. Existing shard handover events are unchanged. ## Why? These system workflows currently have no consistent operation-level event pair, which makes it difficult to correlate a namespace migration request with its final outcome. ## How did you test it? - [x] covered by existing tests - [x] added new unit test(s) `go test -tags test_dep ./common/wideevents ./service/worker/migration` `make fmt-imports` `make lint-code` reports no issues introduced by this change; the repository-wide target still reports existing findings on current `main`. ## Potential risks The terminal event is best effort and cannot run after server-side workflow termination or workflow run timeout because those outcomes do not execute workflow cleanup. --- .../wideevents/namespace_lifecycle_events.go | 63 +++++++++ .../namespace_lifecycle_events_test.go | 26 ++++ service/worker/migration/catchup_workflow.go | 25 +++- .../worker/migration/catchup_workflow_test.go | 14 ++ .../migration/force_replication_workflow.go | 51 ++++++- .../force_replication_workflow_test.go | 37 ++++- service/worker/migration/handover_workflow.go | 17 +++ .../migration/handover_workflow_test.go | 24 ++++ .../migration/workflow_lifecycle_events.go | 128 ++++++++++++++++++ 9 files changed, 372 insertions(+), 13 deletions(-) create mode 100644 service/worker/migration/workflow_lifecycle_events.go diff --git a/common/wideevents/namespace_lifecycle_events.go b/common/wideevents/namespace_lifecycle_events.go index 3d8af8505a4..455b2189f54 100644 --- a/common/wideevents/namespace_lifecycle_events.go +++ b/common/wideevents/namespace_lifecycle_events.go @@ -19,6 +19,69 @@ const ( PhaseHandoverIncomplete = "shard_handover_incomplete" ) +// Namespace migration system workflow lifecycle phases. +const ( + PhaseNamespaceHandoverStarted = "namespace_handover_started" + PhaseNamespaceHandoverFinished = "namespace_handover_finished" + PhaseNamespaceForceReplicationStarted = "namespace_force_replication_started" + PhaseNamespaceForceReplicationFinished = "namespace_force_replication_finished" + PhaseNamespaceCatchupStarted = "namespace_catchup_started" + PhaseNamespaceCatchupFinished = "namespace_catchup_finished" +) + +// Namespace migration system workflow terminal statuses. +const ( + NamespaceMigrationWorkflowSucceeded = "succeeded" + NamespaceMigrationWorkflowCanceled = "canceled" + NamespaceMigrationWorkflowFailed = "failed" +) + +// NamespaceMigrationWorkflowLifecycleInput describes one namespace migration system workflow +// starting or finishing. Input contains only the operation parameters useful for tracing. +type NamespaceMigrationWorkflowLifecycleInput struct { + Phase string + Namespace string + NamespaceID string + WorkflowType string + WorkflowID string + RunID string + FirstRunID string + Input map[string]any + Status string + ErrorMessage string + VerifiedWorkflowCount *int64 +} + +// EmitNamespaceMigrationWorkflowLifecycle emits a system workflow lifecycle event. +func EmitNamespaceMigrationWorkflowLifecycle( + logger otellog.Logger, + in NamespaceMigrationWorkflowLifecycleInput, +) { + details := map[string]any{ + "workflow_type": in.WorkflowType, + "workflow_id": in.WorkflowID, + "run_id": in.RunID, + "first_run_id": in.FirstRunID, + "input": in.Input, + } + if in.Status != "" { + details["status"] = in.Status + } + if in.ErrorMessage != "" { + details["error_message"] = in.ErrorMessage + } + if in.VerifiedWorkflowCount != nil { + details["verified_workflow_count"] = *in.VerifiedWorkflowCount + } + + Emit(logger, NamespaceLifecyclePayload{ + Phase: in.Phase, + Namespace: in.Namespace, + NamespaceID: in.NamespaceID, + Details: details, + }) +} + // Why a shard took or advanced its watermark. const ( // WatermarkAdded: the namespace entered handover. diff --git a/common/wideevents/namespace_lifecycle_events_test.go b/common/wideevents/namespace_lifecycle_events_test.go index bc7f9c3835d..0bea0f9e849 100644 --- a/common/wideevents/namespace_lifecycle_events_test.go +++ b/common/wideevents/namespace_lifecycle_events_test.go @@ -145,6 +145,32 @@ func TestEmitHandoverWatermarkEvents(t *testing.T) { require.Equal(t, true, lagDetails(t, lg.records[1])["deleted_from_db"]) } +func TestEmitNamespaceMigrationWorkflowLifecycle(t *testing.T) { + lg := &captureLogger{} + verifiedCount := int64(42) + + EmitNamespaceMigrationWorkflowLifecycle(lg, NamespaceMigrationWorkflowLifecycleInput{ + Phase: PhaseNamespaceForceReplicationFinished, + Namespace: "ns", + NamespaceID: "ns-id", + WorkflowType: "force-replication-v2", + WorkflowID: "workflow-id", + RunID: "run-id", + FirstRunID: "first-run-id", + Input: map[string]any{"target_cluster": "target"}, + Status: NamespaceMigrationWorkflowSucceeded, + VerifiedWorkflowCount: &verifiedCount, + }) + + require.Len(t, lg.records, 1) + require.Equal(t, PhaseNamespaceForceReplicationFinished, attrValue(lg.records[0], "phase")) + require.Equal(t, "ns-id", attrValue(lg.records[0], "namespace_id")) + details := lagDetails(t, lg.records[0]) + require.Equal(t, "force-replication-v2", details["workflow_type"]) + require.Equal(t, NamespaceMigrationWorkflowSucceeded, details["status"]) + require.EqualValues(t, verifiedCount, details["verified_workflow_count"]) +} + func TestEmitNamespaceRegistered(t *testing.T) { lg := &captureLogger{} diff --git a/service/worker/migration/catchup_workflow.go b/service/worker/migration/catchup_workflow.go index d4b424b74b8..7008aa8a0dd 100644 --- a/service/worker/migration/catchup_workflow.go +++ b/service/worker/migration/catchup_workflow.go @@ -5,6 +5,7 @@ import ( "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/wideevents" ) const ( @@ -21,10 +22,24 @@ type ( CatchUpOutput struct{} ) -func CatchupWorkflow(ctx workflow.Context, params CatchUpParams) (CatchUpOutput, error) { +func CatchupWorkflow(ctx workflow.Context, params CatchUpParams) (_ CatchUpOutput, retErr error) { if err := validateCatchupParams(¶ms); err != nil { return CatchUpOutput{}, err } + if workflow.GetVersion(ctx, migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1) > workflow.DefaultVersion { + lifecycle := newMigrationWorkflowLifecycle( + ctx, + params.Namespace, + wideevents.PhaseNamespaceCatchupStarted, + wideevents.PhaseNamespaceCatchupFinished, + map[string]any{ + "catchup_cluster": params.CatchupCluster, + "target_cluster": params.TargetCluster, + }, + ) + defer func() { lifecycle.emitFinished(ctx, retErr, nil) }() + lifecycle.emitStarted(ctx) + } retryPolicy := &temporal.RetryPolicy{ InitialInterval: time.Second, @@ -39,12 +54,12 @@ func CatchupWorkflow(ctx workflow.Context, params CatchUpParams) (CatchUpOutput, ctx1 := workflow.WithActivityOptions(ctx, ao) var a *activities - err := workflow.ExecuteActivity(ctx1, a.WaitCatchup, params).Get(ctx, nil) - if err != nil { - return CatchUpOutput{}, err + retErr = workflow.ExecuteActivity(ctx1, a.WaitCatchup, params).Get(ctx, nil) + if retErr != nil { + return CatchUpOutput{}, retErr } - return CatchUpOutput{}, err + return CatchUpOutput{}, nil } func validateCatchupParams(params *CatchUpParams) error { diff --git a/service/worker/migration/catchup_workflow_test.go b/service/worker/migration/catchup_workflow_test.go index 840094c5dd1..0934955769e 100644 --- a/service/worker/migration/catchup_workflow_test.go +++ b/service/worker/migration/catchup_workflow_test.go @@ -6,12 +6,22 @@ import ( "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "go.temporal.io/sdk/testsuite" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/wideevents" ) func TestCatchupWorkflow(t *testing.T) { testSuite := &testsuite.WorkflowTestSuite{} env := testSuite.NewTestWorkflowEnvironment() var a *activities + var lifecycleEvents []wideevents.NamespaceMigrationWorkflowLifecycleInput + env.OnGetVersion(migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1).Return(workflow.Version(1)) + env.OnActivity(a.EmitNamespaceMigrationWorkflowLifecycle, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + lifecycleEvents = append(lifecycleEvents, args.Get(1).(wideevents.NamespaceMigrationWorkflowLifecycleInput)) + }). + Return(nil). + Twice() env.OnActivity(a.WaitCatchup, mock.Anything, mock.Anything).Return(nil) @@ -22,5 +32,9 @@ func TestCatchupWorkflow(t *testing.T) { require.True(t, env.IsWorkflowCompleted()) require.NoError(t, env.GetWorkflowError()) + require.Equal(t, []string{ + wideevents.PhaseNamespaceCatchupStarted, + wideevents.PhaseNamespaceCatchupFinished, + }, []string{lifecycleEvents[0].Phase, lifecycleEvents[1].Phase}) env.AssertExpectations(t) } diff --git a/service/worker/migration/force_replication_workflow.go b/service/worker/migration/force_replication_workflow.go index 70a46222692..6ad1f98e087 100644 --- a/service/worker/migration/force_replication_workflow.go +++ b/service/worker/migration/force_replication_workflow.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/wideevents" ) type ( @@ -116,7 +117,7 @@ const ( defaultVerifyIntervalInSeconds = 5 ) -func ForceReplicationWorkflow(ctx workflow.Context, params ForceReplicationParams) error { +func ForceReplicationWorkflow(ctx workflow.Context, params ForceReplicationParams) (retErr error) { // For now, we'll return the initial page token for simplicity. // If we want this to be more precise, we could track processed pages. startPageToken := params.NextPageToken @@ -137,6 +138,8 @@ func ForceReplicationWorkflow(ctx workflow.Context, params ForceReplicationParam if err := validateAndSetForceReplicationParams(ctx, ¶ms); err != nil { return err } + finishLifecycle := startForceReplicationWorkflowLifecycle(ctx, params) + defer func() { finishLifecycle(retErr, verifiedWorkflowCountForEvent(params)) }() if params.TotalForceReplicateWorkflowCount == 0 { wfCount, err := countWorkflowForReplication(ctx, params) @@ -199,7 +202,7 @@ func ForceReplicationWorkflow(ctx workflow.Context, params ForceReplicationParam return workflow.NewContinueAsNewError(ctx, ForceReplicationWorkflow, params) } -func ForceReplicationWorkflowV2(ctx workflow.Context, params ForceReplicationParams) error { +func ForceReplicationWorkflowV2(ctx workflow.Context, params ForceReplicationParams) (retErr error) { // For now, we'll return the initial page token for simplicity. // If we want this to be more precise, we could track processed pages. startPageToken := params.NextPageToken @@ -220,6 +223,8 @@ func ForceReplicationWorkflowV2(ctx workflow.Context, params ForceReplicationPar if err := validateAndSetForceReplicationParams(ctx, ¶ms); err != nil { return err } + finishLifecycle := startForceReplicationWorkflowLifecycle(ctx, params) + defer func() { finishLifecycle(retErr, verifiedWorkflowCountForEvent(params)) }() if params.TotalForceReplicateWorkflowCount == 0 { wfCount, err := countWorkflowForReplication(ctx, params) @@ -277,6 +282,48 @@ func ForceReplicationWorkflowV2(ctx workflow.Context, params ForceReplicationPar return workflow.NewContinueAsNewError(ctx, ForceReplicationWorkflowV2, params) } +func newForceReplicationWorkflowLifecycle( + ctx workflow.Context, + params ForceReplicationParams, +) migrationWorkflowLifecycle { + return newMigrationWorkflowLifecycle( + ctx, + params.Namespace, + wideevents.PhaseNamespaceForceReplicationStarted, + wideevents.PhaseNamespaceForceReplicationFinished, + map[string]any{ + "query": params.Query, + "target_cluster": params.TargetClusterName, + "verification_enabled": params.EnableVerification, + "concurrent_activity_count": params.ConcurrentActivityCount, + "overall_rps": params.OverallRps, + }, + ) +} + +func startForceReplicationWorkflowLifecycle( + ctx workflow.Context, + params ForceReplicationParams, +) func(error, *int64) { + if workflow.GetVersion(ctx, migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1) == workflow.DefaultVersion { + return func(error, *int64) {} + } + lifecycle := newForceReplicationWorkflowLifecycle(ctx, params) + if lifecycle.isFirstRun() { + lifecycle.emitStarted(ctx) + } + return func(err error, verifiedWorkflowCount *int64) { + lifecycle.emitFinished(ctx, err, verifiedWorkflowCount) + } +} + +func verifiedWorkflowCountForEvent(params ForceReplicationParams) *int64 { + if !params.EnableVerification { + return nil + } + return ¶ms.ReplicatedWorkflowCount +} + func maybeKickoffTaskQueueUserDataReplication(ctx workflow.Context, params ForceReplicationParams, onDone func(failureReason string)) error { if workflow.GetVersion(ctx, taskQueueUserDataReplicationVersionMarker, workflow.DefaultVersion, 1) == workflow.DefaultVersion { return nil diff --git a/service/worker/migration/force_replication_workflow_test.go b/service/worker/migration/force_replication_workflow_test.go index bf0ea3ebba0..bb880ba9bda 100644 --- a/service/worker/migration/force_replication_workflow_test.go +++ b/service/worker/migration/force_replication_workflow_test.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" + "go.temporal.io/server/common/wideevents" "go.uber.org/mock/gomock" ) @@ -67,7 +68,15 @@ func (s *ForceReplicationWorkflowTestSuite) TestForceReplicationWorkflow() { namespaceID := uuid.NewString() var a *activities - env.OnActivity(a.CountWorkflow, mock.Anything, mock.Anything).Return(&countWorkflowResponse{WorkflowCount: 4}, nil) + var lifecycleEvents []wideevents.NamespaceMigrationWorkflowLifecycleInput + env.OnGetVersion(migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1).Return(workflow.Version(1)) + env.OnActivity(a.EmitNamespaceMigrationWorkflowLifecycle, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + lifecycleEvents = append(lifecycleEvents, args.Get(1).(wideevents.NamespaceMigrationWorkflowLifecycleInput)) + }). + Return(nil). + Twice() + env.OnActivity(a.CountWorkflow, mock.Anything, mock.Anything).Return(&countWorkflowResponse{WorkflowCount: 10}, nil) env.OnActivity(a.GetMetadata, mock.Anything, MetadataRequest{Namespace: "test-ns"}).Return(&MetadataResponse{ShardCount: 4, NamespaceID: namespaceID}, nil) totalPageCount := 4 @@ -109,6 +118,7 @@ func (s *ForceReplicationWorkflowTestSuite) TestForceReplicationWorkflow() { PageCountPerExecution: 4, EnableVerification: true, TargetClusterEndpoint: "test-target", + ReplicatedWorkflowCount: 6, }) s.True(env.IsWorkflowCompleted()) @@ -126,9 +136,14 @@ func (s *ForceReplicationWorkflowTestSuite) TestForceReplicationWorkflow() { s.Equal(closeTime, status.LastCloseTime) s.True(status.TaskQueueUserDataReplicationStatus.Done) s.Equal("", status.TaskQueueUserDataReplicationStatus.FailureMessage) - s.Equal(int64(4), status.TotalWorkflowCount) - s.Equal(int64(4), status.ReplicatedWorkflowCount) + s.Equal(int64(10), status.TotalWorkflowCount) + s.Equal(int64(10), status.ReplicatedWorkflowCount) s.Equal([]byte(nil), status.PageTokenForRestart) + s.Equal(wideevents.PhaseNamespaceForceReplicationStarted, lifecycleEvents[0].Phase) + s.Equal(wideevents.PhaseNamespaceForceReplicationFinished, lifecycleEvents[1].Phase) + s.Equal(wideevents.NamespaceMigrationWorkflowSucceeded, lifecycleEvents[1].Status) + s.Require().NotNil(lifecycleEvents[1].VerifiedWorkflowCount) + s.Equal(int64(10), *lifecycleEvents[1].VerifiedWorkflowCount) } func (s *ForceReplicationWorkflowTestSuite) TestContinueAsNew() { @@ -184,7 +199,7 @@ func (s *ForceReplicationWorkflowTestSuite) TestContinueAsNew() { expectContinueAsNew := true // Run the workflow once. We should get a continue as new error. - continueAsNewInput, queryStatus := s.testRunForceReplicationForContinueAsNew( + continueAsNewInput, queryStatus, lifecycleEvents := s.testRunForceReplicationForContinueAsNew( mockListWorkflows, ForceReplicationParams{ Namespace: "test-ns", @@ -232,6 +247,8 @@ func (s *ForceReplicationWorkflowTestSuite) TestContinueAsNew() { s.Equal(startTime, queryStatus.LastStartTime) s.Equal(1, queryStatus.ContinuedAsNewCount) s.Equal([]byte("fake-initial-page-token"), queryStatus.PageTokenForRestart) + s.Require().Len(lifecycleEvents, 1) + s.Equal(wideevents.PhaseNamespaceForceReplicationStarted, lifecycleEvents[0].Phase) } func (s *ForceReplicationWorkflowTestSuite) testRunForceReplicationForContinueAsNew( @@ -239,13 +256,21 @@ func (s *ForceReplicationWorkflowTestSuite) testRunForceReplicationForContinueAs input ForceReplicationParams, expectContinueAsNew bool, expMaxPageCountPerExecution int, -) (*ForceReplicationParams, ForceReplicationStatus) { +) (*ForceReplicationParams, ForceReplicationStatus, []wideevents.NamespaceMigrationWorkflowLifecycleInput) { testSuite := &testsuite.WorkflowTestSuite{} env := testSuite.NewTestWorkflowEnvironment() env.RegisterWorkflowWithOptions(ForceTaskQueueUserDataReplicationWorkflow, workflow.RegisterOptions{Name: forceTaskQueueUserDataReplicationWorkflow}) namespaceID := uuid.NewString() var a *activities + var lifecycleEvents []wideevents.NamespaceMigrationWorkflowLifecycleInput + env.OnGetVersion(migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1).Return(workflow.Version(1)) + env.OnActivity(a.EmitNamespaceMigrationWorkflowLifecycle, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + lifecycleEvents = append(lifecycleEvents, args.Get(1).(wideevents.NamespaceMigrationWorkflowLifecycleInput)) + }). + Return(nil). + Once() if input.TotalForceReplicateWorkflowCount == 0 { env.OnActivity(a.CountWorkflow, mock.Anything, mock.Anything).Return(&countWorkflowResponse{WorkflowCount: 10}, nil) } @@ -284,7 +309,7 @@ func (s *ForceReplicationWorkflowTestSuite) testRunForceReplicationForContinueAs var status ForceReplicationStatus s.NoError(envValue.Get(&status)) - return continueAsNewParams, status + return continueAsNewParams, status, lifecycleEvents } func (s *ForceReplicationWorkflowTestSuite) TestInvalidInput() { diff --git a/service/worker/migration/handover_workflow.go b/service/worker/migration/handover_workflow.go index bfe68b18541..3d13422783d 100644 --- a/service/worker/migration/handover_workflow.go +++ b/service/worker/migration/handover_workflow.go @@ -6,6 +6,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/wideevents" ) const ( @@ -63,6 +64,22 @@ func NamespaceHandoverWorkflow(ctx workflow.Context, params NamespaceHandoverPar if err := validateAndSetNamespaceHandoverParams(¶ms); err != nil { return err } + if workflow.GetVersion(ctx, migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1) > workflow.DefaultVersion { + lifecycle := newMigrationWorkflowLifecycle( + ctx, + params.Namespace, + wideevents.PhaseNamespaceHandoverStarted, + wideevents.PhaseNamespaceHandoverFinished, + map[string]any{ + "remote_cluster": params.RemoteCluster, + "allowed_lagging_seconds": params.AllowedLaggingSeconds, + "allowed_lagging_tasks": params.AllowedLaggingTasks, + "handover_timeout_seconds": params.HandoverTimeoutSeconds, + }, + ) + defer func() { lifecycle.emitFinished(ctx, retErr, nil) }() + lifecycle.emitStarted(ctx) + } retryPolicy := &temporal.RetryPolicy{ InitialInterval: time.Second, MaximumInterval: time.Second, diff --git a/service/worker/migration/handover_workflow_test.go b/service/worker/migration/handover_workflow_test.go index 3e66fbbe186..f57126d612d 100644 --- a/service/worker/migration/handover_workflow_test.go +++ b/service/worker/migration/handover_workflow_test.go @@ -11,12 +11,22 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/testsuite" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/wideevents" ) func TestHandoverWorkflow(t *testing.T) { testSuite := &testsuite.WorkflowTestSuite{} env := testSuite.NewTestWorkflowEnvironment() var a *activities + var lifecycleEvents []wideevents.NamespaceMigrationWorkflowLifecycleInput + env.OnGetVersion(migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1).Return(workflow.Version(1)) + env.OnActivity(a.EmitNamespaceMigrationWorkflowLifecycle, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + lifecycleEvents = append(lifecycleEvents, args.Get(1).(wideevents.NamespaceMigrationWorkflowLifecycleInput)) + }). + Return(nil). + Twice() namespaceID := uuid.NewString() @@ -47,6 +57,11 @@ func TestHandoverWorkflow(t *testing.T) { require.True(t, env.IsWorkflowCompleted()) require.NoError(t, env.GetWorkflowError()) + require.Equal(t, []string{ + wideevents.PhaseNamespaceHandoverStarted, + wideevents.PhaseNamespaceHandoverFinished, + }, []string{lifecycleEvents[0].Phase, lifecycleEvents[1].Phase}) + require.Equal(t, wideevents.NamespaceMigrationWorkflowSucceeded, lifecycleEvents[1].Status) env.AssertExpectations(t) } @@ -56,6 +71,14 @@ func TestHandoverWorkflow_CancelAfterHandoverState_ResetsToNormal(t *testing.T) testSuite := &testsuite.WorkflowTestSuite{} env := testSuite.NewTestWorkflowEnvironment() var a *activities + var lifecycleEvents []wideevents.NamespaceMigrationWorkflowLifecycleInput + env.OnGetVersion(migrationWorkflowLifecycleVersion, workflow.DefaultVersion, 1).Return(workflow.Version(1)) + env.OnActivity(a.EmitNamespaceMigrationWorkflowLifecycle, mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + lifecycleEvents = append(lifecycleEvents, args.Get(1).(wideevents.NamespaceMigrationWorkflowLifecycleInput)) + }). + Return(nil). + Twice() namespaceID := uuid.NewString() @@ -96,6 +119,7 @@ func TestHandoverWorkflow_CancelAfterHandoverState_ResetsToNormal(t *testing.T) stateUpdates, "defer must reset state to NORMAL after cancel", ) + require.Equal(t, wideevents.NamespaceMigrationWorkflowCanceled, lifecycleEvents[1].Status) } func TestHandoverWorkflow_SetTimeout(t *testing.T) { diff --git a/service/worker/migration/workflow_lifecycle_events.go b/service/worker/migration/workflow_lifecycle_events.go new file mode 100644 index 00000000000..be5cef6121b --- /dev/null +++ b/service/worker/migration/workflow_lifecycle_events.go @@ -0,0 +1,128 @@ +package migration + +import ( + "context" + "time" + + "go.temporal.io/sdk/temporal" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/wideevents" +) + +const ( + migrationWorkflowLifecycleVersion = "migration-workflow-lifecycle-events-20260819" + maxWorkflowLifecycleErrorLength = 2048 +) + +type migrationWorkflowLifecycle struct { + namespace string + startedPhase string + finishedPhase string + workflowType string + workflowID string + runID string + firstRunID string + input map[string]any +} + +func newMigrationWorkflowLifecycle( + ctx workflow.Context, + namespace string, + startedPhase string, + finishedPhase string, + input map[string]any, +) migrationWorkflowLifecycle { + info := workflow.GetInfo(ctx) + firstRunID := info.FirstRunID + if firstRunID == "" { + firstRunID = info.WorkflowExecution.RunID + } + return migrationWorkflowLifecycle{ + namespace: namespace, + startedPhase: startedPhase, + finishedPhase: finishedPhase, + workflowType: info.WorkflowType.Name, + workflowID: info.WorkflowExecution.ID, + runID: info.WorkflowExecution.RunID, + firstRunID: firstRunID, + input: input, + } +} + +func (l migrationWorkflowLifecycle) isFirstRun() bool { + return l.runID == l.firstRunID +} + +func (l migrationWorkflowLifecycle) emitStarted(ctx workflow.Context) { + l.emit(ctx, l.startedPhase, "", "", nil) +} + +func (l migrationWorkflowLifecycle) emitFinished( + ctx workflow.Context, + err error, + verifiedWorkflowCount *int64, +) { + if workflow.IsContinueAsNewError(err) { + return + } + + status := wideevents.NamespaceMigrationWorkflowSucceeded + errorMessage := "" + if err != nil { + status = wideevents.NamespaceMigrationWorkflowFailed + if temporal.IsCanceledError(err) { + status = wideevents.NamespaceMigrationWorkflowCanceled + } + errorMessage = err.Error() + if len(errorMessage) > maxWorkflowLifecycleErrorLength { + errorMessage = errorMessage[:maxWorkflowLifecycleErrorLength] + } + } + + disconnectedCtx, cancel := workflow.NewDisconnectedContext(ctx) + defer cancel() + l.emit(disconnectedCtx, l.finishedPhase, status, errorMessage, verifiedWorkflowCount) +} + +func (l migrationWorkflowLifecycle) emit( + ctx workflow.Context, + phase string, + status string, + errorMessage string, + verifiedWorkflowCount *int64, +) { + activityCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 10 * time.Second, + RetryPolicy: &temporal.RetryPolicy{MaximumAttempts: 1}, + }) + var a *activities + err := workflow.ExecuteActivity(activityCtx, a.EmitNamespaceMigrationWorkflowLifecycle, + wideevents.NamespaceMigrationWorkflowLifecycleInput{ + Phase: phase, + Namespace: l.namespace, + WorkflowType: l.workflowType, + WorkflowID: l.workflowID, + RunID: l.runID, + FirstRunID: l.firstRunID, + Input: l.input, + Status: status, + ErrorMessage: errorMessage, + VerifiedWorkflowCount: verifiedWorkflowCount, + }, + ).Get(ctx, nil) + if err != nil { + workflow.GetLogger(ctx).Warn("Failed to emit namespace migration workflow lifecycle event", "error", err) + } +} + +func (a *activities) EmitNamespaceMigrationWorkflowLifecycle( + _ context.Context, + in wideevents.NamespaceMigrationWorkflowLifecycleInput, +) error { + if a.emitNamespaceLifecycleEvents == nil || !a.emitNamespaceLifecycleEvents() { + return nil + } + in.NamespaceID = a.namespaceIDForEvent(in.Namespace) + wideevents.EmitNamespaceMigrationWorkflowLifecycle(a.EventLogger, in) + return nil +} From abeeabd4a4c0122117999916b28caed4fca76e73 Mon Sep 17 00:00:00 2001 From: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Date: Thu, 20 Aug 2026 11:43:33 -0700 Subject: [PATCH 027/139] Fix activity timeout regeneration after unpause (#11666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? - Clear an activity’s timer-task status when it is unpaused so timeout tasks are regenerated. - Make ResetActivity with keepPaused=false fully unpause both scheduled and running activities, including clearing pause metadata. - Add and strengthen unit and functional coverage for unpause, reset-unpause, timer regeneration, and keepPaused=true. ## Why? Timeout tasks can fire while an activity is paused and be discarded. Previously, the activity’s timer-task status still indicated that those tasks existed, preventing them from being recreated after unpause and potentially making the timeout ineffective. ResetActivity also bypassed normal unpause handling, and running activities returned early without clearing their paused state. ## How did you test it? - [X] built - [ ] run locally and tested manually - [X] covered by existing tests - [X] added new unit test(s) - [X] added new functional test(s) ## Potential risks Unpausing now invalidates existing timeout tasks and recreates the next applicable timer during transaction close. This is correct, but a behavioral change. Stale queued tasks may still be processed and discarded through the existing stamp validation. --- service/history/workflow/activity.go | 20 +++- service/history/workflow/activity_test.go | 100 +++++++++++++++- tests/activity_api_reset_test.go | 138 ++++++++++++++++++++++ 3 files changed, 250 insertions(+), 8 deletions(-) diff --git a/service/history/workflow/activity.go b/service/history/workflow/activity.go index 287371e0122..cfa662fff5f 100644 --- a/service/history/workflow/activity.go +++ b/service/history/workflow/activity.go @@ -313,6 +313,8 @@ func ResetActivity( } return mutableState.UpdateActivity(ai.ScheduledEventId, func(activityInfo *persistencespb.ActivityInfo, ms historyi.MutableState) error { + wasPaused := activityInfo.Paused + // reset the number of attempts activityInfo.Attempt = 1 activityInfo.ActivityReset = true @@ -339,14 +341,18 @@ func ResetActivity( activityInfo.TimerTaskStatus = TimerTaskStatusNone } - // if activity is running, or it is paused and we don't want to unpause - we don't need to do anything - if GetActivityState(ai) == enumspb.PENDING_ACTIVITY_STATE_STARTED || (ai.Paused && keepPaused) { + if wasPaused && !keepPaused { + unpauseActivityInfo(activityInfo) + } + + // if activity is running, or it remains paused, we don't need to schedule a new run + if GetActivityState(activityInfo) == enumspb.PENDING_ACTIVITY_STATE_STARTED || activityInfo.Paused { return nil } - activityInfo.Stamp++ - if activityInfo.Paused && !keepPaused { - activityInfo.Paused = false + // unpauseActivityInfo already incremented the activity stamp. + if !wasPaused { + activityInfo.Stamp++ } // if activity is not running - we need to regenerate the retry task as schedule activity immediately @@ -376,6 +382,10 @@ func unpauseActivityInfo(ai *persistencespb.ActivityInfo) { ai.PauseInfo = nil ai.Stamp++ + // Timer tasks can be dropped while paused without clearing this mask. Clear all + // bits so transaction close recreates the earliest timer. This can create duplicate + // timer tasks, but after one processes the timeout, the others become no-ops. + ai.TimerTaskStatus = TimerTaskStatusNone } func UnpauseActivity( diff --git a/service/history/workflow/activity_test.go b/service/history/workflow/activity_test.go index 0959de85027..c2f5b31e918 100644 --- a/service/history/workflow/activity_test.go +++ b/service/history/workflow/activity_test.go @@ -370,14 +370,21 @@ func (s *activitySuite) TestResetPausedActivityAcceptance() { s.NotNil(ai.PauseInfo) s.Equal("test_identity", ai.PauseInfo.GetManual().Identity) s.Equal("test_reason", ai.PauseInfo.GetManual().Reason) + timerTaskStatus := int32(TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedScheduleToStart | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat) + ai.TimerTaskStatus = timerTaskStatus prevStamp = ai.Stamp err = ResetActivity(context.Background(), s.mockShard, s.mutableState, ai.ActivityId, false, true, false, 0) - s.NoError(err) + s.Require().NoError(err) s.Equal(int32(1), ai.Attempt, "ActivityInfo.Attempt is not reset") s.Equal(prevStamp, ai.Stamp, "ActivityInfo.Stamp should not change") s.True(ai.Paused, "ActivityInfo.Paused shouldn't change by reset") + s.Equal(pauseInfo, ai.PauseInfo, "ActivityInfo.PauseInfo shouldn't change by reset") + s.Equal(timerTaskStatus, ai.TimerTaskStatus, "ActivityInfo.TimerTaskStatus shouldn't change by reset") } func (s *activitySuite) TestResetAndUnPauseActivityAcceptance() { @@ -400,14 +407,91 @@ func (s *activitySuite) TestResetAndUnPauseActivityAcceptance() { s.NotNil(ai.PauseInfo) s.Equal("test_identity", ai.PauseInfo.GetManual().Identity) s.Equal("test_reason", ai.PauseInfo.GetManual().Reason) + ai.TimerTaskStatus = TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedScheduleToStart | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat prevStamp = ai.Stamp err = ResetActivity(context.Background(), s.mockShard, s.mutableState, ai.ActivityId, false, false, false, 0) - s.NoError(err) + s.Require().NoError(err) s.Equal(int32(1), ai.Attempt, "ActivityInfo.Attempt is not reset") s.NotEqual(prevStamp, ai.Stamp, "ActivityInfo.Stamp should change") - s.False(ai.Paused, "ActivityInfo.Paused shouldn't change by reset") + s.False(ai.Paused, "ActivityInfo.Paused should be cleared by reset") + s.Nil(ai.PauseInfo, "ActivityInfo.PauseInfo should be cleared by reset") + s.Equal(int32(TimerTaskStatusNone), ai.TimerTaskStatus, "ActivityInfo.TimerTaskStatus should be reset to none") + + // Simulate the transaction-close step that recreates activity timers. + timerCreated, err := NewTimerSequence(s.mutableState).CreateNextActivityTimer() + s.Require().NoError(err) + s.True(timerCreated, "activity timeout task should be recreated after reset unpauses the activity") +} + +func (s *activitySuite) TestResetUnpausesRunningActivity() { + ai := s.AddActivityInfo() + ai.StartedEventId = 2 + ai.StartedTime = timestamppb.New(s.mockShard.GetTimeSource().Now()) + + pauseInfo := &persistencespb.ActivityInfo_PauseInfo{ + PauseTime: timestamppb.New(s.mockShard.GetTimeSource().Now()), + PausedBy: &persistencespb.ActivityInfo_PauseInfo_Manual_{ + Manual: &persistencespb.ActivityInfo_PauseInfo_Manual{ + Identity: "test_identity", + Reason: "test_reason", + }, + }, + } + err := PauseActivity(s.mutableState, ai.ActivityId, pauseInfo) + s.Require().NoError(err) + ai.TimerTaskStatus = TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedScheduleToStart | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat + + err = ResetActivity(context.Background(), s.mockShard, s.mutableState, ai.ActivityId, + false, false, false, 0) + s.Require().NoError(err) + s.False(ai.Paused, "ActivityInfo.Paused should be cleared by reset") + s.Nil(ai.PauseInfo, "ActivityInfo.PauseInfo should be cleared by reset") + s.Equal(int32(TimerTaskStatusNone), ai.TimerTaskStatus, "ActivityInfo.TimerTaskStatus should be reset to none") + + // Simulate the transaction-close step that recreates activity timers. + timerCreated, err := NewTimerSequence(s.mutableState).CreateNextActivityTimer() + s.Require().NoError(err) + s.True(timerCreated, "activity timeout task should be recreated after reset unpauses the running activity") +} + +func (s *activitySuite) TestUnpauseActivityAcceptance() { + ai := s.AddActivityInfo() + pauseInfo := &persistencespb.ActivityInfo_PauseInfo{ + PauseTime: timestamppb.New(s.mockShard.GetTimeSource().Now()), + PausedBy: &persistencespb.ActivityInfo_PauseInfo_Manual_{ + Manual: &persistencespb.ActivityInfo_PauseInfo_Manual{ + Identity: "test_identity", + Reason: "test_reason", + }, + }, + } + err := PauseActivity(s.mutableState, ai.ActivityId, pauseInfo) + s.Require().NoError(err) + ai.TimerTaskStatus = TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedScheduleToStart | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat + prevStamp := ai.Stamp + + err = UnpauseActivity(s.mockShard, s.mutableState, ai, false, false, 0) + s.Require().NoError(err) + s.False(ai.Paused, "ActivityInfo.Paused should be cleared by unpause") + s.Nil(ai.PauseInfo, "ActivityInfo.PauseInfo should be cleared by unpause") + s.NotEqual(prevStamp, ai.Stamp, "ActivityInfo.Stamp should change") + s.Equal(int32(TimerTaskStatusNone), ai.TimerTaskStatus, "ActivityInfo.TimerTaskStatus should be reset to none") + + // Simulate the transaction-close step that recreates activity timers. + timerCreated, err := NewTimerSequence(s.mutableState).CreateNextActivityTimer() + s.Require().NoError(err) + s.True(timerCreated, "activity timeout task should be recreated after unpause") } func (s *activitySuite) TestUnpauseActivityWithResumeAcceptance() { @@ -421,6 +505,10 @@ func (s *activitySuite) TestUnpauseActivityWithResumeAcceptance() { s.Equal(int32(1), ai.Attempt, "ActivityInfo.Attempt is shouldn't change") s.NotEqual(prevStamp, ai.Stamp, "ActivityInfo.Stamp should change") s.True(ai.Paused, "ActivityInfo.Paused was not unpaused") + ai.TimerTaskStatus = TimerTaskStatusCreatedScheduleToClose | + TimerTaskStatusCreatedScheduleToStart | + TimerTaskStatusCreatedStartToClose | + TimerTaskStatusCreatedHeartbeat prevStamp = ai.Stamp _, err = UnpauseActivityWithResume(s.mockShard, s.mutableState, ai, false, 0) s.NoError(err) @@ -428,6 +516,12 @@ func (s *activitySuite) TestUnpauseActivityWithResumeAcceptance() { s.Equal(int32(1), ai.Attempt, "ActivityInfo.Attempt is shouldn't change") s.NotEqual(prevStamp, ai.Stamp, "ActivityInfo.Stamp should change") s.False(ai.Paused, "ActivityInfo.Paused was not unpaused") + s.Equal(int32(TimerTaskStatusNone), ai.TimerTaskStatus, "ActivityInfo.TimerTaskStatus should be reset to none") + + // Verify invalidating the stale timer mask allows the timeout task to be regenerated. + timerCreated, err := NewTimerSequence(s.mutableState).CreateNextActivityTimer() + s.Require().NoError(err) + s.True(timerCreated, "activity timeout task should be recreated after unpause") } func (s *activitySuite) TestUnpauseActivityWithNewRun() { diff --git a/tests/activity_api_reset_test.go b/tests/activity_api_reset_test.go index 7172926ef3c..7678508f808 100644 --- a/tests/activity_api_reset_test.go +++ b/tests/activity_api_reset_test.go @@ -232,6 +232,144 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_WhileRunning() { s.Equal(int32(1), startedActivityCount.Load()) } +func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_UnpausesRunningActivity() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + activityCompleteCh := make(chan struct{}) + var startedActivityCount atomic.Int32 + activityFunction := func() (string, error) { + startedActivityCount.Add(1) + s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + return "done!", nil + } + + workflowFn := s.makeWorkflowFunc(activityFunction) + s.SdkWorker().RegisterWorkflow(workflowFn) + s.SdkWorker().RegisterActivity(activityFunction) + + workflowRun, err := s.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: s.tv.WorkflowID(), + TaskQueue: s.TaskQueue(), + }, workflowFn) + s.Require().NoError(err) + + await.Require(ctx, s.T(), func(t *await.T) { + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + require.NoError(t, err) + require.Len(t, description.GetPendingActivities(), 1) + require.Equal(t, enumspb.PENDING_ACTIVITY_STATE_STARTED, description.PendingActivities[0].State) + }, 5*time.Second, 200*time.Millisecond) + + _, err = s.FrontendClient().PauseActivity(ctx, &workflowservice.PauseActivityRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowRun.GetID()}, + Activity: &workflowservice.PauseActivityRequest_Id{Id: "activity-id"}, + }) + s.Require().NoError(err) + + await.Require(ctx, s.T(), func(t *await.T) { + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + require.NoError(t, err) + require.Len(t, description.GetPendingActivities(), 1) + require.Equal(t, enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED, description.PendingActivities[0].State) + require.True(t, description.PendingActivities[0].Paused) + require.NotNil(t, description.PendingActivities[0].PauseInfo) + }, 5*time.Second, 200*time.Millisecond) + + s.Require().NoError(s.resetFn(ctx, workflowRun.GetID(), "activity-id", false, false)) + + // keepPaused=false must clear both the paused flag and its associated metadata. + await.Require(ctx, s.T(), func(t *await.T) { + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + require.NoError(t, err) + require.Len(t, description.GetPendingActivities(), 1) + require.Equal(t, enumspb.PENDING_ACTIVITY_STATE_STARTED, description.PendingActivities[0].State) + require.False(t, description.PendingActivities[0].Paused) + require.Nil(t, description.PendingActivities[0].PauseInfo) + require.Equal(t, int32(1), description.PendingActivities[0].Attempt) + }, 5*time.Second, 200*time.Millisecond) + + activityCompleteCh <- struct{}{} + + s.Require().NoError(workflowRun.Get(ctx, nil)) + s.Equal(int32(1), startedActivityCount.Load()) +} + +func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_TimesOutOnUnpause() { + s.startToCloseTimeout = time.Second + s.activityRetryPolicy = &temporal.RetryPolicy{MaximumAttempts: 1} + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + activityCompleteCh := make(chan struct{}) + defer close(activityCompleteCh) + activityFunction := func() (string, error) { + <-activityCompleteCh + return "done!", nil + } + + workflowFn := s.makeWorkflowFunc(activityFunction) + s.SdkWorker().RegisterWorkflow(workflowFn) + s.SdkWorker().RegisterActivity(activityFunction) + + workflowRun, err := s.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: s.tv.WorkflowID(), + TaskQueue: s.TaskQueue(), + }, workflowFn) + s.Require().NoError(err) + + var activityStartedAt time.Time + await.Require(ctx, s.T(), func(t *await.T) { + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + require.NoError(t, err) + require.Len(t, description.GetPendingActivities(), 1) + pendingActivity := description.PendingActivities[0] + require.Equal(t, enumspb.PENDING_ACTIVITY_STATE_STARTED, pendingActivity.State) + require.NotNil(t, pendingActivity.LastStartedTime) + activityStartedAt = pendingActivity.LastStartedTime.AsTime() + }, 5*time.Second, 200*time.Millisecond) + + _, err = s.FrontendClient().PauseActivity(ctx, &workflowservice.PauseActivityRequest{ + Namespace: s.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowRun.GetID()}, + Activity: &workflowservice.PauseActivityRequest_Id{Id: "activity-id"}, + }) + s.Require().NoError(err) + + await.Require(ctx, s.T(), func(t *await.T) { + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + require.NoError(t, err) + require.Len(t, description.GetPendingActivities(), 1) + require.Equal(t, enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED, description.PendingActivities[0].State) + require.True(t, description.PendingActivities[0].Paused) + }, 5*time.Second, 200*time.Millisecond) + + // Allow the original timeout task to be processed while the activity is paused. + originalDeadline := activityStartedAt.Add(s.startToCloseTimeout) + await.RequireTrue(s.T(), func() bool { + return time.Now().After(originalDeadline.Add(2 * time.Second)) + }, 5*time.Second, 100*time.Millisecond) + + description, err := s.SdkClient().DescribeWorkflowExecution(ctx, workflowRun.GetID(), workflowRun.GetRunID()) + s.Require().NoError(err) + s.Require().Len(description.GetPendingActivities(), 1) + s.True(description.PendingActivities[0].Paused) + + s.Require().NoError(s.resetFn(ctx, workflowRun.GetID(), "activity-id", false, false)) + + workflowResultCtx, workflowResultCancel := context.WithTimeout(ctx, 10*time.Second) + defer workflowResultCancel() + err = workflowRun.Get(workflowResultCtx, nil) + s.Require().Error(err) + var activityErr *temporal.ActivityError + s.Require().ErrorAs(err, &activityErr) + timeoutErr, ok := activityErr.Unwrap().(*temporal.TimeoutError) + s.Require().True(ok) + s.Equal(enumspb.TIMEOUT_TYPE_START_TO_CLOSE, timeoutErr.TimeoutType()) +} + func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_InRetry() { // reset is called while activity is in retry s.initialRetryInterval = 1 * time.Minute From 39fc2c45ec0291de8d580505bf31d8753e09b95b Mon Sep 17 00:00:00 2001 From: Qian Chen Date: Thu, 20 Aug 2026 11:59:15 -0700 Subject: [PATCH 028/139] Emit remote cluster lifecycle wide events (#11672) ## What changed? - Emit `remote_cluster_lifecycle` wide events for successful and failed `AddOrUpdateRemoteCluster` and `RemoveRemoteCluster` calls through both Operator and Admin APIs. - Capture the request, request fingerprint, caller/auth context when available, remote response, persistence request, authoritative pre-mutation persistence state for upserts, cached pre-mutation state for removals, requested connection/replication transitions, mutation, outcome, and terminal error details. - Preserve the `NamespaceLifecyclePayload` envelope and place remote-cluster-specific fields in `details`. - Gate emission with the existing `system.emitNamespaceLifecycleEvents` dynamic config. ## Why? Remote-cluster connection, replication, and removal changes need an auditable record that shows what was requested, what state existed before the mutation, and whether the request succeeded or failed. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] added new unit test(s) - [ ] added new functional test(s) Commands: ```text go test -tags test_dep ./service/frontend ./common/wideevents make lint-code ``` Local E2E against `development-cluster-a.yaml` covered: - Operator: create enabled, update disabled, remove. - Admin: create disabled, update enabled, remove. - Operator removal rejected while an intact global namespace referenced both the local and remote clusters.
Successful Operator update: connection and replication disabled ```json { "details": { "api": "operator", "call_origin": "AddOrUpdateRemoteCluster", "caller_type": "api", "local_cluster": "cluster-a", "mutation": "updated", "outcome": "succeeded", "persisted_before": { "cluster_address": "127.0.0.1:9433", "cluster_id": "5234378e-93a3-42e7-9ef0-b0a68ec20a4f", "cluster_name": "cluster-remote-lifecycle-update-e2e", "failover_version_increment": 100, "history_shard_count": 32, "http_address": "127.0.0.1:9443", "index_search_attributes": null, "initial_failover_version": 21, "is_connection_enabled": true, "is_global_namespace_enabled": true, "is_replication_enabled": true, "tags": { "environment": "local-e2e", "purpose": "remote-cluster-lifecycle-update-wide-events" }, "use_cluster_id_membership": false, "version": 1, "version_info": null }, "persistence_request": { "cluster_metadata": { "cluster_address": "127.0.0.1:9433", "cluster_id": "5234378e-93a3-42e7-9ef0-b0a68ec20a4f", "cluster_name": "cluster-remote-lifecycle-update-e2e", "failover_version_increment": 100, "history_shard_count": 32, "http_address": "127.0.0.1:9443", "index_search_attributes": null, "initial_failover_version": 21, "is_connection_enabled": false, "is_global_namespace_enabled": true, "is_replication_enabled": false, "tags": { "environment": "local-e2e", "purpose": "remote-cluster-lifecycle-update-wide-events" }, "use_cluster_id_membership": false, "version_info": null }, "version": 1 }, "remote_cluster": "cluster-remote-lifecycle-update-e2e", "remote_cluster_id": "5234378e-93a3-42e7-9ef0-b0a68ec20a4f", "request": { "enable_remote_cluster_connection": false, "enable_replication": false, "frontend_address": "127.0.0.1:9433", "frontend_http_address": "127.0.0.1:9443" }, "request_fingerprint": "e356fbe885b9f750f5e9a35ec01c29dfaf8f82f135bbfefd93304b50dcdbcb89", "requested_connection_transition": "disabled", "requested_replication_transition": "disabled" }, "event_name": "remote_cluster_lifecycle", "instrumentation_scope": "go.temporal.io/server/common/wideevents", "namespace": "N/A", "namespace_id": "N/A", "phase": "remote_cluster_upsert" } ```
Failed Operator removal: cluster still referenced by global namespace ```json { "details": { "api": "operator", "cached_before": { "cluster_id": "992aa482-40a4-46ad-a4ec-c15d853cb37c", "cluster_name": "cluster-guard-e2e", "http_address": "127.0.0.1:9543", "initial_failover_version": 31, "is_connection_enabled": true, "is_replication_enabled": true, "rpc_address": "127.0.0.1:9533", "shard_count": 32, "tags": { "environment": "local-e2e", "purpose": "cluster-removal-namespace-guard" } }, "call_origin": "RemoveRemoteCluster", "caller_type": "api", "error": "cannot remove cluster \"cluster-guard-e2e\": still referenced by namespace \"cluster-guard-e2e-ns\"", "error_code": "FailedPrecondition", "error_type": "*serviceerror.FailedPrecondition", "local_cluster": "cluster-a", "mutation": "unknown", "outcome": "failed", "remote_cluster": "cluster-guard-e2e", "remote_cluster_id": "992aa482-40a4-46ad-a4ec-c15d853cb37c", "request": { "cluster_name": "cluster-guard-e2e" }, "request_fingerprint": "9992050714897282663c1cc8a216ad99eff3aee746c24ac1c2a510a5aa9fd23e" }, "event_name": "remote_cluster_lifecycle", "instrumentation_scope": "go.temporal.io/server/common/wideevents", "namespace": "N/A", "namespace_id": "N/A", "phase": "remote_cluster_remove" } ```
--- common/wideevents/remote_cluster_events.go | 22 + .../wideevents/remote_cluster_events_test.go | 22 + service/frontend/admin_handler.go | 73 ++- service/frontend/admin_handler_test.go | 117 +++++ service/frontend/fx.go | 4 + service/frontend/operator_handler.go | 79 ++- service/frontend/operator_handler_test.go | 136 +++++- .../remote_cluster_lifecycle_events.go | 453 ++++++++++++++++++ .../remote_cluster_lifecycle_events_test.go | 307 ++++++++++++ 9 files changed, 1188 insertions(+), 25 deletions(-) create mode 100644 common/wideevents/remote_cluster_events.go create mode 100644 common/wideevents/remote_cluster_events_test.go create mode 100644 service/frontend/remote_cluster_lifecycle_events.go create mode 100644 service/frontend/remote_cluster_lifecycle_events_test.go diff --git a/common/wideevents/remote_cluster_events.go b/common/wideevents/remote_cluster_events.go new file mode 100644 index 00000000000..418cc3a7821 --- /dev/null +++ b/common/wideevents/remote_cluster_events.go @@ -0,0 +1,22 @@ +package wideevents + +import "go.opentelemetry.io/otel/log" + +const RemoteClusterLifecycleEventName = "remote_cluster_lifecycle" + +const ( + PhaseRemoteClusterUpsert = "remote_cluster_upsert" + PhaseRemoteClusterRemove = "remote_cluster_remove" +) + +// RemoteClusterLifecyclePayload preserves the namespace lifecycle event envelope +// while using a distinct event name and remote-cluster-specific details. +type RemoteClusterLifecyclePayload NamespaceLifecyclePayload + +func (p RemoteClusterLifecyclePayload) EventName() string { + return RemoteClusterLifecycleEventName +} + +func (p RemoteClusterLifecyclePayload) Attributes() []log.KeyValue { + return NamespaceLifecyclePayload(p).Attributes() +} diff --git a/common/wideevents/remote_cluster_events_test.go b/common/wideevents/remote_cluster_events_test.go new file mode 100644 index 00000000000..11e75cbb242 --- /dev/null +++ b/common/wideevents/remote_cluster_events_test.go @@ -0,0 +1,22 @@ +package wideevents + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRemoteClusterLifecycleEventName(t *testing.T) { + require.Equal(t, "remote_cluster_lifecycle", RemoteClusterLifecyclePayload{}.EventName()) +} + +func TestRemoteClusterLifecycleFieldSetMatchesNamespaceLifecycle(t *testing.T) { + payload := RemoteClusterLifecyclePayload{ + Phase: "remote_cluster_upsert", + Namespace: "N/A", + NamespaceID: "N/A", + Details: map[string]any{"outcome": "succeeded"}, + } + + require.Equal(t, NamespaceLifecyclePayload(payload).Attributes(), payload.Attributes()) +} diff --git a/service/frontend/admin_handler.go b/service/frontend/admin_handler.go index ba1767d7e58..73b131e8749 100644 --- a/service/frontend/admin_handler.go +++ b/service/frontend/admin_handler.go @@ -12,6 +12,7 @@ import ( "time" "github.com/google/uuid" + otellog "go.opentelemetry.io/otel/log" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" namespacepb "go.temporal.io/api/namespace/v1" @@ -85,6 +86,7 @@ type ( status int32 logger log.Logger + eventLogger otellog.Logger numberOfHistoryShards int32 config *Config namespaceDLQHandler nsreplication.DLQMessageHandler @@ -123,6 +125,7 @@ type ( ReplicatorNamespaceReplicationQueue persistence.NamespaceReplicationQueue visibilityMgr manager.VisibilityManager Logger log.Logger + EventLogger otellog.Logger TaskManager persistence.TaskManager FairTaskManager persistence.FairTaskManager PersistenceExecutionManager persistence.ExecutionManager @@ -173,6 +176,7 @@ func NewAdminHandler( return &AdminHandler{ logger: args.Logger, + eventLogger: args.EventLogger, status: common.DaemonStatusInitialized, numberOfHistoryShards: args.PersistenceConfig.NumHistoryShards, config: args.Config, @@ -864,7 +868,32 @@ func (adh *AdminHandler) AddOrUpdateRemoteCluster( ctx context.Context, request *adminservice.AddOrUpdateRemoteClusterRequest, ) (_ *adminservice.AddOrUpdateRemoteClusterResponse, retError error) { + var ( + lifecycleEvent *remoteClusterLifecycleEvent + remoteResponse *adminservice.DescribeClusterResponse + persistedBefore *persistence.GetClusterMetadataResponse + persistenceRequest *persistence.SaveClusterMetadataRequest + ) + // Bracket deferred emission with panic capture: the inner capture converts handler panics + // into retError for the event, while the outer capture recovers panics from emission itself. + defer log.CapturePanic(adh.logger, &retError) + defer func() { + lifecycleEvent.emitUpsertFailure(retError, remoteResponse, persistedBefore, persistenceRequest) + }() defer log.CapturePanic(adh.logger, &retError) + lifecycleEvent = newRemoteClusterUpsertLifecycleEvent( + ctx, + adh.eventLogger, + adh.clusterMetadata, + adh.config, + remoteClusterAPIAdmin, + remoteClusterUpsertRequestFields{ + FrontendAddress: request.GetFrontendAddress(), + FrontendHTTPAddress: request.GetFrontendHttpAddress(), //nolint:staticcheck // Audit the deprecated API request as received. + EnableRemoteClusterConnection: request.GetEnableRemoteClusterConnection(), + EnableReplication: request.GetEnableReplication(), + }, + ) adminClient := adh.clientFactory.NewRemoteAdminClientWithTimeout( request.GetFrontendAddress(), @@ -874,6 +903,7 @@ func (adh *AdminHandler) AddOrUpdateRemoteCluster( // Fetch cluster metadata from remote cluster resp, err := adminClient.DescribeCluster(ctx, &adminservice.DescribeClusterRequest{}) + remoteResponse = resp if err != nil { return nil, err } @@ -905,14 +935,14 @@ func (adh *AdminHandler) AddOrUpdateRemoteCluster( ) switch err.(type) { case nil: + persistedBefore = clusterData updateRequestVersion = clusterData.Version case *serviceerror.NotFound: updateRequestVersion = 0 default: return nil, err } - - applied, err := clusterMetadataMrg.SaveClusterMetadata(ctx, &persistence.SaveClusterMetadataRequest{ + saveRequest := &persistence.SaveClusterMetadataRequest{ ClusterMetadata: &persistencespb.ClusterMetadata{ ClusterName: resp.GetClusterName(), HistoryShardCount: resp.GetHistoryShardCount(), @@ -927,7 +957,9 @@ func (adh *AdminHandler) AddOrUpdateRemoteCluster( Tags: resp.GetTags(), }, Version: updateRequestVersion, - }) + } + persistenceRequest = saveRequest + applied, err := clusterMetadataMrg.SaveClusterMetadata(ctx, saveRequest) if err != nil { return nil, err } @@ -935,6 +967,7 @@ func (adh *AdminHandler) AddOrUpdateRemoteCluster( return nil, serviceerror.NewInvalidArgument( "Cannot update remote cluster due to update immutable fields") } + lifecycleEvent.emitUpsertSuccess(persistedBefore, persistenceRequest) return &adminservice.AddOrUpdateRemoteClusterResponse{}, nil } @@ -944,18 +977,42 @@ func (adh *AdminHandler) RemoveRemoteCluster( ctx context.Context, request *adminservice.RemoveRemoteClusterRequest, ) (_ *adminservice.RemoveRemoteClusterResponse, retError error) { + var ( + lifecycleEvent *remoteClusterLifecycleEvent + cachedBefore cachedRemoteClusterLookup + persistenceRequest *persistence.DeleteClusterMetadataRequest + ) + // Bracket deferred emission with panic capture: the inner capture converts handler panics + // into retError for the event, while the outer capture recovers panics from emission itself. + defer log.CapturePanic(adh.logger, &retError) + defer func() { + lifecycleEvent.emitRemoveFailure(retError, cachedBefore, persistenceRequest) + }() defer log.CapturePanic(adh.logger, &retError) + clusterName := request.GetClusterName() + lifecycleEvent = newRemoteClusterRemoveLifecycleEvent( + ctx, + adh.eventLogger, + adh.clusterMetadata, + adh.config, + remoteClusterAPIAdmin, + remoteClusterRemoveRequestFields{ClusterName: clusterName}, + ) + if lifecycleEvent != nil { + // Admin does not use the cluster cache for removal; this lookup is audit-only. + cachedBefore = lookupCachedRemoteCluster(adh.clusterMetadata, clusterName) + } - if err := validateClusterNotInUseByNamespaces(adh.namespaceRegistry, adh.clusterMetadata.GetCurrentClusterName(), request.GetClusterName()); err != nil { + if err := validateClusterNotInUseByNamespaces(adh.namespaceRegistry, adh.clusterMetadata.GetCurrentClusterName(), clusterName); err != nil { return nil, err } - if err := adh.clusterMetadataManager.DeleteClusterMetadata( - ctx, - &persistence.DeleteClusterMetadataRequest{ClusterName: request.GetClusterName()}, - ); err != nil { + deleteRequest := &persistence.DeleteClusterMetadataRequest{ClusterName: clusterName} + persistenceRequest = deleteRequest + if err := adh.clusterMetadataManager.DeleteClusterMetadata(ctx, deleteRequest); err != nil { return nil, err } + lifecycleEvent.emitRemoveSuccess(cachedBefore, persistenceRequest) return &adminservice.RemoveRemoteClusterResponse{}, nil } diff --git a/service/frontend/admin_handler_test.go b/service/frontend/admin_handler_test.go index b594c67816d..5c6fe5b1462 100644 --- a/service/frontend/admin_handler_test.go +++ b/service/frontend/admin_handler_test.go @@ -184,6 +184,7 @@ func (s *adminHandlerSuite) SetupTest() { s.mockProducer, s.mockVisibilityMgr, s.mockResource.GetLogger(), + nil, s.mockResource.GetTaskManager(), s.mockResource.GetTaskManager(), s.mockResource.GetExecutionManager(), @@ -231,6 +232,12 @@ func (s *adminHandlerSuite) TearDownTest() { func (s *adminHandlerSuite) Test_RemoveRemoteCluster_Success() { var clusterName = "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{ + clusterName: {ClusterID: "cluster-id"}, + }) s.mockNamespaceCache.EXPECT().GetAllNamespaces().Return(nil) s.mockClusterMetadataManager.EXPECT().DeleteClusterMetadata( gomock.Any(), @@ -239,10 +246,20 @@ func (s *adminHandlerSuite) Test_RemoveRemoteCluster_Success() { _, err := s.handler.RemoveRemoteCluster(context.Background(), &adminservice.RemoveRemoteClusterRequest{ClusterName: clusterName}) s.NoError(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeSucceeded, details["outcome"]) + s.Equal(remoteClusterMutationRemoved, details["mutation"]) + s.Equal("cluster-id", details["remote_cluster_id"]) } func (s *adminHandlerSuite) Test_RemoveRemoteCluster_Error() { var clusterName = "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{ + clusterName: {}, + }) s.mockNamespaceCache.EXPECT().GetAllNamespaces().Return(nil) s.mockClusterMetadataManager.EXPECT().DeleteClusterMetadata( gomock.Any(), @@ -251,6 +268,45 @@ func (s *adminHandlerSuite) Test_RemoveRemoteCluster_Error() { _, err := s.handler.RemoveRemoteCluster(context.Background(), &adminservice.RemoveRemoteClusterRequest{ClusterName: clusterName}) s.Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal(remoteClusterMutationUnknown, details["mutation"]) + s.NotNil(details["persistence_request"]) +} + +func (s *adminHandlerSuite) Test_RemoveRemoteCluster_PanicEmitsFailure() { + clusterName := "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockMetadata.EXPECT().GetAllClusterInfo().DoAndReturn(func() map[string]cluster.ClusterInformation { + panic("test panic") + }) + + _, err := s.handler.RemoveRemoteCluster( + context.Background(), + &adminservice.RemoveRemoteClusterRequest{ClusterName: clusterName}, + ) + s.Require().Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Internal", details["error_code"]) +} + +func (s *adminHandlerSuite) Test_RemoveRemoteCluster_EventEmissionPanicCaptured() { + clusterName := "cluster" + s.handler.eventLogger = &panicRemoteClusterEventLogger{} + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockMetadata.EXPECT().GetAllClusterInfo().DoAndReturn(func() map[string]cluster.ClusterInformation { + panic("handler panic") + }) + + _, err := s.handler.RemoveRemoteCluster( + context.Background(), + &adminservice.RemoveRemoteClusterRequest{ClusterName: clusterName}, + ) + s.Require().Error(err) + s.Contains(err.Error(), "event logger panic") } func (s *adminHandlerSuite) Test_RemoveRemoteCluster_BlockedByGlobalNamespace() { @@ -376,6 +432,9 @@ func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_RecordFound_Success() var clusterName = uuid.NewString() var clusterID = uuid.NewString() var recordVersion int64 = 5 + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockMetadata.EXPECT().GetFailoverVersionIncrement().Return(int64(10)).Times(2) s.mockMetadata.EXPECT().GetAllClusterInfo().Return(make(map[string]cluster.ClusterInformation)) @@ -413,6 +472,11 @@ func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_RecordFound_Success() FrontendAddress: rpcAddress, }) s.NoError(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeSucceeded, details["outcome"]) + s.Equal(remoteClusterMutationUpdated, details["mutation"]) + s.Equal(remoteClusterTransitionUnchanged, details["requested_connection_transition"]) + s.Equal(remoteClusterTransitionUnchanged, details["requested_replication_transition"]) } func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_RecordNotFound_Success() { @@ -672,6 +736,9 @@ func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_ValidationError_EmptyR func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_DescribeCluster_Error() { var rpcAddress = uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( s.mockAdminClient, @@ -682,6 +749,56 @@ func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_DescribeCluster_Error( ) _, err := s.handler.AddOrUpdateRemoteCluster(context.Background(), &adminservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}) s.Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Unknown", details["error_code"]) +} + +func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_PanicEmitsFailure() { + rpcAddress := uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( + s.mockAdminClient, + ) + s.mockAdminClient.EXPECT().DescribeCluster( + gomock.Any(), + &adminservice.DescribeClusterRequest{}, + ).DoAndReturn(func(context.Context, *adminservice.DescribeClusterRequest, ...grpc.CallOption) (*adminservice.DescribeClusterResponse, error) { + panic("test panic") + }) + + _, err := s.handler.AddOrUpdateRemoteCluster( + context.Background(), + &adminservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}, + ) + s.Require().Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Internal", details["error_code"]) +} + +func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_EventEmissionPanicCaptured() { + rpcAddress := uuid.NewString() + s.handler.eventLogger = &panicRemoteClusterEventLogger{} + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( + s.mockAdminClient, + ) + s.mockAdminClient.EXPECT().DescribeCluster( + gomock.Any(), + &adminservice.DescribeClusterRequest{}, + ).DoAndReturn(func(context.Context, *adminservice.DescribeClusterRequest, ...grpc.CallOption) (*adminservice.DescribeClusterResponse, error) { + panic("handler panic") + }) + + _, err := s.handler.AddOrUpdateRemoteCluster( + context.Background(), + &adminservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}, + ) + s.Require().Error(err) + s.Contains(err.Error(), "event logger panic") } func (s *adminHandlerSuite) Test_AddOrUpdateRemoteCluster_GetClusterMetadata_Error() { diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 94cda8a428d..80a80a81037 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -782,6 +782,7 @@ func AdminHandlerProvider( replicatorNamespaceReplicationQueue FEReplicatorNamespaceReplicationQueue, visibilityMgr manager.VisibilityManager, logger log.SnTaggedLogger, + eventLogger otellog.Logger, namespaceReplicationQueue persistence.NamespaceReplicationQueue, taskManager persistence.TaskManager, fairTaskManager persistence.FairTaskManager, @@ -814,6 +815,7 @@ func AdminHandlerProvider( replicatorNamespaceReplicationQueue, visibilityMgr, logger, + eventLogger, taskManager, fairTaskManager, persistenceExecutionManager, @@ -868,6 +870,7 @@ func NamespaceDLQHandlerProvider( func OperatorHandlerProvider( configuration *Config, logger log.SnTaggedLogger, + eventLogger otellog.Logger, sdkClientFactory sdk.ClientFactory, metricsHandler metrics.Handler, visibilityMgr manager.VisibilityManager, @@ -883,6 +886,7 @@ func OperatorHandlerProvider( args := NewOperatorHandlerImplArgs{ configuration, logger, + eventLogger, sdkClientFactory, metricsHandler, visibilityMgr, diff --git a/service/frontend/operator_handler.go b/service/frontend/operator_handler.go index 41d0e5fda24..59ecd64e082 100644 --- a/service/frontend/operator_handler.go +++ b/service/frontend/operator_handler.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "time" + otellog "go.opentelemetry.io/otel/log" enumspb "go.temporal.io/api/enums/v1" namespacepb "go.temporal.io/api/namespace/v1" "go.temporal.io/api/operatorservice/v1" @@ -51,6 +52,7 @@ type ( status int32 logger log.Logger + eventLogger otellog.Logger config *Config sdkClientFactory sdk.ClientFactory metricsHandler metrics.Handler @@ -68,6 +70,7 @@ type ( NewOperatorHandlerImplArgs struct { config *Config Logger log.Logger + EventLogger otellog.Logger sdkClientFactory sdk.ClientFactory MetricsHandler metrics.Handler VisibilityMgr manager.VisibilityManager @@ -95,6 +98,7 @@ func NewOperatorHandlerImpl( handler := &OperatorHandlerImpl{ logger: args.Logger, + eventLogger: args.EventLogger, status: common.DaemonStatusInitialized, config: args.config, sdkClientFactory: args.sdkClientFactory, @@ -605,7 +609,32 @@ func (h *OperatorHandlerImpl) AddOrUpdateRemoteCluster( ctx context.Context, request *operatorservice.AddOrUpdateRemoteClusterRequest, ) (_ *operatorservice.AddOrUpdateRemoteClusterResponse, retError error) { + var ( + lifecycleEvent *remoteClusterLifecycleEvent + remoteResponse *adminservice.DescribeClusterResponse + persistedBefore *persistence.GetClusterMetadataResponse + persistenceRequest *persistence.SaveClusterMetadataRequest + ) + // Bracket deferred emission with panic capture: the inner capture converts handler panics + // into retError for the event, while the outer capture recovers panics from emission itself. + defer log.CapturePanic(h.logger, &retError) + defer func() { + lifecycleEvent.emitUpsertFailure(retError, remoteResponse, persistedBefore, persistenceRequest) + }() defer log.CapturePanic(h.logger, &retError) + lifecycleEvent = newRemoteClusterUpsertLifecycleEvent( + ctx, + h.eventLogger, + h.clusterMetadata, + h.config, + remoteClusterAPIOperator, + remoteClusterUpsertRequestFields{ + FrontendAddress: request.GetFrontendAddress(), + FrontendHTTPAddress: request.GetFrontendHttpAddress(), + EnableRemoteClusterConnection: request.GetEnableRemoteClusterConnection(), + EnableReplication: request.GetEnableReplication(), + }, + ) adminClient := h.clientFactory.NewRemoteAdminClientWithTimeout( request.GetFrontendAddress(), @@ -615,6 +644,7 @@ func (h *OperatorHandlerImpl) AddOrUpdateRemoteCluster( // Fetch cluster metadata from remote cluster resp, err := adminClient.DescribeCluster(ctx, &adminservice.DescribeClusterRequest{}) + remoteResponse = resp if err != nil { return nil, serviceerror.NewUnavailablef( errUnableConnectRemoteClusterMessage, @@ -649,14 +679,14 @@ func (h *OperatorHandlerImpl) AddOrUpdateRemoteCluster( ) switch err.(type) { case nil: + persistedBefore = clusterData updateRequestVersion = clusterData.Version case *serviceerror.NotFound: updateRequestVersion = 0 default: return nil, serviceerror.NewInternalf(errUnableToStoreClusterInfo, err) } - - applied, err := h.clusterMetadataManager.SaveClusterMetadata(ctx, &persistence.SaveClusterMetadataRequest{ + saveRequest := &persistence.SaveClusterMetadataRequest{ ClusterMetadata: &persistencespb.ClusterMetadata{ ClusterName: resp.GetClusterName(), HistoryShardCount: resp.GetHistoryShardCount(), @@ -671,13 +701,16 @@ func (h *OperatorHandlerImpl) AddOrUpdateRemoteCluster( Tags: resp.GetTags(), }, Version: updateRequestVersion, - }) + } + persistenceRequest = saveRequest + applied, err := h.clusterMetadataManager.SaveClusterMetadata(ctx, saveRequest) if err != nil { return nil, serviceerror.NewInternalf(errUnableToStoreClusterInfo, err) } if !applied { return nil, serviceerror.NewInvalidArgumentf(errUnableToStoreClusterInfo, err) } + lifecycleEvent.emitUpsertSuccess(persistedBefore, persistenceRequest) return &operatorservice.AddOrUpdateRemoteClusterResponse{}, nil } @@ -685,29 +718,43 @@ func (h *OperatorHandlerImpl) RemoveRemoteCluster( ctx context.Context, request *operatorservice.RemoveRemoteClusterRequest, ) (_ *operatorservice.RemoveRemoteClusterResponse, retError error) { + var ( + lifecycleEvent *remoteClusterLifecycleEvent + cachedBefore cachedRemoteClusterLookup + persistenceRequest *persistence.DeleteClusterMetadataRequest + ) + // Bracket deferred emission with panic capture: the inner capture converts handler panics + // into retError for the event, while the outer capture recovers panics from emission itself. + defer log.CapturePanic(h.logger, &retError) + defer func() { + lifecycleEvent.emitRemoveFailure(retError, cachedBefore, persistenceRequest) + }() defer log.CapturePanic(h.logger, &retError) + clusterName := request.GetClusterName() + lifecycleEvent = newRemoteClusterRemoveLifecycleEvent( + ctx, + h.eventLogger, + h.clusterMetadata, + h.config, + remoteClusterAPIOperator, + remoteClusterRemoveRequestFields{ClusterName: clusterName}, + ) - var isClusterNameExist bool - for clusterName := range h.clusterMetadata.GetAllClusterInfo() { - if clusterName == request.GetClusterName() { - isClusterNameExist = true - break - } - } - if !isClusterNameExist { + cachedBefore = lookupCachedRemoteCluster(h.clusterMetadata, clusterName) + if !cachedBefore.found { return nil, serviceerror.NewNotFound("The cluster to be deleted cannot be found in clusters cache.") } - if err := validateClusterNotInUseByNamespaces(h.namespaceRegistry, h.clusterMetadata.GetCurrentClusterName(), request.GetClusterName()); err != nil { + if err := validateClusterNotInUseByNamespaces(h.namespaceRegistry, h.clusterMetadata.GetCurrentClusterName(), clusterName); err != nil { return nil, err } - if err := h.clusterMetadataManager.DeleteClusterMetadata( - ctx, - &persistence.DeleteClusterMetadataRequest{ClusterName: request.GetClusterName()}, - ); err != nil { + deleteRequest := &persistence.DeleteClusterMetadataRequest{ClusterName: clusterName} + persistenceRequest = deleteRequest + if err := h.clusterMetadataManager.DeleteClusterMetadata(ctx, deleteRequest); err != nil { return nil, serviceerror.NewInternalf(errUnableToDeleteClusterInfo, err) } + lifecycleEvent.emitRemoveSuccess(cachedBefore, persistenceRequest) return &operatorservice.RemoveRemoteClusterResponse{}, nil } diff --git a/service/frontend/operator_handler_test.go b/service/frontend/operator_handler_test.go index f15fc147e70..d6b323c2417 100644 --- a/service/frontend/operator_handler_test.go +++ b/service/frontend/operator_handler_test.go @@ -34,6 +34,7 @@ import ( delnserrors "go.temporal.io/server/service/worker/deletenamespace/errors" "go.uber.org/mock/gomock" expmaps "golang.org/x/exp/maps" + "google.golang.org/grpc" "google.golang.org/grpc/health" ) @@ -77,6 +78,7 @@ func (s *operatorHandlerSuite) SetupTest() { args := NewOperatorHandlerImplArgs{ &Config{NumHistoryShards: 4}, s.mockResource.Logger, + nil, s.mockResource.GetSDKClientFactory(), s.mockResource.GetMetricsHandler(), s.mockResource.GetVisibilityManager(), @@ -1144,7 +1146,12 @@ func (s *operatorHandlerSuite) Test_DeleteNamespace() { func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_Success() { var clusterName = "cluster" - s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{clusterName: {}}) + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{ + clusterName: {ClusterID: "cluster-id", Enabled: true, ReplicationEnabled: true}, + }) s.mockResource.NamespaceCache.EXPECT().GetAllNamespaces().Return(nil) s.mockResource.ClusterMetadataMgr.EXPECT().DeleteClusterMetadata( gomock.Any(), @@ -1153,10 +1160,17 @@ func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_Success() { _, err := s.handler.RemoveRemoteCluster(context.Background(), &operatorservice.RemoveRemoteClusterRequest{ClusterName: clusterName}) s.NoError(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeSucceeded, details["outcome"]) + s.Equal(remoteClusterMutationRemoved, details["mutation"]) + s.Equal("cluster-id", details["remote_cluster_id"]) } func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_Error() { var clusterName = "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(false) s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{clusterName: {}}) s.mockResource.NamespaceCache.EXPECT().GetAllNamespaces().Return(nil) s.mockResource.ClusterMetadataMgr.EXPECT().DeleteClusterMetadata( @@ -1166,10 +1180,49 @@ func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_Error() { _, err := s.handler.RemoveRemoteCluster(context.Background(), &operatorservice.RemoveRemoteClusterRequest{ClusterName: clusterName}) s.Error(err) + s.Empty(eventLogger.records) +} + +func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_PanicEmitsFailure() { + clusterName := "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().DoAndReturn(func() map[string]cluster.ClusterInformation { + panic("test panic") + }) + + _, err := s.handler.RemoveRemoteCluster( + context.Background(), + &operatorservice.RemoveRemoteClusterRequest{ClusterName: clusterName}, + ) + s.Require().Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Internal", details["error_code"]) +} + +func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_EventEmissionPanicCaptured() { + clusterName := "cluster" + s.handler.eventLogger = &panicRemoteClusterEventLogger{} + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().DoAndReturn(func() map[string]cluster.ClusterInformation { + panic("handler panic") + }) + + _, err := s.handler.RemoveRemoteCluster( + context.Background(), + &operatorservice.RemoveRemoteClusterRequest{ClusterName: clusterName}, + ) + s.Require().Error(err) + s.Contains(err.Error(), "event logger panic") } func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_BlockedByGlobalNamespace() { var clusterName = "cluster" + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) // The namespace lists both the current cluster and the cluster being // removed, so removing clusterName would sever a link the current cluster // relies on -> blocked. @@ -1189,6 +1242,10 @@ func (s *operatorHandlerSuite) Test_RemoveRemoteCluster_BlockedByGlobalNamespace var failedPrecondition *serviceerror.FailedPrecondition s.Require().ErrorAs(err, &failedPrecondition) s.Contains(err.Error(), "global-ns") + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.NotNil(details["cached_before"]) + s.NotContains(details, "persistence_request") } // Test_RemoveRemoteCluster_OrphanedNamespaceDoesNotBlock covers the case that @@ -1340,6 +1397,9 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_RecordNotFound_Succ var httpAddress = uuid.NewString() var clusterName = uuid.NewString() var clusterID = uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockResource.ClusterMetadata.EXPECT().GetFailoverVersionIncrement().Return(int64(10)).Times(2) s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(make(map[string]cluster.ClusterInformation)) @@ -1377,6 +1437,13 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_RecordNotFound_Succ FrontendAddress: rpcAddress, }) s.NoError(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeSucceeded, details["outcome"]) + s.Equal(remoteClusterMutationCreated, details["mutation"]) + s.Equal(remoteClusterTransitionInitializedDisabled, details["requested_connection_transition"]) + s.Equal(remoteClusterTransitionInitializedDisabled, details["requested_replication_transition"]) + s.Nil(details["persisted_before"]) + s.NotNil(details["persistence_request"]) } func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_ValidationError_ClusterNameConflict() { @@ -1595,6 +1662,9 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_ValidationError_Emp func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_DescribeCluster_Error() { var rpcAddress = uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockResource.ClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( s.mockResource.RemoteAdminClient, @@ -1605,12 +1675,67 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_DescribeCluster_Err ) _, err := s.handler.AddOrUpdateRemoteCluster(context.Background(), &operatorservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}) s.Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Unavailable", details["error_code"]) + s.NotContains(details, "persisted_before") + s.NotContains(details, "persistence_request") +} + +func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_PanicEmitsFailure() { + rpcAddress := uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockResource.ClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( + s.mockResource.RemoteAdminClient, + ) + s.mockResource.RemoteAdminClient.EXPECT().DescribeCluster( + gomock.Any(), + &adminservice.DescribeClusterRequest{}, + ).DoAndReturn(func(context.Context, *adminservice.DescribeClusterRequest, ...grpc.CallOption) (*adminservice.DescribeClusterResponse, error) { + panic("test panic") + }) + + _, err := s.handler.AddOrUpdateRemoteCluster( + context.Background(), + &operatorservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}, + ) + s.Require().Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterOutcomeFailed, details["outcome"]) + s.Equal("Internal", details["error_code"]) +} + +func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_EventEmissionPanicCaptured() { + rpcAddress := uuid.NewString() + s.handler.eventLogger = &panicRemoteClusterEventLogger{} + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + s.mockResource.ClientFactory.EXPECT().NewRemoteAdminClientWithTimeout(rpcAddress, gomock.Any(), gomock.Any()).Return( + s.mockResource.RemoteAdminClient, + ) + s.mockResource.RemoteAdminClient.EXPECT().DescribeCluster( + gomock.Any(), + &adminservice.DescribeClusterRequest{}, + ).DoAndReturn(func(context.Context, *adminservice.DescribeClusterRequest, ...grpc.CallOption) (*adminservice.DescribeClusterResponse, error) { + panic("handler panic") + }) + + _, err := s.handler.AddOrUpdateRemoteCluster( + context.Background(), + &operatorservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}, + ) + s.Require().Error(err) + s.Contains(err.Error(), "event logger panic") } func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_GetClusterMetadata_Error() { var rpcAddress = uuid.NewString() var clusterName = uuid.NewString() var clusterID = uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockResource.ClusterMetadata.EXPECT().GetFailoverVersionIncrement().Return(int64(10)).Times(2) s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(make(map[string]cluster.ClusterInformation)) @@ -1632,6 +1757,9 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_GetClusterMetadata_ ) _, err := s.handler.AddOrUpdateRemoteCluster(context.Background(), &operatorservice.AddOrUpdateRemoteClusterRequest{FrontendAddress: rpcAddress}) s.Error(err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.NotContains(details, "persisted_before") + s.NotContains(details, "persistence_request") } func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_SaveClusterMetadata_Error() { @@ -1683,6 +1811,9 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_SaveClusterMetadata var httpAddress = uuid.NewString() var clusterName = uuid.NewString() var clusterID = uuid.NewString() + eventLogger := &captureRemoteClusterEventLogger{} + s.handler.eventLogger = eventLogger + s.handler.config.EmitNamespaceLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) s.mockResource.ClusterMetadata.EXPECT().GetFailoverVersionIncrement().Return(int64(10)).Times(2) s.mockResource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(make(map[string]cluster.ClusterInformation)) @@ -1721,4 +1852,7 @@ func (s *operatorHandlerSuite) Test_AddOrUpdateRemoteCluster_SaveClusterMetadata }) s.Error(err) s.IsType(&serviceerror.InvalidArgument{}, err) + _, details := remoteClusterEventValues(s.T(), eventLogger.records) + s.Equal(remoteClusterMutationCreated, details["mutation"]) + s.NotNil(details["persistence_request"]) } diff --git a/service/frontend/remote_cluster_lifecycle_events.go b/service/frontend/remote_cluster_lifecycle_events.go new file mode 100644 index 00000000000..4e51c300e61 --- /dev/null +++ b/service/frontend/remote_cluster_lifecycle_events.go @@ -0,0 +1,453 @@ +package frontend + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "maps" + + "go.opentelemetry.io/otel/log" + "go.temporal.io/api/serviceerror" + versionpb "go.temporal.io/api/version/v1" + "go.temporal.io/server/api/adminservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/authorization" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/wideevents" +) + +const ( + remoteClusterLifecycleNotApplicable = "N/A" + + remoteClusterAPIAdmin = "admin" + remoteClusterAPIOperator = "operator" + + remoteClusterOutcomeSucceeded = "succeeded" + remoteClusterOutcomeFailed = "failed" + + remoteClusterMutationCreated = "created" + remoteClusterMutationUpdated = "updated" + remoteClusterMutationRemoved = "removed" + remoteClusterMutationUnknown = "unknown" + + remoteClusterTransitionEnabled = "enabled" + remoteClusterTransitionDisabled = "disabled" + remoteClusterTransitionUnchanged = "unchanged" + remoteClusterTransitionInitializedEnabled = "initialized_enabled" + remoteClusterTransitionInitializedDisabled = "initialized_disabled" +) + +type remoteClusterLifecycleEvent struct { + logger log.Logger + phase string + details map[string]any + upsertRequest remoteClusterUpsertRequestFields + removeRequest remoteClusterRemoveRequestFields +} + +func newRemoteClusterUpsertLifecycleEvent( + ctx context.Context, + logger log.Logger, + clusterMetadata cluster.Metadata, + config *Config, + api string, + request remoteClusterUpsertRequestFields, +) *remoteClusterLifecycleEvent { + event := newRemoteClusterLifecycleEvent( + ctx, + logger, + clusterMetadata, + config, + api, + wideevents.PhaseRemoteClusterUpsert, + request, + ) + if event != nil { + event.upsertRequest = request + } + return event +} + +func newRemoteClusterRemoveLifecycleEvent( + ctx context.Context, + logger log.Logger, + clusterMetadata cluster.Metadata, + config *Config, + api string, + request remoteClusterRemoveRequestFields, +) *remoteClusterLifecycleEvent { + event := newRemoteClusterLifecycleEvent( + ctx, + logger, + clusterMetadata, + config, + api, + wideevents.PhaseRemoteClusterRemove, + request, + ) + if event != nil { + event.removeRequest = request + } + return event +} + +type remoteClusterUpsertRequestFields struct { + FrontendAddress string `json:"frontend_address"` + FrontendHTTPAddress string `json:"frontend_http_address"` + EnableRemoteClusterConnection bool `json:"enable_remote_cluster_connection"` + EnableReplication bool `json:"enable_replication"` +} + +type remoteClusterRemoveRequestFields struct { + ClusterName string `json:"cluster_name"` +} + +type cachedRemoteClusterLookup struct { + information cluster.ClusterInformation + lookupPerformed bool + found bool +} + +type remoteClusterMetadataFields struct { + ClusterName string `json:"cluster_name"` + HistoryShardCount int32 `json:"history_shard_count"` + ClusterID string `json:"cluster_id"` + VersionInfo *versionpb.VersionInfo `json:"version_info"` + IndexSearchAttributes map[string]*persistencespb.IndexSearchAttributes `json:"index_search_attributes"` + ClusterAddress string `json:"cluster_address"` + HTTPAddress string `json:"http_address"` + FailoverVersionIncrement int64 `json:"failover_version_increment"` + InitialFailoverVersion int64 `json:"initial_failover_version"` + IsGlobalNamespaceEnabled bool `json:"is_global_namespace_enabled"` + IsConnectionEnabled bool `json:"is_connection_enabled"` + UseClusterIDMembership bool `json:"use_cluster_id_membership"` + Tags map[string]string `json:"tags"` + IsReplicationEnabled bool `json:"is_replication_enabled"` +} + +type persistedRemoteClusterFields struct { + *remoteClusterMetadataFields + Version int64 `json:"version"` +} + +type cachedRemoteClusterFields struct { + ClusterName string `json:"cluster_name"` + ClusterID string `json:"cluster_id"` + RPCAddress string `json:"rpc_address"` + HTTPAddress string `json:"http_address"` + IsConnectionEnabled bool `json:"is_connection_enabled"` + IsReplicationEnabled bool `json:"is_replication_enabled"` + InitialFailoverVersion int64 `json:"initial_failover_version"` + ShardCount int32 `json:"shard_count"` + Tags map[string]string `json:"tags"` +} + +type saveClusterMetadataFields struct { + ClusterMetadata *remoteClusterMetadataFields `json:"cluster_metadata"` + Version int64 `json:"version"` +} + +type deleteClusterMetadataFields struct { + ClusterName string `json:"cluster_name"` +} + +type remoteClusterRequestFingerprintFields struct { + LocalCluster string `json:"local_cluster"` + API string `json:"api"` + Phase string `json:"phase"` + Request any `json:"request"` +} + +func newRemoteClusterLifecycleEvent( + ctx context.Context, + logger log.Logger, + clusterMetadata cluster.Metadata, + config *Config, + api string, + phase string, + request any, +) *remoteClusterLifecycleEvent { + if !remoteClusterLifecycleEventsEnabled(config) { + return nil + } + localCluster := clusterMetadata.GetCurrentClusterName() + details := map[string]any{ + "api": api, + "local_cluster": localCluster, + "mutation": remoteClusterMutationUnknown, + "request": request, + "request_fingerprint": remoteClusterRequestFingerprint(localCluster, api, phase, request), + } + callerInfo := headers.GetCallerInfo(ctx) + setStringDetailIfNotEmpty(details, "caller_type", callerInfo.CallerType) + setStringDetailIfNotEmpty(details, "call_origin", callerInfo.CallOrigin) + if claims, ok := ctx.Value(authorization.MappedClaims).(*authorization.Claims); ok { + setStringDetailIfNotEmpty(details, "auth_subject", claims.Subject) + setStringDetailIfNotEmpty(details, "auth_type", claims.AuthType) + } + return &remoteClusterLifecycleEvent{ + logger: logger, + phase: phase, + details: details, + } +} + +func remoteClusterLifecycleEventsEnabled(config *Config) bool { + // Remote cluster lifecycle events intentionally share the namespace lifecycle gate. + return config != nil && + config.EmitNamespaceLifecycleEvents != nil && + config.EmitNamespaceLifecycleEvents() +} + +func (e *remoteClusterLifecycleEvent) emitUpsertSuccess( + persistedBefore *persistence.GetClusterMetadataResponse, + saveRequest *persistence.SaveClusterMetadataRequest, +) { + if e == nil { + return + } + if saveRequest != nil { + e.setRemoteCluster(saveRequest.GetClusterName(), saveRequest.GetClusterId()) + } + e.populateUpsertPersistence(persistedBefore, saveRequest) + e.emit(nil) +} + +func (e *remoteClusterLifecycleEvent) emitUpsertFailure( + err error, + remoteResponse *adminservice.DescribeClusterResponse, + persistedBefore *persistence.GetClusterMetadataResponse, + saveRequest *persistence.SaveClusterMetadataRequest, +) { + if e == nil || err == nil { + return + } + if remoteResponse != nil { + e.setRemoteCluster(remoteResponse.GetClusterName(), remoteResponse.GetClusterId()) + } else if saveRequest != nil { + e.setRemoteCluster(saveRequest.GetClusterName(), saveRequest.GetClusterId()) + } + e.populateUpsertPersistence(persistedBefore, saveRequest) + e.emit(err) +} + +func (e *remoteClusterLifecycleEvent) populateUpsertPersistence( + persistedBefore *persistence.GetClusterMetadataResponse, + saveRequest *persistence.SaveClusterMetadataRequest, +) { + // A nil persistedBefore only means the mutation is a create once the save request exists. + // If both are nil, the handler failed before determining the persistence mutation. + persistenceMutationKnown := persistedBefore != nil || saveRequest != nil + if !persistenceMutationKnown { + return + } + if persistedBefore == nil { + e.setUpsertCreated() + } else { + e.setUpsertUpdated(persistedBefore) + } + if saveRequest != nil { + e.setSaveRequest(saveRequest) + } +} + +func (e *remoteClusterLifecycleEvent) emitRemoveSuccess( + cachedBefore cachedRemoteClusterLookup, + deleteRequest *persistence.DeleteClusterMetadataRequest, +) { + if e == nil { + return + } + e.populateRemove(cachedBefore, deleteRequest) + e.setRemoved() + e.emit(nil) +} + +func (e *remoteClusterLifecycleEvent) emitRemoveFailure( + err error, + cachedBefore cachedRemoteClusterLookup, + deleteRequest *persistence.DeleteClusterMetadataRequest, +) { + if e == nil || err == nil { + return + } + e.populateRemove(cachedBefore, deleteRequest) + e.emit(err) +} + +func (e *remoteClusterLifecycleEvent) populateRemove( + cachedBefore cachedRemoteClusterLookup, + deleteRequest *persistence.DeleteClusterMetadataRequest, +) { + e.setRemoteCluster(e.removeRequest.ClusterName, "") + if cachedBefore.lookupPerformed { + e.setCachedBefore(e.removeRequest.ClusterName, cachedBefore.information, cachedBefore.found) + } + if deleteRequest != nil { + e.setDeleteRequest(deleteRequest) + } +} + +func lookupCachedRemoteCluster( + clusterMetadata cluster.Metadata, + clusterName string, +) cachedRemoteClusterLookup { + information, found := clusterMetadata.GetAllClusterInfo()[clusterName] + return cachedRemoteClusterLookup{ + information: information, + lookupPerformed: true, + found: found, + } +} + +func (e *remoteClusterLifecycleEvent) emit(err error) { + if err == nil { + e.details["outcome"] = remoteClusterOutcomeSucceeded + } else { + e.details["outcome"] = remoteClusterOutcomeFailed + e.details["error_code"] = serviceerror.ToStatus(err).Code().String() + e.details["error_type"] = fmt.Sprintf("%T", err) + e.details["error"] = err.Error() + } + wideevents.Emit(e.logger, wideevents.RemoteClusterLifecyclePayload{ + Phase: e.phase, + Namespace: remoteClusterLifecycleNotApplicable, + NamespaceID: remoteClusterLifecycleNotApplicable, + Details: e.details, + }) +} + +func (e *remoteClusterLifecycleEvent) setRemoteCluster(clusterName string, clusterID string) { + setStringDetailIfNotEmpty(e.details, "remote_cluster", clusterName) + setStringDetailIfNotEmpty(e.details, "remote_cluster_id", clusterID) +} + +func (e *remoteClusterLifecycleEvent) setUpsertCreated() { + e.details["persisted_before"] = nil + e.details["mutation"] = remoteClusterMutationCreated + e.details["requested_connection_transition"] = transitionForCreate( + e.upsertRequest.EnableRemoteClusterConnection, + ) + e.details["requested_replication_transition"] = transitionForCreate( + e.upsertRequest.EnableReplication, + ) +} + +func (e *remoteClusterLifecycleEvent) setUpsertUpdated( + persistedBefore *persistence.GetClusterMetadataResponse, +) { + e.details["persisted_before"] = persistedRemoteClusterFields{ + remoteClusterMetadataFields: clusterMetadataEventFields(persistedBefore.ClusterMetadata), + Version: persistedBefore.Version, + } + e.details["mutation"] = remoteClusterMutationUpdated + e.details["requested_connection_transition"] = transitionForUpdate( + persistedBefore.GetIsConnectionEnabled(), + e.upsertRequest.EnableRemoteClusterConnection, + ) + e.details["requested_replication_transition"] = transitionForUpdate( + persistedBefore.GetIsReplicationEnabled(), + e.upsertRequest.EnableReplication, + ) +} + +func (e *remoteClusterLifecycleEvent) setSaveRequest(request *persistence.SaveClusterMetadataRequest) { + e.details["persistence_request"] = saveClusterMetadataFields{ + ClusterMetadata: clusterMetadataEventFields(request.ClusterMetadata), + Version: request.Version, + } +} + +func (e *remoteClusterLifecycleEvent) setCachedBefore( + clusterName string, + info cluster.ClusterInformation, + found bool, +) { + e.setRemoteCluster(clusterName, info.ClusterID) + if !found { + e.details["cached_before"] = nil + return + } + e.details["cached_before"] = cachedRemoteClusterFields{ + ClusterName: clusterName, + ClusterID: info.ClusterID, + RPCAddress: info.RPCAddress, + HTTPAddress: info.HTTPAddress, + IsConnectionEnabled: info.Enabled, + IsReplicationEnabled: info.ReplicationEnabled, + InitialFailoverVersion: info.InitialFailoverVersion, + ShardCount: info.ShardCount, + Tags: maps.Clone(info.Tags), + } +} + +func (e *remoteClusterLifecycleEvent) setDeleteRequest(request *persistence.DeleteClusterMetadataRequest) { + e.details["persistence_request"] = deleteClusterMetadataFields{ClusterName: request.ClusterName} +} + +func (e *remoteClusterLifecycleEvent) setRemoved() { + e.details["mutation"] = remoteClusterMutationRemoved +} + +func remoteClusterRequestFingerprint(localCluster string, api string, phase string, request any) string { + encoded, err := json.Marshal(remoteClusterRequestFingerprintFields{ + LocalCluster: localCluster, + API: api, + Phase: phase, + Request: request, + }) + if err != nil { + return "" + } + return fmt.Sprintf("%x", sha256.Sum256(encoded)) +} + +func clusterMetadataEventFields( + metadata *persistencespb.ClusterMetadata, +) *remoteClusterMetadataFields { + if metadata == nil { + return &remoteClusterMetadataFields{} + } + return &remoteClusterMetadataFields{ + ClusterName: metadata.GetClusterName(), + HistoryShardCount: metadata.GetHistoryShardCount(), + ClusterID: metadata.GetClusterId(), + VersionInfo: metadata.GetVersionInfo(), + IndexSearchAttributes: maps.Clone(metadata.GetIndexSearchAttributes()), + ClusterAddress: metadata.GetClusterAddress(), + HTTPAddress: metadata.GetHttpAddress(), + FailoverVersionIncrement: metadata.GetFailoverVersionIncrement(), + InitialFailoverVersion: metadata.GetInitialFailoverVersion(), + IsGlobalNamespaceEnabled: metadata.GetIsGlobalNamespaceEnabled(), + IsConnectionEnabled: metadata.GetIsConnectionEnabled(), + UseClusterIDMembership: metadata.GetUseClusterIdMembership(), + Tags: maps.Clone(metadata.GetTags()), + IsReplicationEnabled: metadata.GetIsReplicationEnabled(), + } +} + +func transitionForCreate(enabled bool) string { + if enabled { + return remoteClusterTransitionInitializedEnabled + } + return remoteClusterTransitionInitializedDisabled +} + +func transitionForUpdate(before bool, requested bool) string { + if before == requested { + return remoteClusterTransitionUnchanged + } + if requested { + return remoteClusterTransitionEnabled + } + return remoteClusterTransitionDisabled +} + +func setStringDetailIfNotEmpty(details map[string]any, key string, value string) { + if value != "" { + details[key] = value + } +} diff --git a/service/frontend/remote_cluster_lifecycle_events_test.go b/service/frontend/remote_cluster_lifecycle_events_test.go new file mode 100644 index 00000000000..0a9b29c1286 --- /dev/null +++ b/service/frontend/remote_cluster_lifecycle_events_test.go @@ -0,0 +1,307 @@ +package frontend + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/stretchr/testify/require" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" + "go.temporal.io/server/api/adminservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/authorization" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/wideevents" + "go.uber.org/mock/gomock" +) + +type captureRemoteClusterEventLogger struct { + embedded.Logger + records []otellog.Record +} + +type panicRemoteClusterEventLogger struct { + embedded.Logger +} + +func (l *captureRemoteClusterEventLogger) Emit(_ context.Context, record otellog.Record) { + l.records = append(l.records, record) +} + +func (l *captureRemoteClusterEventLogger) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + +func (*panicRemoteClusterEventLogger) Emit(context.Context, otellog.Record) { + panic("event logger panic") +} + +func (*panicRemoteClusterEventLogger) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + +func TestRemoteClusterLifecycleUpsertCreated(t *testing.T) { + ctrl := gomock.NewController(t) + metadata := cluster.NewMockMetadata(ctrl) + metadata.EXPECT().GetCurrentClusterName().Return("cluster-a") + logger := &captureRemoteClusterEventLogger{} + ctx := headers.SetCallerInfo(context.Background(), headers.NewCallerInfo("", "operator", "AddOrUpdateRemoteCluster")) + ctx = context.WithValue(ctx, authorization.MappedClaims, &authorization.Claims{ + Subject: "alice", + AuthType: "jwt", + }) + request := remoteClusterUpsertRequestFields{ + FrontendAddress: "cluster-b:7233", + FrontendHTTPAddress: "cluster-b:7243", + EnableRemoteClusterConnection: true, + EnableReplication: true, + } + event := newRemoteClusterUpsertLifecycleEvent( + ctx, + logger, + metadata, + remoteClusterLifecycleTestConfig(true), + remoteClusterAPIOperator, + request, + ) + saveRequest := &persistence.SaveClusterMetadataRequest{ + ClusterMetadata: &persistencespb.ClusterMetadata{ + ClusterName: "cluster-b", + ClusterId: "cluster-b-id", + ClusterAddress: "cluster-b:7233", + IsConnectionEnabled: true, + IsReplicationEnabled: true, + Tags: map[string]string{"environment": "test"}, + }, + Version: 0, + } + event.emitUpsertSuccess(nil, saveRequest) + + attrs, details := remoteClusterEventValues(t, logger.records) + require.Equal(t, wideevents.PhaseRemoteClusterUpsert, attrs["phase"]) + require.Equal(t, "N/A", attrs["namespace"]) + require.Equal(t, "N/A", attrs["namespace_id"]) + require.Equal(t, remoteClusterOutcomeSucceeded, details["outcome"]) + require.Equal(t, remoteClusterMutationCreated, details["mutation"]) + require.Equal(t, remoteClusterTransitionInitializedEnabled, details["requested_connection_transition"]) + require.Equal(t, remoteClusterTransitionInitializedEnabled, details["requested_replication_transition"]) + require.Equal(t, "cluster-b", details["remote_cluster"]) + require.Equal(t, "cluster-b-id", details["remote_cluster_id"]) + require.Nil(t, details["persisted_before"]) + require.Equal(t, "operator", details["caller_type"]) + require.Equal(t, "AddOrUpdateRemoteCluster", details["call_origin"]) + require.Equal(t, "alice", details["auth_subject"]) + require.Equal(t, "jwt", details["auth_type"]) + require.NotEmpty(t, details["request_fingerprint"]) + require.NotNil(t, details["persistence_request"]) +} + +func TestRemoteClusterLifecycleUpsertUpdatedFailure(t *testing.T) { + ctrl := gomock.NewController(t) + metadata := cluster.NewMockMetadata(ctrl) + metadata.EXPECT().GetCurrentClusterName().Return("cluster-a") + logger := &captureRemoteClusterEventLogger{} + event := newRemoteClusterUpsertLifecycleEvent( + context.Background(), + logger, + metadata, + remoteClusterLifecycleTestConfig(true), + remoteClusterAPIAdmin, + remoteClusterUpsertRequestFields{ + EnableRemoteClusterConnection: false, + EnableReplication: true, + }, + ) + persistedBefore := &persistence.GetClusterMetadataResponse{ + ClusterMetadata: &persistencespb.ClusterMetadata{ + IsConnectionEnabled: true, + IsReplicationEnabled: false, + }, + Version: 7, + } + remoteResponse := &adminservice.DescribeClusterResponse{ + ClusterName: "cluster-b", + ClusterId: "cluster-b-id", + } + retError := errors.New("save failed") + event.emitUpsertFailure(retError, remoteResponse, persistedBefore, nil) + + _, details := remoteClusterEventValues(t, logger.records) + require.Equal(t, remoteClusterOutcomeFailed, details["outcome"]) + require.Equal(t, remoteClusterMutationUpdated, details["mutation"]) + require.Equal(t, remoteClusterTransitionDisabled, details["requested_connection_transition"]) + require.Equal(t, remoteClusterTransitionEnabled, details["requested_replication_transition"]) + require.Equal(t, "cluster-b", details["remote_cluster"]) + require.Equal(t, "cluster-b-id", details["remote_cluster_id"]) + require.NotContains(t, details, "failure_stage") + require.Equal(t, "Unknown", details["error_code"]) + require.Equal(t, "save failed", details["error"]) + persistedBeforeDetails, ok := details["persisted_before"].(map[string]any) + require.True(t, ok) + require.InDelta(t, 7, persistedBeforeDetails["version"], 0) +} + +func TestRemoteClusterLifecycleRemoveCachedBefore(t *testing.T) { + ctrl := gomock.NewController(t) + metadata := cluster.NewMockMetadata(ctrl) + metadata.EXPECT().GetCurrentClusterName().Return("cluster-a") + logger := &captureRemoteClusterEventLogger{} + event := newRemoteClusterRemoveLifecycleEvent( + context.Background(), + logger, + metadata, + remoteClusterLifecycleTestConfig(true), + remoteClusterAPIOperator, + remoteClusterRemoveRequestFields{ClusterName: "cluster-b"}, + ) + cachedBefore := cluster.ClusterInformation{ + Enabled: true, + ReplicationEnabled: true, + ClusterID: "cluster-b-id", + Tags: map[string]string{"environment": "test"}, + } + deleteRequest := &persistence.DeleteClusterMetadataRequest{ClusterName: "cluster-b"} + event.emitRemoveSuccess(cachedRemoteClusterLookup{ + information: cachedBefore, + lookupPerformed: true, + found: true, + }, deleteRequest) + + _, details := remoteClusterEventValues(t, logger.records) + require.Equal(t, remoteClusterMutationRemoved, details["mutation"]) + require.Equal(t, "cluster-b", details["remote_cluster"]) + require.Equal(t, "cluster-b-id", details["remote_cluster_id"]) + require.NotNil(t, details["cached_before"]) + require.NotNil(t, details["persistence_request"]) + require.NotContains(t, details, "requested_connection_transition") + require.NotContains(t, details, "requested_replication_transition") +} + +func TestRemoteClusterLifecycleRemoveCacheMiss(t *testing.T) { + ctrl := gomock.NewController(t) + metadata := cluster.NewMockMetadata(ctrl) + metadata.EXPECT().GetCurrentClusterName().Return("cluster-a") + logger := &captureRemoteClusterEventLogger{} + event := newRemoteClusterRemoveLifecycleEvent( + context.Background(), + logger, + metadata, + remoteClusterLifecycleTestConfig(true), + remoteClusterAPIAdmin, + remoteClusterRemoveRequestFields{ClusterName: "missing-cluster"}, + ) + retError := errors.New("blocked") + event.emitRemoveFailure(retError, cachedRemoteClusterLookup{lookupPerformed: true}, nil) + + _, details := remoteClusterEventValues(t, logger.records) + require.Nil(t, details["cached_before"]) + require.Equal(t, remoteClusterMutationUnknown, details["mutation"]) + require.NotContains(t, details, "persistence_request") +} + +func TestRemoteClusterLifecycleRequestFingerprint(t *testing.T) { + request := remoteClusterUpsertRequestFields{ + FrontendAddress: "cluster-b:7233", + EnableRemoteClusterConnection: true, + } + operatorFingerprint := remoteClusterRequestFingerprint( + "cluster-a", + remoteClusterAPIOperator, + wideevents.PhaseRemoteClusterUpsert, + request, + ) + + require.Equal(t, operatorFingerprint, remoteClusterRequestFingerprint( + "cluster-a", + remoteClusterAPIOperator, + wideevents.PhaseRemoteClusterUpsert, + request, + )) + require.NotEqual(t, operatorFingerprint, remoteClusterRequestFingerprint( + "cluster-a", + remoteClusterAPIAdmin, + wideevents.PhaseRemoteClusterUpsert, + request, + )) +} + +func TestRemoteClusterLifecycleEventsEnabled(t *testing.T) { + require.False(t, remoteClusterLifecycleEventsEnabled(nil)) + require.False(t, remoteClusterLifecycleEventsEnabled(&Config{})) + require.False(t, remoteClusterLifecycleEventsEnabled(&Config{ + EmitNamespaceLifecycleEvents: dynamicconfig.GetBoolPropertyFn(false), + })) + require.True(t, remoteClusterLifecycleEventsEnabled(&Config{ + EmitNamespaceLifecycleEvents: dynamicconfig.GetBoolPropertyFn(true), + })) +} + +func TestRemoteClusterLifecycleDisabledEmission(t *testing.T) { + config := remoteClusterLifecycleTestConfig(false) + upsertEvent := newRemoteClusterUpsertLifecycleEvent( + context.Background(), + nil, + nil, + config, + remoteClusterAPIOperator, + remoteClusterUpsertRequestFields{}, + ) + removeEvent := newRemoteClusterRemoveLifecycleEvent( + context.Background(), + nil, + nil, + config, + remoteClusterAPIOperator, + remoteClusterRemoveRequestFields{}, + ) + require.Nil(t, upsertEvent) + require.Nil(t, removeEvent) + require.NotPanics(t, func() { + upsertEvent.emitUpsertSuccess(nil, nil) + upsertEvent.emitUpsertFailure(errors.New("failed"), nil, nil, nil) + removeEvent.emitRemoveSuccess(cachedRemoteClusterLookup{}, nil) + removeEvent.emitRemoveFailure(errors.New("failed"), cachedRemoteClusterLookup{}, nil) + }) +} + +func TestNewRemoteClusterLifecycleEventDisabled(t *testing.T) { + ctrl := gomock.NewController(t) + metadata := cluster.NewMockMetadata(ctrl) + + event := newRemoteClusterUpsertLifecycleEvent( + context.Background(), + &captureRemoteClusterEventLogger{}, + metadata, + remoteClusterLifecycleTestConfig(false), + remoteClusterAPIOperator, + remoteClusterUpsertRequestFields{}, + ) + + require.Nil(t, event) +} + +func remoteClusterLifecycleTestConfig(enabled bool) *Config { + return &Config{EmitNamespaceLifecycleEvents: dynamicconfig.GetBoolPropertyFn(enabled)} +} + +func remoteClusterEventValues( + t *testing.T, + records []otellog.Record, +) (map[string]string, map[string]any) { + t.Helper() + require.Len(t, records, 1) + require.Equal(t, wideevents.RemoteClusterLifecycleEventName, records[0].EventName()) + attrs := make(map[string]string) + records[0].WalkAttributes(func(kv otellog.KeyValue) bool { + attrs[kv.Key] = kv.Value.AsString() + return true + }) + var details map[string]any + require.NoError(t, json.Unmarshal([]byte(attrs["details"]), &details)) + return attrs, details +} From 767162845ef95cb5e4f7157fb980f9c5ce4cbf12 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 13:35:41 -0700 Subject: [PATCH 029/139] Adaptive test timeouts via Await (#10417) ## What changed? Adds `testcontext.EnsureRemaining` and has `await` use it so long await calls can request additional test-scoped context time while still respecting the test context cap. ## Why? Await calls can need more time than the default test context has left (esp after the environment setup). Extending the test timeout in this way allows for (1) stuck tests to fail earlier than the default test timeout and (2) legitimately longer running tests to pass without manually tweaking the test timeout. --------- Co-authored-by: Sean Kane --- common/testing/await/require_ctx.go | 45 ++- common/testing/await/require_ctx_test.go | 11 +- common/testing/parallelsuite/suite.go | 19 +- common/testing/parallelsuite/suite_test.go | 3 + common/testing/testcontext/context.go | 370 ++++++++++++++----- common/testing/testcontext/context_test.go | 396 ++++++++++++++++++++- tests/activity_driver.go | 6 +- tests/activity_driver_test.go | 42 +++ tests/activity_parity_test.go | 4 +- tests/activity_standalone_driver.go | 58 +-- tests/activity_standalone_test.go | 147 ++++---- tests/activity_workflow_driver.go | 57 +-- tests/schedule_migration_test.go | 7 +- tests/testcore/test_env.go | 18 +- 14 files changed, 920 insertions(+), 263 deletions(-) diff --git a/common/testing/await/require_ctx.go b/common/testing/await/require_ctx.go index ad519822499..0ac73763dc6 100644 --- a/common/testing/await/require_ctx.go +++ b/common/testing/await/require_ctx.go @@ -3,9 +3,11 @@ package await import ( "context" "fmt" - "os" "testing" "time" + + "go.temporal.io/server/common/testing/testcontext" + "go.temporal.io/server/common/util" ) const requireMisuseHint = "use the *await.T passed to the callback, not s.T() or suite assertion methods" @@ -20,12 +22,7 @@ const softDeadlockTimeoutEnvVar = "TEMPORAL_AWAIT_SOFT_DEADLOCK_TIMEOUT" const defaultSoftDeadlockTimeout = 30 * time.Second func softDeadlockTimeout() time.Duration { - if s := os.Getenv(softDeadlockTimeoutEnvVar); s != "" { - if d, err := time.ParseDuration(s); err == nil { - return d - } - } - return defaultSoftDeadlockTimeout + return envDuration(softDeadlockTimeoutEnvVar, defaultSoftDeadlockTimeout) } // hardDeadlockTimeoutEnvVar overrides the default hard-deadlock timeout. @@ -40,14 +37,12 @@ const hardDeadlockTimeoutEnvVar = "TEMPORAL_AWAIT_HARD_DEADLOCK_TIMEOUT" const defaultHardDeadlockTimeout = 10 * time.Second func hardDeadlockTimeout() time.Duration { - if s := os.Getenv(hardDeadlockTimeoutEnvVar); s != "" { - if d, err := time.ParseDuration(s); err == nil { - return d - } - } - return defaultHardDeadlockTimeout + return envDuration(hardDeadlockTimeoutEnvVar, defaultHardDeadlockTimeout) } +// postAwaitTimeoutReserve is the minimum time to keep for *after* Await returns. +const postAwaitTimeoutReserve = 10 * time.Second + // Require polls condition until it returns without assertion failures, or // until ctx is canceled or timeout expires (whichever is earliest). // @@ -87,26 +82,26 @@ func run( return } - deadline := time.Now().Add(cfg.totalTimeout) + // Ensure enough context time for the await itself plus post-await reserve. + // This only works for [testcontext]s; other contexts will be left unchanged. + parentCtx = testcontext.EnsureRemaining(parentCtx, tb, cfg.totalTimeout+postAwaitTimeoutReserve) - // Cap at the parent context's deadline if it's earlier than our timeout. - if parentDeadline, hasDeadline := parentCtx.Deadline(); hasDeadline && parentDeadline.Before(deadline) { - deadline = parentDeadline + deadline := time.Now().Add(cfg.totalTimeout) + if parentDeadline, hasDeadline := parentCtx.Deadline(); hasDeadline { + // Cap at the parent context's deadline if it's earlier than our timeout. + deadline = util.MinTime(deadline, parentDeadline) } // Cap at the test's deadline if it's earlier than our deadline. // Ideally, the parent context already accounts for the test's deadline - but we are being defensive. - if d, ok := tb.(interface{ Deadline() (time.Time, bool) }); ok { - if testDeadline, hasDeadline := d.Deadline(); hasDeadline && testDeadline.Before(deadline) { - deadline = testDeadline - } + if testDeadline, hasDeadline := testcontext.GoTestDeadline(tb); hasDeadline { + deadline = util.MinTime(deadline, testDeadline) } - effectiveTimeout := max(0, time.Until(deadline)) awaitCtx, awaitCancel := context.WithDeadline(parentCtx, deadline) defer awaitCancel() - report := timeoutReport{effectiveTimeout: effectiveTimeout} + report := timeoutReport{effectiveTimeout: max(0, time.Until(deadline))} for { // Parent context was canceled while we were sleeping (not our deadline). @@ -143,8 +138,8 @@ func run( report.recordErrors(t.errors) // Attempt-timeout expiry: attemptCtx is done but awaitCtx is not. - // Record nothing special - the attempt's recorded errors (if any) - // already describe what went wrong; otherwise we just retry. + // An attempt timeout is retryable while the await is still active. Track + // it separately so the final report identifies the responsible timeout. attemptHitOwnTimeout := attemptCtx.Err() == context.DeadlineExceeded && awaitCtx.Err() == nil if attemptHitOwnTimeout { report.recordAttemptTimeout() diff --git a/common/testing/await/require_ctx_test.go b/common/testing/await/require_ctx_test.go index 1872ea28571..30df57af4d6 100644 --- a/common/testing/await/require_ctx_test.go +++ b/common/testing/await/require_ctx_test.go @@ -100,8 +100,7 @@ func TestRequire_PropagatesParentContextValues(t *testing.T) { func TestRequire_SetsTimeoutContextDeadline(t *testing.T) { t.Parallel() - longCtx, cancel := context.WithTimeout(testcontext.For(t), time.Minute) - defer cancel() + longCtx := testcontext.For(t) longDeadline, ok := longCtx.Deadline() require.True(t, ok) @@ -233,12 +232,12 @@ func TestRequire_FailureScenarios(t *testing.T) { t.Run("caps attempt context with parent deadline", func(t *testing.T) { t.Parallel() - parentCtx, cancel := context.WithTimeout(testcontext.For(t), time.Second) + ctx, cancel := context.WithTimeout(testcontext.For(t), time.Second) defer cancel() tb := newRecordingTB() tb.run(func() { - await.Require(parentCtx, tb, func(t *await.T) { + await.Require(ctx, tb, func(t *await.T) { deadline, ok := t.Context().Deadline() if !ok { t.Error("missing deadline") @@ -259,13 +258,13 @@ func TestRequire_FailureScenarios(t *testing.T) { t.Run("parent context cancellation stops polling", func(t *testing.T) { t.Parallel() - parentCtx, cancel := context.WithCancel(testcontext.For(t)) + ctx, cancel := context.WithCancel(testcontext.For(t)) defer cancel() var attempts atomic.Int32 tb := newRecordingTB() tb.run(func() { - await.Require(parentCtx, tb, func(t *await.T) { + await.Require(ctx, tb, func(t *await.T) { attempts.Add(1) t.Error("not ready") cancel() diff --git a/common/testing/parallelsuite/suite.go b/common/testing/parallelsuite/suite.go index d2887fccf3a..93eb09d9257 100644 --- a/common/testing/parallelsuite/suite.go +++ b/common/testing/parallelsuite/suite.go @@ -7,7 +7,6 @@ import ( "reflect" "regexp" "strings" - "sync" "testing" "time" @@ -64,13 +63,12 @@ type Suite[T testingSuite] struct { guardT guardT runParallel bool assertT require.TestingT - ctx context.Context // override set in initSuite; lazy-filled by Context() under ctxOnce when nil - ctxOnce sync.Once + ctx context.Context // override set in initSuite; nil means "use the test context" } // copySuite creates a fresh suite instance initialized for the given *testing.T. // assertT overrides which TestingT assertions are bound to; nil means use the copy's own guardT. -// ctx overrides the suite's context; nil means use the default (lazy testcontext.For). +// ctx overrides the suite's context; nil means use the test context (see [Suite.Context]). // //nolint:revive // ctx is last so callers can pass nil to mean "no override"; SA1012 forbids passing nil as the first ctx arg. func (s *Suite[T]) copySuite(t *testing.T, parallel bool, assertT require.TestingT, ctx context.Context) testingSuite { @@ -87,7 +85,6 @@ func (s *Suite[T]) initSuite(t *testing.T, parallel bool, assertT require.Testin g.hasSubtests.Store(false) s.runParallel = parallel s.ctx = ctx - s.ctxOnce = sync.Once{} if s.runParallel { t.Parallel() //nolint:testifylint // parallelsuite intentionally supports parallel tests } @@ -125,13 +122,13 @@ func (s *Suite[T]) AssertionT() require.TestingT { // Context returns the test-scoped context (created from [testcontext]). // Inside an [Await] callback, it returns the await-scoped context. +// +// The result is deliberately not cached; see [testcontext.EnsureRemaining]. func (s *Suite[T]) Context() context.Context { - s.ctxOnce.Do(func() { - if s.ctx == nil { - s.ctx = testcontext.For(s.T()) - } - }) - return s.ctx + if s.ctx != nil { + return s.ctx + } + return testcontext.For(s.T()) } // Run creates a parallel subtest. The callback receives a fresh copy of the diff --git a/common/testing/parallelsuite/suite_test.go b/common/testing/parallelsuite/suite_test.go index 4a88822ca3e..6c0bbecf4d8 100644 --- a/common/testing/parallelsuite/suite_test.go +++ b/common/testing/parallelsuite/suite_test.go @@ -115,6 +115,9 @@ func (s *sealAfterRunSuite) TestAssertionAfterRun() { // T() also panics after Run. require.Panics(t, func() { s.T() }) + + // Context() resolves through T() and also panics after Run. + require.Panics(t, func() { s.Context() }) } func TestRun_AcceptsSuite(t *testing.T) { diff --git a/common/testing/testcontext/context.go b/common/testing/testcontext/context.go index 3a6cdde5d4f..ba5e0e81f50 100644 --- a/common/testing/testcontext/context.go +++ b/common/testing/testcontext/context.go @@ -3,19 +3,32 @@ package testcontext import ( "context" "os" + "slices" "sync" "testing" "time" "go.temporal.io/server/common/debug" + "go.temporal.io/server/common/util" "google.golang.org/grpc/metadata" ) const ( - defaultTimeout = 90 * time.Second + defaultTimeout = 90 * time.Second + // maxTimeout caps the *total* lifetime of a test context that uses the + // default or TEMPORAL_TEST_TIMEOUT-configured timeout, measured from its + // creation - it is not a per-extension budget. [EnsureRemaining] may + // extend such a context until createdAt+max(timeout, maxTimeout) and no + // further. + // + // A timeout requested via [WithTimeout] is its own ceiling and is never + // extended beyond it. + maxTimeout = 2 * time.Minute testNameMetadataKey = "temporal-test-name" + testTimeoutEnvVar = "TEMPORAL_TEST_TIMEOUT" ) +// contextStore tracks one context state per test. type contextStore struct { sync.Mutex byTest map[testing.TB]*contextState @@ -27,165 +40,329 @@ var testContexts = contextStore{ byTest: make(map[testing.TB]*contextState), } +// config records options for creating a test context. type config struct { - timeout time.Duration - timeoutSet bool + // timeout is the explicitly requested timeout, or zero to use the default. + timeout time.Duration } +// contextDecorator records a keyed transformation to replay on replacement contexts. type contextDecorator struct { key any decorate func(context.Context) context.Context } -// DefaultTimeout returns the effective default timeout for test-scoped contexts. +// ownerKey marks a context as belonging to a test's context chain. Context +// values are inherited, so any context derived from a test context - including +// an outdated one - carries the mark too. +type ownerKey struct{} + +// testContext wraps every context this package hands out, so [EnsureRemaining] +// can tell one apart from a context the caller derived further (adding its own +// deadline, cancellation, or values), which is not safe to replace. +type testContext struct { + context.Context +} + +// GoTestDeadline returns the deadline imposed by `go test -timeout`, if any. +// +// It is a hard external cap for any test-scoped deadline: crossing it panics +// the whole test binary - taking every sibling test with it - instead of +// failing just this test. +// +// NOTE: tb.Context() does NOT carry this deadline; only [testing.T.Deadline] has it. +func GoTestDeadline(tb testing.TB) (deadline time.Time, ok bool) { + // testing.TB does not expose Deadline, but *testing.T and *testing.B do. + d, hasDeadline := tb.(interface{ Deadline() (time.Time, bool) }) + if !hasDeadline { + return time.Time{}, false + } + // Inside a synctest bubble the clock is fake and Deadline panics; a + // real-clock deadline would be meaningless there anyway. + defer func() { _ = recover() }() + return d.Deadline() +} + +// newTestContext creates a context for st that expires at deadline, capped by +// the `go test -timeout` deadline. +func newTestContext(tb testing.TB, st *contextState, deadline time.Time) (context.Context, context.CancelFunc) { + if goTestDeadline, ok := GoTestDeadline(tb); ok { + deadline = util.MinTime(deadline, goTestDeadline) + } + + ctx, cancel := context.WithDeadline(tb.Context(), deadline) + ctx = context.WithValue(ctx, ownerKey{}, st) + + // Annotate gRPC requests with the test name for OTEL tracing. + ctx = metadata.AppendToOutgoingContext(ctx, testNameMetadataKey, tb.Name()) + + // Apply context decorators, in order. + for _, decorator := range st.decorators { + ctx = decorator.decorate(ctx) + } + + return &testContext{Context: ctx}, cancel +} + +// DefaultTimeout returns the effective default timeout for test contexts. func DefaultTimeout() time.Duration { - return effectiveTimeout(0) + timeout, _ := effectiveTimeout(0) + return timeout } -// For returns the test-scoped context for tb. The context is canceled -// when the test ends or when the configured test timeout expires. +// For returns the test context for tb. The context is canceled when the test +// ends or when the configured test timeout expires. +// +// The first call creates the test context and fixes its timeout. Later calls +// return the current context, but an explicit different timeout fails instead +// of being silently ignored. // -// The first call creates the per-test context and fixes its timeout. Later calls -// return the same context, but an explicit different timeout fails instead of -// being silently ignored. +// The result is not worth caching: [EnsureRemaining] replaces the context when +// it extends the deadline, so a stored copy keeps the old, shorter one. func For(tb testing.TB, opts ...Option) context.Context { tb.Helper() - cfg := config{timeout: DefaultTimeout()} + var cfg config for _, opt := range opts { opt(&cfg) } - st := getContextState(tb, cfg.timeout) - st.configure(tb, cfg) - return st.context() + st := getOrCreateContextState(tb, cfg) + st.mu.Lock() + defer st.mu.Unlock() + return st.current } -// Option configures the test-scoped context returned by [For]. +// Option configures the test context returned by [For]. type Option func(*config) -// WithTimeout sets a custom timeout for the test-scoped context. +// WithTimeout sets a custom timeout for the test context. An explicitly +// requested timeout is also the ceiling for [EnsureRemaining]. func WithTimeout(timeout time.Duration) Option { return func(cfg *config) { if timeout <= 0 { return } - cfg.timeout = effectiveTimeout(timeout) - cfg.timeoutSet = true + cfg.timeout = timeout } } -// AttachDecorator applies decorator to the test-scoped context once for key. +// AttachDecorator applies decorator to the test context once for key. // Reusing the same key is a no-op. If the test context does not exist yet, // AttachDecorator creates it with the default timeout. Call [For] with [WithTimeout] // first when using a custom timeout. func AttachDecorator[K comparable](tb testing.TB, key K, decorator func(context.Context) context.Context) { tb.Helper() - st := getContextState(tb, DefaultTimeout()) - st.attachDecorator(tb, contextDecorator{ + if any(key) == nil { + tb.Fatal("testcontext: context decorator key must not be nil") + return + } + if decorator == nil { + tb.Fatal("testcontext: context decorator must not be nil") + return + } + + st := getOrCreateContextState(tb, config{}) + + // Decorators may be registered by independent helpers, so apply each keyed + // decorator at most once while preserving call order. + st.mu.Lock() + defer st.mu.Unlock() + + for _, existing := range st.decorators { + if existing.key == key { + return + } + } + next := contextDecorator{ key: key, decorate: decorator, - }) -} - -type contextState struct { - mu sync.Mutex - ctx context.Context - cancel context.CancelFunc - timeout time.Duration - decorators map[any]struct{} + } + st.current = &testContext{Context: next.decorate(st.current)} + st.decorators = append(st.decorators, next) } -func getContextState(tb testing.TB, timeout time.Duration) *contextState { +// EnsureRemaining extends the test context so at least minRemaining remains +// from now, and returns the context to use. +// +// A context created with [WithTimeout] is never extended past that timeout; +// one using the default or a TEMPORAL_TEST_TIMEOUT-configured timeout may +// grow to a total lifetime of max(timeout, [maxTimeout]). +// +// Only the returned context - and later [For] calls - see the extension: a +// context captured earlier keeps its original, shorter deadline, since a +// context's effective deadline can only shrink as it is derived further, never +// grow. +// +// The context to extend is resolved from ctx, not tb, so a subtest's tb can +// extend a context owned by its parent test - the dominant pattern in tests/, +// e.g. await.Require(s.Context(), t, ...) inside a t.Run started from suite +// method s. +// +// ctx comes back unchanged if it is not a test context at all (extension is an +// optimization, so a foreign context is left alone rather than failing the +// caller), or if the caller derived it further - e.g. +// context.WithTimeout(env.Context(), ...) - because swapping it out would +// silently discard that wrapping. The underlying test context is still +// extended for later callers. +func EnsureRemaining(ctx context.Context, tb testing.TB, minRemaining time.Duration) context.Context { tb.Helper() + if ctx == nil { + tb.Fatal("testcontext: nil context") + return nil + } + if minRemaining <= 0 { + tb.Fatalf("testcontext: min remaining must be positive: %v", minRemaining) + return ctx + } - testContexts.Lock() - defer testContexts.Unlock() - - if st, ok := testContexts.byTest[tb]; ok { - return st + st, _ := ctx.Value(ownerKey{}).(*contextState) + if st == nil { + // ctx isn't derived from any test context - e.g. one built directly + // from context.Background(), or tb.Context() itself - so there is + // nothing to extend. + return ctx } - ctx, cancel := context.WithTimeout(tb.Context(), timeout) + st.mu.Lock() + defer st.mu.Unlock() - // Annotate gRPC requests with the test name for OTEL tracing. - ctx = metadata.AppendToOutgoingContext(ctx, testNameMetadataKey, tb.Name()) + testDeadline, ok := st.current.Deadline() + if !ok { + tb.Fatal("testcontext: current context has no deadline") + return ctx + } - st := &contextState{ - ctx: ctx, - cancel: cancel, - timeout: timeout, - decorators: make(map[any]struct{}), - } - testContexts.byTest[tb] = st - - tb.Cleanup(func() { - err := st.err() - st.cancel() - testContexts.Lock() - delete(testContexts.byTest, tb) - testContexts.Unlock() - if err == context.DeadlineExceeded { - tb.Errorf("test exceeded timeout of %v", st.timeout) - } - st.release() - }) - return st -} + // Cap the requested deadline at the context's ceiling. + requestedDeadline := util.MinTime(time.Now().Add(minRemaining), st.maxDeadline()) -func (s *contextState) configure(tb testing.TB, cfg config) { - tb.Helper() + // Only a context this package handed out can be swapped for the extended + // one without dropping state the caller derived onto it. + _, bare := ctx.(*testContext) - s.mu.Lock() - defer s.mu.Unlock() + // Extend the test context if the requested deadline is after the current deadline. + if requestedDeadline.After(testDeadline) { + st.push(newTestContext(st.owner, st, requestedDeadline)) + } - if cfg.timeoutSet && cfg.timeout != s.timeout { - tb.Fatalf("testcontext: test context already exists with timeout %v; cannot change it to %v", s.timeout, cfg.timeout) + if bare { + return st.current } + return ctx +} + +// contextState is the mutable test context state shared by test helpers. +type contextState struct { + // owner is the testing.TB this state was created for; immutable. Used to + // derive replacement contexts even when [EnsureRemaining] is reached + // through a different tb (e.g. a subtest given a parent's context). + owner testing.TB + createdAt time.Time + // timeout is the timeout the context was created with; immutable. + timeout time.Duration + // explicitTimeout records whether timeout was pinned via [WithTimeout], + // making it a hard ceiling. TEMPORAL_TEST_TIMEOUT only raises the + // baseline and remains extendable, like the default. Immutable. + explicitTimeout bool + + mu sync.Mutex + // current is the newest context; [EnsureRemaining] replaces it when the + // deadline is extended. Never nil, so late callers see a canceled context + // instead of a panic. + current context.Context + // cancels tracks every context created for this test so cleanup can release them all. + cancels []context.CancelFunc + decorators []contextDecorator +} +func newContextState(tb testing.TB, timeout time.Duration, explicitTimeout bool) *contextState { + st := &contextState{ + owner: tb, + createdAt: time.Now(), + timeout: timeout, + explicitTimeout: explicitTimeout, + } + st.push(newTestContext(tb, st, st.createdAt.Add(timeout))) + return st } -func (s *contextState) attachDecorator(tb testing.TB, decorator contextDecorator) { +func getOrCreateContextState(tb testing.TB, cfg config) *contextState { tb.Helper() - // Decorators may be registered by independent helpers, so apply each keyed - // decorator at most once while preserving call order. - s.mu.Lock() - defer s.mu.Unlock() + timeout, explicitTimeout := effectiveTimeout(cfg.timeout) - if decorator.key == nil { - tb.Fatal("testcontext: context decorator key must not be nil") - } - if decorator.decorate == nil { - tb.Fatal("testcontext: context decorator must not be nil") + testContexts.Lock() + st, ok := testContexts.byTest[tb] + if !ok { + st = newContextState(tb, timeout, explicitTimeout) + testContexts.byTest[tb] = st + + tb.Cleanup(func() { + // Deregister first: a concurrent helper must not find state that is + // about to be torn down. + testContexts.Lock() + delete(testContexts.byTest, tb) + testContexts.Unlock() + + if timedOut, timeout := st.cleanup(); timedOut { + tb.Errorf("test exceeded timeout of %v", timeout) + } + }) } - if _, ok := s.decorators[decorator.key]; ok { - return + testContexts.Unlock() + + // A freshly created context adopts the requested timeout, so only an + // existing one can conflict with an explicitly requested timeout. + if ok && cfg.timeout != 0 && timeout != st.timeout { + tb.Fatalf("testcontext: test context already exists with timeout %v; cannot change it to %v", st.timeout, timeout) } - s.ctx = decorator.decorate(s.ctx) - s.decorators[decorator.key] = struct{}{} + return st } -func (s *contextState) context() context.Context { - s.mu.Lock() - defer s.mu.Unlock() - return s.ctx +// maxDeadline is the furthest deadline [EnsureRemaining] may extend to. +func (s *contextState) maxDeadline() time.Time { + limit := s.timeout + if !s.explicitTimeout { + // A defaulted or TEMPORAL_TEST_TIMEOUT-configured timeout may grow, + // up to whichever of the two is larger; see [maxTimeout]. + limit = max(limit, maxTimeout*debug.TimeoutMultiplier) + } + return s.createdAt.Add(limit) } -func (s *contextState) err() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.ctx.Err() +func (s *contextState) push(ctx context.Context, cancel context.CancelFunc) { + s.current = ctx + s.cancels = append(s.cancels, cancel) } -func (s *contextState) release() { +// cleanup cancels every context created for the test and reports whether the +// test's context deadline had already fired, and how long after createdAt that was. +func (s *contextState) cleanup() (timedOut bool, timeout time.Duration) { s.mu.Lock() defer s.mu.Unlock() - s.ctx = nil + + timedOut = s.current.Err() == context.DeadlineExceeded + timeout = s.timeout + + if deadline, ok := s.current.Deadline(); ok { + timeout = deadline.Sub(s.createdAt) + } + + for _, cancel := range slices.Backward(s.cancels) { + cancel() + } + + // Keep current: it is canceled now, but callers still racing with cleanup + // must get a context, not a panic. Clearing cancels makes cleanup idempotent. + s.cancels = nil + s.decorators = nil + return timedOut, timeout } -func effectiveTimeout(customTimeout time.Duration) (timeout time.Duration) { +// effectiveTimeout resolves the timeout to use and reports whether it was +// explicitly configured (as opposed to defaulted). +func effectiveTimeout(customTimeout time.Duration) (timeout time.Duration, explicit bool) { defer func() { // Build flag TEMPORAL_DEBUG applies a timeout multiplier to all test timeouts. timeout *= debug.TimeoutMultiplier @@ -193,16 +370,17 @@ func effectiveTimeout(customTimeout time.Duration) (timeout time.Duration) { // 1. Custom timeout (via WithTimeout option). if customTimeout > 0 { - return customTimeout + return customTimeout, true } - // 2. TEMPORAL_TEST_TIMEOUT environment variable. - if envTimeout := os.Getenv("TEMPORAL_TEST_TIMEOUT"); envTimeout != "" { + // 2. TEMPORAL_TEST_TIMEOUT environment variable. Like the default, this + // only raises the baseline - it does not pin a hard ceiling. + if envTimeout := os.Getenv(testTimeoutEnvVar); envTimeout != "" { if dur, err := time.ParseDuration(envTimeout); err == nil && dur > 0 { - return dur + return dur, false } } // 3. Default timeout. - return defaultTimeout + return defaultTimeout, false } diff --git a/common/testing/testcontext/context_test.go b/common/testing/testcontext/context_test.go index 4fe6955f756..c55c8bf5759 100644 --- a/common/testing/testcontext/context_test.go +++ b/common/testing/testcontext/context_test.go @@ -2,13 +2,15 @@ package testcontext import ( "context" + "fmt" + "runtime" + "sync" "sync/atomic" "testing" "testing/synctest" "time" "github.com/stretchr/testify/require" - "go.temporal.io/server/common/debug" "google.golang.org/grpc/metadata" ) @@ -24,7 +26,7 @@ func TestWithTimeout(t *testing.T) { deadline, ok := ctx.Deadline() require.True(t, ok) require.Equal(t, start.Add(DefaultTimeout()), deadline) - require.Equal(t, 90*time.Second*debug.TimeoutMultiplier, DefaultTimeout()) + require.Equal(t, 90*time.Second, DefaultTimeout()) }) }) @@ -138,6 +140,26 @@ func TestCleanupCancelsContext(t *testing.T) { require.ErrorIs(t, ctx.Err(), context.Canceled) } +func TestCleanup(t *testing.T) { + t.Parallel() + + t.Run("reports timeout", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + timeout := time.Millisecond + + tb := newRecordingTB() + tb.run(func() { + ctx := For(tb, WithTimeout(time.Millisecond)) + <-ctx.Done() // let the deadline pass + }) + + require.Equal(t, fmt.Sprintf("test exceeded timeout of %v", timeout), tb.error()) + }) + }) +} + func TestEnvTimeout(t *testing.T) { t.Run("from env", func(t *testing.T) { t.Setenv("TEMPORAL_TEST_TIMEOUT", "10s") @@ -162,4 +184,374 @@ func TestEnvTimeout(t *testing.T) { require.Equal(t, start.Add(time.Second), deadline) }) }) + + t.Run("remains extendable, unlike an explicit WithTimeout", func(t *testing.T) { + t.Setenv("TEMPORAL_TEST_TIMEOUT", "10s") + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t) + originalDeadline, ok := ctx.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(10*time.Second), originalDeadline) + + // TEMPORAL_TEST_TIMEOUT only raises the baseline; it must not pin + // a hard ceiling the way WithTimeout does. + refreshed := EnsureRemaining(ctx, t, time.Minute) + + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(time.Minute), refreshedDeadline) + }) + }) +} + +func TestEnsureRemaining(t *testing.T) { + t.Parallel() + + t.Run("extends when remaining time is too short", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t) + originalDeadline, ok := ctx.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()), originalDeadline) + + refreshed := EnsureRemaining(ctx, t, DefaultTimeout()+10*time.Second) + + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()+10*time.Second), refreshedDeadline) + }) + }) + + t.Run("caps ensured remaining time", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t) + originalDeadline, ok := ctx.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()), originalDeadline) + + refreshed := EnsureRemaining(ctx, t, 10*time.Minute) + + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(maxTimeout), refreshedDeadline) + }) + }) + + t.Run("does not extend beyond an explicitly requested timeout", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t, WithTimeout(100*time.Millisecond)) + + refreshed := EnsureRemaining(ctx, t, 10*time.Minute) + + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(100*time.Millisecond), refreshedDeadline) + require.Same(t, ctx, refreshed, "context should not have been replaced") + }) + }) + + t.Run("keeps a caller-derived context's own deadline", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t) + + // The caller wrapped the test context with its own, tighter + // deadline (e.g. context.WithTimeout(env.Context(), ...)). + // Swapping it for the extended test context would silently + // discard that wrapping, so it must come back unchanged. + derived, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + refreshed := EnsureRemaining(derived, t, DefaultTimeout()+10*time.Second) + + require.Same(t, derived, refreshed, "the caller's own wrapping must not be discarded") + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(time.Second), refreshedDeadline, "the caller's tighter deadline still governs") + + // The underlying test context is still extended for later callers. + extendedDeadline, ok := For(t).Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()+10*time.Second), extendedDeadline) + }) + }) + + t.Run("replays decorators", func(t *testing.T) { + t.Parallel() + + type key struct{} + + AttachDecorator(t, key{}, func(ctx context.Context) context.Context { + return context.WithValue(ctx, key{}, "decorated") + }) + ctx := For(t) + require.Equal(t, "decorated", ctx.Value(key{})) + + refreshed := EnsureRemaining(ctx, t, DefaultTimeout()+10*time.Second) + + require.NotSame(t, ctx, refreshed, "context should have been replaced") + require.Equal(t, "decorated", refreshed.Value(key{})) + }) + + t.Run("preserves test name metadata", func(t *testing.T) { + t.Parallel() + + ctx := For(t) + refreshed := EnsureRemaining(ctx, t, DefaultTimeout()+10*time.Second) + + require.NotSame(t, ctx, refreshed, "context should have been replaced") + md, ok := metadata.FromOutgoingContext(refreshed) + require.True(t, ok) + require.Equal(t, []string{t.Name()}, md.Get(testNameMetadataKey)) + }) + + t.Run("recognizes older context after repeated extensions", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + original := For(t) + + firstRefresh := EnsureRemaining(original, t, DefaultTimeout()+10*time.Second) + firstDeadline, ok := firstRefresh.Deadline() + require.True(t, ok) + require.Equal(t, time.Now().Add(DefaultTimeout()+10*time.Second), firstDeadline) + + // The original context is outdated by now, but still recognized. + refreshed := EnsureRemaining(original, t, DefaultTimeout()+20*time.Second) + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, time.Now().Add(DefaultTimeout()+20*time.Second), refreshedDeadline) + }) + }) + + t.Run("leaves a foreign context unchanged", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + tb := newRecordingTB() + tb.run(func() { + For(tb, WithTimeout(5*time.Millisecond)) + + withDeadline, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + + // Neither context is derived from a test context - e.g. a + // standalone RPC context built from context.Background(). + // Extending is an optimization, so it isn't in a position to + // fail the call; leave the caller's own deadline and + // cancellation intact. + require.Same(t, withDeadline, EnsureRemaining(withDeadline, tb, 10*time.Millisecond)) + // context.Background() isn't a pointer, so require.Same can't + // compare it; require.Equal is identity-equivalent here since + // it's a zero-sized singleton value. + require.Equal(t, context.Background(), EnsureRemaining(context.Background(), tb, 10*time.Millisecond)) + }) + + require.Empty(t, tb.fatal()) + }) + }) + + t.Run("accepts the testing context unchanged", func(t *testing.T) { + t.Parallel() + + For(t) + + // t.Context() is the test context's parent, so it carries no owner + // marker and has no deadline to extend: it comes back unchanged. + require.Same(t, t.Context(), EnsureRemaining(t.Context(), t, DefaultTimeout())) + }) + + t.Run("extends a context belonging to a different tb", func(t *testing.T) { + t.Parallel() + + // Mirrors the dominant pattern in tests/: a suite's context is + // handed to a subtest, whose own tb differs from the one the + // context was created for. Ownership is resolved from ctx, not + // tb, so this must extend the owning (here, parent) state rather + // than failing or silently no-op'ing. + synctest.Test(t, func(t *testing.T) { + start := time.Now() + other := For(t) + + tb := newRecordingTB() + var refreshed context.Context + tb.run(func() { + For(tb) // tb has its own, unrelated state too + + refreshed = EnsureRemaining(other, tb, DefaultTimeout()+10*time.Second) + }) + + require.Empty(t, tb.fatal()) + refreshedDeadline, ok := refreshed.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()+10*time.Second), refreshedDeadline) + + // The extension is visible to the owning test too. + require.Same(t, refreshed, For(t)) + }) + }) + + t.Run("fails for non-positive minimum remaining", func(t *testing.T) { + t.Parallel() + + tb := newRecordingTB() + tb.run(func() { + ctx := For(tb, WithTimeout(5*time.Millisecond)) + + EnsureRemaining(ctx, tb, 0) + }) + + require.Equal(t, + "testcontext: min remaining must be positive: 0s", + tb.fatal(), + ) + }) + + t.Run("safe concurrent calls", func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + start := time.Now() + ctx := For(t) + + var wg sync.WaitGroup + refreshed := make([]context.Context, 8) + for i := range refreshed { + wg.Go(func() { + refreshed[i] = EnsureRemaining(ctx, t, DefaultTimeout()+10*time.Second) + }) + } + wg.Wait() + + // All callers observe the extended deadline, no matter who extended it. + for _, got := range refreshed { + deadline, ok := got.Deadline() + require.True(t, ok) + require.Equal(t, start.Add(DefaultTimeout()+10*time.Second), deadline) + } + }) + }) +} + +// recordingTB records failures instead of failing a real test. +// +// NOTE: it deliberately does not implement Deadline, so it stands in for a +// testing.TB without a `go test -timeout` deadline. +type recordingTB struct { + testing.TB + + ctx context.Context + cancelCtx context.CancelFunc + + mu sync.Mutex + cleanups []func() + fatals []string + errors []string +} + +func newRecordingTB() *recordingTB { + // Like testing.T, hand out a dedicated context that is canceled once the + // "test" is over - not context.Background(). + ctx, cancel := context.WithCancel(context.Background()) + return &recordingTB{ctx: ctx, cancelCtx: cancel} +} + +func (r *recordingTB) Helper() {} + +func (r *recordingTB) Name() string { + return "recordingTB" +} + +func (r *recordingTB) Context() context.Context { + return r.ctx +} + +func (r *recordingTB) Cleanup(fn func()) { + r.mu.Lock() + defer r.mu.Unlock() + r.cleanups = append(r.cleanups, fn) +} + +func (r *recordingTB) Fatal(args ...any) { + r.recordFatal(fmt.Sprint(args...)) +} + +func (r *recordingTB) Fatalf(format string, args ...any) { + r.recordFatal(fmt.Sprintf(format, args...)) +} + +func (r *recordingTB) Error(args ...any) { + r.recordError(fmt.Sprint(args...)) +} + +func (r *recordingTB) Errorf(format string, args ...any) { + r.recordError(fmt.Sprintf(format, args...)) +} + +func (r *recordingTB) recordFatal(msg string) { + r.mu.Lock() + r.fatals = append(r.fatals, msg) + r.mu.Unlock() + runtime.Goexit() +} + +func (r *recordingTB) recordError(msg string) { + r.mu.Lock() + defer r.mu.Unlock() + r.errors = append(r.errors, msg) +} + +func (r *recordingTB) run(fn func()) { + done := make(chan struct{}) + go func() { + defer func() { + r.runCleanups() + close(done) + }() + fn() + }() + <-done +} + +func (r *recordingTB) runCleanups() { + r.mu.Lock() + cleanups := r.cleanups + r.cleanups = nil + r.mu.Unlock() + + for _, cleanup := range cleanups { + cleanup() + } + r.cancelCtx() +} + +func (r *recordingTB) fatal() string { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.fatals) == 0 { + return "" + } + return r.fatals[len(r.fatals)-1] +} + +func (r *recordingTB) error() string { + r.mu.Lock() + defer r.mu.Unlock() + if len(r.errors) == 0 { + return "" + } + return r.errors[len(r.errors)-1] } diff --git a/tests/activity_driver.go b/tests/activity_driver.go index e2b60f99662..5164cdbfd7b 100644 --- a/tests/activity_driver.go +++ b/tests/activity_driver.go @@ -293,7 +293,6 @@ type activityTimeoutInfo struct { // activityDriverState is the state shared by the two drivers. type activityDriverState struct { - ctx context.Context cfg activityConfig token []byte startedAttempt int32 // attempt number returned by the last successful Poll @@ -307,6 +306,9 @@ func (a *activityDriverState) driverState() *activityDriverState { // drivenActivity is what the shared event driver needs from either implementation. type drivenActivity interface { driverState() *activityDriverState + // testContext returns the driver's current test context, fetched fresh per + // call rather than cached, so a later timeout extension is visible. + testContext() context.Context pollForTask(require.TestingT, time.Duration) *workflowservice.PollActivityTaskQueueResponse awaitDispatchDelay(testing.TB, model.Event) timeoutInfo(require.TestingT) activityTimeoutInfo @@ -337,7 +339,7 @@ func awaitActivityTimeout(t testing.TB, a drivenActivity, e model.Event, deadlin state := a.driverState() want := timeoutType(e) var got activityTimeoutInfo - await.Require(state.ctx, t, func(t *await.T) { + await.Require(a.testContext(), t, func(t *await.T) { got = a.timeoutInfo(t) fired := got.timeout == want && (got.terminal || got.attempt > state.startedAttempt) t.Require().Truef(fired, diff --git a/tests/activity_driver_test.go b/tests/activity_driver_test.go index ac0be725433..467b59f6aa4 100644 --- a/tests/activity_driver_test.go +++ b/tests/activity_driver_test.go @@ -3,13 +3,16 @@ package tests // Self-tests for the activity drivers. import ( + "context" "testing" + "testing/synctest" "time" "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/server/chasm/lib/activity/model" "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/testcontext" ) // TestDriversRecognizeTimeoutObservedBeforeWait reproduces a race in awaitTimeout: a retryable timeout @@ -49,3 +52,42 @@ func (s *activityParityTestSuite) TestDriversRecognizeTimeoutObservedBeforeWait( a.awaitTimeout(t, model.StartToCloseElapses, time.Now().Add(waitForDriver)) }) } + +// contextualDriver is the slice of a driver's API this test exercises. +type contextualDriver interface{ testContext() context.Context } + +// TestDriverContextReflectsExtension proves the drivers fetch their context fresh on every call +// instead of caching the one observed at construction, so a timeout extension made after the driver +// starts is visible to later RPCs. env is nil: testContext() never touches it. +// +// Runs inside a synctest bubble so it doesn't need to wait out real minutes of test-context timeout, +// and so `go test -timeout` (a real-clock deadline, meaningless in a fake-clock bubble) can't cap the +// context and mask the very extension this test is checking for. +func TestDriverContextReflectsExtension(t *testing.T) { + t.Parallel() + + for name, newDriver := range map[string]func(*testing.T) contextualDriver{ + "SAA": func(t *testing.T) contextualDriver { return newSAADriver(t, nil, activityConfig{}) }, + "WFA": func(t *testing.T) contextualDriver { return newWFADriver(t, nil, activityConfig{}) }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + d := newDriver(t) + before, ok := d.testContext().Deadline() + require.True(t, ok) + + extended := testcontext.EnsureRemaining(testcontext.For(t), t, testcontext.DefaultTimeout()+time.Minute) + extendedDeadline, ok := extended.Deadline() + require.True(t, ok) + require.True(t, extendedDeadline.After(before), "test setup: EnsureRemaining should have extended the deadline") + + after, ok := d.testContext().Deadline() + require.True(t, ok) + require.Equal(t, extendedDeadline, after, + "driver must observe the extended deadline, not the one captured at construction") + }) + }) + } +} diff --git a/tests/activity_parity_test.go b/tests/activity_parity_test.go index c0ca5961e25..abc7fa3490a 100644 --- a/tests/activity_parity_test.go +++ b/tests/activity_parity_test.go @@ -199,7 +199,7 @@ func (s *activityParityTestSuite) TestTimeoutPreservesUnderlyingFailureCause() { const message = "the terminal timeout must chain the underlying application failure as its Cause" t.Run("WorkflowActivity", func(t *testing.T) { activity := newWFADriver(t, env, cfg).driveTrace(t, trace) - timeout, ok := errors.AsType[*temporal.TimeoutError](activity.run.Get(activity.d.ctx, nil)) + timeout, ok := errors.AsType[*temporal.TimeoutError](activity.run.Get(activity.testContext(), nil)) require.True(t, ok) appErr, ok := errors.AsType[*temporal.ApplicationError](timeout.Unwrap()) require.True(t, ok) @@ -718,7 +718,7 @@ func (s *activityParityTestSuite) TestRetryableFailureTruncation() { status: enumspb.ACTIVITY_EXECUTION_STATUS_FAILED, retryState: enumspb.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED, }, a.terminalOutcome(t)) - appErr, ok := errors.AsType[*temporal.ApplicationError](errors.Unwrap(a.run.Get(a.d.ctx, nil))) + appErr, ok := errors.AsType[*temporal.ApplicationError](errors.Unwrap(a.run.Get(a.testContext(), nil))) require.True(t, ok) require.Equal(t, "TestFailure", appErr.Type()) require.Equal(t, activityLargeFailureMessage, appErr.Message(), diff --git a/tests/activity_standalone_driver.go b/tests/activity_standalone_driver.go index a448ddf0b38..28465aa1e21 100644 --- a/tests/activity_standalone_driver.go +++ b/tests/activity_standalone_driver.go @@ -30,7 +30,7 @@ import ( type saaDriver struct { env *testcore.TestEnv - ctx context.Context + t *testing.T cfg activityConfig numStarted int activityIDPrefix string @@ -40,12 +40,18 @@ type saaDriver struct { func newSAADriver(t *testing.T, env *testcore.TestEnv, cfg activityConfig) *saaDriver { return &saaDriver{ env: env, - ctx: testcontext.For(t), + t: t, cfg: cfg, activityIDPrefix: t.Name(), } } +// testContext returns the driver's current test context, deliberately not cached; +// see [testcontext.EnsureRemaining]. +func (d *saaDriver) testContext() context.Context { + return testcontext.For(d.t) +} + // saaHandle is a handle to an activity instance. type saaHandle struct { activityDriverState @@ -70,6 +76,10 @@ func (a *saaHandle) driveEvent(t testing.TB, e model.Event) { driveActivityEvent(t, a, e) } +func (a *saaHandle) testContext() context.Context { + return a.d.testContext() +} + func (a *saaHandle) awaitTimeout(t testing.TB, e model.Event, deadline time.Time) { awaitActivityTimeout(t, a, e, deadline) } @@ -94,7 +104,7 @@ func (a *saaHandle) timeoutInfo(t require.TestingT) activityTimeoutInfo { // awaitDispatchDelay waits for the public dispatch deadline to become due. A following Poll is what // proves that the task actually reached Matching. func (a *saaHandle) awaitDispatchDelay(t testing.TB, e model.Event) { - awaitActivityDispatchDelay(a.d.ctx, t, e, func(t require.TestingT) (bool, enumspb.PendingActivityState, *timestamppb.Timestamp, any) { + awaitActivityDispatchDelay(a.testContext(), t, e, func(t require.TestingT) (bool, enumspb.PendingActivityState, *timestamppb.Timestamp, any) { info := a.describe(t).GetInfo() return info.GetStatus() == enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING, info.GetRunState(), @@ -106,10 +116,10 @@ func (a *saaHandle) awaitDispatchDelay(t testing.TB, e model.Event) { func (d *saaDriver) start(t require.TestingT, cfg activityConfig) *saaHandle { d.numStarted++ id := fmt.Sprintf("%s-%d", d.activityIDPrefix, d.numStarted) - resp, err := d.env.FrontendClient().StartActivityExecution(d.ctx, d.startRequest(cfg, id, id)) + resp, err := d.env.FrontendClient().StartActivityExecution(d.testContext(), d.startRequest(cfg, id, id)) require.NoError(t, err) return &saaHandle{ - activityDriverState: activityDriverState{ctx: d.ctx, cfg: cfg}, + activityDriverState: activityDriverState{cfg: cfg}, d: d, activityID: id, runID: resp.RunId, @@ -150,7 +160,7 @@ func (d *saaDriver) startRequest(c activityConfig, activityID, taskQueue string) // describe returns the DescribeActivityExecution response, including the outcome, the last failure, // and the heartbeat details. func (a *saaHandle) describe(t require.TestingT) *workflowservice.DescribeActivityExecutionResponse { - resp, err := a.d.env.FrontendClient().DescribeActivityExecution(a.d.ctx, &workflowservice.DescribeActivityExecutionRequest{ + resp, err := a.d.env.FrontendClient().DescribeActivityExecution(a.testContext(), &workflowservice.DescribeActivityExecutionRequest{ Namespace: a.d.env.Namespace().String(), ActivityId: a.activityID, RunId: a.runID, @@ -174,7 +184,7 @@ func (a *saaHandle) activityInfo(t require.TestingT) activityInfo { func (a *saaHandle) terminalOutcome(t require.TestingT) activityTerminalOutcome { deadline := time.Now().Add(activityDriverTimeout) for time.Now().Before(deadline) { - ctx, cancel := context.WithDeadline(a.d.ctx, deadline) + ctx, cancel := context.WithDeadline(a.testContext(), deadline) resp, err := a.d.env.FrontendClient().PollActivityExecution(ctx, &workflowservice.PollActivityExecutionRequest{ Namespace: a.d.env.Namespace().String(), ActivityId: a.activityID, @@ -217,7 +227,7 @@ func saaActivityInfo(i *activitypb.ActivityExecutionInfo) activityInfo { func (a *saaHandle) respondCanceledByID() error { _, err := a.d.env.FrontendClient().RespondActivityTaskCanceledById( - a.d.ctx, + a.testContext(), &workflowservice.RespondActivityTaskCanceledByIdRequest{ Namespace: a.d.env.Namespace().String(), ActivityId: a.activityID, @@ -232,20 +242,21 @@ func (a *saaHandle) respondCanceledByID() error { func (a *saaHandle) rpc(_ testing.TB, e model.Event) error { fc := a.d.env.FrontendClient() ns := a.d.env.Namespace().String() + ctx := a.testContext() switch e.Type { case model.HeartbeatType: - _, err := fc.RecordActivityTaskHeartbeat(a.d.ctx, &workflowservice.RecordActivityTaskHeartbeatRequest{ + _, err := fc.RecordActivityTaskHeartbeat(ctx, &workflowservice.RecordActivityTaskHeartbeatRequest{ Namespace: ns, TaskToken: a.token, Details: activityRecordedHeartbeatDetails, }) return err case model.RespondCompletedType: - _, err := fc.RespondActivityTaskCompleted(a.d.ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + _, err := fc.RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: ns, TaskToken: a.token, Identity: a.d.env.Tv().WorkerIdentity(), Result: payloads.EncodeString("result"), }) return err case model.RespondCompletedByIDType: - _, err := fc.RespondActivityTaskCompletedById(a.d.ctx, &workflowservice.RespondActivityTaskCompletedByIdRequest{ + _, err := fc.RespondActivityTaskCompletedById(ctx, &workflowservice.RespondActivityTaskCompletedByIdRequest{ Namespace: ns, RunId: a.runID, ActivityId: a.activityID, Identity: a.d.env.Tv().WorkerIdentity(), Result: payloads.EncodeString("result"), }) @@ -257,7 +268,7 @@ func (a *saaHandle) rpc(_ testing.TB, e model.Event) error { if e.HasHeartbeatDetails { req.LastHeartbeatDetails = activityHeartbeatDetails } - _, err := fc.RespondActivityTaskFailed(a.d.ctx, req) + _, err := fc.RespondActivityTaskFailed(ctx, req) return err case model.RespondFailedByIDType: req := &workflowservice.RespondActivityTaskFailedByIdRequest{ @@ -267,42 +278,42 @@ func (a *saaHandle) rpc(_ testing.TB, e model.Event) error { if e.HasHeartbeatDetails { req.LastHeartbeatDetails = activityHeartbeatDetails } - _, err := fc.RespondActivityTaskFailedById(a.d.ctx, req) + _, err := fc.RespondActivityTaskFailedById(ctx, req) return err case model.RespondCanceledType: - _, err := fc.RespondActivityTaskCanceled(a.d.ctx, &workflowservice.RespondActivityTaskCanceledRequest{ + _, err := fc.RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: ns, TaskToken: a.token, Identity: a.d.env.Tv().WorkerIdentity(), }) return err case model.RequestCancelType: - _, err := fc.RequestCancelActivityExecution(a.d.ctx, &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := fc.RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), Reason: "drive", RequestId: uuid.NewString(), }) return err case model.TerminateType: - _, err := fc.TerminateActivityExecution(a.d.ctx, &workflowservice.TerminateActivityExecutionRequest{ + _, err := fc.TerminateActivityExecution(ctx, &workflowservice.TerminateActivityExecutionRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), Reason: "drive", RequestId: uuid.NewString(), }) return err case model.PauseType: - _, err := fc.PauseActivityExecution(a.d.ctx, &workflowservice.PauseActivityExecutionRequest{ + _, err := fc.PauseActivityExecution(ctx, &workflowservice.PauseActivityExecutionRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), Reason: "drive", RequestId: uuid.NewString(), }) return err case model.UnpauseType: - _, err := fc.UnpauseActivityExecution(a.d.ctx, &workflowservice.UnpauseActivityExecutionRequest{ + _, err := fc.UnpauseActivityExecution(ctx, &workflowservice.UnpauseActivityExecutionRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), }) return err case model.ResetType: - _, err := fc.ResetActivityExecution(a.d.ctx, &workflowservice.ResetActivityExecutionRequest{ + _, err := fc.ResetActivityExecution(ctx, &workflowservice.ResetActivityExecutionRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), KeepPaused: e.KeepPaused, ResetHeartbeat: e.ResetHeartbeat, }) return err case model.UpdateOptionsType: - _, err := fc.UpdateActivityExecutionOptions(a.d.ctx, &workflowservice.UpdateActivityExecutionOptionsRequest{ + _, err := fc.UpdateActivityExecutionOptions(ctx, &workflowservice.UpdateActivityExecutionOptionsRequest{ Namespace: ns, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), ActivityOptions: &activitypb.ActivityOptions{HeartbeatTimeout: durationpb.New(time.Hour)}, UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, @@ -314,7 +325,8 @@ func (a *saaHandle) rpc(_ testing.TB, e model.Event) error { } func (a *saaHandle) pollForTask(t require.TestingT, timeout time.Duration) *workflowservice.PollActivityTaskQueueResponse { - ctx, cancel := context.WithTimeout(a.d.ctx, timeout) + driverCtx := a.testContext() + ctx, cancel := context.WithTimeout(driverCtx, timeout) defer cancel() resp, err := a.d.env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: a.d.env.Namespace().String(), @@ -324,10 +336,10 @@ func (a *saaHandle) pollForTask(t require.TestingT, timeout time.Duration) *work // Matching signals "waited, found nothing" with an empty response and a nil error, so any error // means the poll did not complete cleanly. if err != nil { - if a.d.ctx.Err() != nil { + if driverCtx.Err() != nil { return nil // teardown } - if deadline, ok := a.d.ctx.Deadline(); ok && time.Until(deadline) < common.MinLongPollTimeout { + if deadline, ok := driverCtx.Deadline(); ok && time.Until(deadline) < common.MinLongPollTimeout { t.Errorf("saaDriver: test context budget exhausted before the poll could run (%.1fs left, need >= %s). "+ "Raise TEMPORAL_TEST_TIMEOUT and `go test -timeout`.\n %v", time.Until(deadline).Seconds(), common.MinLongPollTimeout, err) diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index de4663bc684..3192614b533 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -34,6 +34,7 @@ import ( "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testcontext" "go.temporal.io/server/tests/testcore" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -2106,12 +2107,13 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { } t.Run("ByIDAfterRetry", func(t *testing.T) { + ctx := testcontext.For(t) // Regression test: by-ID cancel must succeed on attempt 2+. activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - _, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + _, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, ActivityType: env.Tv().ActivityType(), @@ -2125,14 +2127,14 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Poll and fail attempt 1 retryably. - attempt1Resp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + attempt1Resp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) require.NoError(t, err) require.EqualValues(t, 1, attempt1Resp.Attempt) - _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + _, err = env.FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ Namespace: env.Namespace().String(), TaskToken: attempt1Resp.TaskToken, Failure: &failurepb.Failure{ @@ -2146,7 +2148,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Poll to start attempt 2. - attempt2Resp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + attempt2Resp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2154,7 +2156,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.EqualValues(t, 2, attempt2Resp.Attempt) // Request cancellation first. - _, err = env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err = env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: attempt2Resp.ActivityRunId, @@ -2165,7 +2167,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Cancel by ID — must succeed on attempt 2. - _, err = env.FrontendClient().RespondActivityTaskCanceledById(s.Context(), &workflowservice.RespondActivityTaskCanceledByIdRequest{ + _, err = env.FrontendClient().RespondActivityTaskCanceledById(ctx, &workflowservice.RespondActivityTaskCanceledByIdRequest{ Namespace: env.Namespace().String(), RunId: attempt2Resp.ActivityRunId, ActivityId: activityID, @@ -2180,17 +2182,18 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("DuplicateRequestIDSucceeds", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp := env.startAndValidateActivity(s.Context(), t, activityID, taskQueue) + startResp := env.startAndValidateActivity(ctx, t, activityID, taskQueue) runID := startResp.RunId - env.pollActivityTaskAndValidate(s.Context(), t, activityID, taskQueue, runID) + env.pollActivityTaskAndValidate(ctx, t, activityID, taskQueue, runID) for range 2 { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2201,7 +2204,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) } - heartbeatResp, err := env.FrontendClient().RecordActivityTaskHeartbeatById(s.Context(), &workflowservice.RecordActivityTaskHeartbeatByIdRequest{ + heartbeatResp, err := env.FrontendClient().RecordActivityTaskHeartbeatById(ctx, &workflowservice.RecordActivityTaskHeartbeatByIdRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: startResp.RunId, @@ -2209,7 +2212,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) require.True(t, heartbeatResp.GetCancelRequested(), "expected CancelRequested to be true but was false") - activityResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + activityResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2228,17 +2231,18 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("DifferentRequestIDFails", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) identity := "client-that-requested-cancellation" - startResp := env.startAndValidateActivity(s.Context(), t, activityID, taskQueue) + startResp := env.startAndValidateActivity(ctx, t, activityID, taskQueue) runID := startResp.RunId - env.pollActivityTaskAndValidate(s.Context(), t, activityID, taskQueue, runID) + env.pollActivityTaskAndValidate(ctx, t, activityID, taskQueue, runID) - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2248,7 +2252,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) require.NoError(t, err) - _, err = env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err = env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2344,12 +2348,13 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { } t.Run("Retry", func(t *testing.T) { + ctx := testcontext.For(t) runTest := func(t *testing.T, requestCancellation bool) { activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + startResp, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, ActivityType: env.Tv().ActivityType(), @@ -2363,7 +2368,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) runID := startResp.RunId - pollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + pollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2371,7 +2376,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.EqualValues(t, 1, pollResp.Attempt) if requestCancellation { - _, err = env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err = env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2387,14 +2392,14 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{NonRetryable: false}, }, } - _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + _, err = env.FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp.TaskToken, Failure: retryableFailure, }) require.NoError(t, err) - activityResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + activityResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2417,9 +2422,10 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("RequestValidations", func(t *testing.T) { + ctx := testcontext.For(t) t.Run("EmptyActivityID", func(t *testing.T) { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), Reason: "Test Cancellation", Identity: "client-that-requested-cancellation", @@ -2432,7 +2438,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { t.Run("ActivityIDTooLong", func(t *testing.T) { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ ActivityId: string(make([]byte, defaultMaxIDLengthLimit+1)), // dynamic config default is 1000 Namespace: env.Namespace().String(), Reason: "Test Cancellation", @@ -2447,7 +2453,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { t.Run("RequestIDTooLong", func(t *testing.T) { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ ActivityId: testcore.RandomizeStr(t.Name()), RequestId: string(make([]byte, defaultMaxIDLengthLimit+1)), // dynamic config default is 1000 Namespace: env.Namespace().String(), @@ -2463,7 +2469,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { t.Run("IdentityTooLong", func(t *testing.T) { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ ActivityId: testcore.RandomizeStr(t.Name()), Namespace: env.Namespace().String(), Reason: "Test Cancellation", @@ -2478,7 +2484,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { t.Run("InvalidRunID", func(t *testing.T) { - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ ActivityId: testcore.RandomizeStr(t.Name()), RunId: "invalid-run-id", Namespace: env.Namespace().String(), @@ -2500,7 +2506,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { ) defer cleanup() - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ ActivityId: testcore.RandomizeStr(t.Name()), Namespace: env.Namespace().String(), Reason: string(make([]byte, blobSizeLimitError+1)), @@ -2514,14 +2520,15 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("ImmediatelyCancelled_WhenInScheduledState", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp := env.startAndValidateActivity(s.Context(), t, activityID, taskQueue) + startResp := env.startAndValidateActivity(ctx, t, activityID, taskQueue) runID := startResp.RunId - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2531,7 +2538,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) require.NoError(t, err) - activityResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + activityResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: runID, @@ -2545,19 +2552,20 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("StaleToken", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - _, err := env.startActivity(s.Context(), activityID, taskQueue) + _, err := env.startActivity(ctx, activityID, taskQueue) require.NoError(t, err) - pollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + pollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) require.NoError(t, err) - _, err = env.FrontendClient().RespondActivityTaskCompleted(s.Context(), &workflowservice.RespondActivityTaskCompletedRequest{ + _, err = env.FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp.TaskToken, Result: defaultResult, @@ -2565,7 +2573,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Fail with stale token (activity already completed) - _, err = env.FrontendClient().RespondActivityTaskCanceled(s.Context(), &workflowservice.RespondActivityTaskCanceledRequest{ + _, err = env.FrontendClient().RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp.TaskToken, }) @@ -2576,13 +2584,14 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("StaleAttemptToken", func(t *testing.T) { + ctx := testcontext.For(t) // Start an activity with retries, fail first attempt, then try to complete with old token. // Use NextRetryDelay=1s to ensure the retry dispatch happens within test timeout. activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + startResp, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, ActivityType: env.Tv().ActivityType(), @@ -2595,7 +2604,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Poll and get task token for attempt 1 - attempt1Resp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + attempt1Resp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2603,7 +2612,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.EqualValues(t, 1, attempt1Resp.Attempt) // Fail the task with NextRetryDelay to control retry timing - _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + _, err = env.FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ Namespace: env.Namespace().String(), TaskToken: attempt1Resp.TaskToken, Failure: &failurepb.Failure{ @@ -2617,14 +2626,14 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Poll to get attempt 2 (ensures retry has happened) - attempt2Resp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + attempt2Resp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) require.NoError(t, err) require.EqualValues(t, 2, attempt2Resp.Attempt) - _, err = env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err = env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: startResp.GetRunId(), @@ -2634,7 +2643,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Try to cancel with the old attempt 1 token - should fail with NotFound - _, err = env.FrontendClient().RespondActivityTaskCanceled(s.Context(), &workflowservice.RespondActivityTaskCanceledRequest{ + _, err = env.FrontendClient().RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: env.Namespace().String(), TaskToken: attempt1Resp.TaskToken, }) @@ -2644,14 +2653,14 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.Contains(t, statusErr.Message(), fmt.Sprintf("activity not found for ID: %s", activityID)) // Heartbeat then cancel with the attempt 2 token and should be no error - heartbeatResp, err := env.FrontendClient().RecordActivityTaskHeartbeat(s.Context(), &workflowservice.RecordActivityTaskHeartbeatRequest{ + heartbeatResp, err := env.FrontendClient().RecordActivityTaskHeartbeat(ctx, &workflowservice.RecordActivityTaskHeartbeatRequest{ Namespace: env.Namespace().String(), TaskToken: attempt2Resp.TaskToken, }) require.NoError(t, err) require.True(t, heartbeatResp.GetCancelRequested(), "expected CancelRequested to be true but was false") - _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + _, err = env.FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ Namespace: env.Namespace().String(), TaskToken: attempt2Resp.TaskToken, Failure: defaultFailure, @@ -2660,14 +2669,15 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("MismatchedTokenNamespace", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) existingNamespace := env.Namespace().String() - _, err := env.startActivity(s.Context(), activityID, taskQueue) + _, err := env.startActivity(ctx, activityID, taskQueue) require.NoError(t, err) - pollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + pollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: existingNamespace, TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2675,7 +2685,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { // Start an activity in a different namespace and try to cancel with existing token externalNamespace := env.ExternalNamespace().String() - _, err = env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + _, err = env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: externalNamespace, ActivityId: activityID, ActivityType: env.Tv().ActivityType(), @@ -2689,7 +2699,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) require.NoError(t, err) - _, err = env.FrontendClient().RespondActivityTaskCanceled(s.Context(), &workflowservice.RespondActivityTaskCanceledRequest{ + _, err = env.FrontendClient().RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: externalNamespace, TaskToken: pollResp.TaskToken, }) @@ -2702,16 +2712,17 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { // The validation ensures that the namespace in the request matches the namespace in the token's // ComponentRef, preventing cross-namespace token reuse attacks. t.Run("MismatchedTokenComponentRef", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) existingNamespace := env.Namespace().String() // Start activity in namespace A and get its task token - _, err := env.startActivity(s.Context(), activityID, taskQueue) + _, err := env.startActivity(ctx, activityID, taskQueue) require.NoError(t, err) - pollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + pollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: existingNamespace, TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2720,7 +2731,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { // Start another activity in namespace B with the same activity ID // (different namespaces allow same activity IDs) externalNamespace := env.ExternalNamespace().String() - _, err = env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + _, err = env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: externalNamespace, ActivityId: activityID, ActivityType: env.Tv().ActivityType(), @@ -2735,7 +2746,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { require.NoError(t, err) // Poll for the task from namespace B - externalPollResp, err := env.FrontendClient().PollActivityTaskQueue(s.Context(), &workflowservice.PollActivityTaskQueueRequest{ + externalPollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: externalNamespace, TaskQueue: &taskqueuepb.TaskQueue{Name: taskQueue, Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, }) @@ -2753,7 +2764,7 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { existingTaskToken, err := tasktoken.NewSerializer().Serialize(existingTask) require.NoError(t, err) - _, err = env.FrontendClient().RespondActivityTaskCanceled(s.Context(), &workflowservice.RespondActivityTaskCanceledRequest{ + _, err = env.FrontendClient().RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: existingNamespace, TaskToken: existingTaskToken, }) @@ -2765,10 +2776,11 @@ func (s *standaloneActivityTestSuite) TestRequestCancel() { }) t.Run("NonExistent", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) - _, err := env.FrontendClient().RequestCancelActivityExecution(s.Context(), &workflowservice.RequestCancelActivityExecutionRequest{ + _, err := env.FrontendClient().RequestCancelActivityExecution(ctx, &workflowservice.RequestCancelActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, Reason: "Test Cancellation", @@ -3743,7 +3755,8 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { require.NoError(t, err) t.Run("MinimalResponse", func(t *testing.T) { - describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + ctx := testcontext.For(t) + describeResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, // Omit RunID to verify that latest run will be used @@ -3759,7 +3772,8 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { }) t.Run("FullResponse", func(t *testing.T) { - describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + ctx := testcontext.For(t) + describeResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: startResp.RunId, @@ -4029,14 +4043,15 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { env := s.newTestEnv() t.Run("Populated", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp, err := env.startActivity(s.Context(), activityID, taskQueue) + startResp, err := env.startActivity(ctx, activityID, taskQueue) require.NoError(t, err) - pollCtx := metadata.NewOutgoingContext(s.Context(), metadata.Pairs( + pollCtx := metadata.NewOutgoingContext(ctx, metadata.Pairs( headers.ClientNameHeaderName, headers.ClientNameGoSDK, headers.ClientVersionHeaderName, temporal.SDKVersion, )) @@ -4050,7 +4065,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { }) require.NoError(t, err) - describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + describeResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: startResp.RunId, @@ -4061,14 +4076,15 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { }) t.Run("EmptyWhenAbsent", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - startResp, err := env.startActivity(s.Context(), activityID, taskQueue) + startResp, err := env.startActivity(ctx, activityID, taskQueue) require.NoError(t, err) - pollCtx := metadata.NewOutgoingContext(s.Context(), metadata.MD{}) + pollCtx := metadata.NewOutgoingContext(ctx, metadata.MD{}) _, err = env.FrontendClient().PollActivityTaskQueue(pollCtx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: env.Namespace().String(), TaskQueue: &taskqueuepb.TaskQueue{ @@ -4079,7 +4095,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { }) require.NoError(t, err) - describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + describeResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, RunId: startResp.RunId, @@ -4092,11 +4108,12 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { // When an attempt fails and a new attempt is scheduled, the next poller's SDK identity must overwrite the // previous attempt's values. t.Run("OverwrittenOnRetry", func(t *testing.T) { + ctx := testcontext.For(t) activityID := testcore.RandomizeStr(t.Name()) taskQueue := testcore.RandomizeStr(t.Name()) - _, err := env.FrontendClient().StartActivityExecution(s.Context(), &workflowservice.StartActivityExecutionRequest{ + _, err := env.FrontendClient().StartActivityExecution(ctx, &workflowservice.StartActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, @@ -4110,7 +4127,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { require.NoError(t, err) // Attempt 1: poll as the Go SDK, then fail retryably. - pollCtx1 := metadata.NewOutgoingContext(s.Context(), metadata.Pairs( + pollCtx1 := metadata.NewOutgoingContext(ctx, metadata.Pairs( headers.ClientNameHeaderName, headers.ClientNameGoSDK, headers.ClientVersionHeaderName, temporal.SDKVersion, )) @@ -4123,7 +4140,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { require.EqualValues(t, 1, pollResp1.Attempt) // Sanity-check that attempt 1's Go SDK identity is set before the retry overwrites it. - describeResp, err := env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + describeResp, err := env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, }) @@ -4131,7 +4148,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { require.Equal(t, headers.ClientNameGoSDK, describeResp.GetInfo().GetSdkName()) require.Equal(t, temporal.SDKVersion, describeResp.GetInfo().GetSdkVersion()) - _, err = env.FrontendClient().RespondActivityTaskFailed(s.Context(), &workflowservice.RespondActivityTaskFailedRequest{ + _, err = env.FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ Namespace: env.Namespace().String(), TaskToken: pollResp1.TaskToken, Failure: &failurepb.Failure{ @@ -4147,7 +4164,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { // Attempt 2: poll as a different SDK to prove the value is overwritten, not appended. // Java version is hardcoded since we don't have the Java SDK imported. javaSdkVersion := "1.35.0" - pollCtx2 := metadata.NewOutgoingContext(s.Context(), metadata.Pairs( + pollCtx2 := metadata.NewOutgoingContext(ctx, metadata.Pairs( headers.ClientNameHeaderName, headers.ClientNameJavaSDK, headers.ClientVersionHeaderName, javaSdkVersion, )) @@ -4159,7 +4176,7 @@ func (s *standaloneActivityTestSuite) TestDescribeActivityExecution() { require.NoError(t, err) require.EqualValues(t, 2, pollResp2.Attempt) - describeResp, err = env.FrontendClient().DescribeActivityExecution(s.Context(), &workflowservice.DescribeActivityExecutionRequest{ + describeResp, err = env.FrontendClient().DescribeActivityExecution(ctx, &workflowservice.DescribeActivityExecutionRequest{ Namespace: env.Namespace().String(), ActivityId: activityID, }) diff --git a/tests/activity_workflow_driver.go b/tests/activity_workflow_driver.go index 208e296c187..c560e8ff1dd 100644 --- a/tests/activity_workflow_driver.go +++ b/tests/activity_workflow_driver.go @@ -35,14 +35,20 @@ import ( type wfaDriver struct { env *testcore.TestEnv - ctx context.Context + t *testing.T cfg activityConfig } // newWFADriver builds a driver. cfg.StartDelay is ignored: a workflow activity has no per-activity // start delay. func newWFADriver(t *testing.T, env *testcore.TestEnv, cfg activityConfig) *wfaDriver { - return &wfaDriver{env: env, ctx: testcontext.For(t), cfg: cfg} + return &wfaDriver{env: env, t: t, cfg: cfg} +} + +// testContext returns the driver's current test context, deliberately not cached; +// see [testcontext.EnsureRemaining]. +func (d *wfaDriver) testContext() context.Context { + return testcontext.For(d.t) } // wfaHandle is a handle to a workflow-scheduled activity. @@ -71,6 +77,10 @@ func (a *wfaHandle) driveEvent(t testing.TB, e model.Event) { driveActivityEvent(t, a, e) } +func (a *wfaHandle) testContext() context.Context { + return a.d.testContext() +} + func (a *wfaHandle) awaitTimeout(t testing.TB, e model.Event, deadline time.Time) { awaitActivityTimeout(t, a, e, deadline) } @@ -86,7 +96,7 @@ func (a *wfaHandle) timeoutInfo(t require.TestingT) activityTimeoutInfo { } } var timeoutErr *temporal.TimeoutError - if errors.As(a.run.Get(a.d.ctx, nil), &timeoutErr) { + if errors.As(a.run.Get(a.testContext(), nil), &timeoutErr) { return activityTimeoutInfo{timeout: timeoutErr.TimeoutType(), terminal: true} } return activityTimeoutInfo{terminal: true} @@ -95,7 +105,7 @@ func (a *wfaHandle) timeoutInfo(t require.TestingT) activityTimeoutInfo { // awaitDispatchDelay waits for the public dispatch deadline to become due. A following Poll is what // proves that the task actually reached Matching. func (a *wfaHandle) awaitDispatchDelay(t testing.TB, e model.Event) { - awaitActivityDispatchDelay(a.d.ctx, t, e, func(t require.TestingT) (bool, enumspb.PendingActivityState, *timestamppb.Timestamp, any) { + awaitActivityDispatchDelay(a.testContext(), t, e, func(t require.TestingT) (bool, enumspb.PendingActivityState, *timestamppb.Timestamp, any) { pa := a.pendingActivityInfo(t) if pa == nil { return false, enumspb.PENDING_ACTIVITY_STATE_UNSPECIFIED, nil, "activity is no longer in progress" @@ -120,12 +130,12 @@ func (d *wfaDriver) start(t *testing.T, cfg activityConfig) *wfaHandle { t.Cleanup(w.Stop) wfID := testcore.RandomizeStr("wfa-run") - run, err := d.env.SdkClient().ExecuteWorkflow(d.ctx, + run, err := d.env.SdkClient().ExecuteWorkflow(d.testContext(), sdkclient.StartWorkflowOptions{ID: wfID, TaskQueue: wfTQ}, wfaSingleActivityWorkflow, wfaActivityParams{Cfg: cfg, ActivityTQ: actTQ, ActivityID: actID}) require.NoError(t, err) a := &wfaHandle{ - activityDriverState: activityDriverState{ctx: d.ctx, cfg: cfg}, + activityDriverState: activityDriverState{cfg: cfg}, d: d, run: run, workflowID: wfID, @@ -134,7 +144,7 @@ func (d *wfaDriver) start(t *testing.T, cfg activityConfig) *wfaHandle { taskQueue: actTQ, } // The workflow schedules the activity, so it does not exist yet when ExecuteWorkflow returns. - await.Require(d.ctx, t, func(t *await.T) { + await.Require(d.testContext(), t, func(t *await.T) { _, activityInProgress := a.activityInfoIfInProgress(t) t.Require().True(activityInProgress, "the workflow has not scheduled its activity") }, activityDriverTimeout, activityDriverPollInterval) @@ -187,7 +197,7 @@ func wfaSingleActivityWorkflow(ctx workflow.Context, params wfaActivityParams) e // pendingActivityInfo is the activity's entry in the workflow's pending set, nil once it is no longer // pending. func (a *wfaHandle) pendingActivityInfo(t require.TestingT) *workflowpb.PendingActivityInfo { - resp, err := a.d.env.SdkClient().DescribeWorkflowExecution(a.d.ctx, a.workflowID, a.runID) + resp, err := a.d.env.SdkClient().DescribeWorkflowExecution(a.testContext(), a.workflowID, a.runID) require.NoError(t, err) for _, pa := range resp.GetPendingActivities() { if pa.GetActivityId() == a.activityID { @@ -208,7 +218,7 @@ func (a *wfaHandle) activityInfo(t require.TestingT) activityInfo { // terminalOutcome waits for the activity to reach a terminal state and reports it. A workflow activity's // terminal status is not in PendingActivities, so it is read from the workflow-result error's cause. func (a *wfaHandle) terminalOutcome(t require.TestingT) activityTerminalOutcome { - err := a.run.Get(a.d.ctx, nil) + err := a.run.Get(a.testContext(), nil) if err == nil { return activityTerminalOutcome{status: enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED} } @@ -252,7 +262,7 @@ func wfaActivityInfo(p *workflowpb.PendingActivityInfo) activityInfo { // waitForCancelRequested waits until the workflow-initiated cancellation reaches the activity. func (a *wfaHandle) waitForCancelRequested(t testing.TB) { - await.Require(a.d.ctx, t, func(t *await.T) { + await.Require(a.testContext(), t, func(t *await.T) { pendingActivity := a.pendingActivityInfo(t) t.Require().NotNil(pendingActivity, "activity is no longer in progress") t.Require().Equal(enumspb.PENDING_ACTIVITY_STATE_CANCEL_REQUESTED, pendingActivity.GetState()) @@ -261,7 +271,7 @@ func (a *wfaHandle) waitForCancelRequested(t testing.TB) { func (a *wfaHandle) respondCanceledByID() error { _, err := a.d.env.FrontendClient().RespondActivityTaskCanceledById( - a.d.ctx, + a.testContext(), &workflowservice.RespondActivityTaskCanceledByIdRequest{ Namespace: a.d.env.Namespace().String(), WorkflowId: a.workflowID, @@ -277,20 +287,21 @@ func (a *wfaHandle) respondCanceledByID() error { func (a *wfaHandle) rpc(t testing.TB, e model.Event) error { fc := a.d.env.FrontendClient() ns := a.d.env.Namespace().String() + ctx := a.testContext() switch e.Type { case model.HeartbeatType: - _, err := fc.RecordActivityTaskHeartbeat(a.d.ctx, &workflowservice.RecordActivityTaskHeartbeatRequest{ + _, err := fc.RecordActivityTaskHeartbeat(ctx, &workflowservice.RecordActivityTaskHeartbeatRequest{ Namespace: ns, TaskToken: a.token, Details: activityRecordedHeartbeatDetails, }) return err case model.RespondCompletedType: - _, err := fc.RespondActivityTaskCompleted(a.d.ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + _, err := fc.RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ Namespace: ns, TaskToken: a.token, Identity: a.d.env.Tv().WorkerIdentity(), Result: payloads.EncodeString("result"), }) return err case model.RespondCompletedByIDType: - _, err := fc.RespondActivityTaskCompletedById(a.d.ctx, &workflowservice.RespondActivityTaskCompletedByIdRequest{ + _, err := fc.RespondActivityTaskCompletedById(ctx, &workflowservice.RespondActivityTaskCompletedByIdRequest{ Namespace: ns, WorkflowId: a.workflowID, RunId: a.runID, ActivityId: a.activityID, Identity: a.d.env.Tv().WorkerIdentity(), Result: payloads.EncodeString("result"), }) @@ -302,7 +313,7 @@ func (a *wfaHandle) rpc(t testing.TB, e model.Event) error { if e.HasHeartbeatDetails { req.LastHeartbeatDetails = activityHeartbeatDetails } - _, err := fc.RespondActivityTaskFailed(a.d.ctx, req) + _, err := fc.RespondActivityTaskFailed(ctx, req) return err case model.RespondFailedByIDType: req := &workflowservice.RespondActivityTaskFailedByIdRequest{ @@ -312,16 +323,16 @@ func (a *wfaHandle) rpc(t testing.TB, e model.Event) error { if e.HasHeartbeatDetails { req.LastHeartbeatDetails = activityHeartbeatDetails } - _, err := fc.RespondActivityTaskFailedById(a.d.ctx, req) + _, err := fc.RespondActivityTaskFailedById(ctx, req) return err case model.RespondCanceledType: - _, err := fc.RespondActivityTaskCanceled(a.d.ctx, &workflowservice.RespondActivityTaskCanceledRequest{ + _, err := fc.RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ Namespace: ns, TaskToken: a.token, Identity: a.d.env.Tv().WorkerIdentity(), }) return err case model.RequestCancelType: if err := a.d.env.SdkClient().SignalWorkflow( - a.d.ctx, + ctx, a.workflowID, a.runID, wfaCancelSignal, @@ -332,22 +343,22 @@ func (a *wfaHandle) rpc(t testing.TB, e model.Event) error { a.waitForCancelRequested(t) return nil case model.PauseType: - _, err := fc.PauseActivityExecution(a.d.ctx, &workflowservice.PauseActivityExecutionRequest{ + _, err := fc.PauseActivityExecution(ctx, &workflowservice.PauseActivityExecutionRequest{ Namespace: ns, WorkflowId: a.workflowID, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), Reason: "drive", RequestId: uuid.NewString(), }) return err case model.UnpauseType: - _, err := fc.UnpauseActivityExecution(a.d.ctx, &workflowservice.UnpauseActivityExecutionRequest{ + _, err := fc.UnpauseActivityExecution(ctx, &workflowservice.UnpauseActivityExecutionRequest{ Namespace: ns, WorkflowId: a.workflowID, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), }) return err case model.ResetType: - _, err := fc.ResetActivityExecution(a.d.ctx, &workflowservice.ResetActivityExecutionRequest{ + _, err := fc.ResetActivityExecution(ctx, &workflowservice.ResetActivityExecutionRequest{ Namespace: ns, WorkflowId: a.workflowID, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), KeepPaused: e.KeepPaused, ResetHeartbeat: e.ResetHeartbeat, }) return err case model.UpdateOptionsType: - _, err := fc.UpdateActivityExecutionOptions(a.d.ctx, &workflowservice.UpdateActivityExecutionOptionsRequest{ + _, err := fc.UpdateActivityExecutionOptions(ctx, &workflowservice.UpdateActivityExecutionOptionsRequest{ Namespace: ns, WorkflowId: a.workflowID, ActivityId: a.activityID, RunId: a.runID, Identity: a.d.env.Tv().ClientIdentity(), ActivityOptions: &activitypb.ActivityOptions{HeartbeatTimeout: durationpb.New(time.Hour)}, UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, @@ -359,7 +370,7 @@ func (a *wfaHandle) rpc(t testing.TB, e model.Event) error { } func (a *wfaHandle) pollForTask(t require.TestingT, timeout time.Duration) *workflowservice.PollActivityTaskQueueResponse { - ctx, cancel := context.WithTimeout(a.d.ctx, timeout) + ctx, cancel := context.WithTimeout(a.testContext(), timeout) defer cancel() resp, err := a.d.env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ Namespace: a.d.env.Namespace().String(), diff --git a/tests/schedule_migration_test.go b/tests/schedule_migration_test.go index 61405179662..e7623ffc848 100644 --- a/tests/schedule_migration_test.go +++ b/tests/schedule_migration_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/common/testing/testcontext" "go.temporal.io/server/service/worker/dummy" "go.temporal.io/server/service/worker/scheduler" "go.temporal.io/server/tests/testcore" @@ -2241,7 +2242,7 @@ func TestScheduleMigration_NoRunningWorkflows_GeneratorStarts(t *testing.T) { ) env.OverrideDynamicConfig(chasmscheduler.CurrentTweakables, tweakables) - ctx := testcore.NewContext() + ctx := testcontext.For(t) sid := testcore.RandomizeStr("sched-migrate-no-running") wid := testcore.RandomizeStr("sched-migrate-no-running-wf") wt := testcore.RandomizeStr("sched-migrate-no-running-wt") @@ -2292,7 +2293,7 @@ func TestScheduleMigration_NoRunningWorkflows_GeneratorStarts(t *testing.T) { } // Confirm the schedule is live on the CHASM stack before polling for closure. - await.Require(testcore.NewContext(), t, func(t *await.T) { + await.Require(ctx, t, func(t *await.T) { _, err := env.GetTestCluster().SchedulerClient().DescribeSchedule(t.Context(), descReq) require.NoError(t, err) }, 6*time.Second, 100*time.Millisecond) @@ -2300,7 +2301,7 @@ func TestScheduleMigration_NoRunningWorkflows_GeneratorStarts(t *testing.T) { // If generator.Generate() was never called, the idle task is never scheduled // and the schedule stays open indefinitely. Closing proves the generator ran. var closedErr *serviceerror.FailedPrecondition - await.Require(testcore.NewContext(), t, func(t *await.T) { + await.Require(ctx, t, func(t *await.T) { _, err := env.GetTestCluster().SchedulerClient().DescribeSchedule(t.Context(), descReq) require.ErrorAs(t, err, &closedErr) }, 6*time.Second, 100*time.Millisecond) diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index b0d68cafe1e..3e0126835e9 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -77,7 +77,6 @@ type TestEnv struct { taskPoller *taskpoller.TaskPoller t *testing.T tv *testvars.TestVars - ctx context.Context dedicatedGuard *dedicatedClusterGuard sdkClientOnce sync.Once @@ -258,6 +257,10 @@ func NewEnv(t *testing.T, opts ...TestOption) *TestEnv { // Check test sharding early, before any expensive operations. checkTestShard(t) + // Create the test context before any expensive setup, so that the deadline + // extension below can compensate for the time setup takes. + testcontext.For(t) + var options testOptions for _, opt := range opts { opt(&options) @@ -311,6 +314,10 @@ func NewEnv(t *testing.T, opts ...TestOption) *TestEnv { // Attach version headers decorator to the test context. testcontext.AttachDecorator(t, versionHeadersContextKey{}, headers.SetVersions) + // Restore as much of the test's timeout budget as the context's ceiling + // allows, now that setup is done. + testcontext.EnsureRemaining(testcontext.For(t), t, testcontext.DefaultTimeout()) + env := &TestEnv{ FunctionalTestBase: base, Assertions: require.New(t), @@ -321,7 +328,6 @@ func NewEnv(t *testing.T, opts ...TestOption) *TestEnv { taskPoller: taskpoller.New(t, cluster.FrontendClient(), ns.String()), t: t, tv: tv, - ctx: testcontext.For(t), sdkWorkerTQ: RandomizeStr("tq-" + t.Name()), dedicatedGuard: dedicatedGuard, } @@ -460,9 +466,11 @@ func (e *TestEnv) Tv() *testvars.TestVars { // ctx, cancel := context.WithTimeout(env.Context(), 10*time.Second) // defer cancel() // +// The context is deliberately not cached; see [testcontext.EnsureRemaining]. +// // Deprecated: use the suite's Context() method instead. func (e *TestEnv) Context() context.Context { - return e.ctx + return testcontext.For(e.t) } // WaitForChannel waits for ch to receive using the TestEnv context. @@ -470,7 +478,7 @@ func (e *TestEnv) WaitForChannel(ch <-chan struct{}) { e.t.Helper() select { case <-ch: - case <-e.ctx.Done(): + case <-e.Context().Done(): e.FailNow("context timeout while waiting for channel") } } @@ -480,7 +488,7 @@ func (e *TestEnv) SendToChannel(ch chan<- struct{}) { e.t.Helper() select { case ch <- struct{}{}: - case <-e.ctx.Done(): + case <-e.Context().Done(): e.FailNow("context timeout while sending to channel") } } From da890c81aabc5e14b810cbb39935cdff96f14623 Mon Sep 17 00:00:00 2001 From: michaely520 Date: Thu, 20 Aug 2026 14:54:13 -0700 Subject: [PATCH 030/139] Improve replication applied lifecycle events (#11645) ## Summary - capture post-transaction mutable state for fresh `NotFound` snapshots and `IsFirstSync` creation - report successful verify history repairs as `outcome=backfilled` - include the repaired event range and `new_run_id` in the verify applied event - add regression coverage for fresh zombie applies and non-current-branch backfills ## Testing - `go test ./service/history/ndc ./service/history/replication ./common/wideevents -count=1` --- common/wideevents/replication_events.go | 3 + common/wideevents/replication_events_test.go | 3 + .../history/ndc/workflow_state_replicator.go | 22 ++- .../ndc/workflow_state_replicator_test.go | 139 ++++++++++++++++-- ...utable_verify_versioned_transition_task.go | 10 +- ...e_verify_versioned_transition_task_test.go | 47 +++++- .../history/replication/replication_events.go | 9 +- 7 files changed, 214 insertions(+), 19 deletions(-) diff --git a/common/wideevents/replication_events.go b/common/wideevents/replication_events.go index f8a425b0c5e..aff0342edfb 100644 --- a/common/wideevents/replication_events.go +++ b/common/wideevents/replication_events.go @@ -362,6 +362,9 @@ func (p ReplicationLifecyclePayload) appendApplied(attrs []log.KeyValue) []log.K if p.Error != "" { attrs = append(attrs, log.String("error", p.Error)) } + if p.NewRunID != "" { + attrs = append(attrs, log.String("new_run_id", p.NewRunID)) + } if p.State == "" { return attrs } diff --git a/common/wideevents/replication_events_test.go b/common/wideevents/replication_events_test.go index c0cd3d30b75..2becfa89e13 100644 --- a/common/wideevents/replication_events_test.go +++ b/common/wideevents/replication_events_test.go @@ -103,6 +103,7 @@ func TestReplicationLifecycleEncodeApplied(t *testing.T) { LastEventID: 9, LastEventVersion: 5, Outcome: "applied", + NewRunID: "continued-run", NewExecutionRunID: "new-run", SignalCount: 6, UpdateCount: 2, @@ -110,6 +111,7 @@ func TestReplicationLifecycleEncodeApplied(t *testing.T) { f := attrMap(p.Attributes()) require.Equal(t, "applied", f["phase"].AsString()) + require.Equal(t, "continued-run", f["new_run_id"].AsString()) require.Equal(t, "new-run", f["new_execution_run_id"].AsString()) require.Equal(t, int64(6), f["signal_count"].AsInt64()) require.Equal(t, int64(2), f["update_count"].AsInt64()) @@ -243,6 +245,7 @@ func TestReplicationLifecycleFieldSetLocked(t *testing.T) { "phase": "applied", "outcome": "applied", "error": "boom", + "new_run_id": "new-run", "state": "Running", "status": "Unspecified", "applied_next_event_id": int64(10), diff --git a/service/history/ndc/workflow_state_replicator.go b/service/history/ndc/workflow_state_replicator.go index d63f97dd6d9..78269aa4aec 100644 --- a/service/history/ndc/workflow_state_replicator.go +++ b/service/history/ndc/workflow_state_replicator.go @@ -224,6 +224,7 @@ func (r *WorkflowStateReplicatorImpl) SyncWorkflowState( nil, false, skipCloseTransferTask, + nil, ) } @@ -255,6 +256,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( // live ms after the lock is released would race with the next writer. var ms historyi.MutableState var appliedMS *persistencespb.WorkflowMutableState + captureMutableState := func(createdMS historyi.MutableState) { ms = createdMS } origin := wideevents.ReplicationTaskOriginFromContext(ctx) defer func() { if emitLifecycle && retError == nil { @@ -302,7 +304,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( if versionedTransitionArtifact.IsFirstSync { // this is the first replication task for this workflow // TODO: Handle reset case to reduce the amount of history events write - continueProcess, err := r.handleFirstReplicationTask(ctx, archetypeID, wfCtx, versionedTransitionArtifact, sourceClusterName) + continueProcess, err := r.handleFirstReplicationTask(ctx, archetypeID, wfCtx, versionedTransitionArtifact, sourceClusterName, captureMutableState) if err != nil || !continueProcess { return err } @@ -311,7 +313,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( ms, err = wfCtx.LoadMutableState(ctx, r.shardContext) switch err.(type) { case *serviceerror.NotFound: - return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, nil, versionedTransitionArtifact, sourceClusterName) + return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, nil, versionedTransitionArtifact, sourceClusterName, captureMutableState) case nil: localTransitionHistory := ms.GetExecutionInfo().TransitionHistory if len(localTransitionHistory) == 0 { @@ -358,7 +360,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( } if localLastWriteVersion < sourceLastWriteVersion || localLastHistoryItem.GetEventId() <= sourceLastHistoryItem.EventId { - return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, ms, versionedTransitionArtifact, sourceClusterName) + return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, ms, versionedTransitionArtifact, sourceClusterName, nil) } return consts.ErrDuplicate } @@ -371,7 +373,7 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( case errors.Is(err, consts.ErrStaleState): // local is stale, try to apply mutable state update if snapshot != nil { - return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, ms, versionedTransitionArtifact, sourceClusterName) + return r.applySnapshot(ctx, namespaceID, wid, rid, archetypeID, wfCtx, releaseFn, ms, versionedTransitionArtifact, sourceClusterName, nil) } return r.applyMutation(ctx, namespaceID, wid, rid, archetypeID, wfCtx, ms, releaseFn, versionedTransitionArtifact, sourceClusterName) case errors.Is(err, consts.ErrStaleReference): @@ -584,6 +586,7 @@ func (r *WorkflowStateReplicatorImpl) handleFirstReplicationTask( wfCtx historyi.WorkflowContext, versionedTransition *replicationspb.VersionedTransitionArtifact, sourceClusterName string, + captureMutableState func(historyi.MutableState), ) (continueProcess bool, retErr error) { mutation, snapshot, executionState, executionInfo, err := parseVersionedTransitionAttributes(versionedTransition) if err != nil { @@ -602,6 +605,7 @@ func (r *WorkflowStateReplicatorImpl) handleFirstReplicationTask( snapshot, versionedTransition, sourceClusterName, + captureMutableState, ) } @@ -626,6 +630,7 @@ func (r *WorkflowStateReplicatorImpl) handleFirstReplicationTaskWithoutNewRun( snapshot *replicationspb.SyncWorkflowStateSnapshotAttributes, versionedTransition *replicationspb.VersionedTransitionArtifact, sourceClusterName string, + captureMutableState func(historyi.MutableState), ) (continueProcess bool, retErr error) { nsEntry, err := r.namespaceRegistry.GetNamespaceByID(namespace.ID(executionInfo.NamespaceId)) if err != nil { @@ -695,6 +700,9 @@ func (r *WorkflowStateReplicatorImpl) handleFirstReplicationTaskWithoutNewRun( if errors.Is(err, consts.ErrDuplicate) { return true, nil } + if err == nil && captureMutableState != nil { + captureMutableState(localMutableState) + } return false, err } @@ -994,6 +1002,7 @@ func (r *WorkflowStateReplicatorImpl) applySnapshot( localMutableState historyi.MutableState, versionedTransition *replicationspb.VersionedTransitionArtifact, sourceClusterName string, + captureMutableState func(historyi.MutableState), ) error { attribute := versionedTransition.GetSyncWorkflowStateSnapshotAttributes() if attribute == nil || attribute.State == nil { @@ -1026,6 +1035,7 @@ func (r *WorkflowStateReplicatorImpl) applySnapshot( versionedTransition.NewRunInfo, true, versionedTransition.IsCloseTransferTaskAcked && versionedTransition.IsForceReplication, + captureMutableState, ) } return r.applySnapshotWhenWorkflowExist( @@ -1678,6 +1688,7 @@ func (r *WorkflowStateReplicatorImpl) applySnapshotWhenWorkflowNotExist( newRunInfo *replicationspb.NewRunInfo, isStateBased bool, skipGenerateCloseTransferTask bool, + captureMutableState func(historyi.MutableState), ) error { var lastWriteVersion int64 executionInfo := sourceMutableState.ExecutionInfo @@ -1745,6 +1756,9 @@ func (r *WorkflowStateReplicatorImpl) applySnapshotWhenWorkflowNotExist( if err != nil { return err } + if captureMutableState != nil { + captureMutableState(mutableState) + } return r.transactionMgr.CreateWorkflow( ctx, archtypeID, diff --git a/service/history/ndc/workflow_state_replicator_test.go b/service/history/ndc/workflow_state_replicator_test.go index 4db69b7cb05..5b1e29796d9 100644 --- a/service/history/ndc/workflow_state_replicator_test.go +++ b/service/history/ndc/workflow_state_replicator_test.go @@ -9,6 +9,8 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" @@ -33,6 +35,7 @@ import ( "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/quotas" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" historyi "go.temporal.io/server/service/history/interfaces" @@ -45,6 +48,11 @@ import ( ) type ( + replicationEventCapture struct { + embedded.Logger + records []otellog.Record + } + workflowReplicatorSuite struct { suite.Suite *require.Assertions @@ -67,6 +75,14 @@ type ( } ) +func (c *replicationEventCapture) Emit(_ context.Context, record otellog.Record) { + c.records = append(c.records, record) +} + +func (*replicationEventCapture) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + func TestWorkflowReplicatorSuite(t *testing.T) { s := new(workflowReplicatorSuite) suite.Run(t, s) @@ -604,6 +620,100 @@ func EqVersionedTransition(expected *persistencespb.VersionedTransition) gomock. return &VersionedTransitionMatcher{expected: expected} } +func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_NotFound_CapturesCreatedState() { + s.mockShard.GetConfig().EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + eventCapture := &replicationEventCapture{} + workflowStateReplicator := s.workflowStateReplicator + workflowStateReplicator.eventLogger = eventCapture + mockTransactionManager := NewMockTransactionManager(s.controller) + workflowStateReplicator.transactionMgr = mockTransactionManager + + namespaceID := uuid.NewString() + versionedTransitionArtifact := &replicationspb.VersionedTransitionArtifact{ + StateAttributes: &replicationspb.VersionedTransitionArtifact_SyncWorkflowStateSnapshotAttributes{ + SyncWorkflowStateSnapshotAttributes: &replicationspb.SyncWorkflowStateSnapshotAttributes{ + State: &persistencespb.WorkflowMutableState{ + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{ + WorkflowId: s.workflowID, + NamespaceId: namespaceID, + TransitionHistory: []*persistencespb.VersionedTransition{ + {NamespaceFailoverVersion: 2, TransitionCount: 10}, + }, + VersionHistories: versionhistory.NewVersionHistories(&historyspb.VersionHistory{}), + WorkflowExecutionTimeout: timestamp.DurationPtr(time.Hour), + WorkflowRunTimeout: timestamp.DurationPtr(time.Hour), + }, + ExecutionState: &persistencespb.WorkflowExecutionState{ + RunId: s.runID, + State: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + }, + NextEventId: 42, + }, + }, + }, + } + + mockWeCtx := historyi.NewMockWorkflowContext(s.controller) + s.mockWorkflowCache.EXPECT().GetOrCreateChasmExecution( + gomock.Any(), + s.mockShard, + namespace.ID(namespaceID), + &commonpb.WorkflowExecution{WorkflowId: s.workflowID, RunId: s.runID}, + chasm.WorkflowArchetypeID, + locks.PriorityHigh, + ).Return(mockWeCtx, wcache.NoopReleaseFn, nil) + mockWeCtx.EXPECT().LoadMutableState(gomock.Any(), s.mockShard). + Return(nil, serviceerror.NewNotFound("mutable state not found")) + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespace.ID(namespaceID)).Return( + namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: "test-namespace"}, + nil, + false, + nil, + int64(100), + ), + nil, + ) + s.mockNamespaceCache.EXPECT().GetNamespaceName(namespace.ID(namespaceID)). + Return(namespace.Name("test-namespace"), nil) + s.mockEventCache.EXPECT().GetEvent(gomock.Any(), gomock.Any(), gomock.Any(), common.FirstEventID, gomock.Any()). + Return(&historypb.HistoryEvent{ + Attributes: &historypb.HistoryEvent_WorkflowExecutionStartedEventAttributes{ + WorkflowExecutionStartedEventAttributes: &historypb.WorkflowExecutionStartedEventAttributes{}, + }, + }, nil).AnyTimes() + mockTransactionManager.EXPECT().CreateWorkflow( + gomock.Any(), + chasm.WorkflowArchetypeID, + gomock.AssignableToTypeOf(&WorkflowImpl{}), + ).DoAndReturn(func(_ context.Context, _ chasm.ArchetypeID, wf Workflow) error { + wf.GetMutableState().GetExecutionState().State = enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE + wf.GetReleaseFn()(nil) + return nil + }) + + err := workflowStateReplicator.ReplicateVersionedTransition( + context.Background(), + chasm.WorkflowArchetypeID, + versionedTransitionArtifact, + "source-cluster", + ) + s.NoError(err) + s.Require().Len(eventCapture.records, 1) + record := eventCapture.records[0] + s.Equal(wideevents.ReplicationLifecycleEventName, record.EventName()) + fields := make(map[string]otellog.Value) + record.WalkAttributes(func(kv otellog.KeyValue) bool { + fields[kv.Key] = kv.Value + return true + }) + s.Equal(string(wideevents.ReplicationApplied), fields["phase"].AsString()) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE.String(), fields["state"].AsString()) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING.String(), fields["status"].AsString()) + s.Equal(int64(42), fields["applied_next_event_id"].AsInt64()) +} + func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_SameBranch_SyncSnapshot() { workflowStateReplicator := NewWorkflowStateReplicator( s.mockShard, @@ -880,7 +990,9 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_SameBranch_S s.NoError(err) } -func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_FirstTask_SyncMutation() { +func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_FirstTask_CapturesCreatedState() { + s.mockShard.GetConfig().EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + eventCapture := &replicationEventCapture{} workflowStateReplicator := NewWorkflowStateReplicator( s.mockShard, s.mockWorkflowCache, @@ -888,7 +1000,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_FirstTask_Sy s.serializer, quotas.NoopRequestRateLimiter, s.logger, - nil, + eventCapture, ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -936,30 +1048,35 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_FirstTask_Sy locks.PriorityHigh, ).Return(mockWeCtx, wcache.NoopReleaseFn, nil) s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespace.ID(namespaceID)).Return(namespace.NewNamespaceForTest( - &persistencespb.NamespaceInfo{}, + &persistencespb.NamespaceInfo{Name: "test-namespace"}, nil, false, nil, int64(100), ), nil).AnyTimes() + s.mockNamespaceCache.EXPECT().GetNamespaceName(namespace.ID(namespaceID)). + Return(namespace.Name("test-namespace"), nil) mockTaskRefresher.EXPECT().Refresh(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1) mockTransactionManager.EXPECT().CreateWorkflow( gomock.Any(), chasm.WorkflowArchetypeID, gomock.AssignableToTypeOf(&WorkflowImpl{}), - ).DoAndReturn(func(ctx context.Context, _ chasm.ArchetypeID, wf Workflow) error { - // Capture localMutableState from the workflow + ).DoAndReturn(func(_ context.Context, _ chasm.ArchetypeID, wf Workflow) error { localMutableState := wf.GetMutableState() - - // Perform your comparisons here s.Equal(localMutableState.GetExecutionInfo().TransitionHistory, transitionHistory) - + localMutableState.GetExecutionState().State = enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE return nil }).Times(1) err := workflowStateReplicator.ReplicateVersionedTransition(context.Background(), chasm.WorkflowArchetypeID, versionedTransitionArtifact, "test") s.NoError(err) - + s.Require().Len(eventCapture.records, 1) + fields := make(map[string]otellog.Value) + eventCapture.records[0].WalkAttributes(func(kv otellog.KeyValue) bool { + fields[kv.Key] = kv.Value + return true + }) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE.String(), fields["state"].AsString()) } func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_MutationProvidedWithGap_ReturnSyncStateError() { @@ -1668,6 +1785,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_WithSnapshot_S mockWeCtx, versionedTransitionArtifact, "test-cluster", + nil, ) s.NoError(err) s.False(continueProcess) @@ -1742,6 +1860,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_WithMutation_S mockWeCtx, versionedTransitionArtifact, "test-cluster", + nil, ) s.NoError(err) s.False(continueProcess) @@ -1767,6 +1886,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_InvalidArtifac mockWeCtx, versionedTransitionArtifact, "test-cluster", + nil, ) s.Error(err) s.Contains(err.Error(), "unknown artifact type") @@ -1843,6 +1963,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_CreateWorkflow mockWeCtx, versionedTransitionArtifact, "test-cluster", + nil, ) s.Error(err) s.Equal(expectedErr, err) diff --git a/service/history/replication/executable_verify_versioned_transition_task.go b/service/history/replication/executable_verify_versioned_transition_task.go index 6aefd18da03..29542705fd7 100644 --- a/service/history/replication/executable_verify_versioned_transition_task.go +++ b/service/history/replication/executable_verify_versioned_transition_task.go @@ -90,9 +90,10 @@ func (e *ExecutableVerifyVersionedTransitionTask) Execute() (retErr error) { // inspectedMS is the mutable-state snapshot examined during verification, captured for the // best-effort "applied" lifecycle event emitted below. var inspectedMS *persistencespb.WorkflowMutableState + var backfillDetails map[string]any defer func() { if emitLifecycle { - e.emitReplicationVerifyApplied(inspectedMS, retErr) + e.emitReplicationVerifyApplied(inspectedMS, retErr, backfillDetails) } }() @@ -207,6 +208,13 @@ func (e *ExecutableVerifyVersionedTransitionTask) Execute() (retErr error) { if err != nil { return err } + backfillDetails = map[string]any{ + "recovery_action": wideevents.ReplRecoveryActionResendHistory, + "first_event_id": lcaItem.EventId + 1, + "first_event_version": startEventVersion, + "last_event_id": lastItem.EventId, + "last_event_version": lastItem.Version, + } return e.BackFillEvents( ctx, e.SourceClusterName(), diff --git a/service/history/replication/executable_verify_versioned_transition_task_test.go b/service/history/replication/executable_verify_versioned_transition_task_test.go index 05d26a5c79d..c282faca02e 100644 --- a/service/history/replication/executable_verify_versioned_transition_task_test.go +++ b/service/history/replication/executable_verify_versioned_transition_task_test.go @@ -1,6 +1,7 @@ package replication import ( + "encoding/json" "errors" "math/rand" "testing" @@ -9,6 +10,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + otellog "go.opentelemetry.io/otel/log" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" enumsspb "go.temporal.io/server/api/enums/v1" @@ -273,9 +275,9 @@ func (s *executableVerifyVersionedTransitionTaskSuite) mockGetMutableState( runId string, mutableState historyi.MutableState, err error, -) { +) *gomock.Call { shardContext := historyi.NewMockShardContext(s.controller) - s.shardController.EXPECT().GetShardByNamespaceWorkflow( + shardCall := s.shardController.EXPECT().GetShardByNamespaceWorkflow( namespace.ID(s.task.NamespaceID), s.task.WorkflowID, ).Return(shardContext, nil) @@ -294,6 +296,7 @@ func (s *executableVerifyVersionedTransitionTaskSuite) mockGetMutableState( chasm.WorkflowArchetypeID, locks.PriorityHigh, ).Return(wfCtx, func(err error) {}, err) + return shardCall } func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_CurrentBranch_NotUpToDate() { @@ -596,6 +599,16 @@ func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_NonCurrentBra } func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_NonCurrentBranch_NotUpToDate() { + s.config.EmitReplicationLifecycleEvents = func() bool { return true } + eventCapture := &eventCaptureLogger{} + eventShard := historyi.NewMockShardContext(s.controller) + eventShard.EXPECT().GetEventLogger().Return(eventCapture).Times(2) + eventShard.EXPECT().GetShardID().Return(int32(1)).Times(2) + s.namespaceCache.EXPECT().GetNamespaceName(namespace.ID(s.namespaceID)). + Return(namespace.Name("test-namespace"), nil).Times(2) + s.executableTask.EXPECT().Attempt().Return(1) + s.executableTask.EXPECT().SourceShardKey().Return(s.sourceShardKey).AnyTimes() + taskNextEvent := int64(10) replicationTask := &replicationspb.ReplicationTask{ TaskType: enumsspb.REPLICATION_TASK_TYPE_VERIFY_VERSIONED_TRANSITION_TASK, @@ -654,7 +667,14 @@ func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_NonCurrentBra }, ).AnyTimes() - s.mockGetMutableState(s.namespaceID, s.workflowID, s.runID, mu, nil) + executingCall := s.shardController.EXPECT().GetShardByNamespaceWorkflow( + namespace.ID(s.namespaceID), s.workflowID, + ).Return(eventShard, nil) + loadCall := s.mockGetMutableState(s.namespaceID, s.workflowID, s.runID, mu, nil) + appliedCall := s.shardController.EXPECT().GetShardByNamespaceWorkflow( + namespace.ID(s.namespaceID), s.workflowID, + ).Return(eventShard, nil) + gomock.InOrder(executingCall, loadCall, appliedCall) task := NewExecutableVerifyVersionedTransitionTask( s.toolBox, @@ -678,6 +698,27 @@ func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_NonCurrentBra err := task.Execute() s.NoError(err) + s.Require().Len(eventCapture.records, 2) + attrs := make(map[string]otellog.Value) + eventCapture.records[1].WalkAttributes(func(kv otellog.KeyValue) bool { + attrs[kv.Key] = kv.Value + return true + }) + s.Equal("backfilled", attrs["outcome"].AsString()) + s.Equal(s.newRunID, attrs["new_run_id"].AsString()) + var details struct { + RecoveryAction string `json:"recovery_action"` + FirstEventID int64 `json:"first_event_id"` + FirstEventVersion int64 `json:"first_event_version"` + LastEventID int64 `json:"last_event_id"` + LastEventVersion int64 `json:"last_event_version"` + } + s.NoError(json.Unmarshal([]byte(attrs["details"].AsString()), &details)) + s.Equal("resend_history", details.RecoveryAction) + s.Equal(int64(9), details.FirstEventID) + s.Equal(int64(1), details.FirstEventVersion) + s.Equal(int64(9), details.LastEventID) + s.Equal(int64(1), details.LastEventVersion) } func (s *executableVerifyVersionedTransitionTaskSuite) TestExecute_Skip_TerminalState() { diff --git a/service/history/replication/replication_events.go b/service/history/replication/replication_events.go index 42f2af70a36..7603e718047 100644 --- a/service/history/replication/replication_events.go +++ b/service/history/replication/replication_events.go @@ -522,12 +522,13 @@ func shippedEventRange( } // emitReplicationVerifyApplied emits a best-effort "applied" ReplicationLifecycle event for a -// verify task. Outcome is "verified" when verification passed (no error) and "resend_needed" when -// the task requested a state resend; any other error is reported with its message. ms may be nil +// verify task. Outcome is "verified" for a no-op success, "backfilled" after repairing history, +// "resend_needed" when requesting state resend, or "error". ms may be nil // (e.g. the workflow was not found) in which case only identity fields are populated. func (e *ExecutableVerifyVersionedTransitionTask) emitReplicationVerifyApplied( ms *persistencespb.WorkflowMutableState, retErr error, + backfillDetails map[string]any, ) { shardContext, err := e.ShardController.GetShardByNamespaceWorkflow(namespace.ID(e.NamespaceID), e.WorkflowID) if err != nil { @@ -548,6 +549,8 @@ func (e *ExecutableVerifyVersionedTransitionTask) emitReplicationVerifyApplied( outcome = "error" errStr = retErr.Error() } + } else if backfillDetails != nil { + outcome = "backfilled" } var nsName string @@ -568,6 +571,7 @@ func (e *ExecutableVerifyVersionedTransitionTask) emitReplicationVerifyApplied( SourceCluster: e.SourceClusterName(), SourceShard: e.SourceShardKey().ShardID, SourceTaskID: e.TaskID(), + NewRunID: e.taskAttr.GetNewRunId(), } if vt := e.ReplicationTask().GetVersionedTransition(); vt != nil { payload.FailoverVersion = vt.GetNamespaceFailoverVersion() @@ -582,6 +586,7 @@ func (e *ExecutableVerifyVersionedTransitionTask) emitReplicationVerifyApplied( if ms != nil { details["actual"] = actualState(ms) } + maps.Copy(details, backfillDetails) payload.Details = details wideevents.Emit(logger, payload) From 042505361519608454d4a016b3d0971c81c724c8 Mon Sep 17 00:00:00 2001 From: mavemuri <74267563+mavemuri@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:43:29 -0700 Subject: [PATCH 031/139] [SDK Ergonomics] NEXUS-519: Support Query-backed Nexus Operations (#11274) ## What changed? Adds server support for WorkflowQuery-backed Nexus Operations ## Why? Part of effort to expose all Temporal primitives as Nexus Operations ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [x] added new functional test(s) ## TODO: - [x] https://github.com/temporalio/api/pull/842 --- common/links/validator.go | 10 ++ common/links/validator_test.go | 33 +++++- common/nexus/links.go | 17 ++- common/nexus/links_test.go | 50 +++++++-- service/history/api/queryworkflow/api.go | 23 ++++- tests/nexus_workflow_query_test.go | 105 +++++++++++++++++++ tests/query_workflow_test.go | 125 +++++++++++++++++++++++ 7 files changed, 351 insertions(+), 12 deletions(-) create mode 100644 tests/nexus_workflow_query_test.go diff --git a/common/links/validator.go b/common/links/validator.go index 40d894e3ed4..e05c25b4d8d 100644 --- a/common/links/validator.go +++ b/common/links/validator.go @@ -72,6 +72,16 @@ func Validate(links []*commonpb.Link, maxAllowedLinks, maxSize int) error { if t.Activity.GetRunId() == "" { return serviceerror.NewInvalidArgument("activity link must not have an empty run ID field") } + case *commonpb.Link_Workflow_: + if t.Workflow.GetNamespace() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty namespace field") + } + if t.Workflow.GetWorkflowId() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty workflow ID field") + } + if t.Workflow.GetRunId() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty run ID field") + } default: return serviceerror.NewInvalidArgument("unsupported link variant") } diff --git a/common/links/validator_test.go b/common/links/validator_test.go index 74c39980620..816de48f597 100644 --- a/common/links/validator_test.go +++ b/common/links/validator_test.go @@ -52,6 +52,15 @@ func TestValidate(t *testing.T) { }, }, } + validWorkflow := &commonpb.Link{ + Variant: &commonpb.Link_Workflow_{ + Workflow: &commonpb.Link_Workflow{ + Namespace: "ns", + WorkflowId: "wid", + RunId: "rid", + }, + }, + } t.Run("HappyPath", func(t *testing.T) { err := links.Validate([]*commonpb.Link{ @@ -59,7 +68,8 @@ func TestValidate(t *testing.T) { validBatchJob, validNexusOperation, validActivity, - }, maxLinks+1, maxSize) + validWorkflow, + }, maxLinks+2, maxSize) require.NoError(t, err) }) @@ -155,4 +165,25 @@ func TestValidate(t *testing.T) { err := links.Validate([]*commonpb.Link{{}}, maxLinks, maxSize) require.ErrorContains(t, err, "unsupported link variant") }) + + t.Run("Workflow/EmptyNamespace", func(t *testing.T) { + l := proto.Clone(validWorkflow).(*commonpb.Link) + l.GetWorkflow().Namespace = "" + err := links.Validate([]*commonpb.Link{l}, maxLinks, maxSize) + require.EqualError(t, err, "workflow link must not have an empty namespace field") + }) + + t.Run("Workflow/EmptyWorkflowID", func(t *testing.T) { + l := proto.Clone(validWorkflow).(*commonpb.Link) + l.GetWorkflow().WorkflowId = "" + err := links.Validate([]*commonpb.Link{l}, maxLinks, maxSize) + require.EqualError(t, err, "workflow link must not have an empty workflow ID field") + }) + + t.Run("Workflow/EmptyRunID", func(t *testing.T) { + l := proto.Clone(validWorkflow).(*commonpb.Link) + l.GetWorkflow().RunId = "" + err := links.Validate([]*commonpb.Link{l}, maxLinks, maxSize) + require.EqualError(t, err, "workflow link must not have an empty run ID field") + }) } diff --git a/common/nexus/links.go b/common/nexus/links.go index f75cf05cfd3..83f253bb3d2 100644 --- a/common/nexus/links.go +++ b/common/nexus/links.go @@ -5,13 +5,14 @@ import ( "github.com/nexus-rpc/sdk-go/nexus" commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/temporalnexus" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" ) // ConvertNexusLinksToProtoLinks converts a slice of Nexus SDK links into Temporal proto links, -// supporting Link_WorkflowEvent and Link_Activity variants. Unsupported or malformed entries -// are skipped with a warning since links are non-essential to execution. +// supporting Link_Workflow, Link_WorkflowEvent, and Link_Activity variants. Unsupported or +// malformed entries are skipped with a warning since links are non-essential to execution. func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) []*commonpb.Link { var out []*commonpb.Link for _, nexusLink := range nexusLinks { @@ -40,6 +41,18 @@ func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) [ out = append(out, &commonpb.Link{ Variant: &commonpb.Link_Activity_{Activity: link}, }) + case string((&commonpb.Link_Workflow{}).ProtoReflect().Descriptor().FullName()): + link, err := temporalnexus.ConvertNexusLinkToLinkWorkflow(nexusLink) + if err != nil { + logger.Warn( + fmt.Sprintf("failed to parse link to %q: %s", nexusLink.Type, nexusLink.URL), + tag.Error(err), + ) + continue + } + out = append(out, &commonpb.Link{ + Variant: &commonpb.Link_Workflow_{Workflow: link}, + }) default: logger.Warn(fmt.Sprintf("invalid link data type: %q", nexusLink.Type)) } diff --git a/common/nexus/links_test.go b/common/nexus/links_test.go index ac841d6105b..9bd981d5cef 100644 --- a/common/nexus/links_test.go +++ b/common/nexus/links_test.go @@ -13,12 +13,10 @@ import ( "go.temporal.io/server/common/testing/protorequire" ) -// TestConvertNexusLinksToProtoLinks_ActivityVariant verifies that the shared -// converter handles both WorkflowEvent and Activity link variants, drops -// unsupported types, and skips malformed entries — exercised by the Nexus task -// handler's start-response flow so a SAA invoked from a Nexus operation can -// surface its Activity link back to the caller. -func TestConvertNexusLinksToProtoLinks_ActivityVariant(t *testing.T) { +// TestConvertNexusLinksToProtoLinks verifies that the converter handles the +// Workflow, WorkflowEvent, and Activity link variants, drops unsupported types, +// and skips malformed entries. +func TestConvertNexusLinksToProtoLinks(t *testing.T) { logger := log.NewTestLogger() workflowEvent := nexus.Link{ @@ -36,6 +34,13 @@ func TestConvertNexusLinksToProtoLinks_ActivityVariant(t *testing.T) { }, Type: "temporal.api.common.v1.Link.Activity", } + workflow := nexus.Link{ + URL: &url.URL{ + Scheme: "temporal", + Path: "/namespaces/ns/workflows/wf-id/run-id", + }, + Type: "temporal.api.common.v1.Link.Workflow", + } unsupported := nexus.Link{ URL: &url.URL{Scheme: "temporal", Path: "/foo"}, Type: "unknown.Type", @@ -44,9 +49,29 @@ func TestConvertNexusLinksToProtoLinks_ActivityVariant(t *testing.T) { URL: &url.URL{Scheme: "temporal", Path: "/namespaces/ns/foo/act-id"}, Type: "temporal.api.common.v1.Link.Activity", } + malformedWorkflows := []nexus.Link{ + { + URL: &url.URL{Scheme: "temporal", Path: "/namespaces//workflows/wid/rid"}, // missing ns + Type: "temporal.api.common.v1.Link.Workflow", + }, + { + URL: &url.URL{Scheme: "temporal", Path: "/namespaces/ns/workflows//rid"}, // missing wid + Type: "temporal.api.common.v1.Link.Workflow", + }, + { + URL: &url.URL{Scheme: "temporal", Path: "/namespaces/ns/workflows/wid/"}, // missing rid + Type: "temporal.api.common.v1.Link.Workflow", + }, + { + URL: &url.URL{Scheme: "temporal", Path: "/foo"}, // incorrect path + Type: "temporal.api.common.v1.Link.Workflow", + }, + } + nexusLinks := []nexus.Link{workflowEvent, activity, workflow, unsupported, malformedActivity} + nexusLinks = append(nexusLinks, malformedWorkflows...) - out := commonnexus.ConvertNexusLinksToProtoLinks([]nexus.Link{workflowEvent, activity, unsupported, malformedActivity}, logger) - require.Len(t, out, 2, "workflow-event and activity links must round-trip; unsupported and malformed entries must be dropped") + out := commonnexus.ConvertNexusLinksToProtoLinks(nexusLinks, logger) + require.Len(t, out, 3, "workflow, workflow-event, and activity links must round-trip; unsupported and malformed entries must be dropped") expected := []*commonpb.Link{ { @@ -73,6 +98,15 @@ func TestConvertNexusLinksToProtoLinks_ActivityVariant(t *testing.T) { }, }, }, + { + Variant: &commonpb.Link_Workflow_{ + Workflow: &commonpb.Link_Workflow{ + Namespace: "ns", + WorkflowId: "wf-id", + RunId: "run-id", + }, + }, + }, } protorequire.ProtoSliceEqual(t, expected, out) } diff --git a/service/history/api/queryworkflow/api.go b/service/history/api/queryworkflow/api.go index 3f4ddfd35dc..b0b85a8daae 100644 --- a/service/history/api/queryworkflow/api.go +++ b/service/history/api/queryworkflow/api.go @@ -37,7 +37,7 @@ func Invoke( workflowConsistencyChecker api.WorkflowConsistencyChecker, rawMatchingClient matchingservice.MatchingServiceClient, matchingClient matchingservice.MatchingServiceClient, -) (_ *historyservice.QueryWorkflowResponse, retError error) { +) (resp *historyservice.QueryWorkflowResponse, retError error) { scope := shardContext.GetMetricsHandler().WithTags(metrics.OperationTag(metrics.HistoryQueryWorkflowScope)) namespaceID := namespace.ID(request.GetNamespaceId()) err := api.ValidateNamespaceUUID(namespaceID) @@ -79,6 +79,27 @@ func Invoke( // Note: QueryWorkflow should not alter mutable state, so it is safe to ignore error and not clear ms. workflowLease.GetReleaseFn()(nil) }() + defer func() { + if retError != nil || resp.GetResponse() == nil { + return + } + // Add link to Workflow regardless of query status. A rejection on the query is not an RPC error, + // so it gets the same link that a processed query would get - only the reason differs. + reason := "Query processed" + if resp.GetResponse().GetQueryRejected() != nil { + reason = "Query rejected" + } + resp.Response.Link = &commonpb.Link{ + Variant: &commonpb.Link_Workflow_{ + Workflow: &commonpb.Link_Workflow{ + Namespace: nsEntry.Name().String(), + WorkflowId: workflowKey.WorkflowID, + RunId: workflowKey.RunID, + Reason: reason, + }, + }, + } + }() // Context metadata is automatically set during mutable state transaction close for operations that mutate state. // Since QueryWorkflow is readonly and never closes the transaction, we explicitly call SetContextMetadata diff --git a/tests/nexus_workflow_query_test.go b/tests/nexus_workflow_query_test.go new file mode 100644 index 00000000000..b120a5f7bad --- /dev/null +++ b/tests/nexus_workflow_query_test.go @@ -0,0 +1,105 @@ +package tests + +import ( + "context" + + "github.com/nexus-rpc/sdk-go/nexus" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + querypb "go.temporal.io/api/query/v1" + apitemporalnexus "go.temporal.io/api/temporalnexus" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/sdk/client" + "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" + "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/payloads" + "go.temporal.io/server/tests/testcore" +) + +func (s *NexusWorkflowTestSuite) TestNexusOperationBackedByQuery(chasmEnabled bool) { + env := s.newTestEnv(chasmEnabled) + ctx := s.Context() + taskQueue := testcore.RandomizeStr(s.T().Name()) + + queryType := "status" + signalName := "done" + handlerWorkflowID := testcore.RandomizeStr(s.T().Name() + "-handler") + handlerWf := func(ctx workflow.Context) error { + _ = workflow.SetQueryHandler(ctx, queryType, func() (string, error) { + return "handler-status", nil + }) + workflow.GetSignalChannel(ctx, signalName).Receive(ctx, nil) + return nil + } + + h := nexustest.Handler{ + OnStartOperation: func( + ctx context.Context, + service, operation string, + input *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + resp, err := env.FrontendClient().QueryWorkflow(ctx, &workflowservice.QueryWorkflowRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: handlerWorkflowID}, + Query: &querypb.WorkflowQuery{QueryType: queryType}, + }) + if err != nil { + return nil, err + } + // simulate the stitching that will be done by the SDKs eventually + nexus.AddHandlerLinks(ctx, apitemporalnexus.ConvertLinkWorkflowToNexusLink(resp.GetLink().GetWorkflow())) + + var result string + if err := payloads.Decode(resp.GetQueryResult(), &result); err != nil { + return nil, err + } + return &nexus.HandlerStartOperationResultSync[any]{Value: result}, nil + }, + } + endpointName := env.createRandomExternalNexusServer(ctx, s.T(), h) + + callerWF := func(ctx workflow.Context) (string, error) { + c := workflow.NewNexusClient(endpointName, "service") + fut := c.ExecuteOperation(ctx, "operation", "input", workflow.NexusOperationOptions{}) + var result string + err := fut.Get(ctx, &result) + return result, err + } + + w := worker.New(env.SdkClient(), taskQueue, worker.Options{}) + w.RegisterWorkflow(callerWF) + w.RegisterWorkflow(handlerWf) + s.NoError(w.Start()) + defer w.Stop() + + handlerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + ID: handlerWorkflowID, + TaskQueue: taskQueue, + }, handlerWf) + s.NoError(err) + + callerRun, err := env.SdkClient().ExecuteWorkflow(ctx, client.StartWorkflowOptions{ + TaskQueue: taskQueue, + }, callerWF) + s.NoError(err) + + var result string + s.NoError(callerRun.Get(ctx, &result)) + s.Equal("handler-status", result) + + // verify the nexus operation completed event carries a link to the handler's workflow + hist := env.GetHistory(env.Namespace().String(), &commonpb.WorkflowExecution{WorkflowId: callerRun.GetID()}) + completedEvent := s.RequireHistoryEvent(hist, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED) + s.Len(completedEvent.GetLinks(), 1) + workflowLink := completedEvent.GetLinks()[0].GetWorkflow() + s.NotNil(workflowLink, "completed event must carry a link of type workflow") + s.Equal(env.Namespace().String(), workflowLink.GetNamespace()) + s.Equal(handlerWorkflowID, workflowLink.GetWorkflowId()) + s.Equal(handlerRun.GetRunID(), workflowLink.GetRunId()) + s.Equal("Query processed", workflowLink.GetReason()) + + s.NoError(env.SdkClient().SignalWorkflow(ctx, handlerWorkflowID, "", signalName, nil)) + s.NoError(handlerRun.Get(ctx, nil)) +} diff --git a/tests/query_workflow_test.go b/tests/query_workflow_test.go index e9c30cb257a..1bf7ac98448 100644 --- a/tests/query_workflow_test.go +++ b/tests/query_workflow_test.go @@ -21,6 +21,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" sdkclient "go.temporal.io/sdk/client" + "go.temporal.io/sdk/converter" "go.temporal.io/sdk/worker" "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/dynamicconfig" @@ -140,6 +141,130 @@ func (s *QueryWorkflowSuite) TestQueryWorkflow_Consistent_PiggybackQuery() { s.Equal("pauseabc", queryResultStr) } +func (s *QueryWorkflowSuite) TestQueryWorkflowResult_ContainsWorkflowLink() { + env := testcore.NewEnv(s.T()) + queryName := "query" + signalName := "test" + workflowFn := func(ctx workflow.Context) error { + orderStatus := "initialized" + _ = workflow.SetQueryHandler(ctx, queryName, func(_ string) (string, error) { + return orderStatus, nil + }) + workflow.GetSignalChannel(ctx, signalName).Receive(ctx, &orderStatus) + return nil + } + ctx := s.Context() + + env.SdkWorker().RegisterWorkflow(workflowFn) + + wid := "nexus-query-workflow-tests" + run, err := env.SdkClient().ExecuteWorkflow( + ctx, + sdkclient.StartWorkflowOptions{ + ID: wid, + TaskQueue: env.WorkerTaskQueue()}, + workflowFn, + ) + s.NoError(err) + + type testCase struct { + name string + query *querypb.WorkflowQuery + err bool + reason string + result string + queryRejectCondition enumspb.QueryRejectCondition + } + + runTestCase := func(tc testCase) { + resp, err := env.FrontendClient().QueryWorkflow(ctx, &workflowservice.QueryWorkflowRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: wid}, + Query: tc.query, + QueryRejectCondition: tc.queryRejectCondition, + }) + if tc.err { + s.Error(err) + return + } + s.NoError(err) + link := resp.GetLink().GetWorkflow() + s.NotNil(link, "query must carry a link of type workflow") + s.Equal(env.Namespace().String(), link.GetNamespace()) + s.Equal(wid, link.GetWorkflowId()) + s.Equal(run.GetRunID(), link.GetRunId()) + s.Equal(tc.reason, link.GetReason()) + if tc.result != "" { + s.Equal(tc.result, testcore.DecodeString(s.T(), resp.QueryResult)) + } + } + + runCases := func(groupName string, cases []testCase) { + for _, tc := range cases { + s.T().Run(groupName+"/"+tc.name, func(t *testing.T) { //nolint:testifylint // subtests need serialized execution(running, completed) + runTestCase(tc) + }) + } + } + + // TCs that are expected to fail on both running and completed workflows identically + invalidTestCases := []testCase{ + { + name: "malformed args", + query: &querypb.WorkflowQuery{ + QueryType: queryName, + QueryArgs: &commonpb.Payloads{ + Payloads: []*commonpb.Payload{{ + Metadata: map[string][]byte{ + converter.MetadataEncoding: []byte(converter.MetadataEncodingJSON), + }, + Data: []byte("dummy data"), // invalid JSON, fails to decode + }}, + }, + }, + err: true, + }, + { + name: "unknown query type", + query: &querypb.WorkflowQuery{QueryType: "some unknown query"}, + err: true, + }, + } + + runningWorkflowTestCases := []testCase{ + { + name: "well-formed query", + query: &querypb.WorkflowQuery{QueryType: queryName}, + reason: "Query processed", + result: "initialized", + }, + } + runCases("running-workflow", runningWorkflowTestCases) + runCases("running-workflow", invalidTestCases) + + // set the query result via signal so that query "reload" on completed workflow is verified inline + s.NoError(env.SdkClient().SignalWorkflow(ctx, wid, "", signalName, "order-delivered")) + s.NoError(run.Get(ctx, nil)) + + completedWorkflowTestCases := []testCase{ + { + name: "well-formed replayable query succeeds", + query: &querypb.WorkflowQuery{QueryType: queryName}, + reason: "Query processed", + result: "order-delivered", + }, + { + name: "well-formed non-replayable query fails", + query: &querypb.WorkflowQuery{QueryType: queryName}, + reason: "Query rejected", + queryRejectCondition: enumspb.QUERY_REJECT_CONDITION_NOT_OPEN, + }, + } + + runCases("completed-workflow", completedWorkflowTestCases) + runCases("completed-workflow", invalidTestCases) +} + func (s *QueryWorkflowSuite) TestQueryWorkflow_QueryWhileBackoff() { env := testcore.NewEnv(s.T()) tv := testvars.New(s.T()) From 20e655da3a5995c2491cb3bfc6948ade96ae8418 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Thu, 20 Aug 2026 16:03:19 -0700 Subject: [PATCH 032/139] Make all force replication tasks low priority (#11692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Every task produced by `GenerateMigrationTasks` is now low priority end to end: - Added a `Priority` field to `HistoryReplicationTask`, `SyncActivityTask` and `SyncHSMTask`. - Round-tripped `Priority` through the replication task serializer for those three, plus `SyncVersionedTransitionTask` (its `Priority` field existed but was never persisted). The proto `ReplicationTaskInfo.Priority` field already existed, so no proto change. - `StreamSenderImpl.getTaskPriority` now honors `Priority` on all five replication task types, defaulting to high when unspecified. - `GenerateMigrationTasks` stamps `TASK_PRIORITY_LOW` on the `HistoryReplicationTask`, the sync activity tasks and the `SyncHSMTask` it returns (`SyncWorkflowStateTask` and `SyncVersionedTransitionTask` already set it). ## Why? `GenerateMigrationTasks` is only reachable through force replication (`GenerateLastHistoryReplicationTasks`, called by the migration/force-replication workflow and by tdbg). That traffic is bulk backfill and should not compete with live replication. Only `SyncWorkflowStateTask` was actually being treated as low priority; everything else fell through to `TASK_PRIORITY_HIGH` in the stream sender, and `SyncVersionedTransitionTask`'s low priority was silently dropped on write. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) `TestTaskGeneratorImpl_GenerateMigrationTasks` now asserts `TASK_PRIORITY_LOW` on every returned task and every task equivalent. `service/history/replication`, `common/persistence/serialization` and `service/history/tasks` pass. Note: `service/history/workflow` has a pre-existing failure in `TestTaskRefresherSuite/TestRefreshSubStateMachineTasks` that reproduces on unmodified `main` at 39fc2c45e and is unrelated to this change. ## Potential risks - `SyncVersionedTransitionTask.Priority` is now persisted where it was previously dropped. Tasks written before this change still deserialize with `TASK_PRIORITY_UNSPECIFIED`. - `getTaskPriority` defaults to high on `UNSPECIFIED` for the newly-handled types, so normal (non-force) replication keeps its current priority. Only tasks explicitly stamped low move to the low priority stream. - Force replication tasks now share the low priority stream and its rate limiting with sync-state traffic, so a large force replication may progress more slowly than before — which is the intent. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../serialization/task_serializers.go | 8 +++++++ .../serialization/task_serializers_test.go | 5 +++++ service/history/replication/stream_sender.go | 15 +++++++++++++ .../tasks/activity_replication_task.go | 1 + .../history/tasks/history_replication_task.go | 1 + service/history/tasks/sync_hsm_task.go | 1 + service/history/workflow/task_generator.go | 16 ++++++++++++-- .../history/workflow/task_generator_test.go | 21 +++++++++++++++++++ 8 files changed, 66 insertions(+), 2 deletions(-) diff --git a/common/persistence/serialization/task_serializers.go b/common/persistence/serialization/task_serializers.go index 652d7baa0fe..beec7718f7d 100644 --- a/common/persistence/serialization/task_serializers.go +++ b/common/persistence/serialization/task_serializers.go @@ -1207,6 +1207,7 @@ func replicationActivityTaskToProto( BranchToken: nil, NewRunBranchToken: nil, VisibilityTime: timestamppb.New(activityTask.VisibilityTimestamp), + Priority: activityTask.Priority, TargetClusters: activityTask.TargetClusters, } } @@ -1228,6 +1229,7 @@ func replicationActivityTaskFromProto( Version: activityTask.Version, TaskID: activityTask.TaskId, ScheduledEventID: activityTask.ScheduledEventId, + Priority: activityTask.Priority, TargetClusters: activityTask.TargetClusters, } } @@ -1249,6 +1251,7 @@ func replicationHistoryTaskToProto( NewRunBranchToken: historyTask.NewRunBranchToken, NewRunId: historyTask.NewRunID, VisibilityTime: timestamppb.New(historyTask.VisibilityTimestamp), + Priority: historyTask.Priority, TargetClusters: historyTask.TargetClusters, } } @@ -1274,6 +1277,7 @@ func replicationHistoryTaskFromProto( BranchToken: historyTask.BranchToken, NewRunBranchToken: historyTask.NewRunBranchToken, NewRunID: historyTask.NewRunId, + Priority: historyTask.Priority, TargetClusters: historyTask.TargetClusters, } } @@ -1360,6 +1364,7 @@ func replicationSyncHSMTaskToProto( TaskType: enumsspb.TASK_TYPE_REPLICATION_SYNC_HSM, TaskId: syncHSMTask.TaskID, VisibilityTime: timestamppb.New(syncHSMTask.VisibilityTimestamp), + Priority: syncHSMTask.Priority, TargetClusters: syncHSMTask.TargetClusters, } } @@ -1379,6 +1384,7 @@ func replicationSyncHSMTaskFromProto( ), VisibilityTimestamp: visibilityTimestamp, TaskID: syncHSMTask.TaskId, + Priority: syncHSMTask.Priority, TargetClusters: syncHSMTask.TargetClusters, } } @@ -1404,6 +1410,7 @@ func replicationSyncVersionedTransitionTaskToProto( TaskId: syncVersionedTransitionTask.TaskID, VisibilityTime: timestamppb.New(syncVersionedTransitionTask.VisibilityTimestamp), ArchetypeId: syncVersionedTransitionTask.ArchetypeID, + Priority: syncVersionedTransitionTask.Priority, VersionedTransition: syncVersionedTransitionTask.VersionedTransition, FirstEventId: syncVersionedTransitionTask.FirstEventID, Version: syncVersionedTransitionTask.FirstEventVersion, @@ -1444,6 +1451,7 @@ func replicationSyncVersionedTransitionTaskFromProto( VisibilityTimestamp: visibilityTimestamp, TaskID: syncVersionedTransitionTask.TaskId, ArchetypeID: syncVersionedTransitionTask.ArchetypeId, + Priority: syncVersionedTransitionTask.Priority, FirstEventID: syncVersionedTransitionTask.FirstEventId, FirstEventVersion: syncVersionedTransitionTask.Version, NextEventID: syncVersionedTransitionTask.NextEventId, diff --git a/common/persistence/serialization/task_serializers_test.go b/common/persistence/serialization/task_serializers_test.go index 44d1ae034b9..34719e38f9c 100644 --- a/common/persistence/serialization/task_serializers_test.go +++ b/common/persistence/serialization/task_serializers_test.go @@ -371,6 +371,7 @@ func (s *taskSerializerSuite) TestSyncActivityTask() { TaskID: rand.Int63(), Version: rand.Int63(), ScheduledEventID: rand.Int63(), + Priority: enumsspb.TASK_PRIORITY_LOW, } s.assertEqualTasks(syncActivityTask) @@ -386,6 +387,7 @@ func (s *taskSerializerSuite) TestHistoryReplicationTask() { NextEventID: rand.Int63(), BranchToken: shuffle.Bytes([]byte("random branch token")), NewRunBranchToken: shuffle.Bytes([]byte("random new branch token")), + Priority: enumsspb.TASK_PRIORITY_LOW, } s.assertEqualTasks(historyReplicationTask) @@ -396,6 +398,7 @@ func (s *taskSerializerSuite) TestSyncHSMTask() { WorkflowKey: s.workflowKey, VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling TaskID: rand.Int63(), + Priority: enumsspb.TASK_PRIORITY_LOW, } s.assertEqualTasks(syncHSMTask) @@ -410,6 +413,7 @@ func (s *taskSerializerSuite) TestSyncVersionedTransitionTask() { FirstEventID: rand.Int63(), NextEventID: rand.Int63(), NewRunID: uuid.New().String(), + Priority: enumsspb.TASK_PRIORITY_LOW, VersionedTransition: &persistencespb.VersionedTransition{ NamespaceFailoverVersion: rand.Int63(), TransitionCount: rand.Int63(), @@ -422,6 +426,7 @@ func (s *taskSerializerSuite) TestSyncVersionedTransitionTask() { NextEventID: rand.Int63(), Version: rand.Int63(), NewRunID: uuid.New().String(), + Priority: enumsspb.TASK_PRIORITY_LOW, }, }, } diff --git a/service/history/replication/stream_sender.go b/service/history/replication/stream_sender.go index 62a4f5ab648..0489fec79f7 100644 --- a/service/history/replication/stream_sender.go +++ b/service/history/replication/stream_sender.go @@ -776,11 +776,26 @@ func (s *StreamSenderImpl) getTaskPriority(task tasks.Task) enumsspb.TaskPriorit return enumsspb.TASK_PRIORITY_LOW } return t.Priority + case *tasks.SyncVersionedTransitionTask: + return defaultHighTaskPriority(t.Priority) + case *tasks.HistoryReplicationTask: + return defaultHighTaskPriority(t.Priority) + case *tasks.SyncActivityTask: + return defaultHighTaskPriority(t.Priority) + case *tasks.SyncHSMTask: + return defaultHighTaskPriority(t.Priority) default: return enumsspb.TASK_PRIORITY_HIGH } } +func defaultHighTaskPriority(priority enumsspb.TaskPriority) enumsspb.TaskPriority { + if priority == enumsspb.TASK_PRIORITY_UNSPECIFIED { + return enumsspb.TASK_PRIORITY_HIGH + } + return priority +} + func (s *StreamSenderImpl) getTaskTargetCluster(task tasks.Task) []string { switch t := task.(type) { case *tasks.SyncWorkflowStateTask: diff --git a/service/history/tasks/activity_replication_task.go b/service/history/tasks/activity_replication_task.go index b83dca6232d..16a1179ab28 100644 --- a/service/history/tasks/activity_replication_task.go +++ b/service/history/tasks/activity_replication_task.go @@ -16,6 +16,7 @@ type ( TaskID int64 Version int64 ScheduledEventID int64 + Priority enumsspb.TaskPriority TargetClusters []string } ) diff --git a/service/history/tasks/history_replication_task.go b/service/history/tasks/history_replication_task.go index f5905c6d199..02c88aa9deb 100644 --- a/service/history/tasks/history_replication_task.go +++ b/service/history/tasks/history_replication_task.go @@ -18,6 +18,7 @@ type ( NextEventID int64 Version int64 NewRunID string + Priority enumsspb.TaskPriority TargetClusters []string // deprecated diff --git a/service/history/tasks/sync_hsm_task.go b/service/history/tasks/sync_hsm_task.go index d96c5ca8205..62e9aa32892 100644 --- a/service/history/tasks/sync_hsm_task.go +++ b/service/history/tasks/sync_hsm_task.go @@ -14,6 +14,7 @@ type ( definition.WorkflowKey VisibilityTimestamp time.Time TaskID int64 + Priority enumsspb.TaskPriority TargetClusters []string } ) diff --git a/service/history/workflow/task_generator.go b/service/history/workflow/task_generator.go index 07853dfd640..3e3cf4c4ded 100644 --- a/service/history/workflow/task_generator.go +++ b/service/history/workflow/task_generator.go @@ -87,6 +87,8 @@ type ( GenerateHistoryReplicationTasks( eventBatches [][]*historypb.HistoryEvent, ) ([]tasks.Task, error) + // GenerateMigrationTasks generates low priority replication tasks and is + // for the force replication path only. Do not call it for live replication. GenerateMigrationTasks(targetClusters []string) ([]tasks.Task, int64, error) // Generate tasks for any updated state machines on mutable state. @@ -767,6 +769,11 @@ func (r *TaskGeneratorImpl) GenerateHistoryReplicationTasks( }, nil } +// GenerateMigrationTasks must only be called from the force replication path +// (i.e. GenerateLastHistoryReplicationTasks), never from live replication. +// Every task it returns is marked TASK_PRIORITY_LOW so that bulk backfill does +// not compete with live replication traffic; using it for live replication +// would silently demote those tasks. func (r *TaskGeneratorImpl) GenerateMigrationTasks(targetClusters []string) ([]tasks.Task, int64, error) { executionInfo := r.mutableState.GetExecutionInfo() versionHistory, err := versionhistory.GetCurrentVersionHistory(executionInfo.GetVersionHistories()) @@ -843,22 +850,27 @@ func (r *TaskGeneratorImpl) GenerateMigrationTasks(targetClusters []string) ([]t FirstEventID: executionInfo.LastFirstEventId, NextEventID: nextEventID, Version: lastItem.GetVersion(), + Priority: enumsspb.TASK_PRIORITY_LOW, TargetClusters: targetClusters, }) activityIDs := make(map[int64]struct{}, len(r.mutableState.GetPendingActivityInfos())) for activityID := range r.mutableState.GetPendingActivityInfos() { activityIDs[activityID] = struct{}{} } - taskEquivalents = append(taskEquivalents, convertSyncActivityInfos( + for _, syncActivityTask := range convertSyncActivityInfos( now, workflowKey, r.mutableState.GetPendingActivityInfos(), activityIDs, targetClusters, - )...) + ) { + syncActivityTask.(*tasks.SyncActivityTask).Priority = enumsspb.TASK_PRIORITY_LOW + taskEquivalents = append(taskEquivalents, syncActivityTask) + } taskEquivalents = append(taskEquivalents, &tasks.SyncHSMTask{ WorkflowKey: workflowKey, // TaskID and VisibilityTimestamp are set by shard + Priority: enumsspb.TASK_PRIORITY_LOW, TargetClusters: targetClusters, }) } diff --git a/service/history/workflow/task_generator_test.go b/service/history/workflow/task_generator_test.go index a48d65bac98..a4e04472830 100644 --- a/service/history/workflow/task_generator_test.go +++ b/service/history/workflow/task_generator_test.go @@ -836,20 +836,41 @@ func TestTaskGeneratorImpl_GenerateMigrationTasks(t *testing.T) { syncVersionTask, ok := resultTasks[0].(*tasks.SyncVersionedTransitionTask) require.True(t, ok) require.Equal(t, chasm.WorkflowArchetypeID, syncVersionTask.GetArchetypeID()) + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, syncVersionTask.Priority) taskEquivalent := syncVersionTask.TaskEquivalents require.Len(t, taskEquivalent, len(tc.expectedTaskEquivalentTypes)) for i, equivalent := range taskEquivalent { require.Equal(t, tc.expectedTaskEquivalentTypes[i], equivalent.GetType()) + requireLowReplicationPriority(t, equivalent) } } else { for i, task := range resultTasks { require.Equal(t, tc.expectedTaskTypes[i], task.GetType()) + requireLowReplicationPriority(t, task) } } }) } } +func requireLowReplicationPriority(t *testing.T, task tasks.Task) { + t.Helper() + switch task := task.(type) { + case *tasks.SyncWorkflowStateTask: + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, task.Priority) + case *tasks.SyncVersionedTransitionTask: + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, task.Priority) + case *tasks.HistoryReplicationTask: + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, task.Priority) + case *tasks.SyncActivityTask: + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, task.Priority) + case *tasks.SyncHSMTask: + require.Equal(t, enumsspb.TASK_PRIORITY_LOW, task.Priority) + default: + require.Fail(t, "unexpected migration task type", "%T", task) + } +} + func TestTaskGeneratorImpl_GenerateDirtySubStateMachineTasks_TrimsTimersForDeletedNodes(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() From 2c48aa571142d743ce236456809bfca276ddca2c Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 16:30:57 -0700 Subject: [PATCH 033/139] Annotate Nexus spans (#11561) ## What changed? Added Temporal Nexus attributes to spans. ## Why? Domain attributes make Nexus traces more useful. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- chasm/lib/nexusoperation/fx.go | 21 ++++++- chasm/lib/nexusoperation/fx_test.go | 1 + common/nexus/nexusrpc/telemetry.go | 51 ++++++++++++++++ common/telemetry/http.go | 38 +++++++++++- common/telemetry/http_test.go | 31 ++++++++++ common/telemetry/tags.go | 6 ++ service/frontend/nexus_handler.go | 15 +++++ tests/nexus_otel_test.go | 93 +++++++++++++++++++++++++---- 8 files changed, 239 insertions(+), 17 deletions(-) create mode 100644 common/nexus/nexusrpc/telemetry.go diff --git a/chasm/lib/nexusoperation/fx.go b/chasm/lib/nexusoperation/fx.go index 96db2bbcabf..53728ee56d7 100644 --- a/chasm/lib/nexusoperation/fx.go +++ b/chasm/lib/nexusoperation/fx.go @@ -15,6 +15,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/persistence" @@ -122,6 +123,7 @@ type clientProviderCacheKey struct { func clientProviderFactory( httpTransportProvider NexusTransportProvider, clusterMetadata cluster.Metadata, + namespaceRegistry namespace.Registry, rpcFactory common.RPCFactory, httpClientTransportInstrumenter telemetry.HTTPClientTransportInstrumenter, ) (ClientProvider, error) { @@ -143,10 +145,10 @@ func clientProviderFactory( return func(ctx context.Context, namespaceID string, entry *persistencespb.NexusEndpointEntry, service string) (*nexusrpc.HTTPClient, error) { var url string + var targetNamespaceID string var httpClient *http.Client // Populate source header for worker targets, and route internally. Callback assumes external target if unset. needsCallbackSourceHeader := false - switch variant := entry.Endpoint.Spec.Target.Variant.(type) { case *persistencespb.NexusEndpointTarget_External_: url = variant.External.GetUrl() @@ -159,6 +161,7 @@ func clientProviderFactory( url = cl.BaseURL() + "/" + commonnexus.RouteDispatchNexusTaskByEndpoint.Path(entry.Id) httpClient = &cl.Client needsCallbackSourceHeader = true + targetNamespaceID = variant.Worker.GetNamespaceId() default: return nil, serviceerror.NewInternal("got unexpected endpoint target") } @@ -176,6 +179,22 @@ func clientProviderFactory( } } + if httpClientTransportInstrumenter != nil { + var targetNamespaceName string + if targetNamespaceID != "" { + if namespaceName, err := namespaceRegistry.GetNamespaceName(namespace.ID(targetNamespaceID)); err == nil { + targetNamespaceName = namespaceName.String() + } + } + + // Add Nexus attributes when the HTTP transport will create a client span. + baseHTTPCaller := httpCaller + httpCaller = func(r *http.Request) (*http.Response, error) { + r = nexusrpc.AnnotateClientRequest(r, targetNamespaceName) + return baseHTTPCaller(r) + } + } + return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ BaseURL: url, Service: service, diff --git a/chasm/lib/nexusoperation/fx_test.go b/chasm/lib/nexusoperation/fx_test.go index 630a1462870..17b4ab9043a 100644 --- a/chasm/lib/nexusoperation/fx_test.go +++ b/chasm/lib/nexusoperation/fx_test.go @@ -105,6 +105,7 @@ func TestClientProviderFactoryUsesSelectedHTTPClient(t *testing.T) { return endpointTransport }, clusterMetadata, + nil, rpcFactory, nil, ) diff --git a/common/nexus/nexusrpc/telemetry.go b/common/nexus/nexusrpc/telemetry.go new file mode 100644 index 00000000000..1576a84b082 --- /dev/null +++ b/common/nexus/nexusrpc/telemetry.go @@ -0,0 +1,51 @@ +package nexusrpc + +import ( + "net/http" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "go.temporal.io/server/common/telemetry" +) + +// ServerSpanAttributes describes Nexus request attributes added to a server trace span. +type ServerSpanAttributes struct { + Endpoint string + Service string + Operation string + RequestID string +} + +// AnnotateServerSpan adds Nexus request attributes to span. +func AnnotateServerSpan(span trace.Span, attrs ServerSpanAttributes) { + // Non-recording spans discard attributes, so avoid constructing them. + if !span.IsRecording() { + return + } + kvs := []attribute.KeyValue{ + attribute.String(telemetry.NexusServiceKey, attrs.Service), + attribute.String(telemetry.NexusOperationKey, attrs.Operation), + } + if attrs.Endpoint != "" { + kvs = append(kvs, attribute.String(telemetry.NexusEndpointKey, attrs.Endpoint)) + } + if attrs.RequestID != "" { + kvs = append(kvs, attribute.String(telemetry.NexusRequestIDKey, attrs.RequestID)) + } + span.SetAttributes(kvs...) +} + +// AnnotateClientRequest returns req with Nexus attributes for its HTTP client span. +func AnnotateClientRequest(req *http.Request, targetNamespaceName string) *http.Request { + attrs := make([]attribute.KeyValue, 0, 2) + if targetNamespaceName != "" { + attrs = append(attrs, attribute.String(telemetry.NexusNamespaceKey, targetNamespaceName)) + } + if requestID := req.Header.Get(headerRequestID); requestID != "" { + attrs = append(attrs, attribute.String(telemetry.NexusRequestIDKey, requestID)) + } + if len(attrs) > 0 { + return telemetry.WithHTTPClientSpanAttributes(req, attrs...) + } + return req +} diff --git a/common/telemetry/http.go b/common/telemetry/http.go index 979cabd141b..9069418ea3b 100644 --- a/common/telemetry/http.go +++ b/common/telemetry/http.go @@ -18,10 +18,14 @@ import ( type HTTPClientTransportInstrumenter func(http.RoundTripper) http.RoundTripper // NewHTTPClientTransportInstrumenter binds a tracer provider and propagator for reuse across HTTP transports. +// It returns nil when tracing is disabled so callers can avoid tracing-only work. func NewHTTPClientTransportInstrumenter( tracerProvider trace.TracerProvider, propagator propagation.TextMapPropagator, ) HTTPClientTransportInstrumenter { + if !isEnabled(tracerProvider) { + return nil + } return func(rt http.RoundTripper) http.RoundTripper { return NewHTTPClientTransport(rt, tracerProvider, propagator) } @@ -39,10 +43,14 @@ func (i HTTPClientTransportInstrumenter) Instrument(rt http.RoundTripper) http.R type HTTPServerHandlerInstrumenter func(handler http.Handler, spanName string) http.Handler // NewHTTPServerHandlerInstrumenter binds a tracer provider and propagator for reuse across HTTP server handlers. +// It returns nil when tracing is disabled so callers can avoid tracing-only work. func NewHTTPServerHandlerInstrumenter( tracerProvider trace.TracerProvider, propagator propagation.TextMapPropagator, ) HTTPServerHandlerInstrumenter { + if !isEnabled(tracerProvider) { + return nil + } return func(handler http.Handler, spanName string) http.Handler { return NewHTTPHandler(handler, spanName, tracerProvider, propagator) } @@ -69,11 +77,12 @@ func NewHTTPClientTransport( if propagator == nil { propagator = propagation.TraceContext{} } + if rt == nil { + rt = http.DefaultTransport + } + rt = &httpClientSpanAttributesTransport{rt: rt} isDebug := DebugMode() if isDebug { - if rt == nil { - rt = http.DefaultTransport - } rt = &debugHTTPClientSpanTransport{rt: rt} } rt = otelhttp.NewTransport( @@ -112,6 +121,29 @@ func NewHTTPHandler( ) } +type httpClientSpanAttributesKey struct{} + +// WithHTTPClientSpanAttributes returns req with attrs for its HTTP client span. +func WithHTTPClientSpanAttributes(req *http.Request, attrs ...attribute.KeyValue) *http.Request { + return req.WithContext(context.WithValue(req.Context(), httpClientSpanAttributesKey{}, attrs)) +} + +type httpClientSpanAttributesTransport struct { + rt http.RoundTripper +} + +var _ http.RoundTripper = (*httpClientSpanAttributesTransport)(nil) + +func (t *httpClientSpanAttributesTransport) RoundTrip(req *http.Request) (*http.Response, error) { + span := trace.SpanFromContext(req.Context()) + if span.IsRecording() { + if attrs, ok := req.Context().Value(httpClientSpanAttributesKey{}).([]attribute.KeyValue); ok { + span.SetAttributes(attrs...) + } + } + return t.rt.RoundTrip(req) +} + type debugHTTPClientRequestStateKey struct{} // debugHTTPClientSpanTransport stores response finalization here through the request context. diff --git a/common/telemetry/http_test.go b/common/telemetry/http_test.go index 450205e0e60..4cb5e2fb48f 100644 --- a/common/telemetry/http_test.go +++ b/common/telemetry/http_test.go @@ -133,6 +133,9 @@ func TestNewHTTPClientTransport(t *testing.T) { var disabled HTTPClientTransportInstrumenter require.Same(t, rt, disabled.Instrument(rt)) + require.Nil(t, NewHTTPClientTransportInstrumenter(nil, nil)) + require.Nil(t, NewHTTPClientTransportInstrumenter(NoopTracerProvider, nil)) + instrumenter := NewHTTPClientTransportInstrumenter(trace.NewTracerProvider(), nil) require.NotSame(t, rt, instrumenter.Instrument(rt)) }) @@ -185,6 +188,31 @@ func TestNewHTTPClientTransport(t *testing.T) { require.NotContains(t, attrs, "http.response.headers.response-header") }) + // Callers can add domain attributes without coupling HTTP instrumentation to their domain. + t.Run("AnnotatesRequest", func(t *testing.T) { + t.Parallel() + + traceEnv := newHTTPTraceEnv(t) + rt := traceEnv.newClientTransport(roundTripperFunc(func(r *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: http.NoBody, + Header: http.Header{}, + Request: r, + }, nil + })) + + req := httptest.NewRequest(http.MethodGet, "http://example.com", nil) + req = WithHTTPClientSpanAttributes(req, attribute.String("test.attribute", "value")) + + resp, err := rt.RoundTrip(req) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + + // Check only the annotated attribute; standard HTTP attributes are covered elsewhere. + require.Equal(t, "value", traceEnv.spanAttrs()["test.attribute"]) + }) + // Debug mode adds diagnostic HTTP headers and payloads to client spans. t.Run("DebugMode", func(t *testing.T) { t.Setenv("TEMPORAL_OTEL_DEBUG", "true") @@ -427,6 +455,9 @@ func TestNewHTTPHandler(t *testing.T) { var disabled HTTPServerHandlerInstrumenter require.Same(t, handler, disabled.Instrument(handler, "test-handler")) + require.Nil(t, NewHTTPServerHandlerInstrumenter(nil, nil)) + require.Nil(t, NewHTTPServerHandlerInstrumenter(NoopTracerProvider, nil)) + recorder := tracetest.NewSpanRecorder() tp := trace.NewTracerProvider(trace.WithSpanProcessor(recorder)) instrumenter := NewHTTPServerHandlerInstrumenter(tp, nil) diff --git a/common/telemetry/tags.go b/common/telemetry/tags.go index 948faaa2b0b..c10d6e16083 100644 --- a/common/telemetry/tags.go +++ b/common/telemetry/tags.go @@ -13,4 +13,10 @@ const ( WorkflowIDKey = "temporalWorkflowID" BusinessIDKey = "temporalBusinessID" RunIDKey = "temporalRunID" + + NexusEndpointKey = "nexus.endpoint" + NexusNamespaceKey = "nexus.namespace" + NexusOperationKey = "nexus.operation" + NexusRequestIDKey = "nexus.request_id" + NexusServiceKey = "nexus.service" ) diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 31e900883f6..f5ab11ceff6 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -15,6 +15,7 @@ import ( "time" "github.com/nexus-rpc/sdk-go/nexus" + "go.opentelemetry.io/otel/trace" enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/serviceerror" @@ -87,6 +88,18 @@ type operationContext struct { cleanupFunctions []func(map[string]string, error) } +func (c *operationContext) annotateServerSpan( + ctx context.Context, + service, operation, requestID string, +) { + nexusrpc.AnnotateServerSpan(trace.SpanFromContext(ctx), nexusrpc.ServerSpanAttributes{ + Endpoint: c.endpointName, + Service: service, + Operation: operation, + RequestID: requestID, + }) +} + // Panic handler and metrics recording function. // Used as a deferred statement in Nexus handler methods. func (c *operationContext) capturePanicAndRecordMetrics(ctxPtr *context.Context, errPtr *error) { @@ -411,6 +424,7 @@ func (h *nexusHandler) StartOperation( ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, options.RequestID) + oc.annotateServerSpan(ctx, service, operation, options.RequestID) defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) var links []*nexuspb.Link @@ -652,6 +666,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, ctx = oc.augmentContext(ctx, options.Header) oc.enrichNexusOperationMetrics(service, operation, options.Header) oc.enrichNexusOperationLogs(service, operation, "") + oc.annotateServerSpan(ctx, service, operation, "") defer oc.capturePanicAndRecordMetrics(&ctx, &retErr) request := oc.matchingRequest(&nexuspb.Request{ diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go index 8b87d74fc43..8322dbbaec3 100644 --- a/tests/nexus_otel_test.go +++ b/tests/nexus_otel_test.go @@ -6,11 +6,13 @@ import ( "net/http/httptest" "net/url" "slices" + "strings" "testing" "time" "github.com/nexus-rpc/sdk-go/nexus" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/propagation" sdktrace "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" @@ -40,6 +42,8 @@ type nexusHTTPSpan struct { ServiceName string Kind oteltrace.SpanKind URLPath string + Status codes.Code + NexusAttrs map[string]any } type NexusOTELSuite struct { @@ -140,10 +144,15 @@ func (s *NexusOTELSuite) TestOperation() { workflow.GetSignalChannel(ctx, "complete").Receive(ctx, nil) return nil, nil } + requestIDs := make(chan string, 1) operation := temporalnexus.NewWorkflowRunOperation( "test-operation", handlerWorkflow, func(_ context.Context, _ nexus.NoValue, options nexus.StartOperationOptions) (client.StartWorkflowOptions, error) { + select { + case requestIDs <- options.RequestID: + default: + } return client.StartWorkflowOptions{ ID: options.RequestID, TaskQueue: handlerTaskQueue, @@ -171,6 +180,12 @@ func (s *NexusOTELSuite) TestOperation() { ScheduleToCloseTimeout: durationpb.New(time.Minute), }) s.NoError(err) + var nexusRequestID string + select { + case nexusRequestID = <-requestIDs: + case <-s.Context().Done(): + s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) + } pollResponse, err := callerEnv.FrontendClient().PollNexusOperationExecution(s.Context(), &workflowservice.PollNexusOperationExecutionRequest{ Namespace: callerEnv.Namespace().String(), @@ -198,6 +213,9 @@ func (s *NexusOTELSuite) TestOperation() { ServiceName: "io.temporal.history", Kind: oteltrace.SpanKindClient, URLPath: operationURLPath, + NexusAttrs: map[string]any{ + "nexus.request_id": nexusRequestID, + }, }, { TraceID: 1, @@ -207,6 +225,12 @@ func (s *NexusOTELSuite) TestOperation() { ServiceName: "io.temporal.frontend", Kind: oteltrace.SpanKindServer, URLPath: operationURLPath, + NexusAttrs: map[string]any{ + "nexus.endpoint": handlerWorkerEndpoint.GetSpec().GetName(), + "nexus.operation": operation.Name(), + "nexus.request_id": nexusRequestID, + "nexus.service": service.Name, + }, }, { TraceID: 2, @@ -224,6 +248,11 @@ func (s *NexusOTELSuite) TestOperation() { ServiceName: "io.temporal.frontend", Kind: oteltrace.SpanKindServer, URLPath: operationURLPath + "/cancel", + NexusAttrs: map[string]any{ + "nexus.endpoint": handlerWorkerEndpoint.GetSpec().GetName(), + "nexus.operation": operation.Name(), + "nexus.service": service.Name, + }, }, }) } @@ -234,10 +263,14 @@ func (s *NexusOTELSuite) TestWorkerOperation() { env := s.newTestEnv(exporter) tv := env.Tv().WithTaskQueue(env.WorkerTaskQueue()) - requestHeaders := make(chan nexus.Header, 1) + type nexusRequest struct { + header nexus.Header + requestID string + } + requests := make(chan nexusRequest, 1) service := nexus.NewService("test-service") operation := nexus.NewSyncOperation("test-operation", func(_ context.Context, _ nexus.NoValue, options nexus.StartOperationOptions) (string, error) { - requestHeaders <- options.Header + requests <- nexusRequest{header: options.Header, requestID: options.RequestID} return tv.Any().String(), nil }) service.MustRegister(operation) @@ -259,9 +292,9 @@ func (s *NexusOTELSuite) TestWorkerOperation() { }) s.NoError(err) - var headers nexus.Header + var request nexusRequest select { - case headers = <-requestHeaders: + case request = <-requests: case <-s.Context().Done(): s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) } @@ -274,6 +307,10 @@ func (s *NexusOTELSuite) TestWorkerOperation() { ServiceName: "io.temporal.history", Kind: oteltrace.SpanKindClient, URLPath: operationURLPath, + NexusAttrs: map[string]any{ + "nexus.namespace": env.Namespace().String(), + "nexus.request_id": request.requestID, + }, }, { TraceID: 1, @@ -283,22 +320,33 @@ func (s *NexusOTELSuite) TestWorkerOperation() { ServiceName: "io.temporal.frontend", Kind: oteltrace.SpanKindServer, URLPath: operationURLPath, + NexusAttrs: map[string]any{ + "nexus.endpoint": endpoint.GetSpec().GetName(), + "nexus.operation": operation.Name(), + "nexus.request_id": request.requestID, + "nexus.service": service.Name, + }, }, }) spanContext := oteltrace.SpanContextFromContext( - propagation.TraceContext{}.Extract(s.Context(), propagation.MapCarrier(headers)), + propagation.TraceContext{}.Extract(s.Context(), propagation.MapCarrier(request.header)), ) s.Require().True(spanContext.IsValid()) s.Require().Equal(spanContext.TraceID(), httpSpans[0].SpanContext.TraceID()) s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) } -// Verifies the namespace and task queue dispatch route is instrumented independently of forwarding. +// Verifies the namespace and task queue route propagates tracing and records handler failures without forwarding. func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { exporter := tracetest.NewInMemoryExporter() env := s.newTestEnv(exporter) taskQueue := env.Tv().TaskQueue().GetName() - pollerErrCh := env.nexusTaskPoller(s.Context(), s.T(), taskQueue, nexusEchoHandler) + pollerErrCh := env.nexusTaskPoller(s.Context(), s.T(), taskQueue, func( + *testing.T, + *workflowservice.PollNexusTaskQueueResponse, + ) (*nexusTaskResponse, error) { + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "deliberate test failure") + }) dispatchURL, err := url.Parse(env.dispatchByTaskQueueURL(taskQueue)) s.NoError(err) nexusClient, err := nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ @@ -313,10 +361,13 @@ func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { traceID = "4bf92f3577b34da6a3ce929d0e0e4736" parentSpanID = "00f067aa0ba902b7" ) + requestID := env.Tv().RequestID() _, err = nexusrpc.StartOperation(s.Context(), nexusClient, op, env.Tv().Any().String(), nexus.StartOperationOptions{ - Header: nexus.Header{"traceparent": "00-" + traceID + "-" + parentSpanID + "-01"}, + Header: nexus.Header{"traceparent": "00-" + traceID + "-" + parentSpanID + "-01"}, + RequestID: requestID, }) - s.NoError(err) + var handlerErr *nexus.HandlerError + s.Require().ErrorAs(err, &handlerErr) s.NoError(<-pollerErrCh) httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ @@ -326,13 +377,19 @@ func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { ServiceName: "io.temporal.frontend", Kind: oteltrace.SpanKindServer, URLPath: dispatchURL.Path + "/test-service/my-operation", + Status: codes.Error, + NexusAttrs: map[string]any{ + "nexus.operation": "my-operation", + "nexus.request_id": requestID, + "nexus.service": "test-service", + }, }}) s.Require().Equal(traceID, httpSpans[0].SpanContext.TraceID().String()) s.Require().Equal(parentSpanID, httpSpans[0].Parent.SpanID().String()) } -// requireNexusHTTPSpans compares all exported HTTP spans after assigning stable local IDs -// and returns the matching raw spans for context propagation assertions. +// requireNexusHTTPSpans compares all exported HTTP spans and their Nexus attributes after +// assigning stable local IDs, then returns the raw spans for context propagation assertions. func (s *NexusOTELSuite) requireNexusHTTPSpans( exporter *tracetest.InMemoryExporter, expected []nexusHTTPSpan, @@ -342,6 +399,7 @@ func (s *NexusOTELSuite) requireNexusHTTPSpans( s.Await(func(s *NexusOTELSuite) { var actual []nexusHTTPSpan actual, httpSpans = s.nexusHTTPSpans(exporter.GetSpans()) + s.Require().Len(actual, len(expected)) s.Require().Equal(expected, actual) }, 10*time.Second, 100*time.Millisecond) return httpSpans @@ -378,12 +436,19 @@ func (s *NexusOTELSuite) nexusHTTPSpans( } } var urlPath string + var nexusAttrs map[string]any for _, attr := range span.Attributes { + key := string(attr.Key) + if strings.HasPrefix(key, "nexus.") { + if nexusAttrs == nil { + nexusAttrs = make(map[string]any) + } + nexusAttrs[key] = attr.Value.AsInterface() + } if attr.Key == semconv.URLPathKey { urlPath = attr.Value.AsString() - break } - if attr.Key == semconv.URLFullKey { + if urlPath == "" && attr.Key == semconv.URLFullKey { if parsedURL, err := url.Parse(attr.Value.AsString()); err == nil { urlPath = parsedURL.Path } @@ -397,6 +462,8 @@ func (s *NexusOTELSuite) nexusHTTPSpans( ServiceName: serviceName, Kind: span.SpanKind, URLPath: urlPath, + Status: span.Status.Code, + NexusAttrs: nexusAttrs, }) } return result, httpSpans From c9b99295b1250cd053e2c1d6466926a2400e7a96 Mon Sep 17 00:00:00 2001 From: Kannan Date: Thu, 20 Aug 2026 16:59:04 -0700 Subject: [PATCH 034/139] Treat missing Worker Deployment Version as deleted (#11696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Treat `NotFound` from the Version workflow delete update as success, so the Deployment workflow can clean up its stale version reference. ## Why When a Version workflow is already closed or its history is gone, the delete update returns `NotFound`, blocking the Deployment workflow from removing the reference. Fixes #11539. ## How did you test it? Unit test covering the `NotFound` → success path and verifying other History errors still propagate. --- common/metrics/metric_defs.go | 1 + service/worker/workerdeployment/activities.go | 21 +++++ .../workerdeployment/activities_test.go | 84 +++++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 service/worker/workerdeployment/activities_test.go diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 4aff005a4c8..74a39eb5688 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1631,6 +1631,7 @@ var ( WorkerDeploymentCreated = NewCounterDef("worker_deployment_created") WorkerDeploymentVersionCreated = NewCounterDef("worker_deployment_version_created") WorkerDeploymentVersionCreatedManagedByController = NewCounterDef("worker_deployment_version_created_managed_by_controller") + WorkerDeploymentVersionNotFoundDuringDelete = NewCounterDef("worker_deployment_version_not_found_during_delete") WorkerDeploymentVersionVisibilityQueryCount = NewCounterDef("worker_deployment_version_visibility_query_count") WorkerDeploymentVersioningOverrideCounter = NewCounterDef("worker_deployment_versioning_override_count") WorkerDeploymentVersioningOneTimeOverrideCounter = NewCounterDef("worker_deployment_versioning_one_time_override_count") diff --git a/service/worker/workerdeployment/activities.go b/service/worker/workerdeployment/activities.go index 1b3385834f0..9a385ace759 100644 --- a/service/worker/workerdeployment/activities.go +++ b/service/worker/workerdeployment/activities.go @@ -14,6 +14,7 @@ import ( "go.temporal.io/sdk/temporal" deploymentspb "go.temporal.io/server/api/deployment/v1" "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/worker_versioning" @@ -174,6 +175,26 @@ func (a *Activities) DeleteWorkerDeploymentVersion(ctx context.Context, args *de }, ) if err != nil { + var notFoundErr *serviceerror.NotFound + // History returns NotFound when the Version workflow is already closed without + // this update or when its history no longer exists. Allow the Deployment workflow + // to remove its stale reference. + if errors.As(err, ¬FoundErr) { + metrics.WorkerDeploymentVersionNotFoundDuringDelete.With(a.MetricsHandler).Record( + 1, + metrics.NamespaceTag(a.namespace.Name().String()), + ) + activity.GetLogger(ctx).Warn( + "version workflow not found during deletion; allowing deployment workflow to remove stale reference", + "namespace", a.namespace.Name().String(), + "deploymentName", args.DeploymentName, + "version", args.Version, + "versionWorkflowID", workflowID, + "requestID", args.RequestId, + "error", err, + ) + return nil + } return err } diff --git a/service/worker/workerdeployment/activities_test.go b/service/worker/workerdeployment/activities_test.go new file mode 100644 index 00000000000..3b6ecbe3c89 --- /dev/null +++ b/service/worker/workerdeployment/activities_test.go @@ -0,0 +1,84 @@ +package workerdeployment + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/sdk/testsuite" + deploymentspb "go.temporal.io/server/api/deployment/v1" + "go.temporal.io/server/api/historyservicemock/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testvars" + "go.uber.org/mock/gomock" +) + +func TestDeleteWorkerDeploymentVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + historyErr error + wantErr bool + }{ + { + name: "version workflow not found", + historyErr: serviceerror.NewNotFound("version workflow not found"), + }, + { + name: "other history error", + historyErr: serviceerror.NewUnavailable("history unavailable"), + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + controller := gomock.NewController(t) + historyClient := historyservicemock.NewMockHistoryServiceClient(controller) + historyClient.EXPECT(). + UpdateWorkflowExecution(gomock.Any(), gomock.Any()). + Return(nil, tt.historyErr) + + tv := testvars.New(t) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + activity := &Activities{ + activityDeps: activityDeps{ + HistoryClient: historyClient, + MetricsHandler: metricsHandler, + }, + namespace: namespace.NewLocalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: tv.NamespaceID().String(), Name: tv.NamespaceName().String()}, + nil, + "", + ), + } + env := (&testsuite.WorkflowTestSuite{}).NewTestActivityEnvironment() + env.RegisterActivity(activity) + + _, err := env.ExecuteActivity(activity.DeleteWorkerDeploymentVersion, &deploymentspb.DeleteVersionActivityArgs{ + DeploymentName: tv.DeploymentSeries(), + Version: tv.DeploymentVersionString(), + RequestId: tv.RequestID(), + }) + if tt.wantErr { + require.Error(t, err) + require.Empty(t, capture.Snapshot()[metrics.WorkerDeploymentVersionNotFoundDuringDelete.Name()]) + } else { + require.NoError(t, err) + recordings := capture.Snapshot()[metrics.WorkerDeploymentVersionNotFoundDuringDelete.Name()] + require.Len(t, recordings, 1) + require.Equal(t, int64(1), recordings[0].Value) + namespaceTag := metrics.NamespaceTag(tv.NamespaceName().String()) + require.Equal(t, namespaceTag.Value, recordings[0].Tags[namespaceTag.Key]) + } + }) + } +} From 83235a9c1da9edef3dfd2d69be7c13ae4790479a Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 20:28:37 -0700 Subject: [PATCH 035/139] Add helpers for testing exported spans (#11655) Introduce reusable test helpers for dealing with OTEL spans. --- common/testing/testtelemetry/spans.go | 104 +++++++++++++++++++++ common/testing/testtelemetry/spans_test.go | 67 +++++++++++++ tests/nexus_otel_test.go | 49 +++++----- 3 files changed, 197 insertions(+), 23 deletions(-) create mode 100644 common/testing/testtelemetry/spans.go create mode 100644 common/testing/testtelemetry/spans_test.go diff --git a/common/testing/testtelemetry/spans.go b/common/testing/testtelemetry/spans.go new file mode 100644 index 00000000000..ecc61846b2d --- /dev/null +++ b/common/testing/testtelemetry/spans.go @@ -0,0 +1,104 @@ +package testtelemetry + +import ( + "slices" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + oteltrace "go.opentelemetry.io/otel/trace" +) + +// LocalSpanID contains stable identifiers for one span and its trace-local relationships. +type LocalSpanID struct { + Trace int + Span int + Parent int +} + +// idGenerator assigns stable, one-based local IDs while preserving zero as no ID. +type idGenerator[T comparable] struct { + ids map[T]int +} + +func (g *idGenerator[T]) get(value T) int { + var zero T + if value == zero { + return 0 + } + if id, ok := g.ids[value]; ok { + return id + } + if g.ids == nil { + g.ids = make(map[T]int) + } + id := len(g.ids) + 1 + g.ids[value] = id + return id +} + +func (g *idGenerator[T]) lookup(value T) int { + return g.ids[value] +} + +func localIDs[T comparable](values []T) []int { + var ids idGenerator[T] + result := make([]int, len(values)) + for i, value := range values { + result[i] = ids.get(value) + } + return result +} + +// LocalAttributeIDs assigns stable IDs to an attribute's values in span order. +func LocalAttributeIDs(spans tracetest.SpanStubs, key attribute.Key) []int { + values := make([]string, len(spans)) + for i, span := range spans { + if value, ok := SpanAttribute(span, key); ok { + values[i] = value.String() + } + } + return localIDs(values) +} + +// LocalSpanIDs assigns stable, one-based IDs in the order spans are provided. +// Callers must sort spans first when they need deterministic ordering. +func LocalSpanIDs(spans tracetest.SpanStubs) []LocalSpanID { + var traceIDs idGenerator[oteltrace.TraceID] + spanIDs := make(map[oteltrace.TraceID]*idGenerator[oteltrace.SpanID]) + for _, span := range spans { + traceID := span.SpanContext.TraceID() + traceIDs.get(traceID) + if spanIDs[traceID] == nil { + spanIDs[traceID] = new(idGenerator[oteltrace.SpanID]) + } + spanIDs[traceID].get(span.SpanContext.SpanID()) + } + + result := make([]LocalSpanID, len(spans)) + for i, span := range spans { + traceID := span.SpanContext.TraceID() + result[i] = LocalSpanID{ + Trace: traceIDs.lookup(traceID), + Span: spanIDs[traceID].lookup(span.SpanContext.SpanID()), + Parent: spanIDs[traceID].lookup(span.Parent.SpanID()), + } + } + return result +} + +func SpanAttribute(span tracetest.SpanStub, key attribute.Key) (attribute.Value, bool) { + for _, attr := range span.Attributes { + if attr.Key == key { + return attr.Value, true + } + } + return attribute.Value{}, false +} + +// FilterSpans returns matching spans without modifying the input slice. +func FilterSpans(spans tracetest.SpanStubs, keep func(tracetest.SpanStub) bool) tracetest.SpanStubs { + filtered := slices.Clone(spans) + return slices.DeleteFunc(filtered, func(span tracetest.SpanStub) bool { + return !keep(span) + }) +} diff --git a/common/testing/testtelemetry/spans_test.go b/common/testing/testtelemetry/spans_test.go new file mode 100644 index 00000000000..b24d72eaea2 --- /dev/null +++ b/common/testing/testtelemetry/spans_test.go @@ -0,0 +1,67 @@ +package testtelemetry + +import ( + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + oteltrace "go.opentelemetry.io/otel/trace" +) + +func TestLocalAttributeIDs(t *testing.T) { + t.Parallel() + + spans := tracetest.SpanStubs{ + {}, + {Attributes: []attribute.KeyValue{attribute.String("key", "first")}}, + {Attributes: []attribute.KeyValue{attribute.String("key", "second")}}, + {Attributes: []attribute.KeyValue{attribute.String("key", "first")}}, + } + require.Equal(t, []int{0, 1, 2, 1}, LocalAttributeIDs(spans, "key")) +} + +func TestLocalSpanIDs(t *testing.T) { + t.Parallel() + + traceID1 := oteltrace.TraceID{1} + traceID2 := oteltrace.TraceID{2} + parentSpanID := oteltrace.SpanID{1} + childSpanID := oteltrace.SpanID{2} + spanContext := func(traceID oteltrace.TraceID, spanID oteltrace.SpanID) oteltrace.SpanContext { + return oteltrace.NewSpanContext(oteltrace.SpanContextConfig{TraceID: traceID, SpanID: spanID}) + } + spans := tracetest.SpanStubs{ + {SpanContext: spanContext(traceID1, childSpanID), Parent: spanContext(traceID1, parentSpanID)}, + {SpanContext: spanContext(traceID1, parentSpanID)}, + {SpanContext: spanContext(traceID2, parentSpanID)}, + } + require.Equal(t, []LocalSpanID{ + {Trace: 1, Span: 1, Parent: 2}, + {Trace: 1, Span: 2}, + {Trace: 2, Span: 1}, + }, LocalSpanIDs(spans)) +} + +func TestSpanAttribute(t *testing.T) { + t.Parallel() + + span := tracetest.SpanStub{Attributes: []attribute.KeyValue{attribute.String("key", "value")}} + value, ok := SpanAttribute(span, "key") + require.True(t, ok) + require.Equal(t, "value", value.AsString()) + + _, ok = SpanAttribute(span, "missing") + require.False(t, ok) +} + +func TestFilterSpans(t *testing.T) { + t.Parallel() + + spans := tracetest.SpanStubs{{Name: "keep"}, {Name: "drop"}} + filtered := FilterSpans(spans, func(span tracetest.SpanStub) bool { + return span.Name == "keep" + }) + require.Equal(t, tracetest.SpanStubs{{Name: "keep"}}, filtered) + require.Len(t, spans, 2) +} diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go index 8322dbbaec3..af5ba6790e5 100644 --- a/tests/nexus_otel_test.go +++ b/tests/nexus_otel_test.go @@ -30,6 +30,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/common/testing/testtelemetry" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" ) @@ -396,39 +397,27 @@ func (s *NexusOTELSuite) requireNexusHTTPSpans( ) tracetest.SpanStubs { s.T().Helper() var httpSpans tracetest.SpanStubs - s.Await(func(s *NexusOTELSuite) { + requireExportedSpans(s, exporter, expected, func(spans tracetest.SpanStubs) []nexusHTTPSpan { var actual []nexusHTTPSpan - actual, httpSpans = s.nexusHTTPSpans(exporter.GetSpans()) - s.Require().Len(actual, len(expected)) - s.Require().Equal(expected, actual) - }, 10*time.Second, 100*time.Millisecond) + actual, httpSpans = s.nexusHTTPSpans(spans) + return actual + }) return httpSpans } func (s *NexusOTELSuite) nexusHTTPSpans( spans tracetest.SpanStubs, ) ([]nexusHTTPSpan, tracetest.SpanStubs) { - httpSpans := slices.DeleteFunc(spans, func(span tracetest.SpanStub) bool { - return span.InstrumentationScope.Name != otelhttp.ScopeName + httpSpans := testtelemetry.FilterSpans(spans, func(span tracetest.SpanStub) bool { + return span.InstrumentationScope.Name == otelhttp.ScopeName }) slices.SortFunc(httpSpans, func(a, b tracetest.SpanStub) int { return a.StartTime.Compare(b.StartTime) }) - - traceIDs := make(map[oteltrace.TraceID]int) - spanIDs := make(map[oteltrace.TraceID]map[oteltrace.SpanID]int) - for _, span := range httpSpans { - traceID := span.SpanContext.TraceID() - if _, ok := traceIDs[traceID]; !ok { - traceIDs[traceID] = len(traceIDs) + 1 - spanIDs[traceID] = make(map[oteltrace.SpanID]int) - } - spanIDs[traceID][span.SpanContext.SpanID()] = len(spanIDs[traceID]) + 1 - } + localIDs := testtelemetry.LocalSpanIDs(httpSpans) result := make([]nexusHTTPSpan, 0, len(httpSpans)) - for _, span := range httpSpans { - traceID := span.SpanContext.TraceID() + for i, span := range httpSpans { var serviceName string if span.Resource != nil { if value, ok := span.Resource.Set().Value(semconv.ServiceNameKey); ok { @@ -455,9 +444,9 @@ func (s *NexusOTELSuite) nexusHTTPSpans( } } result = append(result, nexusHTTPSpan{ - TraceID: traceIDs[traceID], - SpanID: spanIDs[traceID][span.SpanContext.SpanID()], - ParentSpanID: spanIDs[traceID][span.Parent.SpanID()], + TraceID: localIDs[i].Trace, + SpanID: localIDs[i].Span, + ParentSpanID: localIDs[i].Parent, Name: span.Name, ServiceName: serviceName, Kind: span.SpanKind, @@ -468,3 +457,17 @@ func (s *NexusOTELSuite) nexusHTTPSpans( } return result, httpSpans } + +func requireExportedSpans[T any]( + s *NexusOTELSuite, + exporter *tracetest.InMemoryExporter, + expected []T, + project func(tracetest.SpanStubs) []T, +) { + s.T().Helper() + s.Await(func(s *NexusOTELSuite) { + actual := project(exporter.GetSpans()) + s.Require().Len(actual, len(expected)) + s.Require().Equal(expected, actual) + }, 10*time.Second, 100*time.Millisecond) +} From 24434b11e97a5baf7f75806681366bb2c50b2fed Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 20:34:03 -0700 Subject: [PATCH 036/139] Use slices.Backward for reverse iteration (#11676) Go 1.27 prerequisite that applies the `slicesbackward` Go fixer for reverse iteration. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- chasm/tree.go | 3 +-- common/dynamicconfig/memory_client.go | 13 +++++++------ common/effect/buffer.go | 9 ++++++--- common/persistence/history_manager.go | 5 +++-- common/persistence/query_util.go | 5 +++-- .../persistence/versionhistory/version_history.go | 6 ++++-- common/testing/await/require_ctx_test.go | 5 +++-- service/history/workflow/context.go | 5 +++-- service/history/workflow/mutable_state_impl.go | 4 ++-- tests/testcore/cluster_poison_test.go | 5 +++-- tests/testcore/shared_cluster_t.go | 4 ++-- tools/testrunner/log.go | 14 +++++++------- 12 files changed, 44 insertions(+), 34 deletions(-) diff --git a/chasm/tree.go b/chasm/tree.go index 9f490e4e4b8..4a56b867be5 100644 --- a/chasm/tree.go +++ b/chasm/tree.go @@ -2068,8 +2068,7 @@ func (n *Node) closeTransactionUpdateComponentTasks( } sideEffectTasks := componentAttr.GetSideEffectTasks() - for idx := len(sideEffectTasks) - 1; idx >= 0; idx-- { - sideEffectTask := sideEffectTasks[idx] + for _, sideEffectTask := range slices.Backward(sideEffectTasks) { if sideEffectTask.PhysicalTaskStatus == physicalTaskStatusCreated { break } diff --git a/common/dynamicconfig/memory_client.go b/common/dynamicconfig/memory_client.go index 67f4cf55678..9c84c28d7f3 100644 --- a/common/dynamicconfig/memory_client.go +++ b/common/dynamicconfig/memory_client.go @@ -1,6 +1,7 @@ package dynamicconfig import ( + "slices" "sync" "sync/atomic" ) @@ -34,15 +35,15 @@ func (d *MemoryClient) GetValue(key Key) []ConstrainedValue { func (d *MemoryClient) getValueLocked(key Key) []ConstrainedValue { var result []ConstrainedValue - for i := len(d.overrides) - 1; i >= 0; i-- { - if d.overrides[i].valid && d.overrides[i].key == key { - v := d.overrides[i].value - if cvs, ok := v.([]ConstrainedValue); ok { + for _, override := range slices.Backward(d.overrides) { + if override.valid && override.key == key { + value := override.value + if cvs, ok := value.([]ConstrainedValue); ok { result = append(result, cvs...) } else { - result = append(result, ConstrainedValue{Value: v}) + result = append(result, ConstrainedValue{Value: value}) } - if !d.overrides[i].mergeable { + if !override.mergeable { // Non-mergeable: take this value and stop. return result } diff --git a/common/effect/buffer.go b/common/effect/buffer.go index d44d066ac45..00c414936d2 100644 --- a/common/effect/buffer.go +++ b/common/effect/buffer.go @@ -1,6 +1,9 @@ package effect -import "context" +import ( + "context" + "slices" +) // Buffer holds a set of effect and rollback functions that can be invoked as a // batch with a defined order. Once either Apply or Cancel is called, all @@ -43,8 +46,8 @@ func (b *Buffer) Apply(ctx context.Context) bool { func (b *Buffer) Cancel(ctx context.Context) bool { canceled := false b.effects = nil - for i := len(b.cancels) - 1; i >= 0; i-- { - b.cancels[i](ctx) + for _, cancel := range slices.Backward(b.cancels) { + cancel(ctx) canceled = true } b.cancels = nil diff --git a/common/persistence/history_manager.go b/common/persistence/history_manager.go index 705c333f9cd..9777739be73 100644 --- a/common/persistence/history_manager.go +++ b/common/persistence/history_manager.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "slices" "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" @@ -179,8 +180,8 @@ func (m *executionManagerImpl) DeleteHistoryBranch( var deleteRanges []InternalDeleteHistoryBranchRange // for each branch range to delete, we iterate from bottom up, and stop when the range is also used by others findDeleteRanges: - for i := len(brsToDelete) - 1; i >= 0; i-- { - br := brsToDelete[i] + for _, br := range slices.Backward(brsToDelete) { + if maxEndNode, ok := usedBranches[br.GetBranchId()]; ok { // branch is used by others, we can only delete from the maxEndNode if maxEndNode != common.LastEventID { diff --git a/common/persistence/query_util.go b/common/persistence/query_util.go index 7d4a001a77c..d31477c519e 100644 --- a/common/persistence/query_util.go +++ b/common/persistence/query_util.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "slices" "strings" "unicode" ) @@ -235,7 +236,7 @@ func hasWordsBefore(s string, pos int, words ...string) bool { if pos <= 0 || !unicode.IsSpace(rune(s[pos])) { return false } - for i := len(words) - 1; i >= 0; i-- { + for _, word := range slices.Backward(words) { // skip spaces for pos >= 0 && unicode.IsSpace(rune(s[pos])) { pos-- @@ -245,7 +246,7 @@ func hasWordsBefore(s string, pos int, words ...string) bool { pos-- } // check if current pos matches the word - if pos < 0 || !hasWordAt(s, words[i], pos) { + if pos < 0 || !hasWordAt(s, word, pos) { return false } pos-- diff --git a/common/persistence/versionhistory/version_history.go b/common/persistence/versionhistory/version_history.go index 8b3bd3e114a..914a20dc32a 100644 --- a/common/persistence/versionhistory/version_history.go +++ b/common/persistence/versionhistory/version_history.go @@ -1,6 +1,8 @@ package versionhistory import ( + "slices" + "go.temporal.io/api/serviceerror" historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/common" @@ -185,8 +187,8 @@ func SplitVersionHistoryByLastLocalGeneratedItem( initialFailoverVersion int64, failoverVersionIncrement int64, ) (localItems []*historyspb.VersionHistoryItem, remoteItems []*historyspb.VersionHistoryItem) { - for i := len(versionHistoryItems) - 1; i >= 0; i-- { - if versionHistoryItems[i].Version%failoverVersionIncrement == initialFailoverVersion { + for i, versionHistoryItem := range slices.Backward(versionHistoryItems) { + if versionHistoryItem.Version%failoverVersionIncrement == initialFailoverVersion { return versionHistoryItems[:i+1], versionHistoryItems[i+1:] } } diff --git a/common/testing/await/require_ctx_test.go b/common/testing/await/require_ctx_test.go index 30df57af4d6..c9fa0c14f1d 100644 --- a/common/testing/await/require_ctx_test.go +++ b/common/testing/await/require_ctx_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "runtime" + "slices" "strings" "sync" "sync/atomic" @@ -539,8 +540,8 @@ func (r *recordingTB) runCleanups() { r.cleanups = nil r.mu.Unlock() - for i := len(cleanups) - 1; i >= 0; i-- { - cleanups[i]() + for _, cleanup := range slices.Backward(cleanups) { + cleanup() } } diff --git a/service/history/workflow/context.go b/service/history/workflow/context.go index 47f2098ea41..04af98b16b2 100644 --- a/service/history/workflow/context.go +++ b/service/history/workflow/context.go @@ -3,6 +3,7 @@ package workflow import ( "context" "errors" + "slices" "strconv" "go.opentelemetry.io/otel/trace" @@ -1034,9 +1035,9 @@ func (c *ContextImpl) mergeUpdateWithNewReplicationTasks( t.NewRunID = newRunID taskEquivalents := t.TaskEquivalents taskEquivalentsUpdated := false - for idx := len(taskEquivalents) - 1; idx >= 0; idx-- { + for _, taskEquivalent := range slices.Backward(taskEquivalents) { // For state based, we should update a sync versioned transition task and update a history task inside task equivalent. - if historyTask, ok := taskEquivalents[idx].(*tasks.HistoryReplicationTask); ok { + if historyTask, ok := taskEquivalent.(*tasks.HistoryReplicationTask); ok { historyTask.NewRunBranchToken = newRunBranchToken historyTask.NewRunID = newRunID taskEquivalentsUpdated = true diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index af190b9d827..5c73fd1bb93 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7416,8 +7416,8 @@ func (ms *MutableStateImpl) PopTasks() map[tasks.Category][]tasks.Task { } func (ms *MutableStateImpl) DeleteCHASMPureTasks(maxScheduledTime time.Time) { - for lastTaskIdx := len(ms.chasmPureTasks) - 1; lastTaskIdx >= 0; lastTaskIdx-- { - task := ms.chasmPureTasks[lastTaskIdx] + for lastTaskIdx, task := range slices.Backward(ms.chasmPureTasks) { + if !task.GetVisibilityTime().Before(maxScheduledTime) { ms.chasmPureTasks = ms.chasmPureTasks[:lastTaskIdx+1] return diff --git a/tests/testcore/cluster_poison_test.go b/tests/testcore/cluster_poison_test.go index 5381e62fa92..399643e8f58 100644 --- a/tests/testcore/cluster_poison_test.go +++ b/tests/testcore/cluster_poison_test.go @@ -2,6 +2,7 @@ package testcore import ( "fmt" + "slices" "sync" "testing" @@ -54,8 +55,8 @@ func (f *mockSubtestT) finish() { cleanups := f.cleanups f.cleanups = nil f.mu.Unlock() - for i := len(cleanups) - 1; i >= 0; i-- { - cleanups[i]() + for _, cleanup := range slices.Backward(cleanups) { + cleanup() } } diff --git a/tests/testcore/shared_cluster_t.go b/tests/testcore/shared_cluster_t.go index 7a22bf2676e..47abcd39c89 100644 --- a/tests/testcore/shared_cluster_t.go +++ b/tests/testcore/shared_cluster_t.go @@ -144,7 +144,7 @@ func (s *sharedClusterT) doCleanups() { cs := s.cleanups s.cleanups = nil s.mu.Unlock() - for i := len(cs) - 1; i >= 0; i-- { - cs[i]() + for _, c := range slices.Backward(cs) { + c() } } diff --git a/tools/testrunner/log.go b/tools/testrunner/log.go index 75fef7ddf12..17cc6fd6407 100644 --- a/tools/testrunner/log.go +++ b/tools/testrunner/log.go @@ -373,14 +373,14 @@ func normalizedFailureLines(data string) []string { func findLastAssertionFailureBlock(lines []string) (string, bool) { var failLine string - for i := len(lines) - 1; i >= 0; i-- { - line := strings.TrimSpace(lines[i]) - if failLine == "" && strings.HasPrefix(line, goTestFailLinePrefix) { + for i, line := range slices.Backward(lines) { + trimmed := strings.TrimSpace(line) + if failLine == "" && strings.HasPrefix(trimmed, goTestFailLinePrefix) { // Keep the final Go test failure line because it carries the test duration. - failLine = line + failLine = trimmed continue } - if !strings.Contains(lines[i], "Error Trace:") { + if !strings.Contains(line, "Error Trace:") { continue } @@ -422,8 +422,8 @@ func endOfAssertionBlock(lines []string, start int) int { } func findLastTestOutputFailureBlock(lines []string) (start, end int, ok bool) { - for start := len(lines) - 1; start >= 0; start-- { - if !isTestOutputLine(lines[start]) { + for start, line := range slices.Backward(lines) { + if !isTestOutputLine(line) { continue } end := start + 1 From 2f3cbadeec8458266750f8c3ef29528985b86830 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 20:34:31 -0700 Subject: [PATCH 037/139] Tag Nexus completion request logs (#11684) ## What changed The completion (callback) handler's request-scoped logger carried only the namespace, even though a richer one was built just above it. --------- Co-authored-by: Claude Opus 5 (1M context) --- .../frontend/nexus_completion_http_handler.go | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index a6c57d08259..fef7a1134bc 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -40,7 +40,7 @@ import ( ) const nexusCompletionAPIName = configs.CompleteNexusOperation -const nexusCompletionMethodNameForMetrics = "CompleteNexusOperation" +const nexusCompletionMethodName = "CompleteNexusOperation" type nexusCompletionHandler struct { ClusterMetadata cluster.Metadata @@ -157,18 +157,21 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus } logger := log.With( h.Logger, + tag.Operation(nexusCompletionMethodName), tag.WorkflowNamespace(ns.Name().String()), + tag.WorkflowNamespaceID(targetNamespaceID), tag.WorkflowID(targetBusinessID), tag.WorkflowRunID(targetRunID), + tag.RequestID(completion.GetRequestId()), ) rCtx := &requestContext{ nexusCompletionHandler: h, namespace: ns, businessID: targetBusinessID, - logger: log.With(h.Logger, tag.WorkflowNamespace(ns.Name().String())), + logger: logger, metricsHandler: h.MetricsHandler.WithTags(metrics.NamespaceTag(ns.Name().String())), metricsHandlerForInterceptors: h.MetricsHandler.WithTags( - metrics.OperationTag(nexusCompletionMethodNameForMetrics), + metrics.OperationTag(nexusCompletionMethodName), metrics.NamespaceTag(ns.Name().String()), ), requestStartTime: startTime, @@ -183,16 +186,14 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus nsNameEscaped := commonnexus.RouteCompletionCallback.Deserialize(mux.Vars(r.HTTPRequest)) nsName, err := url.PathUnescape(nsNameEscaped) if err != nil { - h.Logger.Error("failed to extract namespace from request", tag.Error(err)) + logger.Error("failed to extract namespace from request", tag.Error(err)) h.preProcessErrorsCounter.Record(1) return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid URL") } if nsName != ns.Name().String() { logger.Error( - "namespace ID in token doesn't match the token", - tag.WorkflowNamespaceID(ns.ID().String()), - tag.Error(err), - tag.String("completion-namespace-id", targetNamespaceID), + "namespace in callback URL doesn't match the completion token", + tag.String("url-namespace", nsName), ) return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid callback token") } @@ -210,7 +211,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "operation token length exceeds allowed limit (%d/%d)", len(r.OperationToken), tokenLimit) } - links := commonnexus.ConvertNexusLinksToProtoLinks(r.Links, h.Logger) + links := commonnexus.ConvertNexusLinksToProtoLinks(r.Links, logger) var successPayload *commonpb.Payload switch r.State { // nolint:exhaustive @@ -223,7 +224,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "invalid result content") } if result.Size() > h.Config.BlobSizeLimitError(ns.Name().String()) { - logger.Error("payload size exceeds error limit for Nexus CompleteOperation request", tag.WorkflowNamespace(ns.Name().String())) + logger.Error("payload size exceeds error limit for Nexus CompleteOperation request") return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "result exceeds size limit") } successPayload = result @@ -403,26 +404,26 @@ func (h *nexusCompletionHandler) completeChasmOperation( func (h *nexusCompletionHandler) forwardCompleteOperation(ctx context.Context, r *nexusrpc.CompletionRequest, rCtx *requestContext) error { targetCluster := rCtx.namespace.ActiveClusterName(namespace.RoutingKey{ID: rCtx.businessID}) + logger := log.With( + rCtx.logger, + tag.SourceCluster(h.ClusterMetadata.GetCurrentClusterName()), + tag.TargetCluster(targetCluster), + ) + client, err := h.ForwardingClients.Get(targetCluster) if err != nil { - h.Logger.Error("unable to get HTTP client for forward request", tag.Operation(nexusCompletionAPIName), tag.WorkflowNamespace(rCtx.namespace.Name().String()), tag.Error(err), tag.SourceCluster(h.ClusterMetadata.GetCurrentClusterName()), tag.TargetCluster(targetCluster)) + logger.Error("unable to get HTTP client for forward request", tag.Error(err)) return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") } forwardURL, err := url.JoinPath(client.BaseURL(), commonnexus.RouteCompletionCallback.Path(rCtx.namespace.Name().String())) if err != nil { - h.Logger.Error("failed to construct forwarding request URL", tag.Operation(nexusCompletionAPIName), tag.WorkflowNamespace(rCtx.namespace.Name().String()), tag.Error(err), tag.TargetCluster(targetCluster)) + logger.Error("failed to construct forwarding request URL", tag.Error(err)) return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") } if h.HTTPTraceProvider != nil { - traceLogger := log.With(h.Logger, - tag.Operation(nexusCompletionAPIName), - tag.WorkflowNamespace(rCtx.namespace.Name().String()), - tag.AttemptStart(time.Now().UTC()), - tag.SourceCluster(h.ClusterMetadata.GetCurrentClusterName()), - tag.TargetCluster(targetCluster), - ) + traceLogger := log.With(logger, tag.AttemptStart(time.Now().UTC())) if trace := h.HTTPTraceProvider.NewForwardingTrace(traceLogger); trace != nil { ctx = httptrace.WithClientTrace(ctx, trace) } @@ -509,7 +510,7 @@ func (c *requestContext) augmentContext(ctx context.Context, header http.Header) ctx = interceptor.PopulateCallerInfo( ctx, func() string { return c.namespace.Name().String() }, - func() string { return nexusCompletionMethodNameForMetrics }, + func() string { return nexusCompletionMethodName }, ) if userAgent := header.Get(headerUserAgent); userAgent != "" { // Preserve original strict behavior: only process if exactly one delimiter present. @@ -624,7 +625,7 @@ func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc if c.namespace.ActiveClusterName(namespace.RoutingKey{ID: c.businessID}) != c.ClusterMetadata.GetCurrentClusterName() { if c.shouldForwardRequest(ctx, request.HTTPRequest.Header, c.businessID) { c.forwarded = true - handler, forwardStartTime := c.RedirectionInterceptor.BeforeCall(nexusCompletionMethodNameForMetrics) + handler, forwardStartTime := c.RedirectionInterceptor.BeforeCall(nexusCompletionMethodName) c.cleanupFunctions = append(c.cleanupFunctions, func(retErr error) { c.RedirectionInterceptor.AfterCall(handler, forwardStartTime, c.namespace.ActiveClusterName(namespace.RoutingKey{ID: c.businessID}), c.namespace.Name().String(), retErr) }) @@ -641,7 +642,7 @@ func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc request, "", c.metricsHandlerForInterceptors, - []tag.Tag{tag.Operation(nexusCompletionMethodNameForMetrics), tag.WorkflowNamespace(c.namespace.Name().String())}, + []tag.Tag{tag.Operation(nexusCompletionMethodName), tag.WorkflowNamespace(c.namespace.Name().String())}, retErr, c.namespace.Name(), ) From 9eadc90f60fd02fa2ee688bf5b29919b7fbb0dda Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 20 Aug 2026 22:26:51 -0700 Subject: [PATCH 038/139] Make UnprocessableTaskError pointer-only (#11678) Go 1.27 prerequisite that makes `UnprocessableTaskError` pointer-only for the stricter errortype vet check. --- service/history/queues/errors/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service/history/queues/errors/errors.go b/service/history/queues/errors/errors.go index 906698b6eed..093b673e7ea 100644 --- a/service/history/queues/errors/errors.go +++ b/service/history/queues/errors/errors.go @@ -42,6 +42,6 @@ func NewUnprocessableTaskError(message string) *UnprocessableTaskError { return &UnprocessableTaskError{Message: message} } -func (e UnprocessableTaskError) Error() string { +func (e *UnprocessableTaskError) Error() string { return "unprocessable task: " + e.Message } From f6772c53c763e15017f410af6d9476fab1b1c6cf Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Fri, 21 Aug 2026 09:09:53 -0700 Subject: [PATCH 039/139] Refactor NamespaceRateLimitInterceptor with functions to consume N tokens (#11582) ## What changed? Refactor NamespaceRateLimitInterceptor with functions to consume N tokens: - removed `tokens` overwrite argument as it's never used - added functions to consume N tokens The changes itself in this PR is no-op since it's introducing new functions to the interface. ## Why? Added flexibility to wrap `NamespaceRateLimitInterceptor`. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../rpc/interceptor/namespace_rate_limit.go | 117 ++++++++++++++---- .../interceptor/namespace_rate_limit_test.go | 1 - service/frontend/fx.go | 1 - .../frontend/nexus_completion_http_handler.go | 7 +- service/frontend/nexus_handler.go | 7 +- service/frontend/nexus_handler_test.go | 1 - service/history/fx.go | 1 - service/matching/fx.go | 1 - 8 files changed, 104 insertions(+), 32 deletions(-) diff --git a/common/rpc/interceptor/namespace_rate_limit.go b/common/rpc/interceptor/namespace_rate_limit.go index 1bcf3b26a20..080e9f667b6 100644 --- a/common/rpc/interceptor/namespace_rate_limit.go +++ b/common/rpc/interceptor/namespace_rate_limit.go @@ -31,15 +31,55 @@ var ( type ( NamespaceRateLimitInterceptor interface { - Intercept(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) - Allow(namespaceName namespace.Name, methodName string, headerGetter headers.HeaderGetter) error - Wait(ctx context.Context, namespaceName namespace.Name, methodName string, headerGetter headers.HeaderGetter) error + Intercept( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + ) (resp any, err error) + + InterceptN( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + numToken int, + ) (resp any, err error) + + Allow( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + ) error + + Wait( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + ) error + + AllowN( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + numToken int, + ) error + + WaitN( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + numToken int, + ) error } NamespaceRateLimitInterceptorImpl struct { namespaceRegistry namespace.Registry rateLimiter quotas.RequestRateLimiter - tokens map[string]int reducePollWorkflowHistoryPriority dynamicconfig.BoolPropertyFn pollMethods map[string]struct{} pollWaitForToken dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -53,7 +93,6 @@ var _ NamespaceRateLimitInterceptor = (*NamespaceRateLimitInterceptorImpl)(nil) func NewNamespaceRateLimitInterceptor( namespaceRegistry namespace.Registry, rateLimiter quotas.RequestRateLimiter, - tokens map[string]int, pollMethods map[string]struct{}, pollWaitForToken dynamicconfig.BoolPropertyFnWithNamespaceFilter, metricsHandler metrics.Handler, @@ -61,7 +100,6 @@ func NewNamespaceRateLimitInterceptor( return &NamespaceRateLimitInterceptorImpl{ namespaceRegistry: namespaceRegistry, rateLimiter: rateLimiter, - tokens: tokens, pollMethods: pollMethods, pollWaitForToken: pollWaitForToken, metricsHandler: metricsHandler, @@ -73,6 +111,16 @@ func (ni *NamespaceRateLimitInterceptorImpl) Intercept( req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, +) (any, error) { + return ni.InterceptN(ctx, req, info, handler, NamespaceRateLimitDefaultToken) +} + +func (ni *NamespaceRateLimitInterceptorImpl) InterceptN( + ctx context.Context, + req any, + info *grpc.UnaryServerInfo, + handler grpc.UnaryHandler, + numToken int, ) (any, error) { if ns := MustGetNamespaceName(ni.namespaceRegistry, req); ns != namespace.EmptyName { method := info.FullMethod @@ -81,15 +129,13 @@ func (ni *NamespaceRateLimitInterceptorImpl) Intercept( } else if IsLongPollDescribeActivityExecutionRequest(req) { method = configs.PollActivityExecutionAPIName } - if ni.pollWaitForToken(ns.String()) { - if _, ok := ni.pollMethods[info.FullMethod]; ok { - if err := ni.Wait(ctx, ns, method, headers.NewGRPCHeaderGetter(ctx)); err != nil { - return nil, err - } - return handler(ctx, req) + if _, ok := ni.pollMethods[info.FullMethod]; ok && ni.pollWaitForToken(ns.String()) { + if err := ni.WaitN(ctx, ns, method, headers.NewGRPCHeaderGetter(ctx), numToken); err != nil { + return nil, err } + return handler(ctx, req) } - if err := ni.Allow(ns, method, headers.NewGRPCHeaderGetter(ctx)); err != nil { + if err := ni.AllowN(ctx, ns, method, headers.NewGRPCHeaderGetter(ctx), numToken); err != nil { return nil, err } } @@ -97,14 +143,25 @@ func (ni *NamespaceRateLimitInterceptorImpl) Intercept( return handler(ctx, req) } -func (ni *NamespaceRateLimitInterceptorImpl) Wait(ctx context.Context, namespaceName namespace.Name, methodName string, headerGetter headers.HeaderGetter) error { - token, ok := ni.tokens[methodName] - if !ok { - token = NamespaceRateLimitDefaultToken - } +func (ni *NamespaceRateLimitInterceptorImpl) Wait( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, +) error { + return ni.WaitN(ctx, namespaceName, methodName, headerGetter, NamespaceRateLimitDefaultToken) +} + +func (ni *NamespaceRateLimitInterceptorImpl) WaitN( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + numToken int, +) error { request := quotas.NewRequest( methodName, - token, + numToken, namespaceName.String(), headerGetter.Get(headers.CallerTypeHeaderName), 0, // this interceptor layer does not throttle based on caller segment @@ -139,15 +196,25 @@ func (ni *NamespaceRateLimitInterceptorImpl) Wait(ctx context.Context, namespace return ctx.Err() } -func (ni *NamespaceRateLimitInterceptorImpl) Allow(namespaceName namespace.Name, methodName string, headerGetter headers.HeaderGetter) error { - token, ok := ni.tokens[methodName] - if !ok { - token = NamespaceRateLimitDefaultToken - } +func (ni *NamespaceRateLimitInterceptorImpl) Allow( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, +) error { + return ni.AllowN(ctx, namespaceName, methodName, headerGetter, NamespaceRateLimitDefaultToken) +} +func (ni *NamespaceRateLimitInterceptorImpl) AllowN( + ctx context.Context, + namespaceName namespace.Name, + methodName string, + headerGetter headers.HeaderGetter, + numToken int, +) error { if !ni.rateLimiter.Allow(time.Now().UTC(), quotas.NewRequest( methodName, - token, + numToken, namespaceName.String(), headerGetter.Get(headers.CallerTypeHeaderName), 0, // this interceptor layer does not throttle based on caller segment diff --git a/common/rpc/interceptor/namespace_rate_limit_test.go b/common/rpc/interceptor/namespace_rate_limit_test.go index dc8d4f90be3..ab90219ccd4 100644 --- a/common/rpc/interceptor/namespace_rate_limit_test.go +++ b/common/rpc/interceptor/namespace_rate_limit_test.go @@ -46,7 +46,6 @@ func (s *namespaceRateLimitInterceptorSuite) newImpl(pollWaitForToken bool) *Nam return &NamespaceRateLimitInterceptorImpl{ namespaceRegistry: s.mockRegistry, rateLimiter: s.mockRateLimiter, - tokens: map[string]int{}, pollMethods: map[string]struct{}{ pollWorkflowTaskQueueMethod: {}, }, diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 80a80a81037..b160db42856 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -640,7 +640,6 @@ func NamespaceRateLimitInterceptorProvider( return interceptor.NewNamespaceRateLimitInterceptor( namespaceRegistry, quotas.NewRoutingRateLimiter(mapping), - map[string]int{}, // no token overrides configs.PollTaskAPISet, serviceConfig.PollWaitForNamespaceRateLimitToken, metricsHandler, diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index fef7a1134bc..1603998b978 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -656,7 +656,12 @@ func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc return commonnexus.ConvertGRPCError(err, false) } - if err := c.NamespaceRateLimitInterceptor.Allow(c.namespace.Name(), nexusCompletionAPIName, request.HTTPRequest.Header); err != nil { + if err := c.NamespaceRateLimitInterceptor.Allow( + ctx, + c.namespace.Name(), + nexusCompletionAPIName, + request.HTTPRequest.Header, + ); err != nil { c.outcomeTag = metrics.OutcomeTag("namespace_rate_limited") return commonnexus.ConvertGRPCError(err, true) } diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index f5ab11ceff6..75e2ab133b9 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -247,7 +247,12 @@ func (c *operationContext) interceptRequest( return commonnexus.ConvertGRPCError(err, false) } - if err := c.namespaceRateLimitInterceptor.Allow(c.namespace.Name(), c.apiName, header); err != nil { + if err := c.namespaceRateLimitInterceptor.Allow( + ctx, + c.namespace.Name(), + c.apiName, + header, + ); err != nil { c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("namespace_rate_limited")) return commonnexus.ConvertGRPCError(err, true) } diff --git a/service/frontend/nexus_handler_test.go b/service/frontend/nexus_handler_test.go index 0f4d13565c0..17891daea3f 100644 --- a/service/frontend/nexus_handler_test.go +++ b/service/frontend/nexus_handler_test.go @@ -145,7 +145,6 @@ func newOperationContext(options contextOptions) *operationContext { oc.namespaceRateLimitInterceptor = interceptor.NewNamespaceRateLimitInterceptor( nil, mockRateLimiter{options.namespaceRateLimitAllow}, - make(map[string]int), map[string]struct{}{}, dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), metrics.NoopMetricsHandler, diff --git a/service/history/fx.go b/service/history/fx.go index 19344a06560..ce20b90be3a 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -352,7 +352,6 @@ func NamespaceRateLimitInterceptorProvider( namespaceRateFn, serviceConfig.OperatorRPSRatio, ), - map[string]int{}, // no token overrides map[string]struct{}{}, // no long polls on history service dynamicconfig.GetBoolPropertyFnFilteredByNamespace(false), // no long poll methods metricsHandler, diff --git a/service/matching/fx.go b/service/matching/fx.go index 1224e753a50..62efeebfd60 100644 --- a/service/matching/fx.go +++ b/service/matching/fx.go @@ -134,7 +134,6 @@ func NamespaceRateLimitInterceptorProvider( namespaceRateFn, serviceConfig.OperatorRPSRatio, ), - map[string]int{}, // no token overrides configs.PollTaskAPISet, // set of APIs that will wait for token instead of immediate rejection serviceConfig.PollWaitForNamespaceRateLimitToken, metricsHandler, From f6b60cb82bd5aa9424af1c3447fe1369780fd14a Mon Sep 17 00:00:00 2001 From: Qian Chen Date: Fri, 21 Aug 2026 11:35:25 -0700 Subject: [PATCH 040/139] Unify namespace lifecycle wide event names (#11720) ## What changed? - emit remote cluster and namespace replication lifecycle records under `namespace_lifecycle` - retain compatibility aliases for specialized event-name constants with TODO cleanup - update tests and shared envelope documentation ## Why? Update schema event names to match namespace lifecycle schema which currently exist so they are properly interpreted. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] added new unit test(s) - [ ] added new functional test(s) --- .../nsreplication/transmission_task_handler_test.go | 3 ++- common/wideevents/namespace_events.go | 5 ++--- common/wideevents/namespace_replication_events.go | 10 ++++++---- common/wideevents/namespace_replication_events_test.go | 4 ++-- common/wideevents/remote_cluster_events.go | 10 ++++++---- common/wideevents/remote_cluster_events_test.go | 2 +- .../frontend/remote_cluster_lifecycle_events_test.go | 2 +- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/common/namespace/nsreplication/transmission_task_handler_test.go b/common/namespace/nsreplication/transmission_task_handler_test.go index b464e636385..29b4f2accf8 100644 --- a/common/namespace/nsreplication/transmission_task_handler_test.go +++ b/common/namespace/nsreplication/transmission_task_handler_test.go @@ -19,6 +19,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" + "go.temporal.io/server/common/wideevents" "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/durationpb" ) @@ -170,7 +171,7 @@ func (s *transmissionTaskSuite) TestHandleTransmissionTask_RegisterNamespaceTask ) s.Require().NoError(err) s.Require().Len(eventLogger.records, 1) - s.Equal("namespace_replication_lifecycle", eventLogger.records[0].EventName()) + s.Equal(wideevents.NamespaceLifecycleEventName, eventLogger.records[0].EventName()) attrs := make(map[string]otellog.Value) eventLogger.records[0].WalkAttributes(func(kv otellog.KeyValue) bool { attrs[kv.Key] = kv.Value diff --git a/common/wideevents/namespace_events.go b/common/wideevents/namespace_events.go index c841833a373..c75c23dc0de 100644 --- a/common/wideevents/namespace_events.go +++ b/common/wideevents/namespace_events.go @@ -4,9 +4,8 @@ import "go.opentelemetry.io/otel/log" // NamespaceLifecycleEventName is the stable event name for the generic, phase-discriminated wide // event describing namespace lifecycle activity (failover, configuration, admission, handover, -// etc.). Namespace replication tasks use NamespaceReplicationLifecyclePayload instead. This package -// owns only the stable envelope: the set of phase values and the -// contents of Details are supplied by the emitter. +// etc.). This package owns only the stable envelope: the set of phase values and the contents of +// Details are supplied by the emitter. const NamespaceLifecycleEventName = "namespace_lifecycle" // NamespaceLifecyclePayload is the NamespaceLifecycle payload. The identity fields are stable; diff --git a/common/wideevents/namespace_replication_events.go b/common/wideevents/namespace_replication_events.go index 2009c2536db..a8e19f0a04f 100644 --- a/common/wideevents/namespace_replication_events.go +++ b/common/wideevents/namespace_replication_events.go @@ -15,7 +15,9 @@ import ( "google.golang.org/protobuf/proto" ) -const NamespaceReplicationLifecycleEventName = "namespace_replication_lifecycle" +// NamespaceReplicationLifecycleEventName aliases NamespaceLifecycleEventName for compatibility. +// TODO: Remove it after callers migrate to NamespaceLifecycleEventName. +const NamespaceReplicationLifecycleEventName = NamespaceLifecycleEventName type NamespaceReplicationPhase string @@ -89,12 +91,12 @@ func (defaultNamespaceReplicationTaskEventDataProvider) Extract( }, true } -// NamespaceReplicationLifecyclePayload uses the namespace lifecycle envelope while retaining a -// distinct event name. Replication-specific data is carried in Details. +// NamespaceReplicationLifecyclePayload uses the namespace lifecycle envelope. Replication-specific +// data is carried in Details. type NamespaceReplicationLifecyclePayload NamespaceLifecyclePayload func (p NamespaceReplicationLifecyclePayload) EventName() string { - return NamespaceReplicationLifecycleEventName + return NamespaceLifecycleEventName } func (p NamespaceReplicationLifecyclePayload) Attributes() []log.KeyValue { diff --git a/common/wideevents/namespace_replication_events_test.go b/common/wideevents/namespace_replication_events_test.go index a8b7e18c135..822bdab6910 100644 --- a/common/wideevents/namespace_replication_events_test.go +++ b/common/wideevents/namespace_replication_events_test.go @@ -20,7 +20,7 @@ import ( ) func TestNamespaceReplicationLifecycleEventName(t *testing.T) { - require.Equal(t, "namespace_replication_lifecycle", NamespaceReplicationLifecyclePayload{}.EventName()) + require.Equal(t, NamespaceLifecycleEventName, NamespaceReplicationLifecyclePayload{}.EventName()) } func TestNamespaceReplicationTaskContext(t *testing.T) { @@ -60,7 +60,7 @@ func TestEmitNamespaceReplicationLifecycle(t *testing.T) { }) require.Len(t, logger.records, 1) - require.Equal(t, NamespaceReplicationLifecycleEventName, logger.records[0].EventName()) + require.Equal(t, NamespaceLifecycleEventName, logger.records[0].EventName()) got := namespaceReplicationRecordValues(logger.records[0]) require.Equal(t, map[string]any{ "phase": "dlqed", diff --git a/common/wideevents/remote_cluster_events.go b/common/wideevents/remote_cluster_events.go index 418cc3a7821..0808048d830 100644 --- a/common/wideevents/remote_cluster_events.go +++ b/common/wideevents/remote_cluster_events.go @@ -2,19 +2,21 @@ package wideevents import "go.opentelemetry.io/otel/log" -const RemoteClusterLifecycleEventName = "remote_cluster_lifecycle" +// RemoteClusterLifecycleEventName aliases NamespaceLifecycleEventName for compatibility. +// TODO: Remove it after callers migrate to NamespaceLifecycleEventName. +const RemoteClusterLifecycleEventName = NamespaceLifecycleEventName const ( PhaseRemoteClusterUpsert = "remote_cluster_upsert" PhaseRemoteClusterRemove = "remote_cluster_remove" ) -// RemoteClusterLifecyclePayload preserves the namespace lifecycle event envelope -// while using a distinct event name and remote-cluster-specific details. +// RemoteClusterLifecyclePayload preserves the namespace lifecycle event envelope with +// remote-cluster-specific details. type RemoteClusterLifecyclePayload NamespaceLifecyclePayload func (p RemoteClusterLifecyclePayload) EventName() string { - return RemoteClusterLifecycleEventName + return NamespaceLifecycleEventName } func (p RemoteClusterLifecyclePayload) Attributes() []log.KeyValue { diff --git a/common/wideevents/remote_cluster_events_test.go b/common/wideevents/remote_cluster_events_test.go index 11e75cbb242..9b1fe5f57ef 100644 --- a/common/wideevents/remote_cluster_events_test.go +++ b/common/wideevents/remote_cluster_events_test.go @@ -7,7 +7,7 @@ import ( ) func TestRemoteClusterLifecycleEventName(t *testing.T) { - require.Equal(t, "remote_cluster_lifecycle", RemoteClusterLifecyclePayload{}.EventName()) + require.Equal(t, NamespaceLifecycleEventName, RemoteClusterLifecyclePayload{}.EventName()) } func TestRemoteClusterLifecycleFieldSetMatchesNamespaceLifecycle(t *testing.T) { diff --git a/service/frontend/remote_cluster_lifecycle_events_test.go b/service/frontend/remote_cluster_lifecycle_events_test.go index 0a9b29c1286..50b7ea72142 100644 --- a/service/frontend/remote_cluster_lifecycle_events_test.go +++ b/service/frontend/remote_cluster_lifecycle_events_test.go @@ -295,7 +295,7 @@ func remoteClusterEventValues( ) (map[string]string, map[string]any) { t.Helper() require.Len(t, records, 1) - require.Equal(t, wideevents.RemoteClusterLifecycleEventName, records[0].EventName()) + require.Equal(t, wideevents.NamespaceLifecycleEventName, records[0].EventName()) attrs := make(map[string]string) records[0].WalkAttributes(func(kv otellog.KeyValue) bool { attrs[kv.Key] = kv.Value.AsString() From fc530c118478e81ac1a416a2e7a992d7bceca9db Mon Sep 17 00:00:00 2001 From: Brian VanLoo Date: Fri, 21 Aug 2026 12:10:54 -0700 Subject: [PATCH 041/139] Add observability for multicursor queue-state resolution loss (#11695) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Adds metrics for how often and by how much queue slices fail to narrow their predicate, and how large persisted queue state actually is. - `queue_slice_pending_keys` — histogram, recorded on every narrowing attempt (declined or succeeded). This is the distribution `queueShrinkPredicateMaxPendingKeys` should be sized against. - `shard_info_size` / `queue_state_size` — histograms recorded when a shard record is actually written, giving whole-record and per-category size. - `queue_state_size_total` / `queue_slice_count_total` — counters paired with the histograms above (and with the existing `queue_slice_count`), so an exact bytes-per-slice ratio is possible. - `queue_slice_count` gains a `task_category` tag (previously untagged beyond `operation`). These are only metrics changes - no behavior changes. ## Why? A slice only narrows its predicate below `queueShrinkPredicateMaxPendingKeys` (10) pending namespaces; above that it stays universal and re-reads the whole range every time. Raising that threshold safely requires knowing the pending-key distribution and the persisted size. This PR is the baseline for evaluating a follow-on encoding. There are two counters because this server's tally-backed Prometheus reporter doesn't preserve the true recorded value when a histogram flushes — it replays each sample as its bucket's upper bound, so a histogram's `_sum` has no more precision than its buckets. ## How did you test it? - [x] built - [x] covered by existing tests - [x] added new unit test(s) - [x] run locally and tested manually (`queue_predicate_resolution_loss` confirmed live against a local server under forced narrowing-decline conditions) ## Potential risks --- common/metrics/metric_defs.go | 5 ++ service/history/queues/queue_base.go | 7 +- service/history/queues/queue_base_test.go | 97 +++++++++++++++++++++++ service/history/queues/slice.go | 1 + service/history/queues/slice_test.go | 75 ++++++++++++++++++ service/history/shard/context_impl.go | 15 ++++ service/history/shard/context_test.go | 96 ++++++++++++++++++++++ 7 files changed, 295 insertions(+), 1 deletion(-) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 74a39eb5688..61eab914afe 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -724,6 +724,9 @@ var ( HistoryCount = NewDimensionlessHistogramDef("history_count") TasksCompletedPerShardInfoUpdate = NewDimensionlessHistogramDef("tasks_per_shardinfo_update") TimeBetweenShardInfoUpdates = NewTimerDef("time_between_shardinfo_update") + ShardInfoSize = NewBytesHistogramDef("shard_info_size") + QueueStateSize = NewBytesHistogramDef("queue_state_size") + QueueStateSizeTotal = NewCounterDef("queue_state_size_total") SearchAttributesSize = NewBytesHistogramDef("search_attributes_size") MemoSize = NewBytesHistogramDef("memo_size") TooManyPendingChildWorkflows = NewCounterDef( @@ -953,6 +956,8 @@ var ( QueueScheduleLatency = NewTimerDef("queue_latency_schedule") // latency for scheduling 100 tasks in one task channel QueueReaderCountHistogram = NewDimensionlessHistogramDef("queue_reader_count") QueueSliceCountHistogram = NewDimensionlessHistogramDef("queue_slice_count") + QueueSliceCountTotal = NewCounterDef("queue_slice_count_total") + QueueSlicePendingKeys = NewDimensionlessHistogramDef("queue_slice_pending_keys") QueueActionCounter = NewCounterDef("queue_actions") QueueAlertShadowCounter = NewCounterDef("queue_alert_shadow") QueuePredicateResolutionLoss = NewCounterDef( diff --git a/service/history/queues/queue_base.go b/service/history/queues/queue_base.go index 974e66e4745..16beb869921 100644 --- a/service/history/queues/queue_base.go +++ b/service/history/queues/queue_base.go @@ -329,7 +329,12 @@ func (p *queueBase) checkpoint() { } } metrics.QueueReaderCountHistogram.With(p.metricsHandler).Record(int64(len(readerScopes))) - metrics.QueueSliceCountHistogram.With(p.metricsHandler).Record(int64(p.monitor.GetTotalSliceCount())) + sliceCount := int64(p.monitor.GetTotalSliceCount()) + categoryTag := metrics.TaskCategoryTag(p.category.Name()) + // The counter is a true accumulator; the histogram's _sum is not, since tally's Prometheus + // reporter replays each sample as its bucket's upper bound, not the recorded value. + metrics.QueueSliceCountHistogram.With(p.metricsHandler).Record(sliceCount, categoryTag) + metrics.QueueSliceCountTotal.With(p.metricsHandler).Record(sliceCount, categoryTag) metrics.PendingTasksCounter.With(p.metricsHandler).Record(int64(p.monitor.GetTotalPendingTaskCount())) // NOTE: Must range-complete task first. diff --git a/service/history/queues/queue_base_test.go b/service/history/queues/queue_base_test.go index 313120987e6..a7c0891f806 100644 --- a/service/history/queues/queue_base_test.go +++ b/service/history/queues/queue_base_test.go @@ -17,6 +17,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/predicates" @@ -456,6 +457,102 @@ func (s *queueBaseSuite) TestCheckPoint_NoPendingTasks() { s.True(exclusiveReaderHighWatermark.CompareTo(base.exclusiveDeletionHighWatermark) == 0) } +func (s *queueBaseSuite) TestCheckPoint_RecordsSliceCountWithTaskCategoryTag() { + numSlices := 3 + scopes := NewRandomScopes(numSlices) + queueState := &queueState{ + readerScopes: map[int64][]Scope{ + DefaultReaderId: scopes, + }, + exclusiveReaderHighWatermark: tasks.MaximumKey, + } + persistenceState := ToPersistenceQueueState(queueState) + + mockShard := shard.NewTestContext( + s.controller, + &persistencespb.ShardInfo{ + ShardId: 0, + RangeId: 10, + QueueStates: map[int32]*persistencespb.QueueState{ + int32(tasks.CategoryIDTimer): persistenceState, + }, + }, + s.config, + ) + mockShard.Resource.ClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() + mockShard.Resource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() + + captureHandler := metricstest.NewCaptureHandler() + capture := captureHandler.StartCapture() + defer captureHandler.StopCapture(capture) + s.metricsHandler = captureHandler + + base := s.newQueueBase(mockShard, tasks.CategoryTimer, nil) + base.checkpointTimer = time.NewTimer(s.options.CheckpointInterval()) + + // set to a smaller value so that delete will be triggered, matching TestCheckPoint_SlicePredicateAction + base.exclusiveDeletionHighWatermark = tasks.MinimumKey + + mockShard.Resource.ExecutionMgr.EXPECT().RangeCompleteHistoryTasks(gomock.Any(), gomock.Any()).Return(nil).Times(1) + mockShard.Resource.ShardMgr.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Return(nil).Times(1) + + base.checkpoint() + + snapshot := capture.Snapshot() + recordings := snapshot[metrics.QueueSliceCountHistogram.Name()] + s.Require().Len(recordings, 1) + s.Equal(int64(numSlices), recordings[0].Value) + s.Equal(tasks.CategoryTimer.Name(), recordings[0].Tags["task_category"]) +} + +func (s *queueBaseSuite) TestCheckPoint_RecordsSliceCountTotal() { + numSlices := 3 + scopes := NewRandomScopes(numSlices) + queueState := &queueState{ + readerScopes: map[int64][]Scope{ + DefaultReaderId: scopes, + }, + exclusiveReaderHighWatermark: tasks.MaximumKey, + } + persistenceState := ToPersistenceQueueState(queueState) + + mockShard := shard.NewTestContext( + s.controller, + &persistencespb.ShardInfo{ + ShardId: 0, + RangeId: 10, + QueueStates: map[int32]*persistencespb.QueueState{ + int32(tasks.CategoryIDTimer): persistenceState, + }, + }, + s.config, + ) + mockShard.Resource.ClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() + mockShard.Resource.ClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() + + captureHandler := metricstest.NewCaptureHandler() + capture := captureHandler.StartCapture() + defer captureHandler.StopCapture(capture) + s.metricsHandler = captureHandler + + base := s.newQueueBase(mockShard, tasks.CategoryTimer, nil) + base.checkpointTimer = time.NewTimer(s.options.CheckpointInterval()) + + // set to a smaller value so that delete will be triggered, matching TestCheckPoint_SlicePredicateAction + base.exclusiveDeletionHighWatermark = tasks.MinimumKey + + mockShard.Resource.ExecutionMgr.EXPECT().RangeCompleteHistoryTasks(gomock.Any(), gomock.Any()).Return(nil).Times(1) + mockShard.Resource.ShardMgr.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Return(nil).Times(1) + + base.checkpoint() + + snapshot := capture.Snapshot() + recordings := snapshot[metrics.QueueSliceCountTotal.Name()] + s.Require().Len(recordings, 1) + s.Equal(int64(numSlices), recordings[0].Value) + s.Equal(tasks.CategoryTimer.Name(), recordings[0].Tags["task_category"]) +} + func (s *queueBaseSuite) TestCheckPoint_SlicePredicateAction() { exclusiveReaderHighWatermark := tasks.MaximumKey scopes := NewRandomScopes(3) diff --git a/service/history/queues/slice.go b/service/history/queues/slice.go index e07b9c238a5..bd1bfa83df2 100644 --- a/service/history/queues/slice.go +++ b/service/history/queues/slice.go @@ -348,6 +348,7 @@ func (s *SliceImpl) shrinkPredicate() { // TODO: this should be generic enough to shrink any predicate type, probably doesn't belong here. pendingPerKey := s.pendingPerKey + metrics.QueueSlicePendingKeys.With(s.metricsHandler).Record(int64(len(pendingPerKey))) if len(pendingPerKey) > s.maxPendingKeysFn() { // only shrink predicate if there're few keys left metrics.QueuePredicateResolutionLoss.With(s.metricsHandler).Record( diff --git a/service/history/queues/slice_test.go b/service/history/queues/slice_test.go index 15f4f68fa57..3a47a4e5b93 100644 --- a/service/history/queues/slice_test.go +++ b/service/history/queues/slice_test.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/predicates" ctasks "go.temporal.io/server/common/tasks" @@ -450,6 +451,80 @@ func (s *sliceSuite) TestShrinkScope_ShrinkPredicate() { } } +func (s *sliceSuite) TestShrinkScope_RecordsPendingKeysHistogram() { + testCases := []struct { + name string + numPendingNamespaces int + leaveIteratorsOpen bool + expectDeclined bool + }{ + { + name: "narrowing succeeds", + numPendingNamespaces: 3, + }, + { + name: "narrowing declines", + numPendingNamespaces: 12, + expectDeclined: true, + }, + { + name: "slice still reading its range: no sample recorded", + numPendingNamespaces: 3, + leaveIteratorsOpen: true, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + r := NewRandomRange() + predicate := predicates.Universal[tasks.Task]() + + handler := metricstest.NewCaptureHandler() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + + slice := NewSlice(nil, s.executableFactory, s.monitor, NewScope(r, predicate), GrouperNamespaceID{}, noPredicateSizeLimit, defaultMaxPendingKeys, handler) + if !tc.leaveIteratorsOpen { + slice.iterators = []Iterator{} // manually set iterators to be empty to trigger predicate update + } + + // One pending executable per namespace, each with its own distinct namespace ID, + // so the pending-key count is exact rather than a coincidence of random assignment + // across a shared, smaller pool of namespace IDs. + executables := s.randomExecutablesInRange(r, tc.numPendingNamespaces) + for _, executable := range executables { + mockExecutable := executable.(*MockExecutable) + mockExecutable.EXPECT().GetTask().Return(mockExecutable).AnyTimes() + mockExecutable.EXPECT().GetNamespaceID().Return(uuid.NewString()).AnyTimes() + mockExecutable.EXPECT().State().Return(ctasks.TaskStatePending).MaxTimes(1) + slice.add(executable) + } + + slice.ShrinkScope() + s.validateSliceState(slice) + + snapshot := capture.Snapshot() + + if tc.leaveIteratorsOpen { + s.Empty(snapshot[metrics.QueueSlicePendingKeys.Name()]) + s.Empty(snapshot[metrics.QueuePredicateResolutionLoss.Name()]) + return + } + + pendingKeysRecordings := snapshot[metrics.QueueSlicePendingKeys.Name()] + s.Require().Len(pendingKeysRecordings, 1) + s.Equal(int64(tc.numPendingNamespaces), pendingKeysRecordings[0].Value) + + lossRecordings := snapshot[metrics.QueuePredicateResolutionLoss.Name()] + if tc.expectDeclined { + s.Len(lossRecordings, 1) + } else { + s.Empty(lossRecordings) + } + }) + } +} + func (s *sliceSuite) TestSelectTasks_NoError() { r := NewRandomRange() namespaceIDs := []string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()} diff --git a/service/history/shard/context_impl.go b/service/history/shard/context_impl.go index f1369261f8c..b55b29c23e0 100644 --- a/service/history/shard/context_impl.go +++ b/service/history/shard/context_impl.go @@ -1257,6 +1257,21 @@ func (s *ContextImpl) updateShardInfo( s.tasksCompletedSinceLastUpdate = 0 updatedShardInfo := trimShardInfo(s.config, s.clusterMetadata.GetAllClusterInfo(), s.copyShardInfo(s.shardInfo)) + + metrics.ShardInfoSize.With(s.metricsHandler).Record(int64(updatedShardInfo.Size())) + for categoryID, queueState := range updatedShardInfo.QueueStates { + category, ok := s.taskCategoryRegistry.GetCategoryByID(int(categoryID)) + if !ok { + continue + } + sizeBytes := int64(queueState.Size()) + categoryTag := metrics.TaskCategoryTag(category.Name()) + // The counter is a true accumulator; the histogram's _sum is not, since tally's Prometheus + // reporter replays each sample as its bucket's upper bound, not the recorded value. + metrics.QueueStateSize.With(s.metricsHandler).Record(sizeBytes, categoryTag) + metrics.QueueStateSizeTotal.With(s.metricsHandler).Record(sizeBytes, categoryTag) + } + request := &persistence.UpdateShardRequest{ ShardInfo: updatedShardInfo, PreviousRangeID: s.shardInfo.GetRangeId(), diff --git a/service/history/shard/context_test.go b/service/history/shard/context_test.go index 8b369b99928..abaef132d95 100644 --- a/service/history/shard/context_test.go +++ b/service/history/shard/context_test.go @@ -994,6 +994,102 @@ func (s *contextSuite) TestUpdateShardInfo_FirstUpdate() { s.Equal(0, s.mockShard.tasksCompletedSinceLastUpdate) } +func (s *contextSuite) TestUpdateShardInfo_RecordsSizeMetrics() { + s.mockShard.state = contextStateAcquired + s.setImmediateAckLevels(map[int32]int64{ + int32(tasks.CategoryIDTransfer): 100, + int32(tasks.CategoryIDTimer): 200, + }) + + expectedTransferSize := int64(s.mockShard.shardInfo.QueueStates[int32(tasks.CategoryIDTransfer)].Size()) + expectedTimerSize := int64(s.mockShard.shardInfo.QueueStates[int32(tasks.CategoryIDTimer)].Size()) + + s.mockShardManager.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Return(nil).Times(1) + + captureHandler := metricstest.NewCaptureHandler() + s.mockShard.SetMetricsHandler(captureHandler) + capture := captureHandler.StartCapture() + defer captureHandler.StopCapture(capture) + + err := s.mockShard.updateShardInfo(0, func() {}) + s.NoError(err) + + snapshot := capture.Snapshot() + + shardInfoSizeRecordings := snapshot[metrics.ShardInfoSize.Name()] + s.Require().Len(shardInfoSizeRecordings, 1) + s.GreaterOrEqual(shardInfoSizeRecordings[0].Value.(int64), expectedTransferSize) + s.GreaterOrEqual(shardInfoSizeRecordings[0].Value.(int64), expectedTimerSize) + + queueStateSizeRecordings := snapshot[metrics.QueueStateSize.Name()] + s.Require().Len(queueStateSizeRecordings, 2) + + sizeByCategory := make(map[string]int64, len(queueStateSizeRecordings)) + for _, recording := range queueStateSizeRecordings { + sizeByCategory[recording.Tags["task_category"]] = recording.Value.(int64) + } + s.Equal(map[string]int64{ + tasks.CategoryTransfer.Name(): expectedTransferSize, + tasks.CategoryTimer.Name(): expectedTimerSize, + }, sizeByCategory) +} + +func (s *contextSuite) TestUpdateShardInfo_RecordsQueueStateSizeTotal() { + s.mockShard.state = contextStateAcquired + s.setImmediateAckLevels(map[int32]int64{ + int32(tasks.CategoryIDTransfer): 100, + int32(tasks.CategoryIDTimer): 200, + }) + + expectedTransferSize := int64(s.mockShard.shardInfo.QueueStates[int32(tasks.CategoryIDTransfer)].Size()) + expectedTimerSize := int64(s.mockShard.shardInfo.QueueStates[int32(tasks.CategoryIDTimer)].Size()) + + s.mockShardManager.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Return(nil).Times(1) + + captureHandler := metricstest.NewCaptureHandler() + s.mockShard.SetMetricsHandler(captureHandler) + capture := captureHandler.StartCapture() + defer captureHandler.StopCapture(capture) + + err := s.mockShard.updateShardInfo(0, func() {}) + s.NoError(err) + + snapshot := capture.Snapshot() + recordings := snapshot[metrics.QueueStateSizeTotal.Name()] + s.Require().Len(recordings, 2) + + sizeByCategory := make(map[string]int64, len(recordings)) + for _, recording := range recordings { + sizeByCategory[recording.Tags["task_category"]] = recording.Value.(int64) + } + s.Equal(map[string]int64{ + tasks.CategoryTransfer.Name(): expectedTransferSize, + tasks.CategoryTimer.Name(): expectedTimerSize, + }, sizeByCategory) +} + +func (s *contextSuite) TestUpdateShardInfo_DoesNotRecordSizeMetrics_WhenThrottled() { + s.mockShard.state = contextStateAcquired + + // First call always persists, establishing lastUpdated. + s.mockShardManager.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Return(nil).Times(1) + s.NoError(s.mockShard.updateShardInfo(0, func() {})) + + captureHandler := metricstest.NewCaptureHandler() + s.mockShard.SetMetricsHandler(captureHandler) + capture := captureHandler.StartCapture() + defer captureHandler.StopCapture(capture) + + // No time has passed and too few tasks completed: shouldn't persist, and shouldn't record size. + s.mockShardManager.EXPECT().UpdateShard(gomock.Any(), gomock.Any()).Times(0) + s.NoError(s.mockShard.updateShardInfo(0, func() {})) + + snapshot := capture.Snapshot() + s.Empty(snapshot[metrics.ShardInfoSize.Name()]) + s.Empty(snapshot[metrics.QueueStateSize.Name()]) + s.Empty(snapshot[metrics.QueueStateSizeTotal.Name()]) +} + // setImmediateAckLevels replaces the shard's queue states so each given immediate category has its // ack level at the given task id, i.e. a backlog of everything above it. func (s *contextSuite) setImmediateAckLevels(ackLevelByCategoryID map[int32]int64) { From ceb1cc1071b1fd37f8a2cd8ef5ed3e4333245cc2 Mon Sep 17 00:00:00 2001 From: Prathyush PV Date: Fri, 21 Aug 2026 13:41:54 -0700 Subject: [PATCH 042/139] Validate history pagination branch token against mutable state (#11723) ## What changed? `GetWorkflowExecutionHistory` and `GetWorkflowExecutionHistoryReverse` now check `branch_token` in the page token against the token in mutable state. ## Why? To confirm if it is still the correct branch after conflict resolution. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- common/dynamicconfig/constants.go | 14 ++ common/log/tag/tags.go | 5 + common/metrics/metric_defs.go | 1 + .../api/getworkflowexecutionhistory/api.go | 13 ++ .../getworkflowexecutionhistoryreverse/api.go | 17 ++ service/history/api/token.go | 125 +++++++++++ service/history/api/token_test.go | 117 ++++++++++ service/history/configs/config.go | 13 +- service/history/history_engine_test.go | 207 ++++++++++++++++++ tests/gethistory_test.go | 129 +++++++++++ 10 files changed, 638 insertions(+), 3 deletions(-) create mode 100644 service/history/api/token_test.go diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 91ab4903c51..c253135b776 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1717,6 +1717,20 @@ leaves the membership ring, giving in-flight long-polls time to drain before the // keys for history + EnablePaginationTokenBranchValidation = NewGlobalBoolSetting( + "history.enablePaginationTokenBranchValidation", + true, + `EnablePaginationTokenBranchValidation enables checking that the branch token in a +GetWorkflowExecutionHistory(Reverse) page token is the current branch token of the requested +execution.`, + ) + EnablePaginationTokenBranchValidationShadowMode = NewGlobalBoolSetting( + "history.enablePaginationTokenBranchValidationShadowMode", + false, + `EnablePaginationTokenBranchValidationShadowMode logs and emits metrics for a page token whose +branch token is not the execution's current one, but still serves the read.`, + ) + EnableReplicationStream = NewGlobalBoolSetting( "history.enableReplicationStream", true, diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index fa9c16346fa..96747429378 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -259,6 +259,11 @@ func WorkflowBranchToken(branchToken []byte) ZapTag { return NewBinaryTag("wf-branch-token", branchToken) } +// WorkflowRequestBranchToken returns tag for a branch token supplied by the caller +func WorkflowRequestBranchToken(branchToken []byte) ZapTag { + return NewBinaryTag("wf-request-branch-token", branchToken) +} + // WorkflowTreeID returns tag for WorkflowTreeID func WorkflowTreeID(treeID string) ZapTag { return NewStringTag("wf-tree-id", treeID) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 61eab914afe..92808f0bbd4 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1026,6 +1026,7 @@ var ( FailedWorkflowTasksCounter = NewCounterDef("failed_workflow_tasks") WorkflowTaskAttempt = NewDimensionlessHistogramDef("workflow_task_attempt") StaleMutableStateCounter = NewCounterDef("stale_mutable_state") + PaginationTokenBranchMismatchCounter = NewCounterDef("pagination_token_branch_mismatch") AutoResetPointsLimitExceededCounter = NewCounterDef("auto_reset_points_exceed_limit") AutoResetPointCorruptionCounter = NewCounterDef("auto_reset_point_corruption") BatchableTaskBatchCount = NewGaugeDef("batchable_task_batch_count") diff --git a/service/history/api/getworkflowexecutionhistory/api.go b/service/history/api/getworkflowexecutionhistory/api.go index 64ae77b2c59..dfeab72a241 100644 --- a/service/history/api/getworkflowexecutionhistory/api.go +++ b/service/history/api/getworkflowexecutionhistory/api.go @@ -256,6 +256,19 @@ func Invoke( continuationToken.FirstEventId = continuationToken.GetNextEventId() continuationToken.NextEventId = nextEventID continuationToken.IsWorkflowRunning = isWorkflowRunning + } else { + if err = api.ValidateBranchTokenForExecution( + ctx, + shardContext, + workflowConsistencyChecker, + eventNotifier, + namespaceName, + namespaceID, + execution, + continuationToken.BranchToken, + ); err != nil { + return nil, err + } } } else { continuationToken = &tokenspb.HistoryContinuation{} diff --git a/service/history/api/getworkflowexecutionhistoryreverse/api.go b/service/history/api/getworkflowexecutionhistoryreverse/api.go index ebf282b22cc..4f87eee5eac 100644 --- a/service/history/api/getworkflowexecutionhistoryreverse/api.go +++ b/service/history/api/getworkflowexecutionhistoryreverse/api.go @@ -103,6 +103,23 @@ func Invoke( } execution.RunId = continuationToken.GetRunId() + + var namespaceName namespace.Name + if entry, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespaceID); err == nil { + namespaceName = entry.Name() + } + if err = api.ValidateBranchTokenForExecution( + ctx, + shardContext, + workflowConsistencyChecker, + eventNotifier, + namespaceName, + namespaceID, + execution, + continuationToken.BranchToken, + ); err != nil { + return nil, err + } } // TODO below is a temporal solution to guard against invalid event batch diff --git a/service/history/api/token.go b/service/history/api/token.go index de8e770bb2b..4d233bd0c01 100644 --- a/service/history/api/token.go +++ b/service/history/api/token.go @@ -1,10 +1,20 @@ package api import ( + "bytes" + "context" + + commonpb "go.temporal.io/api/common/v1" historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" tokenspb "go.temporal.io/server/api/token/v1" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/service/history/consts" + "go.temporal.io/server/service/history/events" + historyi "go.temporal.io/server/service/history/interfaces" ) // NOTE: DO NOT MODIFY UNLESS ALSO APPLIED TO ./service/frontend/token_deprecated.go @@ -116,3 +126,118 @@ func ValidatePaginationToken( } return nil } + +const ( + // branchTokenMismatchReasonNonCurrent is a branch this execution records but no longer reads from. + branchTokenMismatchReasonNonCurrent metrics.ReasonString = "non_current_branch" + branchTokenMismatchReasonForeign metrics.ReasonString = "foreign_branch" +) + +// maxLoggedBranchTokenLen bounds caller-supplied bytes reaching the log. +const maxLoggedBranchTokenLen = 4096 + +func branchTokenMismatchReason( + currentBranchToken []byte, + requestBranchToken []byte, + versionHistories *historyspb.VersionHistories, +) metrics.ReasonString { + if bytes.Equal(requestBranchToken, currentBranchToken) { + return "" + } + for _, versionHistory := range versionHistories.GetHistories() { + if bytes.Equal(versionHistory.GetBranchToken(), requestBranchToken) { + return branchTokenMismatchReasonNonCurrent + } + } + return branchTokenMismatchReasonForeign +} + +func reportBranchTokenMismatch( + shardContext historyi.ShardContext, + namespaceName string, + execution *commonpb.WorkflowExecution, + reason metrics.ReasonString, + currentBranchToken []byte, + requestBranchToken []byte, +) { + if namespaceName == "" { + namespaceName = metrics.NamespaceUnknownTag().Value + } + metrics.PaginationTokenBranchMismatchCounter.With(shardContext.GetMetricsHandler()).Record( + 1, + metrics.NamespaceTag(namespaceName), + metrics.ReasonTag(reason), + ) + loggedRequestToken := requestBranchToken[:min(len(requestBranchToken), maxLoggedBranchTokenLen)] + shardContext.GetLogger().Warn("Pagination branch token is not the execution's current branch token", + tag.WorkflowNamespace(namespaceName), + tag.WorkflowID(execution.GetWorkflowId()), + tag.WorkflowRunID(execution.GetRunId()), + tag.NewStringTag("reason", string(reason)), + tag.WorkflowBranchToken(currentBranchToken), + tag.WorkflowRequestBranchToken(loggedRequestToken), + ) +} + +// ValidateBranchTokenForExecution rejects a paging branch token that is not the execution's current +// one. +func ValidateBranchTokenForExecution( + ctx context.Context, + shardContext historyi.ShardContext, + workflowConsistencyChecker WorkflowConsistencyChecker, + eventNotifier events.Notifier, + namespaceName namespace.Name, + namespaceID namespace.ID, + execution *commonpb.WorkflowExecution, + requestBranchToken []byte, +) error { + config := shardContext.GetConfig() + if !config.EnablePaginationTokenBranchValidation() { + return nil + } + if len(requestBranchToken) == 0 { + return consts.ErrInvalidNextPageToken + } + + response, err := GetOrPollWorkflowMutableState( + ctx, + shardContext, + &historyservice.GetMutableStateRequest{ + NamespaceId: namespaceID.String(), + Execution: execution, + }, + workflowConsistencyChecker, + eventNotifier, + ) + if err != nil { + return err + } + + currentBranchToken := response.GetCurrentBranchToken() + mismatchReason := branchTokenMismatchReason( + currentBranchToken, + requestBranchToken, + response.GetVersionHistories(), + ) + if mismatchReason == "" { + return nil + } + + reportBranchTokenMismatch( + shardContext, + namespaceName.String(), + execution, + mismatchReason, + currentBranchToken, + requestBranchToken, + ) + if config.EnablePaginationTokenBranchValidationShadowMode() { + return nil + } + return serviceerrors.NewCurrentBranchChanged( + currentBranchToken, + requestBranchToken, + nil, + nil, + ) +} diff --git a/service/history/api/token_test.go b/service/history/api/token_test.go new file mode 100644 index 00000000000..e774c5731eb --- /dev/null +++ b/service/history/api/token_test.go @@ -0,0 +1,117 @@ +package api + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + historyspb "go.temporal.io/server/api/history/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/primitives" + "go.temporal.io/server/service/history/configs" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" + "go.uber.org/mock/gomock" +) + +func newTestBranchToken( + t *testing.T, + treeID string, + branchID string, + ancestors []*persistencespb.HistoryBranchRange, +) []byte { + t.Helper() + branchUtil := persistence.NewHistoryBranchUtil(serialization.NewSerializer()) + token, err := branchUtil.NewHistoryBranch( + "namespace-id", "workflow-id", "run-id", treeID, &branchID, ancestors, 0, 0, 0, + ) + require.NoError(t, err) + return token +} + +func testVersionHistories(tokens ...[]byte) *historyspb.VersionHistories { + versionHistories := &historyspb.VersionHistories{} + for _, token := range tokens { + versionHistories.Histories = append( + versionHistories.Histories, + &historyspb.VersionHistory{BranchToken: token}, + ) + } + return versionHistories +} + +func TestBranchTokenMismatchReason(t *testing.T) { + treeID := primitives.NewUUID().String() + branchID := primitives.NewUUID().String() + otherTreeID := primitives.NewUUID().String() + otherBranchID := primitives.NewUUID().String() + + current := newTestBranchToken(t, treeID, branchID, nil) + nonCurrent := newTestBranchToken(t, otherTreeID, otherBranchID, nil) + + t.Run("matches the current branch token", func(t *testing.T) { + got := branchTokenMismatchReason(current, current, testVersionHistories(current)) + require.Empty(t, got) + }) + + t.Run("matches the current token when an identical token is recorded twice", func(t *testing.T) { + got := branchTokenMismatchReason(current, current, testVersionHistories(current, current)) + require.Empty(t, got) + }) + + t.Run("matches opaque tokens the branch parser cannot read", func(t *testing.T) { + opaque := []byte{1, 2, 3} + got := branchTokenMismatchReason(opaque, opaque, testVersionHistories(opaque)) + require.Empty(t, got) + }) + + t.Run("reports a non-current branch when the token names an older history", func(t *testing.T) { + histories := testVersionHistories(nonCurrent, current) + got := branchTokenMismatchReason(current, nonCurrent, histories) + require.Equal(t, branchTokenMismatchReasonNonCurrent, got) + }) + + t.Run("reports foreign for a different branch in the same tree", func(t *testing.T) { + request := newTestBranchToken(t, treeID, otherBranchID, nil) + got := branchTokenMismatchReason(current, request, testVersionHistories(current)) + require.Equal(t, branchTokenMismatchReasonForeign, got) + }) + + t.Run("reports foreign for the current branch carrying injected ancestors", func(t *testing.T) { + request := newTestBranchToken(t, treeID, branchID, []*persistencespb.HistoryBranchRange{ + {BranchId: otherBranchID, BeginNodeId: 1, EndNodeId: 1000}, + }) + got := branchTokenMismatchReason(current, request, testVersionHistories(current)) + require.Equal(t, branchTokenMismatchReasonForeign, got) + }) + + t.Run("reports foreign when there are no version histories", func(t *testing.T) { + got := branchTokenMismatchReason(current, nonCurrent, nil) + require.Equal(t, branchTokenMismatchReasonForeign, got) + }) +} + +func TestValidateBranchTokenForExecution_EmptyRequestToken(t *testing.T) { + for _, tc := range []struct { + name string + validation bool + wantErr error + }{ + {name: "rejected while validating", validation: true, wantErr: consts.ErrInvalidNextPageToken}, + {name: "served once validation is disabled", validation: false}, + } { + t.Run(tc.name, func(t *testing.T) { + shardContext := historyi.NewMockShardContext(gomock.NewController(t)) + shardContext.EXPECT().GetConfig().Return(&configs.Config{ + EnablePaginationTokenBranchValidation: dynamicconfig.GetBoolPropertyFn(tc.validation), + }).AnyTimes() + + err := ValidateBranchTokenForExecution( + context.Background(), shardContext, nil, nil, "", "", nil, nil) + require.ErrorIs(t, err, tc.wantErr) + }) + } +} diff --git a/service/history/configs/config.go b/service/history/configs/config.go index aeddbb54159..e59e3989aa9 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -424,6 +424,9 @@ type Config struct { SendRawHistoryBytesToMatchingService dynamicconfig.BoolPropertyFn SendRawWorkflowHistory dynamicconfig.BoolPropertyFnWithNamespaceFilter + EnablePaginationTokenBranchValidation dynamicconfig.BoolPropertyFn + EnablePaginationTokenBranchValidationShadowMode dynamicconfig.BoolPropertyFn + WorkflowIdReuseMinimalInterval dynamicconfig.DurationPropertyFnWithNamespaceFilter EnableWorkflowIdReuseStartTimeValidation dynamicconfig.BoolPropertyFnWithNamespaceFilter BusinessIDReuseRate dynamicconfig.IntPropertyFnWithNamespaceFilter @@ -827,9 +830,13 @@ func NewConfig( EnableUpdateWithStartRetryOnClosedWorkflowAbort: dynamicconfig.EnableUpdateWithStartRetryOnClosedWorkflowAbort.Get(dc), EnableUpdateWithStartRetryableErrorOnClosedWorkflowAbort: dynamicconfig.EnableUpdateWithStartRetryableErrorOnClosedWorkflowAbort.Get(dc), - SendRawHistoryBetweenInternalServices: dynamicconfig.SendRawHistoryBetweenInternalServices.Get(dc), - SendRawHistoryBytesToMatchingService: dynamicconfig.SendRawHistoryBytesToMatchingService.Get(dc), - SendRawWorkflowHistory: dynamicconfig.SendRawWorkflowHistory.Get(dc), + SendRawHistoryBetweenInternalServices: dynamicconfig.SendRawHistoryBetweenInternalServices.Get(dc), + SendRawHistoryBytesToMatchingService: dynamicconfig.SendRawHistoryBytesToMatchingService.Get(dc), + SendRawWorkflowHistory: dynamicconfig.SendRawWorkflowHistory.Get(dc), + + EnablePaginationTokenBranchValidation: dynamicconfig.EnablePaginationTokenBranchValidation.Get(dc), + EnablePaginationTokenBranchValidationShadowMode: dynamicconfig.EnablePaginationTokenBranchValidationShadowMode.Get(dc), + WorkflowIdReuseMinimalInterval: dynamicconfig.WorkflowIdReuseMinimalInterval.Get(dc), EnableWorkflowIdReuseStartTimeValidation: dynamicconfig.EnableWorkflowIdReuseStartTimeValidation.Get(dc), BusinessIDReuseRate: dynamicconfig.BusinessIDReuseRate.Get(dc), diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index 1e41a803970..ee0ad9e73e0 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -49,12 +49,14 @@ import ( "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" + serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/service/history/api" @@ -7028,3 +7030,208 @@ func addFailWorkflowEvent( ) return event } + +func (s *engineSuite) mockExecutionWithForeignBranchToken( + we *commonpb.WorkflowExecution, +) (ownedToken []byte, foreignToken []byte) { + branchUtil := persistence.NewHistoryBranchUtil(serialization.NewSerializer()) + treeID := uuid.NewString() + ownedBranchID := uuid.NewString() + foreignBranchID := uuid.NewString() + + ownedBranchToken, err := branchUtil.NewHistoryBranch( + tests.NamespaceID.String(), we.WorkflowId, we.RunId, treeID, &ownedBranchID, nil, 0, 0, 0) + s.NoError(err) + foreignBranchToken, err := branchUtil.NewHistoryBranch( + tests.NamespaceID.String(), we.WorkflowId, we.RunId, treeID, &foreignBranchID, nil, 0, 0, 0) + s.NoError(err) + + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{ + State: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{ + State: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + RunId: we.RunId, + }, + NextEventId: 5, + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{ + NamespaceId: tests.NamespaceID.String(), + WorkflowId: we.WorkflowId, + VersionHistories: &historyspb.VersionHistories{ + CurrentVersionHistoryIndex: 0, + Histories: []*historyspb.VersionHistory{ + { + BranchToken: ownedBranchToken, + Items: []*historyspb.VersionHistoryItem{ + {EventId: 4, Version: 0}, + }, + }, + }, + }, + }, + }, + MutableStateStats: persistence.MutableStateStatistics{}, + }, nil).AnyTimes() + + return ownedBranchToken, foreignBranchToken +} + +func (s *engineSuite) getHistoryRequestWithPageToken( + we *commonpb.WorkflowExecution, + continuation *tokenspb.HistoryContinuation, + waitNewEvent bool, +) *historyservice.GetWorkflowExecutionHistoryRequest { + nextPageToken, err := api.SerializeHistoryToken(continuation) + s.NoError(err) + return &historyservice.GetWorkflowExecutionHistoryRequest{ + NamespaceId: tests.NamespaceID.String(), + Request: &workflowservice.GetWorkflowExecutionHistoryRequest{ + Execution: we, + MaximumPageSize: 10, + NextPageToken: nextPageToken, + WaitNewEvent: waitNewEvent, + HistoryEventFilterType: enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT, + SkipArchival: true, + }, + } +} + +// expectHistoryReadWithBranchToken allows the history read and pins the branch token it is issued with. +func (s *engineSuite) expectHistoryReadWithBranchToken(branchToken []byte) { + s.mockSearchAttributesProvider.EXPECT().GetSearchAttributes(gomock.Any(), false). + Return(searchattribute.TestNameTypeMap(), nil).AnyTimes() + s.mockVisibilityMgr.EXPECT().GetIndexName().Return(esIndexName).AnyTimes() + s.mockExecutionMgr.EXPECT().ReadHistoryBranch(gomock.Any(), gomock.Any()).DoAndReturn( + func(_ context.Context, request *persistence.ReadHistoryBranchRequest) (*persistence.ReadHistoryBranchResponse, error) { + s.Equal(branchToken, request.BranchToken) + return &persistence.ReadHistoryBranchResponse{HistoryEvents: []*historypb.HistoryEvent{}}, nil + }, + ).MinTimes(1) +} + +func (s *engineSuite) TestGetWorkflowExecutionHistory_BranchTokenNotOwnedByExecution() { + we := commonpb.WorkflowExecution{WorkflowId: "wid-foreign-branch", RunId: uuid.NewString()} + + engine, err := s.historyEngine.shardContext.GetEngine(context.Background()) + s.NoError(err) + + s.config.EnablePaginationTokenBranchValidationShadowMode = dynamicconfig.GetBoolPropertyFn(false) + _, foreignBranchToken := s.mockExecutionWithForeignBranchToken(&we) + req := s.getHistoryRequestWithPageToken(&we, &tokenspb.HistoryContinuation{ + RunId: we.GetRunId(), + FirstEventId: common.FirstEventID, + NextEventId: 5, + PersistenceToken: []byte("some random persistence token"), + BranchToken: foreignBranchToken, + IsWorkflowRunning: true, + }, false) + + // The history read must never be attempted, on either the decoded or the raw path. + s.mockExecutionMgr.EXPECT().ReadHistoryBranch(gomock.Any(), gomock.Any()).Times(0) + s.mockExecutionMgr.EXPECT().ReadRawHistoryBranch(gomock.Any(), gomock.Any()).Times(0) + + for _, sendRawHistory := range []bool{false, true} { + s.config.SendRawWorkflowHistory = func(string) bool { return sendRawHistory } + _, err = engine.GetWorkflowExecutionHistory(context.Background(), req) + var branchErr *serviceerrors.CurrentBranchChanged + s.ErrorAs(err, &branchErr, "sendRawHistory=%v", sendRawHistory) + } +} + +func (s *engineSuite) TestGetWorkflowExecutionHistory_ForeignBranchTokenServedWhenNotEnforcing() { + we := commonpb.WorkflowExecution{WorkflowId: "wid-foreign-branch-served", RunId: uuid.NewString()} + + engine, err := s.historyEngine.shardContext.GetEngine(context.Background()) + s.NoError(err) + + s.config.SendRawWorkflowHistory = func(string) bool { return false } + _, foreignBranchToken := s.mockExecutionWithForeignBranchToken(&we) + // Not enforcing reads with the caller's token, exactly as before validation existed. + s.expectHistoryReadWithBranchToken(foreignBranchToken) + + for _, tc := range []struct { + name string + validation bool + shadow bool + }{ + {name: "shadow mode", validation: true, shadow: true}, + {name: "validation disabled", validation: false, shadow: false}, + } { + s.config.EnablePaginationTokenBranchValidation = dynamicconfig.GetBoolPropertyFn(tc.validation) + s.config.EnablePaginationTokenBranchValidationShadowMode = dynamicconfig.GetBoolPropertyFn(tc.shadow) + _, err = engine.GetWorkflowExecutionHistory( + context.Background(), + s.getHistoryRequestWithPageToken(&we, &tokenspb.HistoryContinuation{ + RunId: we.GetRunId(), + FirstEventId: common.FirstEventID, + NextEventId: 5, + PersistenceToken: []byte("some random persistence token"), + BranchToken: foreignBranchToken, + IsWorkflowRunning: true, + }, false), + ) + s.NoError(err, tc.name) + } +} + +func (s *engineSuite) TestGetWorkflowExecutionHistory_LongPollDiscardsRequestBranchToken() { + we := commonpb.WorkflowExecution{WorkflowId: "wid-longpoll-overwrite", RunId: uuid.NewString()} + + engine, err := s.historyEngine.shardContext.GetEngine(context.Background()) + s.NoError(err) + + s.config.EnablePaginationTokenBranchValidationShadowMode = dynamicconfig.GetBoolPropertyFn(false) + s.config.SendRawWorkflowHistory = func(string) bool { return false } + ownedBranchToken, foreignBranchToken := s.mockExecutionWithForeignBranchToken(&we) + s.expectHistoryReadWithBranchToken(ownedBranchToken) + + // An empty persistence token and a next event ID behind mutable state select the long poll + // refresh, which replaces the caller's branch token instead of validating it. + req := s.getHistoryRequestWithPageToken(&we, &tokenspb.HistoryContinuation{ + RunId: we.GetRunId(), + FirstEventId: common.FirstEventID, + NextEventId: 2, + PersistenceToken: nil, + BranchToken: foreignBranchToken, + IsWorkflowRunning: true, + }, true) + + _, err = engine.GetWorkflowExecutionHistory(context.Background(), req) + s.NoError(err) +} + +func (s *engineSuite) TestGetWorkflowExecutionHistoryReverse_BranchTokenNotOwnedByExecution() { + we := commonpb.WorkflowExecution{WorkflowId: "wid-foreign-branch-reverse", RunId: uuid.NewString()} + + engine, err := s.historyEngine.shardContext.GetEngine(context.Background()) + s.NoError(err) + + s.config.EnablePaginationTokenBranchValidationShadowMode = dynamicconfig.GetBoolPropertyFn(false) + _, foreignBranchToken := s.mockExecutionWithForeignBranchToken(&we) + + nextPageToken, err := api.SerializeHistoryToken(&tokenspb.HistoryContinuation{ + RunId: we.GetRunId(), + FirstEventId: common.FirstEventID, + NextEventId: 5, + PersistenceToken: []byte("some random persistence token"), + BranchToken: foreignBranchToken, + }) + s.NoError(err) + + // The history read must never be attempted. + s.mockExecutionMgr.EXPECT().ReadHistoryBranchReverse(gomock.Any(), gomock.Any()).Times(0) + + _, err = engine.GetWorkflowExecutionHistoryReverse( + context.Background(), + &historyservice.GetWorkflowExecutionHistoryReverseRequest{ + NamespaceId: tests.NamespaceID.String(), + Request: &workflowservice.GetWorkflowExecutionHistoryReverseRequest{ + Execution: &we, + MaximumPageSize: 10, + NextPageToken: nextPageToken, + }, + }, + ) + var branchErr *serviceerrors.CurrentBranchChanged + s.ErrorAs(err, &branchErr) +} diff --git a/tests/gethistory_test.go b/tests/gethistory_test.go index 0a7d6845f28..e4428d2c621 100644 --- a/tests/gethistory_test.go +++ b/tests/gethistory_test.go @@ -2,21 +2,25 @@ package tests import ( "context" + "errors" "slices" "testing" "time" "github.com/google/uuid" + "github.com/stretchr/testify/require" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" + "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" sdkclient "go.temporal.io/sdk/client" "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/testing/parallelsuite" @@ -747,3 +751,128 @@ func (s *GetHistorySuite) getHistory( return responseInner.History.Events, responseInner.NextPageToken } + +// startMultiBatchWorkflow starts a workflow and signals it repeatedly. Each signal is its own +// transaction and therefore its own history node, so a page size of 1 yields many pages and a +// continuation taken from an early page is far from the last page. +func startMultiBatchWorkflow( + ctx context.Context, + assertions *require.Assertions, + env *testcore.TestEnv, +) { + _, err := env.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowId: env.Tv().WorkflowID(), + WorkflowType: env.Tv().WorkflowType(), + TaskQueue: env.Tv().TaskQueue(), + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: env.Tv().WorkerIdentity(), + }) + assertions.NoError(err) + + for range 6 { + _, err = env.FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: env.Tv().WorkflowID(), + }, + SignalName: "signal", + Identity: env.Tv().WorkerIdentity(), + }) + assertions.NoError(err) + } +} + +func (s *GetHistorySuite) TestGetWorkflowExecutionHistory_ContinuationFromAnotherNamespace( + enableTransitionHistory bool, +) { + // A single-shard cluster so both namespaces map to the same history shard. On SQL backends + // shard_id is part of the history_node primary key, so without this the replay would be + // rejected for the wrong reason; on Cassandra it is not part of the key at all. + env := s.newTestEnv( + enableTransitionHistory, + testcore.WithHistoryShardCount(1), + // Shadow mode reports the mismatch but still serves the page. + testcore.WithDynamicConfig(dynamicconfig.EnablePaginationTokenBranchValidationShadowMode, false), + ) + + otherNamespace := namespace.Name(testcore.RandomizeStr("other-namespace")) + _, err := env.RegisterNamespace(otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + s.Require().NoError(err) + + startMultiBatchWorkflow(s.Context(), s.Require(), env) + + // A genuine, non-final continuation obtained with legitimate access to the first namespace. + firstPage, err := env.FrontendClient().GetWorkflowExecutionHistory( + s.Context(), + &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: env.Tv().WorkflowID()}, + MaximumPageSize: 1, + HistoryEventFilterType: enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT, + }, + ) + s.Require().NoError(err) + s.Require().NotEmpty(firstPage.NextPageToken, "need a non-final continuation for this test") + + // Replayed against the other namespace. No run ID is sent, so nothing in the request itself + // refers to the workflow whose history the token points at. + resp, err := env.FrontendClient().GetWorkflowExecutionHistory( + s.Context(), + &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: otherNamespace.String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: env.Tv().WorkflowID()}, + MaximumPageSize: 1, + NextPageToken: firstPage.NextPageToken, + HistoryEventFilterType: enumspb.HISTORY_EVENT_FILTER_TYPE_ALL_EVENT, + }, + ) + s.Error(err) + s.Empty(resp.GetHistory().GetEvents()) + var invalidArgument *serviceerror.InvalidArgument + var notFound *serviceerror.NotFound + s.True( + errors.As(err, &invalidArgument) || errors.As(err, ¬Found), + "expected InvalidArgument or NotFound, got %T: %v", err, err, + ) +} + +func (s *RawHistorySuite) TestGetWorkflowExecutionHistoryReverse_ContinuationFromAnotherNamespace() { + env := s.newTestEnv( + testcore.WithHistoryShardCount(1), + // Shadow mode reports the mismatch but still serves the page. + testcore.WithDynamicConfig(dynamicconfig.EnablePaginationTokenBranchValidationShadowMode, false), + ) + + otherNamespace := namespace.Name(testcore.RandomizeStr("other-namespace")) + _, err := env.RegisterNamespace(otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + s.Require().NoError(err) + + startMultiBatchWorkflow(s.Context(), s.Require(), env) + + firstPage, err := env.FrontendClient().GetWorkflowExecutionHistoryReverse( + s.Context(), + &workflowservice.GetWorkflowExecutionHistoryReverseRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: env.Tv().WorkflowID()}, + MaximumPageSize: 1, + }, + ) + s.Require().NoError(err) + s.Require().NotEmpty(firstPage.NextPageToken, "need a non-final continuation for this test") + + resp, err := env.FrontendClient().GetWorkflowExecutionHistoryReverse( + s.Context(), + &workflowservice.GetWorkflowExecutionHistoryReverseRequest{ + Namespace: otherNamespace.String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: env.Tv().WorkflowID()}, + MaximumPageSize: 1, + NextPageToken: firstPage.NextPageToken, + }, + ) + s.Error(err) + s.Empty(resp.GetHistory().GetEvents()) +} From 6805caea59e35d0611ed96be69641b8d86bff11c Mon Sep 17 00:00:00 2001 From: akbala Date: Fri, 21 Aug 2026 14:12:38 -0700 Subject: [PATCH 043/139] Return forked branch token from storage layer to OSS so both components have the same view of the branch token. (#11704) ## What changed? ForkHistoryBranchResponse now carries BaseBranchToken, and the two reset paths rebuild through it when the store set it. A store that doesn't (Cassandra, SQL) leaves it nil and both call sites fall back to the token the caller already had, so behavior is unchanged everywhere else. ## Why? ForkHistoryBranch can modify the base branch token in ways the caller may not have visibility into. This PR fixes it so that the changed token is returned to the caller. ## How did you test it? - [ ] built - [ ] added new unit test(s) - [ ] added new functional test(s) --------- Co-authored-by: Claude Opus 5 (1M context) --- common/persistence/data_interfaces.go | 9 +- service/history/ndc/resetter.go | 15 ++- service/history/ndc/resetter_test.go | 94 ++++++++++++++++++- service/history/ndc/workflow_resetter.go | 17 +++- service/history/ndc/workflow_resetter_test.go | 91 +++++++++++++++++- 5 files changed, 213 insertions(+), 13 deletions(-) diff --git a/common/persistence/data_interfaces.go b/common/persistence/data_interfaces.go index 1312c902654..c818b447866 100644 --- a/common/persistence/data_interfaces.go +++ b/common/persistence/data_interfaces.go @@ -925,6 +925,12 @@ type ( ForkHistoryBranchResponse struct { // branchToken to represent the new branch NewBranchToken []byte + // The storage layer might have changed the base branch token. This + // response represents the changed base branch token to be used for further + // processing in a workflow rebuild operation. + // An empty value indicates that the caller's base branch token + // is still correct. + BaseBranchToken []byte } // CompleteForkBranchRequest is used to complete forking @@ -1165,7 +1171,8 @@ type ( // ReadRawHistoryBranch returns history node raw data for a branch ByBatch // NOTE: this API should only be used by 3+DC ReadRawHistoryBranch(ctx context.Context, request *ReadHistoryBranchRequest) (*ReadRawHistoryBranchResponse, error) - // ForkHistoryBranch forks a new branch from a old branch + // ForkHistoryBranch forks a new branch from an old branch. The response may contain a different/rewritten base token that + // should be used for further action. ForkHistoryBranch(ctx context.Context, request *ForkHistoryBranchRequest) (*ForkHistoryBranchResponse, error) // DeleteHistoryBranch removes a branch // If this is the last branch to delete, it will also remove the root node diff --git a/service/history/ndc/resetter.go b/service/history/ndc/resetter.go index aaf27cb8cc7..9147c171d16 100644 --- a/service/history/ndc/resetter.go +++ b/service/history/ndc/resetter.go @@ -98,7 +98,7 @@ func (r *resetterImpl) resetWorkflow( return nil, err } - resetBranchToken, err := r.getResetBranchToken(ctx, baseBranchToken, baseLastEventID) + resetBranchToken, newBaseBranchToken, err := r.getResetBranchToken(ctx, baseBranchToken, baseLastEventID) if err != nil { return nil, err } @@ -112,7 +112,7 @@ func (r *resetterImpl) resetWorkflow( r.workflowID, r.baseRunID, ), - baseBranchToken, + newBaseBranchToken, baseLastEventID, new(baseLastEventVersion), definition.NewWorkflowKey( @@ -205,7 +205,7 @@ func (r *resetterImpl) getResetBranchToken( ctx context.Context, baseBranchToken []byte, baseLastEventID int64, -) ([]byte, error) { +) (newBranchToken []byte, forkedBaseBranchToken []byte, err error) { // fork a new history branch shardID := r.shard.GetShardID() @@ -218,8 +218,13 @@ func (r *resetterImpl) getResetBranchToken( NewRunID: r.newRunID, }) if err != nil { - return nil, err + return nil, nil, err } - return resp.NewBranchToken, nil + // An empty resp.BaseBranchToken means the storage layer did not rewrite the + // token. So it is still valid. + if len(resp.BaseBranchToken) == 0 { + return resp.NewBranchToken, baseBranchToken, nil + } + return resp.NewBranchToken, resp.BaseBranchToken, nil } diff --git a/service/history/ndc/resetter_test.go b/service/history/ndc/resetter_test.go index d42d56fcaed..69d397b3be3 100644 --- a/service/history/ndc/resetter_test.go +++ b/service/history/ndc/resetter_test.go @@ -126,6 +126,98 @@ func (s *resetterSuite) TestResetWorkflow_NoError() { incomingFirstEventID := baseEventID + 12 incomingVersion := baseVersion + 3 + rebuildStats := RebuildStats{ + HistorySize: 9999, + ExternalPayloadSize: 1234, + ExternalPayloadCount: 56, + } + newBranchToken := []byte("other random branch token") + resurrectedBranchToken := []byte("resurrected branch token") + + s.mockBaseMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{VersionHistories: versionHistories}).AnyTimes() + + mockBaseWorkflowReleaseFnCalled := false + mockBaseWorkflowReleaseFn := func(err error) { + mockBaseWorkflowReleaseFnCalled = true + } + mockBaseWorkflow := NewMockWorkflow(s.controller) + mockBaseWorkflow.EXPECT().GetMutableState().Return(s.mockBaseMutableState).AnyTimes() + mockBaseWorkflow.EXPECT().GetReleaseFn().Return(mockBaseWorkflowReleaseFn) + + s.mockTransactionMgr.EXPECT().LoadWorkflow( + ctx, + s.namespaceID, + s.workflowID, + s.baseRunID, + chasm.WorkflowArchetypeID, + ).Return(mockBaseWorkflow, nil) + + s.mockStateBuilder.EXPECT().Rebuild( + ctx, + now, + definition.NewWorkflowKey( + s.namespaceID.String(), + s.workflowID, + s.baseRunID, + ), + resurrectedBranchToken, + baseEventID, + new(baseVersion), + definition.NewWorkflowKey( + s.namespaceID.String(), + s.workflowID, + s.newRunID, + ), + newBranchToken, + gomock.Any(), + ).Return(s.mockRebuiltMutableState, rebuildStats, nil) + s.mockRebuiltMutableState.EXPECT().AddHistorySize(rebuildStats.HistorySize) + s.mockRebuiltMutableState.EXPECT().AddExternalPayloadSize(rebuildStats.ExternalPayloadSize) + s.mockRebuiltMutableState.EXPECT().AddExternalPayloadCount(rebuildStats.ExternalPayloadCount) + + shardID := s.mockShard.GetShardID() + s.mockExecManager.EXPECT().ForkHistoryBranch(gomock.Any(), &persistence.ForkHistoryBranchRequest{ + ForkBranchToken: branchToken, + ForkNodeID: baseEventID + 1, + Info: persistence.BuildHistoryGarbageCleanupInfo(s.namespaceID.String(), s.workflowID, s.newRunID), + ShardID: shardID, + NamespaceID: s.namespaceID.String(), + NewRunID: s.newRunID, + }).Return(&persistence.ForkHistoryBranchResponse{NewBranchToken: newBranchToken, BaseBranchToken: resurrectedBranchToken}, nil) + + s.mockRebuiltMutableState.EXPECT().RefreshExpirationTimeoutTask(gomock.Any()).Return(nil) + + rebuiltMutableState, err := s.workflowResetter.resetWorkflow( + ctx, + now, + baseEventID, + baseVersion, + incomingFirstEventID, + incomingVersion, + ) + s.NoError(err) + s.Equal(s.mockRebuiltMutableState, rebuiltMutableState) + s.True(mockBaseWorkflowReleaseFnCalled) +} + +func (s *resetterSuite) TestResetWorkflow_NoError_BaseBranchToken_Nil() { + ctx := context.Background() + now := time.Now().UTC() + + branchToken := []byte("some random branch token") + lastEventID := int64(500) + version := int64(123) + versionHistory := versionhistory.NewVersionHistory( + branchToken, + []*historyspb.VersionHistoryItem{versionhistory.NewVersionHistoryItem(lastEventID, version)}, + ) + versionHistories := versionhistory.NewVersionHistories(versionHistory) + + baseEventID := lastEventID - 100 + baseVersion := version + incomingFirstEventID := baseEventID + 12 + incomingVersion := baseVersion + 3 + rebuildStats := RebuildStats{ HistorySize: 9999, ExternalPayloadSize: 1234, @@ -182,7 +274,7 @@ func (s *resetterSuite) TestResetWorkflow_NoError() { ShardID: shardID, NamespaceID: s.namespaceID.String(), NewRunID: s.newRunID, - }).Return(&persistence.ForkHistoryBranchResponse{NewBranchToken: newBranchToken}, nil) + }).Return(&persistence.ForkHistoryBranchResponse{NewBranchToken: newBranchToken, BaseBranchToken: nil}, nil) s.mockRebuiltMutableState.EXPECT().RefreshExpirationTimeoutTask(gomock.Any()).Return(nil) diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go index 939b45ceea8..9698de784db 100644 --- a/service/history/ndc/workflow_resetter.go +++ b/service/history/ndc/workflow_resetter.go @@ -135,6 +135,10 @@ func (r *workflowResetterImpl) ResetWorkflow( var currentWorkflowMutation *persistence.WorkflowMutation var currentWorkflowEventsSeq []*persistence.WorkflowEvents + // Every reapply path below deliberately uses the pre-fork baseBranchToken, not the rewritten + // token that forkAndGenerateBranchToken returns. A storage layer that rewrites the base token + // only guarantees the range below the fork point is readable through it; these reads start at + // the fork point and go up, so they must keep the caller's original token. var reapplyEventsFn workflowResetReapplyEventsFn // currentWorkflow is nil only for a reset by explicit runId whose workflow has no current // execution (the current run was deleted while older runs survive). This is exclusive to the @@ -510,7 +514,7 @@ func (r *workflowResetterImpl) replayResetWorkflow( resetRequestID string, ) (Workflow, error) { - resetBranchToken, err := r.forkAndGenerateBranchToken( + resetBranchToken, newBaseBranchToken, err := r.forkAndGenerateBranchToken( ctx, namespaceID, workflowID, @@ -544,7 +548,7 @@ func (r *workflowResetterImpl) replayResetWorkflow( workflowID, baseRunID, ), - baseBranchToken, + newBaseBranchToken, baseRebuildLastEventID, new(baseRebuildLastEventVersion), definition.NewWorkflowKey( @@ -677,7 +681,7 @@ func (r *workflowResetterImpl) forkAndGenerateBranchToken( forkBranchToken []byte, forkNodeID int64, resetRunID string, -) ([]byte, error) { +) (newBranchToken []byte, forkedBaseBranchToken []byte, err error) { // fork a new history branch shardID := r.shardContext.GetShardID() resp, err := r.executionMgr.ForkHistoryBranch(ctx, &persistence.ForkHistoryBranchRequest{ @@ -689,10 +693,13 @@ func (r *workflowResetterImpl) forkAndGenerateBranchToken( NewRunID: resetRunID, }) if err != nil { - return nil, err + return nil, nil, err } - return resp.NewBranchToken, nil + if len(resp.BaseBranchToken) == 0 { + return resp.NewBranchToken, forkBranchToken, nil + } + return resp.NewBranchToken, resp.BaseBranchToken, nil } func (r *workflowResetterImpl) terminateWorkflow( diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index a9b7e8870e4..7d1f69b2350 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -372,6 +372,69 @@ func (s *workflowResetterSuite) TestPersistToDB_CurrentExecutionMissing() { } func (s *workflowResetterSuite) TestReplayResetWorkflow() { + ctx := context.Background() + baseBranchToken := []byte("some random base branch token") + forkedBaseBranchToken := []byte("some random forked base branch token") + baseRebuildLastEventID := int64(1233) + baseRebuildLastEventVersion := int64(12) + + resetBranchToken := []byte("some random reset branch token") + resetRequestID := uuid.NewString() + resetStats := RebuildStats{ + HistorySize: 4411, + ExternalPayloadSize: 1234, + ExternalPayloadCount: 56, + } + resetMutableState := historyi.NewMockMutableState(s.controller) + + s.mockExecutionMgr.EXPECT().ForkHistoryBranch(gomock.Any(), gomock.Any()).Return( + &persistence.ForkHistoryBranchResponse{NewBranchToken: resetBranchToken, BaseBranchToken: forkedBaseBranchToken}, nil, + ) + + s.mockStateRebuilder.EXPECT().Rebuild( + ctx, + gomock.Any(), + definition.NewWorkflowKey( + s.namespaceID.String(), + s.workflowID, + s.baseRunID, + ), + forkedBaseBranchToken, + baseRebuildLastEventID, + new(baseRebuildLastEventVersion), + definition.NewWorkflowKey( + s.namespaceID.String(), + s.workflowID, + s.resetRunID, + ), + resetBranchToken, + resetRequestID, + ).Return(resetMutableState, resetStats, nil) + resetMutableState.EXPECT().SetBaseWorkflow( + s.baseRunID, + baseRebuildLastEventID, + baseRebuildLastEventVersion, + ) + resetMutableState.EXPECT().AddHistorySize(resetStats.HistorySize) + resetMutableState.EXPECT().AddExternalPayloadSize(resetStats.ExternalPayloadSize) + resetMutableState.EXPECT().AddExternalPayloadCount(resetStats.ExternalPayloadCount) + + resetWorkflow, err := s.workflowResetter.replayResetWorkflow( + ctx, + s.namespaceID, + s.workflowID, + s.baseRunID, + baseBranchToken, + baseRebuildLastEventID, + baseRebuildLastEventVersion, + s.resetRunID, + resetRequestID, + ) + s.NoError(err) + s.Equal(resetMutableState, resetWorkflow.GetMutableState()) +} + +func (s *workflowResetterSuite) TestReplayResetWorkflow_EmptyBaseBranchToken() { ctx := context.Background() baseBranchToken := []byte("some random base branch token") baseRebuildLastEventID := int64(1233) @@ -597,6 +660,31 @@ func (s *workflowResetterSuite) TestFailInflightActivity() { } func (s *workflowResetterSuite) TestGenerateBranchToken() { + baseBranchToken := []byte("some random base branch token") + resurrectedBaseBranchToken := []byte("some random resurrected new base branch token") + baseNodeID := int64(1234) + + resetBranchToken := []byte("some random reset branch token") + + shardID := s.mockShard.GetShardID() + s.mockExecutionMgr.EXPECT().ForkHistoryBranch(gomock.Any(), &persistence.ForkHistoryBranchRequest{ + ForkBranchToken: baseBranchToken, + ForkNodeID: baseNodeID, + Info: persistence.BuildHistoryGarbageCleanupInfo(s.namespaceID.String(), s.workflowID, s.resetRunID), + ShardID: shardID, + NamespaceID: s.namespaceID.String(), + NewRunID: s.resetRunID, + }).Return(&persistence.ForkHistoryBranchResponse{NewBranchToken: resetBranchToken, BaseBranchToken: resurrectedBaseBranchToken}, nil) + + newBranchToken, newBaseBranchToken, err := s.workflowResetter.forkAndGenerateBranchToken( + context.Background(), s.namespaceID, s.workflowID, baseBranchToken, baseNodeID, s.resetRunID, + ) + s.NoError(err) + s.Equal(resetBranchToken, newBranchToken) + s.Equal(resurrectedBaseBranchToken, newBaseBranchToken) +} + +func (s *workflowResetterSuite) TestGenerateBranchToken_EmptyBaseBranchToken() { baseBranchToken := []byte("some random base branch token") baseNodeID := int64(1234) @@ -612,11 +700,12 @@ func (s *workflowResetterSuite) TestGenerateBranchToken() { NewRunID: s.resetRunID, }).Return(&persistence.ForkHistoryBranchResponse{NewBranchToken: resetBranchToken}, nil) - newBranchToken, err := s.workflowResetter.forkAndGenerateBranchToken( + newBranchToken, newBaseBranchToken, err := s.workflowResetter.forkAndGenerateBranchToken( context.Background(), s.namespaceID, s.workflowID, baseBranchToken, baseNodeID, s.resetRunID, ) s.NoError(err) s.Equal(resetBranchToken, newBranchToken) + s.Equal(baseBranchToken, newBaseBranchToken) } func (s *workflowResetterSuite) TestTerminateWorkflow() { From 46f09425bcdf58b27c116a566fe8f33c611c3474 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 08:28:54 -0700 Subject: [PATCH 044/139] Use integer range loops (#11677) Go 1.27 prerequisite that applies the `rangeint` Go fixer to use integer range loops. --- common/finalizer/finalizer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/finalizer/finalizer_test.go b/common/finalizer/finalizer_test.go index 52d134403c9..aae50bb5904 100644 --- a/common/finalizer/finalizer_test.go +++ b/common/finalizer/finalizer_test.go @@ -65,7 +65,7 @@ func TestFinalizer(t *testing.T) { capture := mh.StartCapture() var completed atomic.Int32 - for i := 0; i < 5; i += 1 { + for i := range 5 { require.NoError(t, f.Register( fmt.Sprintf("%v", i), func(ctx context.Context) error { From 19634d38e62b344a141fd75fe5645d9ed5b91c14 Mon Sep 17 00:00:00 2001 From: Dan Davison Date: Mon, 24 Aug 2026 11:35:07 -0400 Subject: [PATCH 045/139] Reorganize CHASM activity codebase (#11446) ## What changed? - Reorganize `chasm/lib/activity` ## Why? - Improve navigability and codebase comprehensibility ## How did you test it? - [x] covered by existing tests --- chasm/lib/activity/activity.go | 1680 +---------------- chasm/lib/activity/activity_test.go | 973 ---------- chasm/lib/activity/attempt.go | 447 +++++ chasm/lib/activity/attempt_test.go | 266 +++ chasm/lib/activity/metrics.go | 194 ++ chasm/lib/activity/operator_commands.go | 592 ++++++ chasm/lib/activity/operator_commands_test.go | 709 +++++++ chasm/lib/activity/responses.go | 292 +++ chasm/lib/activity/responses_test.go | 45 + chasm/lib/activity/statemachine.go | 857 +++++---- .../activity/{activity_tasks.go => tasks.go} | 0 .../{activity_tasks_test.go => tasks_test.go} | 0 12 files changed, 3115 insertions(+), 2940 deletions(-) create mode 100644 chasm/lib/activity/attempt.go create mode 100644 chasm/lib/activity/attempt_test.go create mode 100644 chasm/lib/activity/metrics.go create mode 100644 chasm/lib/activity/operator_commands.go create mode 100644 chasm/lib/activity/operator_commands_test.go create mode 100644 chasm/lib/activity/responses.go create mode 100644 chasm/lib/activity/responses_test.go rename chasm/lib/activity/{activity_tasks.go => tasks.go} (100%) rename chasm/lib/activity/{activity_tasks_test.go => tasks_test.go} (100%) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index e4e56159f5e..e13784b1311 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -1,66 +1,31 @@ -// A note on times and terminology: -// -// We name 3 times in the lifecycle of an activity attempt: -// -// schedule_time - the time at which the activity entered SCHEDULED state -// dispatch_time - the time at which the activity task is due to be dispatched to Matching (AddActivityTask) -// start_time - the time at which the activity enters STARTED state (Matching task picked up by poller) -// -// They are always ordered as: (schedule_time) <= (dispatch_time) < (start_time). -// -// A ScheduleToStart timeout applies to the time between dispatch and start. If there is a delay -// before dispatch (i.e. a start delay on the first attempt, or a backoff interval / next retry -// delay on a second or subsequent attempt) then schedule_time < dispatch_time. Otherwise, they are -// equal. -// -// The main Activity struct has a.ScheduleTime which is the schedule time of the first -// attempt; i.e. the time at which the activity was created. This is never changed. -// -// The naming situation is not perfectly clean. See e.g. the comment below on -// nextAttemptDispatchTime (which is called next_attempt_schedule_time in the public API). - package activity import ( "errors" "fmt" - "math/rand" - "slices" - "strconv" - "time" "github.com/nexus-rpc/sdk-go/nexus" apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas - callbackpb "go.temporal.io/api/callback/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" - failurepb "go.temporal.io/api/failure/v1" historypb "go.temporal.io/api/history/v1" sdkpb "go.temporal.io/api/sdk/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservice/v1" - "go.temporal.io/server/api/matchingservice/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/chasm/lib/callback" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common" - "go.temporal.io/server/common/activityoptions" - "go.temporal.io/server/common/backoff" "go.temporal.io/server/common/contextutil" - commonfailure "go.temporal.io/server/common/failure" "go.temporal.io/server/common/metrics" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "go.temporal.io/server/common/payload" "go.temporal.io/server/common/retrypolicy" serviceerrors "go.temporal.io/server/common/serviceerror" - "go.temporal.io/server/common/tqid" - "go.temporal.io/server/common/util" "go.temporal.io/server/service/history/consts" - "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -256,25 +221,6 @@ func NewEmbeddedActivity( ) { } -func (a *Activity) createAddActivityTaskRequest(ctx chasm.Context, namespaceID string) (*matchingservice.AddActivityTaskRequest, error) { - // Get latest component ref and unmarshal into proto ref - componentRef, err := ctx.Ref(a) - if err != nil { - return nil, err - } - - // Note: No need to set the vector clock here, as the components track version conflicts for read/write - // TODO: Need to fill in VersionDirective once we decide how to handle versioning for standalone activities - return &matchingservice.AddActivityTaskRequest{ - NamespaceId: namespaceID, - ScheduleToStartTimeout: a.ScheduleToStartTimeout, - TaskQueue: a.GetTaskQueue(), - Priority: a.GetPriority(), - ComponentRef: componentRef, - Stamp: a.LastAttempt.Get(ctx).GetStamp(), - }, nil -} - // HandleStarted updates the activity on recording activity task started and populates the response. func (a *Activity) HandleStarted(ctx chasm.MutableContext, request *historyservice.RecordActivityTaskStartedRequest) ( *historyservice.RecordActivityTaskStartedResponse, error, @@ -301,21 +247,6 @@ func (a *Activity) HandleStarted(ctx chasm.MutableContext, request *historyservi return a.GenerateRecordActivityTaskStartedResponse(ctx, request.GetPollRequest().GetNamespace()) } -func (a *Activity) taskScheduleToStartMetricsHandler(ctx chasm.Context) metrics.Handler { - actCtx := activityContextFromChasm(ctx) - namespaceEntry := ctx.NamespaceEntry() - namespaceName := namespaceEntry.Name().String() - taskQueue := a.GetTaskQueue().GetName() - return metrics.GetPerTaskQueuePartitionTypeScope( - a.baseMetricsHandler(ctx, metrics.HistoryRecordActivityTaskStartedScope), - namespaceName, - tqid.UnsafeTaskQueueFamily(namespaceEntry.ID().String(), taskQueue). - TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY). - RootPartition(), - actCtx.config.BreakdownMetricsByTaskQueue(namespaceName, taskQueue, enumspb.TASK_QUEUE_TYPE_ACTIVITY), - ) -} - // GenerateRecordActivityTaskStartedResponse generates the response for HandleStarted. func (a *Activity) GenerateRecordActivityTaskStartedResponse( ctx chasm.Context, @@ -357,65 +288,6 @@ func (a *Activity) GenerateRecordActivityTaskStartedResponse( }, nil } -// dispatchTimeForAttempt returns the dispatch time of the given attempt. -func (a *Activity) dispatchTimeForAttempt(attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { - if dispatchTime := attempt.GetDispatchTime(); dispatchTime != nil { - return dispatchTime - } - if attempt.GetCount() == 1 { - return timestamppb.New(a.firstDispatchTime()) - } - return dispatchTimeForRetry(attempt) -} - -// dispatchTimeForRetry computes the time a retried attempt will be dispatched to Matching, -// as complete_time + retry_interval. Returns nil if either field is missing or zero. -func dispatchTimeForRetry(attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { - retryInterval := attempt.GetCurrentRetryInterval() - completeTime := attempt.GetCompleteTime() - if retryInterval != nil && retryInterval.AsDuration() > 0 && completeTime != nil { - return timestamppb.New(completeTime.AsTime().Add(retryInterval.AsDuration())) - } - return nil -} - -// nextAttemptDispatchTime is the dispatch_time of the attempt that is currently being waited for. -// It is null when the dispatch time has passed, in terminal states, and when paused or when an -// attempt is in progress, since in those states the dispatch time of a future attempt is unknown: -// we do not even know if there will be a next attempt. -// -// In the public Describe API response of SAA and WFA, this has the name next_attempt_schedule_time. -// In that field name, the term "schedule_time" is actually a dispatch time; specifically, the -// dispatch time defined by this method. -// -// For WFA, next_attempt_schedule_time is null prior to the first attempt since start delay is not -// supported, hence the activity is due to be dispatched to Matching as soon as the activity is -// created. But for SAA, if there's a start delay, then next_attempt_schedule_time is the -// dispatch_time (non-null). -func (a *Activity) nextAttemptDispatchTime(ctx chasm.Context, attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { - if a.hasAttemptInProgress() || a.isPaused() || a.isTerminal() { - return nil - } - if t := a.dispatchTimeForAttempt(attempt); t != nil { - if t.AsTime().After(ctx.Now(a)) { - return t - } - } - return nil -} - -// currentRetryInterval is the retry interval if the activity is currently waiting for a retry; nil otherwise. -func (a *Activity) currentRetryInterval(ctx chasm.Context, attempt *activitypb.ActivityAttemptState) *durationpb.Duration { - if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { - if t := a.dispatchTimeForAttempt(attempt); t != nil { - if t.AsTime().After(ctx.Now(a)) { - return attempt.GetCurrentRetryInterval() - } - } - } - return nil -} - // RecordCompleted applies the provided function to record activity completion. // For standalone activities, it also triggers any registered completion callbacks. func (a *Activity) RecordCompleted(ctx chasm.MutableContext, applyFn func(ctx chasm.MutableContext) error) error { @@ -706,217 +578,6 @@ func (a *Activity) Terminate( }) } -func (a *Activity) UpdateActivityExecutionOptions( - ctx chasm.MutableContext, - req *activitypb.UpdateActivityExecutionOptionsRequest, -) (*activitypb.UpdateActivityExecutionOptionsResponse, error) { - frontendReq := req.GetFrontendRequest() - requestID := frontendReq.GetRequestId() - if requestID != "" && requestID == a.GetLastUpdateOptionsRequestId() { - // A repeated request ID returns the current options, which may differ from the original response. - return a.updateActivityExecutionOptionsResponse(), nil - } - - switch a.Status { - case activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, - activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - activitypb.ACTIVITY_EXECUTION_STATUS_FAILED, - activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED, - activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, - activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: - return nil, serviceerror.NewFailedPreconditionf("Cannot update options for activity in state %s", a.Status.String()) - default: - } - - if a.ResetRestoreOptions { - return nil, serviceerror.NewFailedPrecondition( - "cannot update options while a deferred Reset(RestoreOriginalOptions) is pending") - } - - if frontendReq.GetRestoreOriginal() { - if err := validateOriginalOptionsRestorable(a.GetOriginalOptions()); err != nil { - return nil, err - } - } - - updateFields := map[string]struct{}{} - if mask := frontendReq.GetUpdateMask(); mask != nil { - updateFields = util.ParseFieldMask(mask) - } - - _, hasStartDelayInMask := updateFields["startDelay"] - if hasStartDelayInMask { - newDelay := frontendReq.GetActivityOptions().GetStartDelay() - if err := validateStartDelay(newDelay); err != nil { - return nil, err - } - if newDelay.AsDuration() > 0 { - actCtx := activityContextFromChasm(ctx) - if !actCtx.config.StartDelayEnabled(frontendReq.GetNamespace()) { - return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") - } - } - inDelayWindow := a.firstDispatchTime().After(ctx.Now(a)) - pausedBeforeFirstAttempt := a.GetFirstAttemptStartedTime() == nil && a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED - if !(inDelayWindow || pausedBeforeFirstAttempt) { //nolint:staticcheck // QF1001: DeMorgan rearrangement would not be an improvement - return nil, serviceerror.NewFailedPrecondition( - "cannot update start_delay: the first activity attempt has already been dispatched") - } - } - - attempt := a.LastAttempt.Get(ctx) - - if frontendReq.GetRestoreOriginal() { - ogOptions := a.GetOriginalOptions() - a.TaskQueue = common.CloneProto(ogOptions.GetTaskQueue()) - a.ScheduleToCloseTimeout = common.CloneProto(ogOptions.GetScheduleToCloseTimeout()) - a.ScheduleToStartTimeout = common.CloneProto(ogOptions.GetScheduleToStartTimeout()) - a.StartToCloseTimeout = common.CloneProto(ogOptions.GetStartToCloseTimeout()) - a.HeartbeatTimeout = common.CloneProto(ogOptions.GetHeartbeatTimeout()) - a.RetryPolicy = common.CloneProto(ogOptions.GetRetryPolicy()) - a.Priority = common.CloneProto(ogOptions.GetPriority()) - // start_delay only governs the first dispatch. Once the first attempt has started, restoring - // the original value would shift ScheduleToClose without affecting dispatch timing. - if a.GetFirstAttemptStartedTime() == nil { - a.StartDelay = common.CloneProto(ogOptions.GetStartDelay()) - } - } else { - if err := a.mergeActivityOptions(frontendReq); err != nil { - return nil, err - } - } - - // Recalculate policy-derived retry intervals based on the (possibly updated) retry policy. - // Worker-provided NextRetryDelay values are preserved for their already-scheduled retry. - if a.shouldRecalculateCurrentRetryInterval(attempt, frontendReq.GetRestoreOriginal(), updateFields) { - newInterval := backoff.CalculateExponentialRetryInterval(a.RetryPolicy, attempt.GetCount()-1) - attempt.CurrentRetryInterval = durationpb.New(newInterval) - } - - // Recreate the ScheduleToClose task at the (possibly updated) deadline. - a.reissueScheduleToClose(ctx) - - attempt.Stamp++ - - a.reissueRunningAttemptTimers(ctx, attempt) - if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { - a.reissueDispatchAndScheduleToStart(ctx, attempt) - } - - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityUpdateOptionsScope) - a.emitOnUpdateOptionsMetrics(metricsHandler) - - if requestID != "" { - a.LastUpdateOptionsRequestId = requestID - } - return a.updateActivityExecutionOptionsResponse(), nil -} - -func (a *Activity) updateActivityExecutionOptionsResponse() *activitypb.UpdateActivityExecutionOptionsResponse { - return &activitypb.UpdateActivityExecutionOptionsResponse{ - FrontendResponse: &workflowservice.UpdateActivityExecutionOptionsResponse{ - ActivityOptions: &apiactivitypb.ActivityOptions{ - TaskQueue: a.GetTaskQueue(), - ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), - ScheduleToStartTimeout: a.GetScheduleToStartTimeout(), - StartToCloseTimeout: a.GetStartToCloseTimeout(), - HeartbeatTimeout: a.GetHeartbeatTimeout(), - RetryPolicy: a.GetRetryPolicy(), - Priority: a.GetPriority(), - StartDelay: a.GetStartDelay(), - }, - }, - } -} - -// shouldRecalculateCurrentRetryInterval reports whether the pending retry's CurrentRetryInterval -// should be re-derived from the (possibly updated) retry policy. All of the following must hold: -// -// - the activity is waiting to retry, i.e. in SCHEDULED or PAUSED state (a running attempt has no -// pending backoff to recalculate); -// - a retry is actually pending (CurrentRetryInterval is set); -// - the update touches the retry policy: either RestoreOriginal (which replaces the whole policy), -// or the update mask includes "retryPolicy" or one of its subfields; -// - the pending interval is still policy-derived (CurrentRetryIntervalSource is RETRY_POLICY). -// A worker-provided NextRetryDelay override is preserved regardless of its value. An -// UNSPECIFIED source (attempt state persisted before this field existed) is treated the same -// as an override, since we can no longer tell whether it was policy-derived or a worker -// override, and preserving it is the safer default. -func (a *Activity) shouldRecalculateCurrentRetryInterval( - attempt *activitypb.ActivityAttemptState, - restoreOriginal bool, - updateFields map[string]struct{}, -) bool { - status := a.GetStatus() - if status != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED && - status != activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED { - return false - } - - if attempt.GetCurrentRetryInterval() == nil { - return false - } - - if !restoreOriginal { - if !util.FieldMaskHasPathOrSubPath(updateFields, "retryPolicy") { - return false - } - } - - return attempt.GetCurrentRetryIntervalSource() == activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY -} - -// mergeActivityOptions applies the field mask from the request to the activity state. -// The structure mirrors the field-mask logic in service/history/api/updateactivityoptions/api.go -func (a *Activity) mergeActivityOptions( - req *workflowservice.UpdateActivityExecutionOptionsRequest, -) error { - updateFields := util.ParseFieldMask(req.GetUpdateMask()) - - // Build an ActivityOptions view of the current Activity state so we can use the shared merge function. - ao := &apiactivitypb.ActivityOptions{ - TaskQueue: common.CloneProto(a.TaskQueue), - ScheduleToCloseTimeout: common.CloneProto(a.ScheduleToCloseTimeout), - ScheduleToStartTimeout: common.CloneProto(a.ScheduleToStartTimeout), - StartToCloseTimeout: common.CloneProto(a.StartToCloseTimeout), - HeartbeatTimeout: common.CloneProto(a.HeartbeatTimeout), - Priority: common.CloneProto(a.Priority), - RetryPolicy: common.CloneProto(a.RetryPolicy), - StartDelay: common.CloneProto(a.StartDelay), - } - - if err := activityoptions.MergeActivityOptions(ao, common.CloneProto(req.GetActivityOptions()), updateFields); err != nil { - return err - } - - if util.FieldMaskHasSubPath(updateFields, "retryPolicy") { - if err := retrypolicy.Validate(ao.GetRetryPolicy()); err != nil { - return err - } - } - - // Re-normalize timeouts after the update so that relationships like - // start_to_close <= schedule_to_close and heartbeat <= start_to_close are preserved. - // This mirrors adjustActivityOptions for workflow-embedded activities. - if err := validateAndNormalizeTimeouts(req.GetActivityId(), a.GetActivityType().GetName(), durationpb.New(0), ao); err != nil { - return err - } - - // Write the merged and normalized options back to the Activity state fields. - a.TaskQueue = ao.TaskQueue - a.ScheduleToCloseTimeout = ao.ScheduleToCloseTimeout - a.ScheduleToStartTimeout = ao.ScheduleToStartTimeout - a.StartToCloseTimeout = ao.StartToCloseTimeout - a.HeartbeatTimeout = ao.HeartbeatTimeout - a.Priority = ao.Priority - a.RetryPolicy = ao.RetryPolicy - a.StartDelay = ao.StartDelay - - return nil -} - // getOrCreateLastHeartbeat retrieves the last heartbeat state, initializing it if not present. The heartbeat is lazily created // to avoid unnecessary writes when heartbeats are not used. func (a *Activity) getOrCreateLastHeartbeat(ctx chasm.MutableContext) *activitypb.ActivityHeartbeatState { @@ -928,1291 +589,114 @@ func (a *Activity) getOrCreateLastHeartbeat(ctx chasm.MutableContext) *activityp return heartbeat } -func (a *Activity) handleCancellationRequested(ctx chasm.MutableContext, request *activitypb.RequestCancelActivityExecutionRequest) ( - *activitypb.RequestCancelActivityExecutionResponse, error, -) { - req := request.GetFrontendRequest() - newReqID := req.GetRequestId() - existingReqID := a.GetCancelState().GetRequestId() - - // Deduplicate first because a retry may arrive after the activity transitions to Canceled. - if newReqID != "" && existingReqID == newReqID { - return &activitypb.RequestCancelActivityExecutionResponse{}, nil - } - - if a.isTerminal() { - return nil, serviceerror.NewFailedPreconditionf("activity is in terminal state %v", a.GetStatus()) - } - - // Reject a second cancellation request with a different request ID. - if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED { - return nil, serviceerror.NewFailedPrecondition( - fmt.Sprintf("cancellation already requested with request ID %s", existingReqID)) +// lastHeartbeatDetails returns the details recorded by the most recent heartbeat, or nil if +// the activity never heartbeated. +func (a *Activity) lastHeartbeatDetails(ctx chasm.Context) *commonpb.Payloads { + heartbeat, ok := a.LastHeartbeat.TryGet(ctx) + if !ok { + return nil } + return heartbeat.GetDetails() +} - hasAttemptInProgress := a.hasAttemptInProgress() - originalStatus := a.GetStatus() - - // Always transition to CancelRequested - // TODO: this is questionable, since CancelRequested is otherwise a state that implies an attempt - // is in progress. - if err := TransitionCancelRequested.Apply(a, ctx, req); err != nil { +// RecordHeartbeat records a heartbeat for the activity. +func (a *Activity) RecordHeartbeat( + ctx chasm.MutableContext, + input WithToken[*historyservice.RecordActivityTaskHeartbeatRequest], +) (*historyservice.RecordActivityTaskHeartbeatResponse, error) { + err := a.validateActivityTaskToken(ctx, input.Token, input.Request.GetNamespaceId(), false) + if err != nil { return nil, err } - - // Transition to Canceled if no attempt in progress; otherwise wait for worker response. - if !hasAttemptInProgress { - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) - err := TransitionCanceled.Apply(a, ctx, cancelEvent{ - metricsHandler: metricsHandler, - fromStatus: originalStatus, - details: &commonpb.Payloads{ - Payloads: []*commonpb.Payload{ - payload.EncodeString(req.GetReason()), - }, + details := input.Request.GetHeartbeatRequest().GetDetails() + prevHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) + a.LastHeartbeat = chasm.NewDataField(ctx, &activitypb.ActivityHeartbeatState{ + RecordedTime: timestamppb.New(ctx.Now(a)), + Details: details, + TotalHeartbeatCount: prevHeartbeat.GetTotalHeartbeatCount() + 1, + }) + if heartbeatTimeout := a.GetHeartbeatTimeout().AsDuration(); heartbeatTimeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: ctx.Now(a).Add(heartbeatTimeout), }, - }) - if err != nil { - return nil, err - } - } - - return &activitypb.RequestCancelActivityExecutionResponse{}, nil -} - -func (a *Activity) handlePauseRequested(ctx chasm.MutableContext, req *activitypb.PauseActivityExecutionRequest) ( - *activitypb.PauseActivityExecutionResponse, error, -) { - // Deduplicate a replay of a request that already paused this activity, even if the - // activity has since been unpaused. Without this check, a delayed replay of an old - // Pause request would silently re-pause an activity that a later Unpause resumed. - newReqID := req.GetFrontendRequest().GetRequestId() - if newReqID != "" && a.LastPauseState.GetRequestId() == newReqID { - return &activitypb.PauseActivityExecutionResponse{}, nil - } - - canPause := TransitionPaused.Possible(a) - canRequestPause := TransitionPauseRequested.Possible(a) - if !canPause && !canRequestPause { - return nil, serviceerror.NewFailedPreconditionf("activity is in non-pausable state %v", a.GetStatus()) - } - - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityPausedScope) - - event := pauseEvent{req: req.GetFrontendRequest(), metricsHandler: metricsHandler} - if canPause { - if err := TransitionPaused.Apply(a, ctx, event); err != nil { - return nil, err - } - } else { - if err := TransitionPauseRequested.Apply(a, ctx, event); err != nil { - return nil, err - } - } - return &activitypb.PauseActivityExecutionResponse{}, nil -} - -func (a *Activity) handleUnpauseRequested(ctx chasm.MutableContext, req *activitypb.UnpauseActivityExecutionRequest) ( - _ *activitypb.UnpauseActivityExecutionResponse, retErr error, -) { - frontendReq := req.GetFrontendRequest() - requestID := frontendReq.GetRequestId() - if requestID != "" && requestID == a.GetLastUnpauseRequestId() { - return &activitypb.UnpauseActivityExecutionResponse{}, nil - } - if requestID != "" { - defer func() { - if retErr == nil { - a.LastUnpauseRequestId = requestID - } - }() - } - - if a.isTerminal() { - return nil, serviceerror.NewFailedPreconditionf("activity is in terminal state %v", a.GetStatus()) - } - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityUnpausedScope) - - event := unpauseEvent{req: frontendReq, metricsHandler: metricsHandler} - switch { - case TransitionUnpaused.Possible(a): - if err := TransitionUnpaused.Apply(a, ctx, event); err != nil { - return nil, err - } - case TransitionUnpausedWhilePauseRequested.Possible(a): - if err := TransitionUnpausedWhilePauseRequested.Apply(a, ctx, event); err != nil { - return nil, err - } - default: - return nil, serviceerror.NewFailedPreconditionf("activity is in non-unpausable state %v", a.GetStatus()) + &activitypb.HeartbeatTimeoutTask{ + Stamp: a.LastAttempt.Get(ctx).GetStamp(), + }, + ) } - a.emitOnUnpausedMetrics(metricsHandler) - return &activitypb.UnpauseActivityExecutionResponse{}, nil -} + a.emitHeartbeatMetrics(ctx, details) -// isPaused reports whether the activity is currently paused (waiting) or has a pending pause request -// (worker still running). -func (a *Activity) isPaused() bool { - switch a.GetStatus() { - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, - activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: - return true + response := &historyservice.RecordActivityTaskHeartbeatResponse{} + switch a.Status { + case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: + response.CancelRequested = true case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: - return a.GetResetShouldPause() + response.ActivityReset = true + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: + response.ActivityPaused = true default: - return false + // no-op } + return response, nil } -func (a *Activity) unpause( - ctx chasm.MutableContext, - event unpauseEvent, -) { - attempt := a.LastAttempt.Get(ctx) - dispatchTime := a.unpauseDispatchTime(ctx, event) - attempt.DispatchTime = timestamppb.New(dispatchTime) - - attempt.Stamp++ - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, - &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}) +// StoreOrSelf returns the store for the activity. If the store is not set as a field (e.g. +// standalone activities), it returns the activity itself. +func (a *Activity) StoreOrSelf(ctx chasm.Context) ActivityStore { + store, ok := a.Store.TryGet(ctx) + if ok { + return store } - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime}, - a.newActivityDispatchTask(ctx)) + return a } -// unpauseDispatchTime computes when an unpaused attempt should be dispatched -func (a *Activity) unpauseDispatchTime(ctx chasm.MutableContext, event unpauseEvent) time.Time { - unpauseTime := ctx.Now(a) - if jitter := event.req.GetJitter().AsDuration(); jitter > 0 { - unpauseTime = unpauseTime.Add(time.Duration(rand.Int63n(int64(jitter)))) //nolint:gosec +// validateActivityTaskToken validates a task token against the current activity state. +// +// allowForceCompleteWithNoAttempt permits a by-ID token to pass even though the activity has no +// attempt in progress (Scheduled or Paused). Only HandleCompleted sets this, mirroring the +// workflow-activity behavior of letting RespondActivityTaskCompletedById force-complete an +// activity before any worker has started it. +func (a *Activity) validateActivityTaskToken( + ctx chasm.Context, + token *tokenspb.Task, + requestNamespaceID string, + allowForceCompleteWithNoAttempt bool, +) error { + forceCompleteWithNoAttempt := allowForceCompleteWithNoAttempt && + token.Attempt == ByIDTokenAttempt && + !a.hasAttemptInProgress() + if !a.hasAttemptInProgress() && !forceCompleteWithNoAttempt { + return serviceerror.NewNotFound("activity task not found") } - dispatchTime := a.dispatchTimeRespectingStartDelay(unpauseTime) - retryDispatchTime := dispatchTimeForRetry(a.LastAttempt.Get(ctx)) - if retryDispatchTime != nil && retryDispatchTime.AsTime().After(dispatchTime) { - return retryDispatchTime.AsTime() + if token.Attempt != ByIDTokenAttempt && token.Attempt != a.LastAttempt.Get(ctx).GetCount() { + return serviceerror.NewNotFound("activity task not found") } - return dispatchTime -} - -func (a *Activity) recordPauseState( - ctx chasm.MutableContext, - event pauseEvent, -) { - a.LastPauseState = &activitypb.ActivityPauseState{ - PauseTime: timestamppb.New(ctx.Now(a)), - Identity: event.req.GetIdentity(), - Reason: event.req.GetReason(), - RequestId: event.req.GetRequestId(), + tokenStamp := token.GetActivityAttemptStamp() + startedStamp := a.LastAttempt.Get(ctx).GetStartedStamp() + // Matching versions without stamped tokens leave tokenStamp zero; + // History versions without StartedStamp persistence leave startedStamp zero. + requiresLegacyStampCompatibility := tokenStamp == 0 || startedStamp == 0 + if !requiresLegacyStampCompatibility && tokenStamp != startedStamp { + return serviceerror.NewNotFound("activity task not found") } - a.emitOnPausedMetrics(event.metricsHandler) -} -func (a *Activity) clearHeartbeatDetails(ctx chasm.MutableContext) { - if hb, ok := a.LastHeartbeat.TryGet(ctx); ok { - hb.Details = nil - hb.RecordedTime = nil + ref, err := chasm.DeserializeComponentRef(token.GetComponentRef()) + if err != nil { + return serviceerror.NewInvalidArgument("malformed token") } -} -func (a *Activity) reset(ctx chasm.MutableContext, event resetEvent) { - attempt := a.LastAttempt.Get(ctx) - attempt.Count = 1 - attempt.Stamp++ - attempt.CurrentRetryInterval = nil - attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - if event.req.GetResetHeartbeat() { - a.clearHeartbeatDetails(ctx) - } - dispatchTime := a.dispatchTimeRespectingStartDelay(event.resetTime) - attempt.DispatchTime = timestamppb.New(dispatchTime) - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, - &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}, - ) - } - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime}, - a.newActivityDispatchTask(ctx), - ) - a.emitOnResetMetrics(event.metricsHandler) -} - -// handleReset handles the activity execution reset. -// -// For SCHEDULED and PAUSED activities (no worker running): re-dispatches at attempt 1. Any pending -// retry backoff is discarded (reset clears CurrentRetryInterval), but a pending start_delay is -// honored so the re-dispatched attempt 1 does not fire before its original requested start time. A -// PAUSED activity is unpaused first — unless keepPaused is set, in which case the counter is reset -// to 1 but the activity stays PAUSED until a later unpause. -// -// For STARTED activities: transitions to RESET_REQUESTED. The worker is notified via -// ActivityReset=true on its next heartbeat response and continues to use its existing task token. -// If the attempt fails, the activity transitions back to SCHEDULED at attempt 1 via -// TransitionResetAttemptFailedToScheduled. -// -// RestoreOriginalOptions follows the same split: applied immediately for a non-running activity, -// and deferred for a running one, so the in-flight attempt is not disturbed — every restored option -// takes effect on the new attempt 1. -// -// For CANCEL_REQUESTED activities: rejected with FailedPrecondition; cancel takes precedence. -func (a *Activity) handleReset( - ctx chasm.MutableContext, - req *activitypb.ResetActivityExecutionRequest, -) (_ *activitypb.ResetActivityExecutionResponse, retErr error) { - frontendReq := req.GetFrontendRequest() - requestID := frontendReq.GetRequestId() - if requestID != "" && requestID == a.GetLastResetRequestId() { - return &activitypb.ResetActivityExecutionResponse{}, nil - } - if requestID != "" { - defer func() { - if retErr == nil { - a.LastResetRequestId = requestID - } - }() - } - - if frontendReq.GetRestoreOriginalOptions() { - if err := validateOriginalOptionsRestorable(a.GetOriginalOptions()); err != nil { - return nil, err - } - } - - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityResetScope) - - switch a.Status { - case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: - return nil, serviceerror.NewFailedPrecondition("cannot reset an activity with a pending cancellation") - case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: - // A reset is already pending on this running attempt. For now we reject a second reset - // request clearly. - // TODO (dan): define desired behavior and implement - return nil, serviceerror.NewFailedPrecondition("cannot reset an activity with a pending reset") - case activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: - return a.deferResetWhileRunning(ctx, frontendReq, metricsHandler) - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: - // No worker is running; restore takes effect immediately. - if frontendReq.GetRestoreOriginalOptions() { - a.restoreOriginalOptions(ctx) - } - if frontendReq.GetKeepPaused() { - return a.resetKeepPaused(ctx, frontendReq, metricsHandler) - } - // No keepPaused: perform an immediate reset. restoreOriginalOptions (if requested) already - // ran above, so skip it in resetImmediately to avoid restoring twice. - return a.resetImmediately(ctx, frontendReq, metricsHandler, false) - case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED: - return a.resetImmediately(ctx, frontendReq, metricsHandler, frontendReq.GetRestoreOriginalOptions()) - default: - // Terminal or unspecified state. - return nil, serviceerror.NewFailedPrecondition("activity execution is not running") - } -} - -// deferResetWhileRunning defers reset mutations (option restore, heartbeat details clear, -// attempt-count rewind) while a worker is still executing (STARTED or PAUSE_REQUESTED), so that the -// in-flight attempt is undisturbed. The deferred reset applies when the worker yields. -func (a *Activity) deferResetWhileRunning( - ctx chasm.MutableContext, - frontendReq *workflowservice.ResetActivityExecutionRequest, - metricsHandler metrics.Handler, -) (*activitypb.ResetActivityExecutionResponse, error) { - keepPaused := frontendReq.GetKeepPaused() - pauseRequested := a.Status == activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED - - if pauseRequested && !keepPaused { - // Unpause; the deferred reset will apply on the next retry via STARTED->SCHEDULED. - if err := TransitionUnpausedWhilePauseRequested.Apply(a, ctx, unpauseEvent{ - req: &workflowservice.UnpauseActivityExecutionRequest{}, - metricsHandler: metricsHandler, - }); err != nil { - return nil, err - } - } - if frontendReq.GetRestoreOriginalOptions() { - a.ResetRestoreOptions = true - } - if frontendReq.GetResetHeartbeat() { - a.ResetShouldClearHeartbeat = true - } - // keepPaused on a paused (PAUSE_REQUESTED) activity preserves the pause: when the worker - // yields the activity lands back in PAUSED rather than SCHEDULED. - a.ResetShouldPause = keepPaused && pauseRequested - if err := TransitionResetRequested.Apply(a, ctx, nil); err != nil { - return nil, err - } - a.emitOnResetMetrics(metricsHandler) - return &activitypb.ResetActivityExecutionResponse{}, nil -} - -func (a *Activity) resetKeepPaused( - ctx chasm.MutableContext, - frontendReq *workflowservice.ResetActivityExecutionRequest, - metricsHandler metrics.Handler, -) (*activitypb.ResetActivityExecutionResponse, error) { - attempt := a.LastAttempt.Get(ctx) - attempt.Count = 1 - attempt.Stamp++ - attempt.CurrentRetryInterval = nil - attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - attempt.DispatchTime = nil - if frontendReq.GetResetHeartbeat() { - a.clearHeartbeatDetails(ctx) - } - a.emitOnResetMetrics(metricsHandler) - return &activitypb.ResetActivityExecutionResponse{}, nil -} - -func (a *Activity) resetImmediately( - ctx chasm.MutableContext, - frontendReq *workflowservice.ResetActivityExecutionRequest, - metricsHandler metrics.Handler, - restore bool, -) (*activitypb.ResetActivityExecutionResponse, error) { - if restore { - a.restoreOriginalOptions(ctx) - } - resetTime := ctx.Now(a) - if jitter := frontendReq.GetJitter().AsDuration(); jitter > 0 { - resetTime = resetTime.Add(time.Duration(rand.Int63n(int64(jitter)))) //nolint:gosec - } - if err := TransitionReset.Apply(a, ctx, resetEvent{ - req: frontendReq, - resetTime: resetTime, - metricsHandler: metricsHandler, - }); err != nil { - return nil, err - } - return &activitypb.ResetActivityExecutionResponse{}, nil -} - -// recordScheduleToStartOrCloseTimeoutFailure records schedule-to-start or schedule-to-close timeout outcomes. Such -// timeouts are not retried, so we set the outcome failure directly and leave the attempt failure as is. -func (a *Activity) recordScheduleToStartOrCloseTimeoutFailure( - ctx chasm.MutableContext, - timeoutType enumspb.TimeoutType, - message string, - cause *failurepb.Failure, -) error { - failure := &failurepb.Failure{ - Message: message, - Cause: cause, - FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ - TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ - TimeoutType: timeoutType, - LastHeartbeatDetails: a.lastHeartbeatDetails(ctx), - }, - }, - } - - a.Outcome.Get(ctx).Variant = &activitypb.ActivityOutcome_Failed_{ - Failed: &activitypb.ActivityOutcome_Failed{ - Failure: failure, - }, - } - - return nil -} - -// applyFailedAttempt mutates activity state when a worker yields with retries remaining. -func (a *Activity) applyFailedAttempt(ctx chasm.MutableContext, event rescheduleEvent) error { - attempt := a.LastAttempt.Get(ctx) - attempt.Count++ - attempt.Stamp++ - return a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, ctx.Now(a), false) -} - -// recordFailedAttempt records any failures resulting from a tried attempt, including worker application failures and -// start-to-close timeouts. Since the calls come from retried attempts we update the attempt failure info but leave -// the outcome failure empty to avoid duplication. -func (a *Activity) recordFailedAttempt( - ctx chasm.MutableContext, - retryInterval time.Duration, - retryIntervalSource activitypb.ActivityRetryIntervalSource, - failure *failurepb.Failure, - currentTime time.Time, - noRetriesLeft bool, -) error { - attempt := a.LastAttempt.Get(ctx) - - attemptFailure := failure - if !noRetriesLeft { - // Similar to workflow activity, truncate only retryable failure, not the final one. - attemptFailure = truncateRetryableFailure(ctx, failure) - } - - attempt.LastFailureDetails = &activitypb.ActivityAttemptState_LastFailureDetails{ - Failure: attemptFailure, - Time: timestamppb.New(currentTime), - } - attempt.CompleteTime = timestamppb.New(currentTime) - - if noRetriesLeft { - attempt.CurrentRetryInterval = nil - attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - } else { - attempt.CurrentRetryInterval = durationpb.New(retryInterval) - attempt.CurrentRetryIntervalSource = retryIntervalSource - } - return nil -} - -// truncateRetryableFailure caps the size of a failure retained in the activity's state while it -// retries, mirroring MutableStateImpl.truncateRetryableActivityFailure for workflow activities. -func truncateRetryableFailure(ctx chasm.Context, attemptFailure *failurepb.Failure) *failurepb.Failure { - actCtx := activityContextFromChasm(ctx) - sizeLimit := actCtx.config.MutableStateActivityFailureSizeLimitError(ctx.NamespaceEntry().Name().String()) - if attemptFailure.Size() <= sizeLimit { - return attemptFailure - } - - // nonRetryable is set to false here as only failures of attempts that will be retried are - // truncated, so the value is only for visibility/debugging purposes. - serverFailure := commonfailure.NewServerFailure(common.FailureReasonFailureExceedsLimit, false) - serverFailure.Cause = commonfailure.Truncate(attemptFailure, sizeLimit) - return serverFailure -} - -// tryReschedule attempts to reschedule the activity for retry. It handles the cases of pause and -// reset requests that were received while the last attempt was in progress. failureRetryable -// reports whether the failure itself permits a retry (timeouts always do; RespondActivityTaskFailed -// depends on the failure). -func (a *Activity) tryReschedule( - ctx chasm.MutableContext, - failureRetryable bool, - overridingRetryInterval time.Duration, - failure *failurepb.Failure, -) (enumspb.RetryState, error) { - status := a.GetStatus() - if status == activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED { - event := rescheduleEvent{failure: failure} - if a.ResetShouldPause { - return enumspb.RETRY_STATE_IN_PROGRESS, TransitionResetAttemptFailedToPaused.Apply(a, ctx, event) - } - return enumspb.RETRY_STATE_IN_PROGRESS, TransitionResetAttemptFailedToScheduled.Apply(a, ctx, event) - } - if a.GetRetryPolicy() == nil { - return enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, nil - } - if status == activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED { - return enumspb.RETRY_STATE_CANCEL_REQUESTED, nil - } - retryState, retryInterval := a.shouldRetry(ctx, overridingRetryInterval) - if !failureRetryable { - retryState = enumspb.RETRY_STATE_NON_RETRYABLE_FAILURE - } - if retryState != enumspb.RETRY_STATE_IN_PROGRESS { - return retryState, nil - } - retryIntervalSource := activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY - if overridingRetryInterval > 0 { - retryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_WORKER_OVERRIDE - } - event := rescheduleEvent{retryInterval: retryInterval, retryIntervalSource: retryIntervalSource, failure: failure} - switch status { - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: - return enumspb.RETRY_STATE_IN_PROGRESS, TransitionAttemptFailedWhilePauseRequested.Apply(a, ctx, event) - default: - return enumspb.RETRY_STATE_IN_PROGRESS, TransitionRescheduled.Apply(a, ctx, event) - } -} - -func (a *Activity) shouldRetry(ctx chasm.Context, overridingRetryInterval time.Duration) (enumspb.RetryState, time.Duration) { - if !TransitionRescheduled.Possible(a) && - !TransitionAttemptFailedWhilePauseRequested.Possible(a) { - return enumspb.RETRY_STATE_UNSPECIFIED, 0 - } - attempt := a.LastAttempt.Get(ctx) - retryPolicy := a.RetryPolicy - enoughTime, retryInterval := a.hasEnoughTimeForRetry(ctx, overridingRetryInterval) - - if retryPolicy.GetMaximumAttempts() > 0 && attempt.GetCount() >= retryPolicy.GetMaximumAttempts() { - return enumspb.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED, retryInterval - } - if !enoughTime { - return enumspb.RETRY_STATE_TIMEOUT, retryInterval - } - return enumspb.RETRY_STATE_IN_PROGRESS, retryInterval -} - -// timeoutRetryable reports whether a StartToClose or Heartbeat timeout may be retried under the retry -// policy. -func (a *Activity) timeoutRetryable(timeoutType enumspb.TimeoutType) bool { - return !slices.Contains( - a.GetRetryPolicy().GetNonRetryableErrorTypes(), - retrypolicy.TimeoutFailureTypePrefix+timeoutType.String(), - ) -} - -// hasEnoughTimeForRetry checks if there is enough time left in the schedule-to-close timeout. If sufficient time -// remains, it will also return a valid retry interval. -func (a *Activity) hasEnoughTimeForRetry(ctx chasm.Context, overridingRetryInterval time.Duration) (bool, time.Duration) { - attempt := a.LastAttempt.Get(ctx) - - // Use overriding retry interval if provided, else calculate based on retry policy - retryInterval := overridingRetryInterval - if retryInterval <= 0 { - retryInterval = backoff.CalculateExponentialRetryInterval(a.RetryPolicy, attempt.Count) - } - - scheduleToClose := a.GetScheduleToCloseTimeout().AsDuration() - if scheduleToClose == 0 { - return true, retryInterval - } - - deadline := a.scheduleToCloseDeadline() - return ctx.Now(a).Add(retryInterval).Before(deadline), retryInterval -} - -func (a *Activity) firstDispatchTime() time.Time { - return a.ScheduleTime.AsTime().Add(a.GetStartDelay().AsDuration()) -} - -func (a *Activity) newActivityDispatchTask(ctx chasm.Context) *activitypb.ActivityDispatchTask { - dispatchReason := activitypb.DISPATCH_REASON_IMMEDIATE - if a.GetFirstAttemptStartedTime() != nil { - dispatchReason = activitypb.DISPATCH_REASON_RETRY - } else if a.GetStartDelay().AsDuration() > 0 { - dispatchReason = activitypb.DISPATCH_REASON_START_DELAY - } - - return &activitypb.ActivityDispatchTask{ - Stamp: a.LastAttempt.Get(ctx).GetStamp(), - DispatchReason: dispatchReason, - StartDelayBucket: startDelayBucket(a.GetStartDelay().AsDuration()), - } -} - -func startDelayBucket(delay time.Duration) activitypb.StartDelayBucket { - switch { - case delay <= 0: - return activitypb.START_DELAY_BUCKET_NONE - case delay < time.Minute: - return activitypb.START_DELAY_BUCKET_LT_1M - case delay < 10*time.Minute: - return activitypb.START_DELAY_BUCKET_1M_10M - case delay < time.Hour: - return activitypb.START_DELAY_BUCKET_10M_1H - case delay < 6*time.Hour: - return activitypb.START_DELAY_BUCKET_1H_6H - case delay < 24*time.Hour: - return activitypb.START_DELAY_BUCKET_6H_1D - case delay < 7*24*time.Hour: - return activitypb.START_DELAY_BUCKET_1D_7D - case delay <= 30*24*time.Hour: - return activitypb.START_DELAY_BUCKET_7D_30D - default: - return activitypb.START_DELAY_BUCKET_GT_30D - } -} - -// reissueDispatchAndScheduleToStart re-emits the ActivityDispatchTask and ScheduleToStart timeout task for -// a SCHEDULED activity. Retries fire at the retry time; first attempts dispatch now, lifted to -// honor any pending start_delay. -func (a *Activity) reissueDispatchAndScheduleToStart(ctx chasm.MutableContext, attempt *activitypb.ActivityAttemptState) { - var dispatchTime time.Time - if retryDispatchTime := dispatchTimeForRetry(attempt); retryDispatchTime != nil { - dispatchTime = retryDispatchTime.AsTime() - } else { - dispatchTime = a.dispatchTimeRespectingStartDelay(ctx.Now(a)) - } - attempt.DispatchTime = timestamppb.New(dispatchTime) - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime}, - a.newActivityDispatchTask(ctx), - ) - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, - &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}, - ) - } -} - -// reissueRunningAttemptTimers re-emits the StartToClose and Heartbeat timeout tasks for the -// currently-running attempt, anchored to the attempt's StartedTime. Called from options-update -// paths after stamp bump so the old tasks are invalidated and replaced with the (possibly -// updated) timeouts. No-op unless the activity is in a status where a worker holds the task token -// (STARTED / CANCEL_REQUESTED / PAUSE_REQUESTED / RESET_REQUESTED). -func (a *Activity) reissueRunningAttemptTimers(ctx chasm.MutableContext, attempt *activitypb.ActivityAttemptState) { - if !a.hasAttemptInProgress() { - return - } - if timeout := a.GetStartToCloseTimeout().AsDuration(); timeout > 0 { - deadline := attempt.GetStartedTime().AsTime().Add(timeout) - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: deadline}, - &activitypb.StartToCloseTimeoutTask{Stamp: attempt.GetStamp()}, - ) - } - if hbTimeout := a.GetHeartbeatTimeout().AsDuration(); hbTimeout > 0 { - // Next heartbeat fires at max(last recorded heartbeat, current attempt start) + heartbeat timeout. - lastHb, _ := a.LastHeartbeat.TryGet(ctx) - lastHbTime := util.MaxTime( - lastHb.GetRecordedTime().AsTime(), - attempt.GetStartedTime().AsTime(), - ).Add(hbTimeout) - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: lastHbTime}, - &activitypb.HeartbeatTimeoutTask{Stamp: attempt.GetStamp()}, - ) - } -} - -// dispatchTimeRespectingStartDelay advances a candidate dispatch time t to the first dispatch time -// (ScheduleTime + start_delay) while the activity has not yet been picked up by a worker, so that -// pre-dispatch re-scheduling (unpause, reset, options update) honors any remaining start_delay. -// Returns t unchanged if the first attempt has already started. -func (a *Activity) dispatchTimeRespectingStartDelay(t time.Time) time.Time { - if a.GetFirstAttemptStartedTime() != nil { - return t - } - if dispatchTime := a.firstDispatchTime(); dispatchTime.After(t) { - return dispatchTime - } - return t -} - -// reissueScheduleToClose bumps the ScheduleToCloseStamp and re-emits the ScheduleToClose timeout task -// at the current deadline. -func (a *Activity) reissueScheduleToClose(ctx chasm.MutableContext) { - if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { - a.ScheduleToCloseStamp++ - ctx.AddTask( - a, - chasm.TaskAttributes{ScheduledTime: deadline}, - &activitypb.ScheduleToCloseTimeoutTask{Stamp: a.GetScheduleToCloseStamp()}, - ) - } -} - -// applyDeferredOptionRestore applies a Reset(RestoreOriginalOptions) that was deferred because a -// worker was running an attempt at reset time (see handleReset). -func (a *Activity) applyDeferredOptionRestore(ctx chasm.MutableContext) { - if !a.ResetRestoreOptions { - return - } - a.ResetRestoreOptions = false - a.restoreOriginalOptions(ctx) -} - -// applyDeferredHeartbeatClear applies a Reset(ResetHeartbeat) that was deferred because a worker was -// running an attempt at reset time (see handleReset). -func (a *Activity) applyDeferredHeartbeatClear(ctx chasm.MutableContext) { - if !a.ResetShouldClearHeartbeat { - return - } - a.ResetShouldClearHeartbeat = false - a.clearHeartbeatDetails(ctx) -} - -// restoreOriginalOptions resets the activity's options to the values it was originally scheduled -// with and reissues the ScheduleToClose timer at the resulting deadline. start_delay is restored -// only if the activity has never started. -func (a *Activity) restoreOriginalOptions(ctx chasm.MutableContext) { - og := a.GetOriginalOptions() - a.TaskQueue = common.CloneProto(og.GetTaskQueue()) - a.ScheduleToCloseTimeout = common.CloneProto(og.GetScheduleToCloseTimeout()) - a.ScheduleToStartTimeout = common.CloneProto(og.GetScheduleToStartTimeout()) - a.StartToCloseTimeout = common.CloneProto(og.GetStartToCloseTimeout()) - a.HeartbeatTimeout = common.CloneProto(og.GetHeartbeatTimeout()) - a.RetryPolicy = common.CloneProto(og.GetRetryPolicy()) - a.Priority = common.CloneProto(og.GetPriority()) - if a.GetFirstAttemptStartedTime() == nil { - a.StartDelay = common.CloneProto(og.GetStartDelay()) - } - a.reissueScheduleToClose(ctx) -} - -// scheduleToCloseDeadline returns the absolute time at which the ScheduleToClose timeout expires, -// accounting for start delay. Returns zero time if no ScheduleToClose timeout is set. -func (a *Activity) scheduleToCloseDeadline() time.Time { - timeout := a.GetScheduleToCloseTimeout().AsDuration() - if timeout == 0 { - return time.Time{} - } - return a.firstDispatchTime().Add(timeout) -} - -func createStartToCloseTimeoutFailure() *failurepb.Failure { - return &failurepb.Failure{ - Message: fmt.Sprintf(common.FailureReasonActivityTimeout, enumspb.TIMEOUT_TYPE_START_TO_CLOSE.String()), - FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ - TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ - TimeoutType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, - }, - }, - } -} - -func createHeartbeatTimeoutFailure() *failurepb.Failure { - return &failurepb.Failure{ - Message: fmt.Sprintf(common.FailureReasonActivityTimeout, enumspb.TIMEOUT_TYPE_HEARTBEAT.String()), - FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ - TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ - TimeoutType: enumspb.TIMEOUT_TYPE_HEARTBEAT, - }, - }, - } -} - -// lastHeartbeatDetails returns the details recorded by the most recent heartbeat, or nil if -// the activity never heartbeated. -func (a *Activity) lastHeartbeatDetails(ctx chasm.Context) *commonpb.Payloads { - heartbeat, ok := a.LastHeartbeat.TryGet(ctx) - if !ok { - return nil - } - return heartbeat.GetDetails() -} - -// RecordHeartbeat records a heartbeat for the activity. -func (a *Activity) RecordHeartbeat( - ctx chasm.MutableContext, - input WithToken[*historyservice.RecordActivityTaskHeartbeatRequest], -) (*historyservice.RecordActivityTaskHeartbeatResponse, error) { - err := a.validateActivityTaskToken(ctx, input.Token, input.Request.GetNamespaceId(), false) - if err != nil { - return nil, err - } - details := input.Request.GetHeartbeatRequest().GetDetails() - prevHeartbeat, _ := a.LastHeartbeat.TryGet(ctx) - a.LastHeartbeat = chasm.NewDataField(ctx, &activitypb.ActivityHeartbeatState{ - RecordedTime: timestamppb.New(ctx.Now(a)), - Details: details, - TotalHeartbeatCount: prevHeartbeat.GetTotalHeartbeatCount() + 1, - }) - if heartbeatTimeout := a.GetHeartbeatTimeout().AsDuration(); heartbeatTimeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: ctx.Now(a).Add(heartbeatTimeout), - }, - &activitypb.HeartbeatTimeoutTask{ - Stamp: a.LastAttempt.Get(ctx).GetStamp(), - }, - ) - } - a.emitHeartbeatMetrics(ctx, details) - - response := &historyservice.RecordActivityTaskHeartbeatResponse{} - switch a.Status { - case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: - response.CancelRequested = true - case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: - response.ActivityReset = true - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: - response.ActivityPaused = true - default: - // no-op - } - return response, nil -} - -// InternalStatusToAPIStatus converts internal activity execution status to API status. -func InternalStatusToAPIStatus(status activitypb.ActivityExecutionStatus) enumspb.ActivityExecutionStatus { - switch status { - case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED, - activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: - return enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: - return enumspb.ACTIVITY_EXECUTION_STATUS_PAUSED - case activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED: - return enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED - case activitypb.ACTIVITY_EXECUTION_STATUS_FAILED: - return enumspb.ACTIVITY_EXECUTION_STATUS_FAILED - case activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED: - return enumspb.ACTIVITY_EXECUTION_STATUS_CANCELED - case activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED: - return enumspb.ACTIVITY_EXECUTION_STATUS_TERMINATED - case activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT: - return enumspb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT - case activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: - return enumspb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED - default: - panic(fmt.Sprintf("unknown activity execution status: %v", status)) //nolint:forbidigo - } -} - -func (a *Activity) runState() enumspb.PendingActivityState { - status := a.GetStatus() - switch status { - case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED: - return enumspb.PENDING_ACTIVITY_STATE_SCHEDULED - case activitypb.ACTIVITY_EXECUTION_STATUS_STARTED: - return enumspb.PENDING_ACTIVITY_STATE_STARTED - case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: - // The worker is still executing under its existing task token; the public PendingActivityState - // enum does not have a RESET_REQUESTED variant. The reset is surfaced to the worker via - // ActivityReset=true on its next heartbeat response. - if a.isPaused() { - return enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED - } - return enumspb.PENDING_ACTIVITY_STATE_STARTED - case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: - return enumspb.PENDING_ACTIVITY_STATE_CANCEL_REQUESTED - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: - return enumspb.PENDING_ACTIVITY_STATE_PAUSED - case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: - return enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED - case activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - activitypb.ACTIVITY_EXECUTION_STATUS_FAILED, - activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, - activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, - activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: - return enumspb.PENDING_ACTIVITY_STATE_UNSPECIFIED - default: - panic(fmt.Sprintf("unknown activity execution status: %v", status)) //nolint:forbidigo - } -} - -func (a *Activity) buildActivityExecutionInfo( - ctx chasm.Context, - request *workflowservice.DescribeActivityExecutionRequest, -) *apiactivitypb.ActivityExecutionInfo { - status := InternalStatusToAPIStatus(a.GetStatus()) - runState := a.runState() - - requestData := a.RequestData.Get(ctx) - attempt := a.LastAttempt.Get(ctx) - heartbeat, _ := a.LastHeartbeat.TryGet(ctx) - key := ctx.ExecutionKey() - executionInfo := ctx.ExecutionInfo() - var closeTime *timestamppb.Timestamp - var executionDuration *durationpb.Duration - if a.LifecycleState(ctx) != chasm.LifecycleStateRunning { - executionDuration = durationpb.New(executionInfo.CloseTime.Sub(a.GetScheduleTime().AsTime())) - closeTime = timestamppb.New(executionInfo.CloseTime) - } - - var expirationTime *timestamppb.Timestamp - if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { - expirationTime = timestamppb.New(deadline) - } - - sa := &commonpb.SearchAttributes{ - IndexedFields: a.Visibility.Get(ctx).CustomSearchAttributes(ctx), - } - - info := &apiactivitypb.ActivityExecutionInfo{ - ActivityId: key.BusinessID, - ActivityType: a.GetActivityType(), - Attempt: attempt.GetCount(), - CanceledReason: a.CancelState.GetReason(), - CloseTime: closeTime, - CurrentRetryInterval: a.currentRetryInterval(ctx, attempt), - ExecutionDuration: executionDuration, - ExecutionTime: timestamppb.New(a.firstDispatchTime()), - ExpirationTime: expirationTime, - Header: requestData.GetHeader(), - HeartbeatTimeout: a.GetHeartbeatTimeout(), - Links: ctx.Links(a), - TotalHeartbeatCount: heartbeat.GetTotalHeartbeatCount(), - LastAttemptCompleteTime: attempt.GetCompleteTime(), - LastHeartbeatTime: heartbeat.GetRecordedTime(), - LastStartedTime: attempt.GetStartedTime(), - LastWorkerIdentity: attempt.GetLastWorkerIdentity(), - LastDeploymentVersion: attempt.GetLastDeploymentVersion(), - SdkName: attempt.GetSdkName(), - SdkVersion: attempt.GetSdkVersion(), - NextAttemptScheduleTime: a.nextAttemptDispatchTime(ctx, attempt), - Priority: a.GetPriority(), - RetryPolicy: a.GetRetryPolicy(), - RunId: key.RunID, - RunState: runState, - ScheduleTime: a.GetScheduleTime(), - ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), - ScheduleToStartTimeout: a.GetScheduleToStartTimeout(), - StartDelay: a.GetStartDelay(), - StartToCloseTimeout: a.GetStartToCloseTimeout(), - StateSizeBytes: int64(executionInfo.ApproximateStateSize), - StateTransitionCount: executionInfo.StateTransitionCount, - SearchAttributes: sa, - Status: status, - TaskQueue: a.GetTaskQueue().GetName(), - UserMetadata: a.effectiveUserMetadata(ctx), - } - if request.GetIncludeHeartbeatDetails() { - info.HeartbeatDetails = heartbeat.GetDetails() - } - if request.GetIncludeLastFailure() { - info.LastFailure = attempt.GetLastFailureDetails().GetFailure() - } - - return info -} - -func (a *Activity) buildDescribeActivityExecutionResponse( - ctx chasm.Context, - req *activitypb.DescribeActivityExecutionRequest, -) (*activitypb.DescribeActivityExecutionResponse, error) { - request := req.GetFrontendRequest() - - token, err := ctx.Ref(a) - if err != nil { - return nil, err - } - - info := a.buildActivityExecutionInfo(ctx, request) - - var input *commonpb.Payloads - if request.GetIncludeInput() { - input = a.RequestData.Get(ctx).GetInput() - } - - callbackInfos, err := a.buildCallbackInfos(ctx) - if err != nil { - return nil, err - } - - response := &workflowservice.DescribeActivityExecutionResponse{ - Info: info, - RunId: ctx.ExecutionKey().RunID, - Input: input, - LongPollToken: token, - Callbacks: callbackInfos, - } - - if request.GetIncludeOutcome() { - response.Outcome = a.outcome(ctx) - } - - return &activitypb.DescribeActivityExecutionResponse{ - FrontendResponse: response, - }, nil -} - -func (a *Activity) buildCallbackInfos(ctx chasm.Context) ([]*apiactivitypb.CallbackInfo, error) { - if len(a.Callbacks) == 0 { - return nil, nil - } - - cbInfos := make([]*apiactivitypb.CallbackInfo, 0, len(a.Callbacks)) - for _, field := range a.Callbacks { - cb := field.Get(ctx) - - cbSpec, err := cb.ToAPICallback() - if err != nil { - return nil, err - } - - var state enumspb.CallbackState - switch cb.Status { - case callbackspb.CALLBACK_STATUS_UNSPECIFIED: - return nil, serviceerror.NewInternal("callback with UNSPECIFIED state") - case callbackspb.CALLBACK_STATUS_STANDBY: - state = enumspb.CALLBACK_STATE_STANDBY - case callbackspb.CALLBACK_STATUS_SCHEDULED: - state = enumspb.CALLBACK_STATE_SCHEDULED - case callbackspb.CALLBACK_STATUS_BACKING_OFF: - state = enumspb.CALLBACK_STATE_BACKING_OFF - case callbackspb.CALLBACK_STATUS_FAILED: - state = enumspb.CALLBACK_STATE_FAILED - case callbackspb.CALLBACK_STATUS_SUCCEEDED: - state = enumspb.CALLBACK_STATE_SUCCEEDED - default: - return nil, serviceerror.NewInternalf("unknown callback state: %v", cb.Status) - } - - cbInfos = append(cbInfos, &apiactivitypb.CallbackInfo{ - Trigger: &apiactivitypb.CallbackInfo_Trigger{ - Variant: &apiactivitypb.CallbackInfo_Trigger_ActivityClosed{}, - }, - Info: &callbackpb.CallbackInfo{ - Callback: cbSpec, - RegistrationTime: cb.RegistrationTime, - State: state, - Attempt: cb.Attempt, - LastAttemptCompleteTime: cb.LastAttemptCompleteTime, - LastAttemptFailure: cb.LastAttemptFailure, - NextAttemptScheduleTime: cb.NextAttemptScheduleTime, - }, - }) - } - return cbInfos, nil -} - -func (a *Activity) buildPollActivityExecutionResponse( - ctx chasm.Context, -) *activitypb.PollActivityExecutionResponse { - return &activitypb.PollActivityExecutionResponse{ - FrontendResponse: &workflowservice.PollActivityExecutionResponse{ - RunId: ctx.ExecutionKey().RunID, - Outcome: a.outcome(ctx), - }, - } -} - -// outcome retrieves the activity outcome (result or failure) if the activity has completed. -// Returns nil if the activity has not completed. -func (a *Activity) outcome(ctx chasm.Context) *apiactivitypb.ActivityExecutionOutcome { - if !a.LifecycleState(ctx).IsClosed() { - return nil - } - activityOutcome := a.Outcome.Get(ctx) - if successful := activityOutcome.GetSuccessful(); successful != nil { - return &apiactivitypb.ActivityExecutionOutcome{ - Value: &apiactivitypb.ActivityExecutionOutcome_Result{Result: successful.GetOutput()}, - } - } - if failure := a.terminalFailure(ctx); failure != nil { - return &apiactivitypb.ActivityExecutionOutcome{ - Value: &apiactivitypb.ActivityExecutionOutcome_Failure{Failure: failure}, - RetryState: activityOutcome.GetRetryState(), - } - } - return nil -} - -// terminalFailure returns the failure for a closed activity. The failure may be stored in Outcome.Failed -// (terminated, canceled, timed out) or in LastAttempt.LastFailureDetails (failed after exhausting retries). -// Returns nil if no failure is found. -func (a *Activity) terminalFailure(ctx chasm.Context) *failurepb.Failure { - if f := a.Outcome.Get(ctx).GetFailed(); f != nil { - return f.GetFailure() - } - if details := a.LastAttempt.Get(ctx).GetLastFailureDetails(); details != nil { - return details.GetFailure() - } - return nil -} - -// StoreOrSelf returns the store for the activity. If the store is not set as a field (e.g. -// standalone activities), it returns the activity itself. -func (a *Activity) StoreOrSelf(ctx chasm.Context) ActivityStore { - store, ok := a.Store.TryGet(ctx) - if ok { - return store - } - return a -} - -// validateActivityTaskToken validates a task token against the current activity state. -// -// allowForceCompleteWithNoAttempt permits a by-ID token to pass even though the activity has no -// attempt in progress (Scheduled or Paused). Only HandleCompleted sets this, mirroring the -// workflow-activity behavior of letting RespondActivityTaskCompletedById force-complete an -// activity before any worker has started it. -func (a *Activity) validateActivityTaskToken( - ctx chasm.Context, - token *tokenspb.Task, - requestNamespaceID string, - allowForceCompleteWithNoAttempt bool, -) error { - forceCompleteWithNoAttempt := allowForceCompleteWithNoAttempt && - token.Attempt == ByIDTokenAttempt && - !a.hasAttemptInProgress() - if !a.hasAttemptInProgress() && !forceCompleteWithNoAttempt { - return serviceerror.NewNotFound("activity task not found") - } - if token.Attempt != ByIDTokenAttempt && token.Attempt != a.LastAttempt.Get(ctx).GetCount() { - return serviceerror.NewNotFound("activity task not found") - } - tokenStamp := token.GetActivityAttemptStamp() - startedStamp := a.LastAttempt.Get(ctx).GetStartedStamp() - // Matching versions without stamped tokens leave tokenStamp zero; - // History versions without StartedStamp persistence leave startedStamp zero. - requiresLegacyStampCompatibility := tokenStamp == 0 || startedStamp == 0 - if !requiresLegacyStampCompatibility && tokenStamp != startedStamp { - return serviceerror.NewNotFound("activity task not found") - } - - ref, err := chasm.DeserializeComponentRef(token.GetComponentRef()) - if err != nil { - return serviceerror.NewInvalidArgument("malformed token") - } - - // Validate that the request namespace matches the token's namespace. - // This prevents cross-namespace token reuse attacks where an attacker could use a valid token from namespace B to - // complete an activity in namespace A. - if requestNamespaceID != ref.NamespaceID { - return serviceerror.NewInvalidArgument("token does not match namespace") + // Validate that the request namespace matches the token's namespace. + // This prevents cross-namespace token reuse attacks where an attacker could use a valid token from namespace B to + // complete an activity in namespace A. + if requestNamespaceID != ref.NamespaceID { + return serviceerror.NewInvalidArgument("token does not match namespace") } return nil } -// baseMetricsHandler adds only the operation tag. -func (a *Activity) baseMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { - return ctx.MetricsHandler().WithTags(metrics.OperationTag(operation)) -} - -// enrichedMetricsHandler adds standard activity tags in addition to the operation tag. -func (a *Activity) enrichedMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { - namespaceName := ctx.NamespaceEntry().Name() - // activityContextFromChasm panics if the context value is missing; this is intentional and - // indicates a library registration bug rather than a runtime error. - actCtx := activityContextFromChasm(ctx) - breakdownMetricsByTaskQueue := actCtx.config.BreakdownMetricsByTaskQueue - taskQueueFamily := a.GetTaskQueue().GetName() - return metrics.GetPerTaskQueueFamilyScope( - ctx.MetricsHandler(), - namespaceName.String(), - tqid.UnsafeTaskQueueFamily(namespaceName.String(), taskQueueFamily), - breakdownMetricsByTaskQueue(namespaceName.String(), taskQueueFamily, enumspb.TASK_QUEUE_TYPE_ACTIVITY), - metrics.OperationTag(operation), - metrics.ActivityTypeTag(a.GetActivityType().GetName()), - metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), - metrics.WorkflowTypeTag(WorkflowTypeTag), - ) -} - -func (a *Activity) emitOnAttemptTimedOutMetrics(metricsHandler metrics.Handler, timeoutType enumspb.TimeoutType) { - timeoutTag := metrics.StringTag("timeout_type", timeoutType.String()) - metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) -} - -func (a *Activity) emitOnAttemptFailedMetrics(ctx chasm.Context, metricsHandler metrics.Handler) { - attempt := a.LastAttempt.Get(ctx) - startedTime := attempt.GetStartedTime().AsTime() - - latency := time.Since(startedTime) - metrics.ActivityStartToCloseLatency.With(metricsHandler).Record(latency) - - metrics.ActivityTaskFail.With(metricsHandler).Record(1) -} - -// emitPayloadSizeMetric records the serialized size of a user payload. -func emitPayloadSizeMetric(metricsHandler metrics.Handler, size int) { - if size > 0 { - metrics.ActivityPayloadSize.With(metricsHandler).Record(int64(size)) - } -} - -// emitHeartbeatMetrics records the heartbeat count and payload size for heartbeat details. -func (a *Activity) emitHeartbeatMetrics(ctx chasm.Context, details *commonpb.Payloads) { - metricsHandler := a.baseMetricsHandler(ctx, metrics.HistoryRecordActivityTaskHeartbeatScope) - detailsSize := details.Size() - emitPayloadSizeMetric(metricsHandler, detailsSize) - metrics.ActivityHeartbeatCount.With(metricsHandler).Record( - 1, - metrics.StringTag("has_details", strconv.FormatBool(detailsSize > 0)), - ) -} - -func (a *Activity) emitOnCompletedMetrics( - ctx chasm.Context, - baseHandler metrics.Handler, - enrichedHandler metrics.Handler, - result *commonpb.Payloads, - attemptWasStarted bool, -) { - attempt := a.LastAttempt.Get(ctx) - startedTime := attempt.GetStartedTime().AsTime() - - if attemptWasStarted { - startToCloseLatency := time.Since(startedTime) - metrics.ActivityStartToCloseLatency.With(enrichedHandler).Record(startToCloseLatency) - } - - scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) - metrics.ActivityScheduleToCloseLatency.With(enrichedHandler).Record(scheduleToCloseLatency) - - metrics.ActivitySuccess.With(enrichedHandler).Record(1) - emitPayloadSizeMetric(baseHandler, result.Size()) -} - -func (a *Activity) emitOnFailedMetrics( - ctx chasm.Context, - baseHandler metrics.Handler, - enrichedHandler metrics.Handler, - failure *failurepb.Failure, -) { - attempt := a.LastAttempt.Get(ctx) - startedTime := attempt.GetStartedTime().AsTime() - - startToCloseLatency := time.Since(startedTime) - metrics.ActivityStartToCloseLatency.With(enrichedHandler).Record(startToCloseLatency) - - scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) - metrics.ActivityScheduleToCloseLatency.With(enrichedHandler).Record(scheduleToCloseLatency) - - metrics.ActivityTaskFail.With(enrichedHandler).Record(1) - metrics.ActivityFail.With(enrichedHandler).Record(1) - emitPayloadSizeMetric(baseHandler, failure.Size()) -} - -func (a *Activity) emitOnTerminatedMetrics( - metricsHandler metrics.Handler, -) { - // Terminated activities do not count as properly finished activities so we do not - // record any of the latency metrics. - metrics.ActivityTerminate.With(metricsHandler).Record(1) -} - -func (a *Activity) emitOnCanceledMetrics( - ctx chasm.Context, - metricsHandler metrics.Handler, - fromStatus activitypb.ActivityExecutionStatus, -) { - // Record start-to-close latency only while an attempt is running. SCHEDULED (incl. retry - // backoff) and PAUSED have no running attempt and a possibly-stale StartedTime. - attemptRunning := fromStatus != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED && - fromStatus != activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED - if attemptRunning { - if startedTime := a.LastAttempt.Get(ctx).GetStartedTime(); startedTime != nil { - metrics.ActivityStartToCloseLatency.With(metricsHandler).Record(time.Since(startedTime.AsTime())) - } - } - - scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) - metrics.ActivityScheduleToCloseLatency.With(metricsHandler).Record(scheduleToCloseLatency) - - metrics.ActivityCancel.With(metricsHandler).Record(1) -} - -func (a *Activity) emitOnTimedOutMetrics( - metricsHandler metrics.Handler, - timeoutType enumspb.TimeoutType, -) { - scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) - metrics.ActivityScheduleToCloseLatency.With(metricsHandler).Record(scheduleToCloseLatency) - - timeoutTag := metrics.StringTag("timeout_type", timeoutType.String()) - metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) - metrics.ActivityTimeout.With(metricsHandler).Record(1, timeoutTag) -} - -func (a *Activity) emitOnPausedMetrics( - metricsHandler metrics.Handler, -) { - metrics.ActivityPause.With(metricsHandler).Record(1) -} - -func (a *Activity) emitOnUpdateOptionsMetrics( - metricsHandler metrics.Handler, -) { - metrics.ActivityUpdateOptions.With(metricsHandler).Record(1) -} - -func (a *Activity) emitOnUnpausedMetrics( - metricsHandler metrics.Handler, -) { - metrics.ActivityUnpause.With(metricsHandler).Record(1) -} - -func (a *Activity) emitOnResetMetrics( - metricsHandler metrics.Handler, -) { - metrics.ActivityReset.With(metricsHandler).Record(1) -} - // SearchAttributes implements chasm.VisibilitySearchAttributesProvider interface. // Returns the current search attribute values for this activity execution. func (a *Activity) SearchAttributes(_ chasm.Context) []chasm.SearchAttributeKeyValue { diff --git a/chasm/lib/activity/activity_test.go b/chasm/lib/activity/activity_test.go index b0f7d73d211..df704edda0b 100644 --- a/chasm/lib/activity/activity_test.go +++ b/chasm/lib/activity/activity_test.go @@ -9,9 +9,6 @@ import ( "github.com/stretchr/testify/require" apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas commonpb "go.temporal.io/api/common/v1" - deploymentpb "go.temporal.io/api/deployment/v1" - enumspb "go.temporal.io/api/enums/v1" - failurepb "go.temporal.io/api/failure/v1" sdkpb "go.temporal.io/api/sdk/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" @@ -21,83 +18,17 @@ import ( tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" - "go.temporal.io/server/common" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute/sadefs" serviceerrors "go.temporal.io/server/common/serviceerror" - "go.temporal.io/server/common/testing/protorequire" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" ) -func TestTryRescheduleRetryStatePrecedence(t *testing.T) { - testCases := []struct { - name string - status activitypb.ActivityExecutionStatus - retryPolicy *commonpb.RetryPolicy - expected enumspb.RetryState - finalStatus activitypb.ActivityExecutionStatus - }{ - { - name: "retry policy not set", - status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - expected: enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, - finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - }, - { - name: "retry policy not set takes precedence over cancellation", - status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - expected: enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, - finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - }, - { - name: "cancellation requested", - status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - retryPolicy: defaultRetryPolicy, - expected: enumspb.RETRY_STATE_CANCEL_REQUESTED, - finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - }, - { - name: "reset requested takes precedence over retry policy not set", - status: activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED, - expected: enumspb.RETRY_STATE_IN_PROGRESS, - finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { - return defaultTime.Add(2 * time.Second) - }, - }, - } - injectActivityContext(t, ctx) // needed for last test case - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: tc.status, - RetryPolicy: tc.retryPolicy, - ScheduleTime: timestamppb.New(defaultTime), - ScheduleToCloseTimeout: durationpb.New(time.Second), - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 1}), - } - - retryState, err := activity.tryReschedule(ctx, false, 0, &failurepb.Failure{}) - - require.NoError(t, err) - require.Equal(t, tc.expected, retryState) - require.Equal(t, tc.finalStatus, activity.GetStatus()) - }) - } -} - func TestSearchAttributesIncludesExecutionTime(t *testing.T) { testTime := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) testCases := []struct { @@ -141,85 +72,6 @@ func TestSearchAttributesIncludesExecutionTime(t *testing.T) { } } -func TestStartDelayBucket(t *testing.T) { - testCases := []struct { - name string - delay time.Duration - expected activitypb.StartDelayBucket - }{ - {name: "negative", delay: -time.Second, expected: activitypb.START_DELAY_BUCKET_NONE}, - {name: "zero", delay: 0, expected: activitypb.START_DELAY_BUCKET_NONE}, - {name: "less than one minute", delay: time.Minute - time.Nanosecond, expected: activitypb.START_DELAY_BUCKET_LT_1M}, - {name: "one minute", delay: time.Minute, expected: activitypb.START_DELAY_BUCKET_1M_10M}, - {name: "ten minutes", delay: 10 * time.Minute, expected: activitypb.START_DELAY_BUCKET_10M_1H}, - {name: "one hour", delay: time.Hour, expected: activitypb.START_DELAY_BUCKET_1H_6H}, - {name: "six hours", delay: 6 * time.Hour, expected: activitypb.START_DELAY_BUCKET_6H_1D}, - {name: "one day", delay: 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_1D_7D}, - {name: "seven days", delay: 7 * 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_7D_30D}, - {name: "thirty days", delay: 30 * 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_7D_30D}, - {name: "more than thirty days", delay: 30*24*time.Hour + time.Nanosecond, expected: activitypb.START_DELAY_BUCKET_GT_30D}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.expected, startDelayBucket(tc.delay)) - }) - } -} - -func TestNewActivityDispatchTask(t *testing.T) { - testCases := []struct { - name string - startDelay time.Duration - firstAttemptStartedTime *timestamppb.Timestamp - expectedReason activitypb.DispatchReason - expectedBucket activitypb.StartDelayBucket - }{ - { - name: "immediate", - expectedReason: activitypb.DISPATCH_REASON_IMMEDIATE, - expectedBucket: activitypb.START_DELAY_BUCKET_NONE, - }, - { - name: "start delay", - startDelay: time.Hour, - expectedReason: activitypb.DISPATCH_REASON_START_DELAY, - expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, - }, - { - name: "retry without configured start delay", - firstAttemptStartedTime: timestamppb.Now(), - expectedReason: activitypb.DISPATCH_REASON_RETRY, - expectedBucket: activitypb.START_DELAY_BUCKET_NONE, - }, - { - name: "retry preserves configured start delay bucket", - startDelay: time.Hour, - firstAttemptStartedTime: timestamppb.Now(), - expectedReason: activitypb.DISPATCH_REASON_RETRY, - expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - StartDelay: durationpb.New(tc.startDelay), - FirstAttemptStartedTime: tc.firstAttemptStartedTime, - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Stamp: 1}), - } - - task := activity.newActivityDispatchTask(ctx) - require.Equal(t, int32(1), task.GetStamp()) - require.Equal(t, tc.expectedReason, task.GetDispatchReason()) - require.Equal(t, tc.expectedBucket, task.GetStartDelayBucket()) - }) - } -} - func TestHandleStarted(t *testing.T) { testTime := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) testRequestID := "test-request-id" @@ -546,207 +398,6 @@ func TestActivityTerminate(t *testing.T) { } } -func TestRequestDeduplicationAfterTerminalState(t *testing.T) { - t.Run("CancellationAfterCanceled", func(t *testing.T) { - const requestID = "cancel-request-id" - - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, - CancelState: &activitypb.ActivityCancelState{ - RequestId: requestID, - }, - }, - } - - _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ - FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{ - RequestId: requestID, - }, - }) - require.NoError(t, err) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) - }) - - t.Run("NewCancellationAfterCanceled", func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, - CancelState: &activitypb.ActivityCancelState{ - RequestId: "original-request-id", - }, - }, - } - - _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ - FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{ - RequestId: "new-request-id", - }, - }) - require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) - require.EqualError(t, err, "activity is in terminal state Canceled") - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) - }) - - t.Run("TerminationAfterTerminated", func(t *testing.T) { - const requestID = "terminate-request-id" - - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - TerminateState: &activitypb.ActivityTerminateState{ - RequestId: requestID, - }, - }, - } - - _, err := activity.Terminate(ctx, chasm.TerminateComponentRequest{ - RequestID: requestID, - }) - require.NoError(t, err) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) - }) - - t.Run("NewTerminationAfterTerminated", func(t *testing.T) { - const requestID = "terminate-request-id" - - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - TerminateState: &activitypb.ActivityTerminateState{ - RequestId: requestID, - }, - }, - } - - _, err := activity.Terminate(ctx, chasm.TerminateComponentRequest{ - RequestID: "new-request-id", - }) - require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) - require.EqualError(t, err, "already terminated with request ID "+requestID) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) - }) - - t.Run("PauseAfterTerminated", func(t *testing.T) { - const requestID = "pause-request-id" - - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - LastPauseState: &activitypb.ActivityPauseState{ - RequestId: requestID, - }, - }, - } - - _, err := activity.handlePauseRequested(ctx, &activitypb.PauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ - RequestId: requestID, - }, - }) - require.NoError(t, err) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) - }) - - t.Run("NewPauseAfterTerminated", func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, - LastPauseState: &activitypb.ActivityPauseState{ - RequestId: "pause-request-id", - }, - }, - } - - _, err := activity.handlePauseRequested(ctx, &activitypb.PauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ - RequestId: "new-request-id", - }, - }) - require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) - require.EqualError(t, err, "activity is in non-pausable state Terminated") - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) - }) -} - -// Check that we do not emit a StartToCloseLatency metric when cancelling an activity that has no -// attempt in progress. Cancelling a SCHEDULED or PAUSED activity transitions straight to Canceled, -// and the status captured before the CancelRequested transition must be forwarded so that the -// metric is not emitted. -func TestHandleCancellationRequestedDirectCancelMetrics(t *testing.T) { - testCases := []struct { - name string - status activitypb.ActivityExecutionStatus - }{ - { - name: "scheduled during retry backoff, stale StartedTime", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - }, - { - name: "paused during retry backoff, stale StartedTime", - status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return defaultTime }, - HandleMetricsHandler: func() metrics.Handler { return metricsHandler }, - HandleNamespaceEntry: testNamespaceEntry, - GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ - config: &Config{ - BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), - }, - }), - }, - } - - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - RetryPolicy: defaultRetryPolicy, - Status: tc.status, - ScheduleTime: timestamppb.New(defaultTime), - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - ScheduleToCloseTimeout: durationpb.New(defaultScheduleToCloseTimeout), - ScheduleToStartTimeout: durationpb.New(defaultScheduleToStartTimeout), - StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), - }, - // A stale StartedTime from a prior attempt: no attempt is currently running, so - // StartToCloseLatency must not be recorded on cancellation. - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ - Count: 1, - StartedTime: timestamppb.New(defaultTime), - }), - Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), - } - - _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ - FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{Reason: "test"}, - }) - require.NoError(t, err) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) - - snapshot := capture.Snapshot() - require.NotEmpty(t, snapshot[metrics.ActivityCancel.Name()]) - require.NotEmpty(t, snapshot[metrics.ActivityScheduleToCloseLatency.Name()]) - require.Empty(t, snapshot[metrics.ActivityStartToCloseLatency.Name()], - "no attempt was running; StartToCloseLatency must not be recorded") - }) - } -} - func TestRecordHeartbeatPauseResetCancelFlags(t *testing.T) { testTime := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) const ( @@ -1126,296 +777,6 @@ func TestActivityTaskTokenLegacyStampCompatibility(t *testing.T) { } } -func TestUpdateStartedActivityExecutionOptionsDoesNotBumpStartedStamp(t *testing.T) { - testTime := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return testTime }, - HandleNamespaceEntry: testNamespaceEntry, - GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ - config: &Config{ - BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), - }, - }), - }, - } - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - ScheduleToCloseTimeout: durationpb.New(10 * time.Minute), - ScheduleToStartTimeout: durationpb.New(2 * time.Minute), - StartToCloseTimeout: durationpb.New(3 * time.Minute), - HeartbeatTimeout: durationpb.New(time.Minute), - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ - Count: 1, - Stamp: 7, - StartedStamp: 7, - }), - Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), - } - attempt := activity.LastAttempt.Get(ctx) - originalStamp := attempt.GetStamp() - originalStartedStamp := attempt.GetStartedStamp() - - _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ - FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ - ActivityId: "test-activity-id", - RequestId: "update-request-id", - ActivityOptions: &apiactivitypb.ActivityOptions{ - HeartbeatTimeout: durationpb.New(2 * time.Minute), - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, - }, - }) - require.NoError(t, err) - - require.Equal(t, originalStamp+1, attempt.GetStamp()) - require.Equal(t, originalStartedStamp, attempt.GetStartedStamp()) -} - -func TestHandlePauseRequestedDedupBeforeValidation(t *testing.T) { - for _, status := range []activitypb.ActivityExecutionStatus{ - activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, - } { - t.Run(status.String(), func(t *testing.T) { - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: status, - LastPauseState: &activitypb.ActivityPauseState{ - RequestId: "pause-request-id", - }, - }, - } - - _, err := activity.handlePauseRequested( - &chasm.MockMutableContext{}, - &activitypb.PauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ - RequestId: "pause-request-id", - }, - }, - ) - require.NoError(t, err) - require.Equal(t, status, activity.GetStatus()) - }) - } -} - -func TestHandleUnpauseRequestedRequestID(t *testing.T) { - t.Run("deduplicates latest request ID", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - LastUnpauseRequestId: "unpause-request-id", - ScheduleToStartTimeout: durationpb.New(time.Minute), - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), - } - - _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ - RequestId: "unpause-request-id", - }, - }) - require.NoError(t, err) - require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, activity.GetStatus()) - require.Equal(t, int32(3), activity.LastAttempt.Get(ctx).GetCount()) - }) - - t.Run("does not persist request ID when unpause fails", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - }, - } - - _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ - RequestId: "unpause-request-id", - }, - }) - var failedPreconditionErr *serviceerror.FailedPrecondition - require.ErrorAs(t, err, &failedPreconditionErr) - require.Empty(t, activity.GetLastUnpauseRequestId(), "a failed unpause must not be recorded for de-dup") - }) - - t.Run("does not record failed request ID", func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - LastUnpauseRequestId: "previous-unpause-request-id", - }, - } - - _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ - FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ - RequestId: "failed-unpause-request-id", - }, - }) - require.Error(t, err) - require.Equal(t, "previous-unpause-request-id", activity.GetLastUnpauseRequestId()) - }) -} - -func TestHandleResetRequestID(t *testing.T) { - t.Run("deduplicates latest request ID", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - LastResetRequestId: "reset-request-id", - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), - } - - _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ - FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ - RequestId: "reset-request-id", - }, - }) - require.NoError(t, err) - require.Equal(t, int32(3), activity.LastAttempt.Get(ctx).GetCount()) - }) - - t.Run("records successful request ID", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), - } - - _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ - FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ - RequestId: "reset-request-id", - }, - }) - require.NoError(t, err) - require.Equal(t, "reset-request-id", activity.GetLastResetRequestId()) - }) - - t.Run("does not record failed request ID", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - LastResetRequestId: "previous-reset-request-id", - }, - } - - _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ - FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ - RequestId: "failed-reset-request-id", - }, - }) - require.Error(t, err) - require.Equal(t, "previous-reset-request-id", activity.GetLastResetRequestId()) - }) -} - -func TestUpdateActivityExecutionOptionsRequestID(t *testing.T) { - t.Run("deduplicates latest request ID", func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, - LastUpdateOptionsRequestId: "update-request-id", - }, - } - - resp, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ - FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ - RequestId: "update-request-id", - }, - }) - require.NoError(t, err) - require.Equal(t, "current-task-queue", resp.GetFrontendResponse().GetActivityOptions().GetTaskQueue().GetName()) - }) - - t.Run("records successful request ID", func(t *testing.T) { - ctx := newOperatorCommandTestContext(t) - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - Status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, - ScheduleToCloseTimeout: durationpb.New(10 * time.Minute), - ScheduleToStartTimeout: durationpb.New(2 * time.Minute), - StartToCloseTimeout: durationpb.New(3 * time.Minute), - HeartbeatTimeout: durationpb.New(time.Minute), - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ - Count: 1, - Stamp: 7, - StartedStamp: 7, - }), - Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), - } - - _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ - FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ - RequestId: "update-request-id", - ActivityOptions: &apiactivitypb.ActivityOptions{ - HeartbeatTimeout: durationpb.New(2 * time.Minute), - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, - }, - }) - require.NoError(t, err) - require.Equal(t, "update-request-id", activity.GetLastUpdateOptionsRequestId()) - }) - - t.Run("does not record failed request ID", func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, - LastUpdateOptionsRequestId: "previous-update-request-id", - }, - } - - _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ - FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ - RequestId: "failed-update-request-id", - }, - }) - require.Error(t, err) - require.Equal(t, "previous-update-request-id", activity.GetLastUpdateOptionsRequestId()) - }) -} - -func newOperatorCommandTestContext(t *testing.T) *chasm.MockMutableContext { - t.Helper() - return &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, - HandleNamespaceEntry: testNamespaceEntry, - GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ - config: &Config{ - BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), - }, - }), - }, - } -} - func TestContextMetadata(t *testing.T) { t.Run("returns activity type and task queue", func(t *testing.T) { ctx := &chasm.MockMutableContext{} @@ -1566,35 +927,6 @@ func TestNewStandaloneActivity_OriginalOptionsUnaffectedBySubfieldUpdate(t *test require.Equal(t, "original-fairness-key", originalOptions.GetPriority().GetFairnessKey()) } -func TestMergeActivityOptionsRejectsInvalidMergedRetryPolicy(t *testing.T) { - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "T"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: "Q"}, - ScheduleToCloseTimeout: durationpb.New(30 * time.Second), - ScheduleToStartTimeout: durationpb.New(20 * time.Second), - StartToCloseTimeout: durationpb.New(10 * time.Second), - RetryPolicy: &commonpb.RetryPolicy{ - InitialInterval: durationpb.New(10 * time.Second), - BackoffCoefficient: 2, - MaximumInterval: durationpb.New(30 * time.Second), - MaximumAttempts: 5, - }, - }, - } - - err := activity.mergeActivityOptions(&workflowservice.UpdateActivityExecutionOptionsRequest{ - ActivityId: "act", - ActivityOptions: &apiactivitypb.ActivityOptions{ - RetryPolicy: &commonpb.RetryPolicy{ - InitialInterval: durationpb.New(60 * time.Second), - }, - }, - UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"retry_policy.initial_interval"}}, - }) - require.ErrorContains(t, err, "MaximumInterval cannot be less than InitialInterval") -} - // TestEffectiveUserMetadata_PrefersFrameworkLocation ensures the helper used by // readers (Describe, etc.) returns the framework-level user metadata when both // the new and legacy locations are populated. @@ -1781,308 +1113,3 @@ func TestEffectiveUserMetadata_FallsBackToLegacy(t *testing.T) { got := activity.effectiveUserMetadata(ctx) require.Same(t, legacyMD, got) } - -func TestShouldRecalculateCurrentRetryInterval(t *testing.T) { - retryInterval := 2 * time.Second - - testCases := []struct { - name string - status activitypb.ActivityExecutionStatus - restoreOriginal bool - updateFields map[string]struct{} - currentRetryInterval *durationpb.Duration - retryIntervalSource activitypb.ActivityRetryIntervalSource - expectRecalculate bool - }{ - { - name: "retry policy subfield update with policy-derived interval", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, - expectRecalculate: true, - }, - { - name: "retry policy replacement with policy-derived interval", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"retryPolicy": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, - expectRecalculate: true, - }, - { - name: "restore original with policy-derived interval", - status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, - restoreOriginal: true, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, - expectRecalculate: true, - }, - { - name: "unrelated update preserves retry interval", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"heartbeatTimeout": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, - }, - { - name: "worker override is preserved regardless of value", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_WORKER_OVERRIDE, - }, - { - name: "unspecified source (pre-existing attempt state) is preserved", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, - }, - { - name: "started activity is not in retry backoff", - status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, - updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, - currentRetryInterval: durationpb.New(retryInterval), - retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, - }, - { - name: "missing retry interval", - status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: tc.status, - }, - } - attempt := &activitypb.ActivityAttemptState{ - CurrentRetryInterval: tc.currentRetryInterval, - CurrentRetryIntervalSource: tc.retryIntervalSource, - } - - got := activity.shouldRecalculateCurrentRetryInterval( - attempt, - tc.restoreOriginal, - tc.updateFields, - ) - - require.Equal(t, tc.expectRecalculate, got) - }) - } -} - -// TestUpdateActivityExecutionOptions_RestoreOriginal_RejectsMissingOriginalOptions mimics an -// activity persisted before original_options existed (e.g. created by a binary predating this -// field's introduction). Restoring such an activity's options has nothing valid to fall back to -// and must be rejected rather than silently wiping TaskQueue and both close/start timeouts. -func TestUpdateActivityExecutionOptions_RestoreOriginal_RejectsMissingOriginalOptions(t *testing.T) { - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, - }, - } - - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - ActivityType: &commonpb.ActivityType{Name: "T"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, - ScheduleToCloseTimeout: durationpb.New(30 * time.Second), - StartToCloseTimeout: durationpb.New(10 * time.Second), - OriginalOptions: nil, - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{}), - } - - _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ - FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ - ActivityId: "act", - RestoreOriginal: true, - RequestId: "failed-update-request-id", - }, - }) - - require.Error(t, err) - require.Empty(t, activity.GetLastUpdateOptionsRequestId()) - require.Equal(t, "current-task-queue", activity.GetTaskQueue().GetName()) - require.Equal(t, 30*time.Second, activity.GetScheduleToCloseTimeout().AsDuration()) - require.Equal(t, 10*time.Second, activity.GetStartToCloseTimeout().AsDuration()) -} - -// TestHandleReset_RestoreOriginalOptions_RejectsMissingOriginalOptions covers the equivalent gap -// on the Reset(RestoreOriginalOptions=true) path, for an activity with no original_options -// snapshot (see the Update test above for the scenario this mimics). -func TestHandleReset_RestoreOriginalOptions_RejectsMissingOriginalOptions(t *testing.T) { - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, - }, - } - - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - ActivityType: &commonpb.ActivityType{Name: "T"}, - TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, - ScheduleToCloseTimeout: durationpb.New(30 * time.Second), - StartToCloseTimeout: durationpb.New(10 * time.Second), - OriginalOptions: nil, - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{}), - } - - _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ - FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ - ActivityId: "act", - RestoreOriginalOptions: true, - RequestId: "failed-reset-request-id", - }, - }) - - require.Error(t, err) - require.Empty(t, activity.GetLastResetRequestId()) - require.Equal(t, "current-task-queue", activity.GetTaskQueue().GetName()) -} - -func TestBuildActivityExecutionInfo_IncludeLastDeploymentVersion(t *testing.T) { - ctx := &chasm.MockMutableContext{ - MockContext: chasm.MockContext{ - HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, - }, - } - - activity := &Activity{ - ActivityState: &activitypb.ActivityState{ - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ - LastDeploymentVersion: &deploymentpb.WorkerDeploymentVersion{ - DeploymentName: "test-deployment", - BuildId: "test-build-1", - }, - }), - RequestData: chasm.NewDataField(ctx, &activitypb.ActivityRequestData{}), - Visibility: chasm.NewComponentField(ctx, &chasm.Visibility{}), - } - - resp, err := activity.buildDescribeActivityExecutionResponse(ctx, &activitypb.DescribeActivityExecutionRequest{ - FrontendRequest: &workflowservice.DescribeActivityExecutionRequest{}, - }) - - require.NoError(t, err) - protorequire.ProtoEqual(t, &deploymentpb.WorkerDeploymentVersion{ - DeploymentName: "test-deployment", - BuildId: "test-build-1", - }, resp.FrontendResponse.GetInfo().GetLastDeploymentVersion()) -} - -// oversizedActivityFailure returns an application failure whose details push it past the retained -// failure size limit, mirroring the failure used by the workflow activity truncation tests. -func oversizedActivityFailure(t *testing.T) *failurepb.Failure { - activityFailure := &failurepb.Failure{ - Message: "activity failure with large details", - Source: "application", - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: "application-failure-type", - NonRetryable: false, - Details: &commonpb.Payloads{ - Payloads: []*commonpb.Payload{ - { - Data: make([]byte, defaultFailureSizeLimit*2), - }, - }, - }, - }}, - } - require.Greater(t, activityFailure.Size(), defaultFailureSizeLimit) - return activityFailure -} - -// withinLimitActivityFailure returns an application failure small enough to be retained whole. -func withinLimitActivityFailure(t *testing.T) *failurepb.Failure { - activityFailure := &failurepb.Failure{ - Message: "activity failure", - Source: "application", - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: "application-failure-type", - }}, - } - require.LessOrEqual(t, activityFailure.Size(), defaultFailureSizeLimit) - return activityFailure -} - -// recordFailedAttempt truncates the retryable failure, and not the final failure. -func TestRecordFailedAttempt_FailureTruncation(t *testing.T) { - requireTruncated := func(t *testing.T, sent, retained *failurepb.Failure) { - require.LessOrEqual(t, retained.Size(), defaultFailureSizeLimit) - require.Equal(t, common.FailureReasonFailureExceedsLimit, retained.GetMessage()) - require.NotNil(t, retained.GetServerFailureInfo()) - cause := retained.GetCause() - require.Equal(t, sent.GetMessage(), cause.GetMessage()) - require.Equal(t, "application-failure-type", cause.GetApplicationFailureInfo().GetType()) - require.Nil(t, cause.GetApplicationFailureInfo().GetDetails()) - } - requireKeptWhole := func(t *testing.T, sent, retained *failurepb.Failure) { - protorequire.ProtoEqual(t, sent, retained) - } - - testCases := []struct { - name string - failure func(*testing.T) *failurepb.Failure - noRetriesLeft bool - requireResult func(t *testing.T, sent, retained *failurepb.Failure) - }{ - { - name: "OversizedRetryableFailureIsTruncated", - failure: oversizedActivityFailure, - requireResult: requireTruncated, - }, - { - name: "RetryableFailureWithinLimitIsKeptWhole", - failure: withinLimitActivityFailure, - requireResult: requireKeptWhole, - }, - { - name: "OversizedTerminalFailureIsKeptWhole", - failure: oversizedActivityFailure, - noRetriesLeft: true, - requireResult: requireKeptWhole, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - ctx := &chasm.MockMutableContext{} - ctx.HandleNow = func(chasm.Component) time.Time { return defaultTime } - injectActivityContext(t, ctx) - - attemptState := &activitypb.ActivityAttemptState{Count: 1} - activity := &Activity{ - LastAttempt: chasm.NewDataField(ctx, attemptState), - } - activityFailure := tc.failure(t) - - retryInterval, retryIntervalSource := time.Second, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY - if tc.noRetriesLeft { - retryInterval, retryIntervalSource = 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - } - - err := activity.recordFailedAttempt( - ctx, - retryInterval, - retryIntervalSource, - activityFailure, - defaultTime, - tc.noRetriesLeft, - ) - require.NoError(t, err) - - tc.requireResult(t, activityFailure, attemptState.GetLastFailureDetails().GetFailure()) - }) - } -} diff --git a/chasm/lib/activity/attempt.go b/chasm/lib/activity/attempt.go new file mode 100644 index 00000000000..59646982c31 --- /dev/null +++ b/chasm/lib/activity/attempt.go @@ -0,0 +1,447 @@ +// A note on times and terminology: +// +// We name 3 times in the lifecycle of an activity attempt: +// +// schedule_time - the time at which the activity entered SCHEDULED state +// dispatch_time - the time at which the activity task is due to be dispatched to Matching (AddActivityTask) +// start_time - the time at which the activity enters STARTED state (Matching task picked up by poller) +// +// They are always ordered as: (schedule_time) <= (dispatch_time) < (start_time). +// +// A ScheduleToStart timeout applies to the time between dispatch and start. If there is a delay +// before dispatch (i.e. a start delay on the first attempt, or a backoff interval / next retry +// delay on a second or subsequent attempt) then schedule_time < dispatch_time. Otherwise, they are +// equal. +// +// The main Activity struct has a.ScheduleTime which is the schedule time of the first +// attempt; i.e. the time at which the activity was created. This is never changed. +// +// The naming situation is not perfectly clean. See e.g. the comment below on +// nextAttemptDispatchTime (which is called next_attempt_schedule_time in the public API). + +package activity + +import ( + "fmt" + "slices" + "time" + + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/backoff" + commonfailure "go.temporal.io/server/common/failure" + "go.temporal.io/server/common/retrypolicy" + "go.temporal.io/server/common/util" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func (a *Activity) createAddActivityTaskRequest(ctx chasm.Context, namespaceID string) (*matchingservice.AddActivityTaskRequest, error) { + // Get latest component ref and unmarshal into proto ref + componentRef, err := ctx.Ref(a) + if err != nil { + return nil, err + } + + // Note: No need to set the vector clock here, as the components track version conflicts for read/write + // TODO: Need to fill in VersionDirective once we decide how to handle versioning for standalone activities + return &matchingservice.AddActivityTaskRequest{ + NamespaceId: namespaceID, + ScheduleToStartTimeout: a.ScheduleToStartTimeout, + TaskQueue: a.GetTaskQueue(), + Priority: a.GetPriority(), + ComponentRef: componentRef, + Stamp: a.LastAttempt.Get(ctx).GetStamp(), + }, nil +} + +// dispatchTimeForAttempt returns the dispatch time of the given attempt. +func (a *Activity) dispatchTimeForAttempt(attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { + if dispatchTime := attempt.GetDispatchTime(); dispatchTime != nil { + return dispatchTime + } + if attempt.GetCount() == 1 { + return timestamppb.New(a.firstDispatchTime()) + } + return dispatchTimeForRetry(attempt) +} + +// dispatchTimeForRetry computes the time a retried attempt will be dispatched to Matching, +// as complete_time + retry_interval. Returns nil if either field is missing or zero. +func dispatchTimeForRetry(attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { + retryInterval := attempt.GetCurrentRetryInterval() + completeTime := attempt.GetCompleteTime() + if retryInterval != nil && retryInterval.AsDuration() > 0 && completeTime != nil { + return timestamppb.New(completeTime.AsTime().Add(retryInterval.AsDuration())) + } + return nil +} + +// nextAttemptDispatchTime is the dispatch_time of the attempt that is currently being waited for. +// It is null when the dispatch time has passed, in terminal states, and when paused or when an +// attempt is in progress, since in those states the dispatch time of a future attempt is unknown: +// we do not even know if there will be a next attempt. +// +// In the public Describe API response of SAA and WFA, this has the name next_attempt_schedule_time. +// In that field name, the term "schedule_time" is actually a dispatch time; specifically, the +// dispatch time defined by this method. +// +// For WFA, next_attempt_schedule_time is null prior to the first attempt since start delay is not +// supported, hence the activity is due to be dispatched to Matching as soon as the activity is +// created. But for SAA, if there's a start delay, then next_attempt_schedule_time is the +// dispatch_time (non-null). +func (a *Activity) nextAttemptDispatchTime(ctx chasm.Context, attempt *activitypb.ActivityAttemptState) *timestamppb.Timestamp { + if a.hasAttemptInProgress() || a.isPaused() || a.isTerminal() { + return nil + } + if t := a.dispatchTimeForAttempt(attempt); t != nil { + if t.AsTime().After(ctx.Now(a)) { + return t + } + } + return nil +} + +// currentRetryInterval is the retry interval if the activity is currently waiting for a retry; nil otherwise. +func (a *Activity) currentRetryInterval(ctx chasm.Context, attempt *activitypb.ActivityAttemptState) *durationpb.Duration { + if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { + if t := a.dispatchTimeForAttempt(attempt); t != nil { + if t.AsTime().After(ctx.Now(a)) { + return attempt.GetCurrentRetryInterval() + } + } + } + return nil +} + +// recordFailedAttempt records any failures resulting from a tried attempt, including worker application failures and +// start-to-close timeouts. Since the calls come from retried attempts we update the attempt failure info but leave +// the outcome failure empty to avoid duplication. +func (a *Activity) recordFailedAttempt( + ctx chasm.MutableContext, + retryInterval time.Duration, + retryIntervalSource activitypb.ActivityRetryIntervalSource, + failure *failurepb.Failure, + currentTime time.Time, + noRetriesLeft bool, +) error { + attempt := a.LastAttempt.Get(ctx) + + attemptFailure := failure + if !noRetriesLeft { + // Similar to workflow activity, truncate only retryable failure, not the final one. + attemptFailure = truncateRetryableFailure(ctx, failure) + } + + attempt.LastFailureDetails = &activitypb.ActivityAttemptState_LastFailureDetails{ + Failure: attemptFailure, + Time: timestamppb.New(currentTime), + } + attempt.CompleteTime = timestamppb.New(currentTime) + + if noRetriesLeft { + attempt.CurrentRetryInterval = nil + attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + } else { + attempt.CurrentRetryInterval = durationpb.New(retryInterval) + attempt.CurrentRetryIntervalSource = retryIntervalSource + } + return nil +} + +// recordScheduleToStartOrCloseTimeoutFailure records schedule-to-start or schedule-to-close timeout outcomes. Such +// timeouts are not retried, so we set the outcome failure directly and leave the attempt failure as is. +func (a *Activity) recordScheduleToStartOrCloseTimeoutFailure( + ctx chasm.MutableContext, + timeoutType enumspb.TimeoutType, + message string, + cause *failurepb.Failure, +) error { + failure := &failurepb.Failure{ + Message: message, + Cause: cause, + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: timeoutType, + LastHeartbeatDetails: a.lastHeartbeatDetails(ctx), + }, + }, + } + + a.Outcome.Get(ctx).Variant = &activitypb.ActivityOutcome_Failed_{ + Failed: &activitypb.ActivityOutcome_Failed{ + Failure: failure, + }, + } + + return nil +} + +// truncateRetryableFailure caps the size of a failure retained in the activity's state while it +// retries, mirroring MutableStateImpl.truncateRetryableActivityFailure for workflow activities. +func truncateRetryableFailure(ctx chasm.Context, attemptFailure *failurepb.Failure) *failurepb.Failure { + actCtx := activityContextFromChasm(ctx) + sizeLimit := actCtx.config.MutableStateActivityFailureSizeLimitError(ctx.NamespaceEntry().Name().String()) + if attemptFailure.Size() <= sizeLimit { + return attemptFailure + } + + // nonRetryable is set to false here as only failures of attempts that will be retried are + // truncated, so the value is only for visibility/debugging purposes. + serverFailure := commonfailure.NewServerFailure(common.FailureReasonFailureExceedsLimit, false) + serverFailure.Cause = commonfailure.Truncate(attemptFailure, sizeLimit) + return serverFailure +} + +// tryReschedule attempts to reschedule the activity for retry. It handles the cases of pause and +// reset requests that were received while the last attempt was in progress. failureRetryable +// reports whether the failure itself permits a retry (timeouts always do; RespondActivityTaskFailed +// depends on the failure). +func (a *Activity) tryReschedule( + ctx chasm.MutableContext, + failureRetryable bool, + overridingRetryInterval time.Duration, + failure *failurepb.Failure, +) (enumspb.RetryState, error) { + status := a.GetStatus() + if status == activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED { + event := rescheduleEvent{failure: failure} + if a.ResetShouldPause { + return enumspb.RETRY_STATE_IN_PROGRESS, TransitionResetAttemptFailedToPaused.Apply(a, ctx, event) + } + return enumspb.RETRY_STATE_IN_PROGRESS, TransitionResetAttemptFailedToScheduled.Apply(a, ctx, event) + } + if a.GetRetryPolicy() == nil { + return enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, nil + } + if status == activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED { + return enumspb.RETRY_STATE_CANCEL_REQUESTED, nil + } + retryState, retryInterval := a.shouldRetry(ctx, overridingRetryInterval) + if !failureRetryable { + retryState = enumspb.RETRY_STATE_NON_RETRYABLE_FAILURE + } + if retryState != enumspb.RETRY_STATE_IN_PROGRESS { + return retryState, nil + } + retryIntervalSource := activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY + if overridingRetryInterval > 0 { + retryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_WORKER_OVERRIDE + } + event := rescheduleEvent{retryInterval: retryInterval, retryIntervalSource: retryIntervalSource, failure: failure} + switch status { + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: + return enumspb.RETRY_STATE_IN_PROGRESS, TransitionAttemptFailedWhilePauseRequested.Apply(a, ctx, event) + default: + return enumspb.RETRY_STATE_IN_PROGRESS, TransitionRescheduled.Apply(a, ctx, event) + } +} + +func (a *Activity) shouldRetry(ctx chasm.Context, overridingRetryInterval time.Duration) (enumspb.RetryState, time.Duration) { + if !TransitionRescheduled.Possible(a) && + !TransitionAttemptFailedWhilePauseRequested.Possible(a) { + return enumspb.RETRY_STATE_UNSPECIFIED, 0 + } + attempt := a.LastAttempt.Get(ctx) + retryPolicy := a.RetryPolicy + enoughTime, retryInterval := a.hasEnoughTimeForRetry(ctx, overridingRetryInterval) + + if retryPolicy.GetMaximumAttempts() > 0 && attempt.GetCount() >= retryPolicy.GetMaximumAttempts() { + return enumspb.RETRY_STATE_MAXIMUM_ATTEMPTS_REACHED, retryInterval + } + if !enoughTime { + return enumspb.RETRY_STATE_TIMEOUT, retryInterval + } + return enumspb.RETRY_STATE_IN_PROGRESS, retryInterval +} + +// timeoutRetryable reports whether a StartToClose or Heartbeat timeout may be retried under the retry +// policy. +func (a *Activity) timeoutRetryable(timeoutType enumspb.TimeoutType) bool { + return !slices.Contains( + a.GetRetryPolicy().GetNonRetryableErrorTypes(), + retrypolicy.TimeoutFailureTypePrefix+timeoutType.String(), + ) +} + +// hasEnoughTimeForRetry checks if there is enough time left in the schedule-to-close timeout. If sufficient time +// remains, it will also return a valid retry interval. +func (a *Activity) hasEnoughTimeForRetry(ctx chasm.Context, overridingRetryInterval time.Duration) (bool, time.Duration) { + attempt := a.LastAttempt.Get(ctx) + + // Use overriding retry interval if provided, else calculate based on retry policy + retryInterval := overridingRetryInterval + if retryInterval <= 0 { + retryInterval = backoff.CalculateExponentialRetryInterval(a.RetryPolicy, attempt.Count) + } + + scheduleToClose := a.GetScheduleToCloseTimeout().AsDuration() + if scheduleToClose == 0 { + return true, retryInterval + } + + deadline := a.scheduleToCloseDeadline() + return ctx.Now(a).Add(retryInterval).Before(deadline), retryInterval +} + +func (a *Activity) firstDispatchTime() time.Time { + return a.ScheduleTime.AsTime().Add(a.GetStartDelay().AsDuration()) +} + +func (a *Activity) newActivityDispatchTask(ctx chasm.Context) *activitypb.ActivityDispatchTask { + dispatchReason := activitypb.DISPATCH_REASON_IMMEDIATE + if a.GetFirstAttemptStartedTime() != nil { + dispatchReason = activitypb.DISPATCH_REASON_RETRY + } else if a.GetStartDelay().AsDuration() > 0 { + dispatchReason = activitypb.DISPATCH_REASON_START_DELAY + } + + return &activitypb.ActivityDispatchTask{ + Stamp: a.LastAttempt.Get(ctx).GetStamp(), + DispatchReason: dispatchReason, + StartDelayBucket: startDelayBucket(a.GetStartDelay().AsDuration()), + } +} + +func startDelayBucket(delay time.Duration) activitypb.StartDelayBucket { + switch { + case delay <= 0: + return activitypb.START_DELAY_BUCKET_NONE + case delay < time.Minute: + return activitypb.START_DELAY_BUCKET_LT_1M + case delay < 10*time.Minute: + return activitypb.START_DELAY_BUCKET_1M_10M + case delay < time.Hour: + return activitypb.START_DELAY_BUCKET_10M_1H + case delay < 6*time.Hour: + return activitypb.START_DELAY_BUCKET_1H_6H + case delay < 24*time.Hour: + return activitypb.START_DELAY_BUCKET_6H_1D + case delay < 7*24*time.Hour: + return activitypb.START_DELAY_BUCKET_1D_7D + case delay <= 30*24*time.Hour: + return activitypb.START_DELAY_BUCKET_7D_30D + default: + return activitypb.START_DELAY_BUCKET_GT_30D + } +} + +// reissueDispatchAndScheduleToStart re-emits the ActivityDispatchTask and ScheduleToStart timeout task for +// a SCHEDULED activity. Retries fire at the retry time; first attempts dispatch now, lifted to +// honor any pending start_delay. +func (a *Activity) reissueDispatchAndScheduleToStart(ctx chasm.MutableContext, attempt *activitypb.ActivityAttemptState) { + var dispatchTime time.Time + if retryDispatchTime := dispatchTimeForRetry(attempt); retryDispatchTime != nil { + dispatchTime = retryDispatchTime.AsTime() + } else { + dispatchTime = a.dispatchTimeRespectingStartDelay(ctx.Now(a)) + } + attempt.DispatchTime = timestamppb.New(dispatchTime) + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime}, + a.newActivityDispatchTask(ctx), + ) + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, + &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}, + ) + } +} + +// reissueRunningAttemptTimers re-emits the StartToClose and Heartbeat timeout tasks for the +// currently-running attempt, anchored to the attempt's StartedTime. Called from options-update +// paths after stamp bump so the old tasks are invalidated and replaced with the (possibly +// updated) timeouts. No-op unless the activity is in a status where a worker holds the task token +// (STARTED / CANCEL_REQUESTED / PAUSE_REQUESTED / RESET_REQUESTED). +func (a *Activity) reissueRunningAttemptTimers(ctx chasm.MutableContext, attempt *activitypb.ActivityAttemptState) { + if !a.hasAttemptInProgress() { + return + } + if timeout := a.GetStartToCloseTimeout().AsDuration(); timeout > 0 { + deadline := attempt.GetStartedTime().AsTime().Add(timeout) + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: deadline}, + &activitypb.StartToCloseTimeoutTask{Stamp: attempt.GetStamp()}, + ) + } + if hbTimeout := a.GetHeartbeatTimeout().AsDuration(); hbTimeout > 0 { + // Next heartbeat fires at max(last recorded heartbeat, current attempt start) + heartbeat timeout. + lastHb, _ := a.LastHeartbeat.TryGet(ctx) + lastHbTime := util.MaxTime( + lastHb.GetRecordedTime().AsTime(), + attempt.GetStartedTime().AsTime(), + ).Add(hbTimeout) + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: lastHbTime}, + &activitypb.HeartbeatTimeoutTask{Stamp: attempt.GetStamp()}, + ) + } +} + +// dispatchTimeRespectingStartDelay advances a candidate dispatch time t to the first dispatch time +// (ScheduleTime + start_delay) while the activity has not yet been picked up by a worker, so that +// pre-dispatch re-scheduling (unpause, reset, options update) honors any remaining start_delay. +// Returns t unchanged if the first attempt has already started. +func (a *Activity) dispatchTimeRespectingStartDelay(t time.Time) time.Time { + if a.GetFirstAttemptStartedTime() != nil { + return t + } + if dispatchTime := a.firstDispatchTime(); dispatchTime.After(t) { + return dispatchTime + } + return t +} + +// reissueScheduleToClose bumps the ScheduleToCloseStamp and re-emits the ScheduleToClose timeout task +// at the current deadline. +func (a *Activity) reissueScheduleToClose(ctx chasm.MutableContext) { + if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { + a.ScheduleToCloseStamp++ + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: deadline}, + &activitypb.ScheduleToCloseTimeoutTask{Stamp: a.GetScheduleToCloseStamp()}, + ) + } +} + +// scheduleToCloseDeadline returns the absolute time at which the ScheduleToClose timeout expires, +// accounting for start delay. Returns zero time if no ScheduleToClose timeout is set. +func (a *Activity) scheduleToCloseDeadline() time.Time { + timeout := a.GetScheduleToCloseTimeout().AsDuration() + if timeout == 0 { + return time.Time{} + } + return a.firstDispatchTime().Add(timeout) +} + +func createStartToCloseTimeoutFailure() *failurepb.Failure { + return &failurepb.Failure{ + Message: fmt.Sprintf(common.FailureReasonActivityTimeout, enumspb.TIMEOUT_TYPE_START_TO_CLOSE.String()), + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, + }, + }, + } +} + +func createHeartbeatTimeoutFailure() *failurepb.Failure { + return &failurepb.Failure{ + Message: fmt.Sprintf(common.FailureReasonActivityTimeout, enumspb.TIMEOUT_TYPE_HEARTBEAT.String()), + FailureInfo: &failurepb.Failure_TimeoutFailureInfo{ + TimeoutFailureInfo: &failurepb.TimeoutFailureInfo{ + TimeoutType: enumspb.TIMEOUT_TYPE_HEARTBEAT, + }, + }, + } +} diff --git a/chasm/lib/activity/attempt_test.go b/chasm/lib/activity/attempt_test.go new file mode 100644 index 00000000000..38e5c506324 --- /dev/null +++ b/chasm/lib/activity/attempt_test.go @@ -0,0 +1,266 @@ +package activity + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/testing/protorequire" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestTryRescheduleRetryStatePrecedence(t *testing.T) { + testCases := []struct { + name string + status activitypb.ActivityExecutionStatus + retryPolicy *commonpb.RetryPolicy + expected enumspb.RetryState + finalStatus activitypb.ActivityExecutionStatus + }{ + { + name: "retry policy not set", + status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + expected: enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, + finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + }, + { + name: "retry policy not set takes precedence over cancellation", + status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + expected: enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, + finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + }, + { + name: "cancellation requested", + status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + retryPolicy: defaultRetryPolicy, + expected: enumspb.RETRY_STATE_CANCEL_REQUESTED, + finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + }, + { + name: "reset requested takes precedence over retry policy not set", + status: activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED, + expected: enumspb.RETRY_STATE_IN_PROGRESS, + finalStatus: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { + return defaultTime.Add(2 * time.Second) + }, + }, + } + injectActivityContext(t, ctx) // needed for last test case + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: tc.status, + RetryPolicy: tc.retryPolicy, + ScheduleTime: timestamppb.New(defaultTime), + ScheduleToCloseTimeout: durationpb.New(time.Second), + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 1}), + } + + retryState, err := activity.tryReschedule(ctx, false, 0, &failurepb.Failure{}) + + require.NoError(t, err) + require.Equal(t, tc.expected, retryState) + require.Equal(t, tc.finalStatus, activity.GetStatus()) + }) + } +} + +func TestStartDelayBucket(t *testing.T) { + testCases := []struct { + name string + delay time.Duration + expected activitypb.StartDelayBucket + }{ + {name: "negative", delay: -time.Second, expected: activitypb.START_DELAY_BUCKET_NONE}, + {name: "zero", delay: 0, expected: activitypb.START_DELAY_BUCKET_NONE}, + {name: "less than one minute", delay: time.Minute - time.Nanosecond, expected: activitypb.START_DELAY_BUCKET_LT_1M}, + {name: "one minute", delay: time.Minute, expected: activitypb.START_DELAY_BUCKET_1M_10M}, + {name: "ten minutes", delay: 10 * time.Minute, expected: activitypb.START_DELAY_BUCKET_10M_1H}, + {name: "one hour", delay: time.Hour, expected: activitypb.START_DELAY_BUCKET_1H_6H}, + {name: "six hours", delay: 6 * time.Hour, expected: activitypb.START_DELAY_BUCKET_6H_1D}, + {name: "one day", delay: 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_1D_7D}, + {name: "seven days", delay: 7 * 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_7D_30D}, + {name: "thirty days", delay: 30 * 24 * time.Hour, expected: activitypb.START_DELAY_BUCKET_7D_30D}, + {name: "more than thirty days", delay: 30*24*time.Hour + time.Nanosecond, expected: activitypb.START_DELAY_BUCKET_GT_30D}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expected, startDelayBucket(tc.delay)) + }) + } +} + +func TestNewActivityDispatchTask(t *testing.T) { + testCases := []struct { + name string + startDelay time.Duration + firstAttemptStartedTime *timestamppb.Timestamp + expectedReason activitypb.DispatchReason + expectedBucket activitypb.StartDelayBucket + }{ + { + name: "immediate", + expectedReason: activitypb.DISPATCH_REASON_IMMEDIATE, + expectedBucket: activitypb.START_DELAY_BUCKET_NONE, + }, + { + name: "start delay", + startDelay: time.Hour, + expectedReason: activitypb.DISPATCH_REASON_START_DELAY, + expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, + }, + { + name: "retry without configured start delay", + firstAttemptStartedTime: timestamppb.Now(), + expectedReason: activitypb.DISPATCH_REASON_RETRY, + expectedBucket: activitypb.START_DELAY_BUCKET_NONE, + }, + { + name: "retry preserves configured start delay bucket", + startDelay: time.Hour, + firstAttemptStartedTime: timestamppb.Now(), + expectedReason: activitypb.DISPATCH_REASON_RETRY, + expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + StartDelay: durationpb.New(tc.startDelay), + FirstAttemptStartedTime: tc.firstAttemptStartedTime, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Stamp: 1}), + } + + task := activity.newActivityDispatchTask(ctx) + require.Equal(t, int32(1), task.GetStamp()) + require.Equal(t, tc.expectedReason, task.GetDispatchReason()) + require.Equal(t, tc.expectedBucket, task.GetStartDelayBucket()) + }) + } +} + +// oversizedActivityFailure returns an application failure whose details push it past the retained +// failure size limit, mirroring the failure used by the workflow activity truncation tests. +func oversizedActivityFailure(t *testing.T) *failurepb.Failure { + activityFailure := &failurepb.Failure{ + Message: "activity failure with large details", + Source: "application", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "application-failure-type", + NonRetryable: false, + Details: &commonpb.Payloads{ + Payloads: []*commonpb.Payload{ + { + Data: make([]byte, defaultFailureSizeLimit*2), + }, + }, + }, + }}, + } + require.Greater(t, activityFailure.Size(), defaultFailureSizeLimit) + return activityFailure +} + +// withinLimitActivityFailure returns an application failure small enough to be retained whole. +func withinLimitActivityFailure(t *testing.T) *failurepb.Failure { + activityFailure := &failurepb.Failure{ + Message: "activity failure", + Source: "application", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "application-failure-type", + }}, + } + require.LessOrEqual(t, activityFailure.Size(), defaultFailureSizeLimit) + return activityFailure +} + +// recordFailedAttempt truncates the retryable failure, and not the final failure. +func TestRecordFailedAttempt_FailureTruncation(t *testing.T) { + requireTruncated := func(t *testing.T, sent, retained *failurepb.Failure) { + require.LessOrEqual(t, retained.Size(), defaultFailureSizeLimit) + require.Equal(t, common.FailureReasonFailureExceedsLimit, retained.GetMessage()) + require.NotNil(t, retained.GetServerFailureInfo()) + cause := retained.GetCause() + require.Equal(t, sent.GetMessage(), cause.GetMessage()) + require.Equal(t, "application-failure-type", cause.GetApplicationFailureInfo().GetType()) + require.Nil(t, cause.GetApplicationFailureInfo().GetDetails()) + } + requireKeptWhole := func(t *testing.T, sent, retained *failurepb.Failure) { + protorequire.ProtoEqual(t, sent, retained) + } + + testCases := []struct { + name string + failure func(*testing.T) *failurepb.Failure + noRetriesLeft bool + requireResult func(t *testing.T, sent, retained *failurepb.Failure) + }{ + { + name: "OversizedRetryableFailureIsTruncated", + failure: oversizedActivityFailure, + requireResult: requireTruncated, + }, + { + name: "RetryableFailureWithinLimitIsKeptWhole", + failure: withinLimitActivityFailure, + requireResult: requireKeptWhole, + }, + { + name: "OversizedTerminalFailureIsKeptWhole", + failure: oversizedActivityFailure, + noRetriesLeft: true, + requireResult: requireKeptWhole, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + ctx.HandleNow = func(chasm.Component) time.Time { return defaultTime } + injectActivityContext(t, ctx) + + attemptState := &activitypb.ActivityAttemptState{Count: 1} + activity := &Activity{ + LastAttempt: chasm.NewDataField(ctx, attemptState), + } + activityFailure := tc.failure(t) + + retryInterval, retryIntervalSource := time.Second, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY + if tc.noRetriesLeft { + retryInterval, retryIntervalSource = 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + } + + err := activity.recordFailedAttempt( + ctx, + retryInterval, + retryIntervalSource, + activityFailure, + defaultTime, + tc.noRetriesLeft, + ) + require.NoError(t, err) + + tc.requireResult(t, activityFailure, attemptState.GetLastFailureDetails().GetFailure()) + }) + } +} diff --git a/chasm/lib/activity/metrics.go b/chasm/lib/activity/metrics.go new file mode 100644 index 00000000000..9b979346d4a --- /dev/null +++ b/chasm/lib/activity/metrics.go @@ -0,0 +1,194 @@ +package activity + +import ( + "strconv" + "time" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/tqid" +) + +func (a *Activity) taskScheduleToStartMetricsHandler(ctx chasm.Context) metrics.Handler { + actCtx := activityContextFromChasm(ctx) + namespaceEntry := ctx.NamespaceEntry() + namespaceName := namespaceEntry.Name().String() + taskQueue := a.GetTaskQueue().GetName() + return metrics.GetPerTaskQueuePartitionTypeScope( + a.baseMetricsHandler(ctx, metrics.HistoryRecordActivityTaskStartedScope), + namespaceName, + tqid.UnsafeTaskQueueFamily(namespaceEntry.ID().String(), taskQueue). + TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY). + RootPartition(), + actCtx.config.BreakdownMetricsByTaskQueue(namespaceName, taskQueue, enumspb.TASK_QUEUE_TYPE_ACTIVITY), + ) +} + +// baseMetricsHandler adds only the operation tag. +func (a *Activity) baseMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { + return ctx.MetricsHandler().WithTags(metrics.OperationTag(operation)) +} + +// enrichedMetricsHandler adds standard activity tags in addition to the operation tag. +func (a *Activity) enrichedMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { + namespaceName := ctx.NamespaceEntry().Name() + // activityContextFromChasm panics if the context value is missing; this is intentional and + // indicates a library registration bug rather than a runtime error. + actCtx := activityContextFromChasm(ctx) + breakdownMetricsByTaskQueue := actCtx.config.BreakdownMetricsByTaskQueue + taskQueueFamily := a.GetTaskQueue().GetName() + return metrics.GetPerTaskQueueFamilyScope( + ctx.MetricsHandler(), + namespaceName.String(), + tqid.UnsafeTaskQueueFamily(namespaceName.String(), taskQueueFamily), + breakdownMetricsByTaskQueue(namespaceName.String(), taskQueueFamily, enumspb.TASK_QUEUE_TYPE_ACTIVITY), + metrics.OperationTag(operation), + metrics.ActivityTypeTag(a.GetActivityType().GetName()), + metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), + metrics.WorkflowTypeTag(WorkflowTypeTag), + ) +} + +func (a *Activity) emitOnAttemptTimedOutMetrics(metricsHandler metrics.Handler, timeoutType enumspb.TimeoutType) { + timeoutTag := metrics.StringTag("timeout_type", timeoutType.String()) + metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) +} + +func (a *Activity) emitOnAttemptFailedMetrics(ctx chasm.Context, metricsHandler metrics.Handler) { + attempt := a.LastAttempt.Get(ctx) + startedTime := attempt.GetStartedTime().AsTime() + + latency := time.Since(startedTime) + metrics.ActivityStartToCloseLatency.With(metricsHandler).Record(latency) + + metrics.ActivityTaskFail.With(metricsHandler).Record(1) +} + +// emitPayloadSizeMetric records the serialized size of a user payload. +func emitPayloadSizeMetric(metricsHandler metrics.Handler, size int) { + if size > 0 { + metrics.ActivityPayloadSize.With(metricsHandler).Record(int64(size)) + } +} + +// emitHeartbeatMetrics records the heartbeat count and payload size for heartbeat details. +func (a *Activity) emitHeartbeatMetrics(ctx chasm.Context, details *commonpb.Payloads) { + metricsHandler := a.baseMetricsHandler(ctx, metrics.HistoryRecordActivityTaskHeartbeatScope) + detailsSize := details.Size() + emitPayloadSizeMetric(metricsHandler, detailsSize) + metrics.ActivityHeartbeatCount.With(metricsHandler).Record( + 1, + metrics.StringTag("has_details", strconv.FormatBool(detailsSize > 0)), + ) +} + +func (a *Activity) emitOnCompletedMetrics( + ctx chasm.Context, + baseHandler metrics.Handler, + enrichedHandler metrics.Handler, + result *commonpb.Payloads, + attemptWasStarted bool, +) { + attempt := a.LastAttempt.Get(ctx) + startedTime := attempt.GetStartedTime().AsTime() + + if attemptWasStarted { + startToCloseLatency := time.Since(startedTime) + metrics.ActivityStartToCloseLatency.With(enrichedHandler).Record(startToCloseLatency) + } + + scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) + metrics.ActivityScheduleToCloseLatency.With(enrichedHandler).Record(scheduleToCloseLatency) + + metrics.ActivitySuccess.With(enrichedHandler).Record(1) + emitPayloadSizeMetric(baseHandler, result.Size()) +} + +func (a *Activity) emitOnFailedMetrics( + ctx chasm.Context, + baseHandler metrics.Handler, + enrichedHandler metrics.Handler, + failure *failurepb.Failure, +) { + attempt := a.LastAttempt.Get(ctx) + startedTime := attempt.GetStartedTime().AsTime() + + startToCloseLatency := time.Since(startedTime) + metrics.ActivityStartToCloseLatency.With(enrichedHandler).Record(startToCloseLatency) + + scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) + metrics.ActivityScheduleToCloseLatency.With(enrichedHandler).Record(scheduleToCloseLatency) + + metrics.ActivityTaskFail.With(enrichedHandler).Record(1) + metrics.ActivityFail.With(enrichedHandler).Record(1) + emitPayloadSizeMetric(baseHandler, failure.Size()) +} + +func (a *Activity) emitOnTerminatedMetrics( + metricsHandler metrics.Handler, +) { + // Terminated activities do not count as properly finished activities so we do not + // record any of the latency metrics. + metrics.ActivityTerminate.With(metricsHandler).Record(1) +} + +func (a *Activity) emitOnCanceledMetrics( + ctx chasm.Context, + metricsHandler metrics.Handler, + fromStatus activitypb.ActivityExecutionStatus, +) { + // Record start-to-close latency only while an attempt is running. SCHEDULED (incl. retry + // backoff) and PAUSED have no running attempt and a possibly-stale StartedTime. + attemptRunning := fromStatus != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED && + fromStatus != activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED + if attemptRunning { + if startedTime := a.LastAttempt.Get(ctx).GetStartedTime(); startedTime != nil { + metrics.ActivityStartToCloseLatency.With(metricsHandler).Record(time.Since(startedTime.AsTime())) + } + } + + scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) + metrics.ActivityScheduleToCloseLatency.With(metricsHandler).Record(scheduleToCloseLatency) + + metrics.ActivityCancel.With(metricsHandler).Record(1) +} + +func (a *Activity) emitOnTimedOutMetrics( + metricsHandler metrics.Handler, + timeoutType enumspb.TimeoutType, +) { + scheduleToCloseLatency := time.Since(a.GetScheduleTime().AsTime()) + metrics.ActivityScheduleToCloseLatency.With(metricsHandler).Record(scheduleToCloseLatency) + + timeoutTag := metrics.StringTag("timeout_type", timeoutType.String()) + metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) + metrics.ActivityTimeout.With(metricsHandler).Record(1, timeoutTag) +} + +func (a *Activity) emitOnPausedMetrics( + metricsHandler metrics.Handler, +) { + metrics.ActivityPause.With(metricsHandler).Record(1) +} + +func (a *Activity) emitOnUpdateOptionsMetrics( + metricsHandler metrics.Handler, +) { + metrics.ActivityUpdateOptions.With(metricsHandler).Record(1) +} + +func (a *Activity) emitOnUnpausedMetrics( + metricsHandler metrics.Handler, +) { + metrics.ActivityUnpause.With(metricsHandler).Record(1) +} + +func (a *Activity) emitOnResetMetrics( + metricsHandler metrics.Handler, +) { + metrics.ActivityReset.With(metricsHandler).Record(1) +} diff --git a/chasm/lib/activity/operator_commands.go b/chasm/lib/activity/operator_commands.go new file mode 100644 index 00000000000..62b66a310e7 --- /dev/null +++ b/chasm/lib/activity/operator_commands.go @@ -0,0 +1,592 @@ +package activity + +import ( + "fmt" + "math/rand" + "time" + + apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/activityoptions" + "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/retrypolicy" + "go.temporal.io/server/common/util" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +//revive:disable-next-line:cognitive-complexity +func (a *Activity) UpdateActivityExecutionOptions( + ctx chasm.MutableContext, + req *activitypb.UpdateActivityExecutionOptionsRequest, +) (*activitypb.UpdateActivityExecutionOptionsResponse, error) { + frontendReq := req.GetFrontendRequest() + requestID := frontendReq.GetRequestId() + if requestID != "" && requestID == a.GetLastUpdateOptionsRequestId() { + // A repeated request ID returns the current options, which may differ from the original response. + return a.updateActivityExecutionOptionsResponse(), nil + } + + switch a.Status { + case activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, + activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + activitypb.ACTIVITY_EXECUTION_STATUS_FAILED, + activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED, + activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, + activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: + return nil, serviceerror.NewFailedPreconditionf("Cannot update options for activity in state %s", a.Status.String()) + default: + } + + if a.ResetRestoreOptions { + return nil, serviceerror.NewFailedPrecondition( + "cannot update options while a deferred Reset(RestoreOriginalOptions) is pending") + } + + if frontendReq.GetRestoreOriginal() { + if err := validateOriginalOptionsRestorable(a.GetOriginalOptions()); err != nil { + return nil, err + } + } + + updateFields := map[string]struct{}{} + if mask := frontendReq.GetUpdateMask(); mask != nil { + updateFields = util.ParseFieldMask(mask) + } + + _, hasStartDelayInMask := updateFields["startDelay"] + if hasStartDelayInMask { + newDelay := frontendReq.GetActivityOptions().GetStartDelay() + if err := validateStartDelay(newDelay); err != nil { + return nil, err + } + if newDelay.AsDuration() > 0 { + actCtx := activityContextFromChasm(ctx) + if !actCtx.config.StartDelayEnabled(frontendReq.GetNamespace()) { + return nil, serviceerror.NewInvalidArgument("start_delay is not enabled for this namespace") + } + } + inDelayWindow := a.firstDispatchTime().After(ctx.Now(a)) + pausedBeforeFirstAttempt := a.GetFirstAttemptStartedTime() == nil && a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED + if !(inDelayWindow || pausedBeforeFirstAttempt) { //nolint:staticcheck // QF1001: DeMorgan rearrangement would not be an improvement + return nil, serviceerror.NewFailedPrecondition( + "cannot update start_delay: the first activity attempt has already been dispatched") + } + } + + attempt := a.LastAttempt.Get(ctx) + + if frontendReq.GetRestoreOriginal() { + ogOptions := a.GetOriginalOptions() + a.TaskQueue = common.CloneProto(ogOptions.GetTaskQueue()) + a.ScheduleToCloseTimeout = common.CloneProto(ogOptions.GetScheduleToCloseTimeout()) + a.ScheduleToStartTimeout = common.CloneProto(ogOptions.GetScheduleToStartTimeout()) + a.StartToCloseTimeout = common.CloneProto(ogOptions.GetStartToCloseTimeout()) + a.HeartbeatTimeout = common.CloneProto(ogOptions.GetHeartbeatTimeout()) + a.RetryPolicy = common.CloneProto(ogOptions.GetRetryPolicy()) + a.Priority = common.CloneProto(ogOptions.GetPriority()) + // start_delay only governs the first dispatch. Once the first attempt has started, restoring + // the original value would shift ScheduleToClose without affecting dispatch timing. + if a.GetFirstAttemptStartedTime() == nil { + a.StartDelay = common.CloneProto(ogOptions.GetStartDelay()) + } + } else { + if err := a.mergeActivityOptions(frontendReq); err != nil { + return nil, err + } + } + + // Recalculate policy-derived retry intervals based on the (possibly updated) retry policy. + // Worker-provided NextRetryDelay values are preserved for their already-scheduled retry. + if a.shouldRecalculateCurrentRetryInterval(attempt, frontendReq.GetRestoreOriginal(), updateFields) { + newInterval := backoff.CalculateExponentialRetryInterval(a.RetryPolicy, attempt.GetCount()-1) + attempt.CurrentRetryInterval = durationpb.New(newInterval) + } + + // Recreate the ScheduleToClose task at the (possibly updated) deadline. + a.reissueScheduleToClose(ctx) + + attempt.Stamp++ + + a.reissueRunningAttemptTimers(ctx, attempt) + if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED { + a.reissueDispatchAndScheduleToStart(ctx, attempt) + } + + metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityUpdateOptionsScope) + a.emitOnUpdateOptionsMetrics(metricsHandler) + + if requestID != "" { + a.LastUpdateOptionsRequestId = requestID + } + return a.updateActivityExecutionOptionsResponse(), nil +} + +func (a *Activity) updateActivityExecutionOptionsResponse() *activitypb.UpdateActivityExecutionOptionsResponse { + return &activitypb.UpdateActivityExecutionOptionsResponse{ + FrontendResponse: &workflowservice.UpdateActivityExecutionOptionsResponse{ + ActivityOptions: &apiactivitypb.ActivityOptions{ + TaskQueue: a.GetTaskQueue(), + ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), + ScheduleToStartTimeout: a.GetScheduleToStartTimeout(), + StartToCloseTimeout: a.GetStartToCloseTimeout(), + HeartbeatTimeout: a.GetHeartbeatTimeout(), + RetryPolicy: a.GetRetryPolicy(), + Priority: a.GetPriority(), + StartDelay: a.GetStartDelay(), + }, + }, + } +} + +// shouldRecalculateCurrentRetryInterval reports whether the pending retry's CurrentRetryInterval +// should be re-derived from the (possibly updated) retry policy. All of the following must hold: +// +// - the activity is waiting to retry, i.e. in SCHEDULED or PAUSED state (a running attempt has no +// pending backoff to recalculate); +// - a retry is actually pending (CurrentRetryInterval is set); +// - the update touches the retry policy: either RestoreOriginal (which replaces the whole policy), +// or the update mask includes "retryPolicy" or one of its subfields; +// - the pending interval is still policy-derived (CurrentRetryIntervalSource is RETRY_POLICY). +// A worker-provided NextRetryDelay override is preserved regardless of its value. An +// UNSPECIFIED source (attempt state persisted before this field existed) is treated the same +// as an override, since we can no longer tell whether it was policy-derived or a worker +// override, and preserving it is the safer default. +func (a *Activity) shouldRecalculateCurrentRetryInterval( + attempt *activitypb.ActivityAttemptState, + restoreOriginal bool, + updateFields map[string]struct{}, +) bool { + status := a.GetStatus() + if status != activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED && + status != activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED { + return false + } + + if attempt.GetCurrentRetryInterval() == nil { + return false + } + + if !restoreOriginal { + if !util.FieldMaskHasPathOrSubPath(updateFields, "retryPolicy") { + return false + } + } + + return attempt.GetCurrentRetryIntervalSource() == activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY +} + +// mergeActivityOptions applies the field mask from the request to the activity state. +// The structure mirrors the field-mask logic in service/history/api/updateactivityoptions/api.go +func (a *Activity) mergeActivityOptions( + req *workflowservice.UpdateActivityExecutionOptionsRequest, +) error { + updateFields := util.ParseFieldMask(req.GetUpdateMask()) + + // Build an ActivityOptions view of the current Activity state so we can use the shared merge function. + ao := &apiactivitypb.ActivityOptions{ + TaskQueue: common.CloneProto(a.TaskQueue), + ScheduleToCloseTimeout: common.CloneProto(a.ScheduleToCloseTimeout), + ScheduleToStartTimeout: common.CloneProto(a.ScheduleToStartTimeout), + StartToCloseTimeout: common.CloneProto(a.StartToCloseTimeout), + HeartbeatTimeout: common.CloneProto(a.HeartbeatTimeout), + Priority: common.CloneProto(a.Priority), + RetryPolicy: common.CloneProto(a.RetryPolicy), + StartDelay: common.CloneProto(a.StartDelay), + } + + if err := activityoptions.MergeActivityOptions(ao, common.CloneProto(req.GetActivityOptions()), updateFields); err != nil { + return err + } + + if util.FieldMaskHasSubPath(updateFields, "retryPolicy") { + if err := retrypolicy.Validate(ao.GetRetryPolicy()); err != nil { + return err + } + } + + // Re-normalize timeouts after the update so that relationships like + // start_to_close <= schedule_to_close and heartbeat <= start_to_close are preserved. + // This mirrors adjustActivityOptions for workflow-embedded activities. + if err := validateAndNormalizeTimeouts(req.GetActivityId(), a.GetActivityType().GetName(), durationpb.New(0), ao); err != nil { + return err + } + + // Write the merged and normalized options back to the Activity state fields. + a.TaskQueue = ao.TaskQueue + a.ScheduleToCloseTimeout = ao.ScheduleToCloseTimeout + a.ScheduleToStartTimeout = ao.ScheduleToStartTimeout + a.StartToCloseTimeout = ao.StartToCloseTimeout + a.HeartbeatTimeout = ao.HeartbeatTimeout + a.Priority = ao.Priority + a.RetryPolicy = ao.RetryPolicy + a.StartDelay = ao.StartDelay + + return nil +} + +func (a *Activity) handleCancellationRequested(ctx chasm.MutableContext, request *activitypb.RequestCancelActivityExecutionRequest) ( + *activitypb.RequestCancelActivityExecutionResponse, error, +) { + req := request.GetFrontendRequest() + newReqID := req.GetRequestId() + existingReqID := a.GetCancelState().GetRequestId() + + // Deduplicate first because a retry may arrive after the activity transitions to Canceled. + if newReqID != "" && existingReqID == newReqID { + return &activitypb.RequestCancelActivityExecutionResponse{}, nil + } + + if a.isTerminal() { + return nil, serviceerror.NewFailedPreconditionf("activity is in terminal state %v", a.GetStatus()) + } + + // Reject a second cancellation request with a different request ID. + if a.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED { + return nil, serviceerror.NewFailedPrecondition( + fmt.Sprintf("cancellation already requested with request ID %s", existingReqID)) + } + + hasAttemptInProgress := a.hasAttemptInProgress() + originalStatus := a.GetStatus() + + // Always transition to CancelRequested + // TODO: this is questionable, since CancelRequested is otherwise a state that implies an attempt + // is in progress. + if err := TransitionCancelRequested.Apply(a, ctx, req); err != nil { + return nil, err + } + + // Transition to Canceled if no attempt in progress; otherwise wait for worker response. + if !hasAttemptInProgress { + metricsHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) + err := TransitionCanceled.Apply(a, ctx, cancelEvent{ + metricsHandler: metricsHandler, + fromStatus: originalStatus, + details: &commonpb.Payloads{ + Payloads: []*commonpb.Payload{ + payload.EncodeString(req.GetReason()), + }, + }, + }) + if err != nil { + return nil, err + } + } + + return &activitypb.RequestCancelActivityExecutionResponse{}, nil +} + +func (a *Activity) handlePauseRequested(ctx chasm.MutableContext, req *activitypb.PauseActivityExecutionRequest) ( + *activitypb.PauseActivityExecutionResponse, error, +) { + // Deduplicate a replay of a request that already paused this activity, even if the + // activity has since been unpaused. Without this check, a delayed replay of an old + // Pause request would silently re-pause an activity that a later Unpause resumed. + newReqID := req.GetFrontendRequest().GetRequestId() + if newReqID != "" && a.LastPauseState.GetRequestId() == newReqID { + return &activitypb.PauseActivityExecutionResponse{}, nil + } + + canPause := TransitionPaused.Possible(a) + canRequestPause := TransitionPauseRequested.Possible(a) + if !canPause && !canRequestPause { + return nil, serviceerror.NewFailedPreconditionf("activity is in non-pausable state %v", a.GetStatus()) + } + + metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityPausedScope) + + event := pauseEvent{req: req.GetFrontendRequest(), metricsHandler: metricsHandler} + if canPause { + if err := TransitionPaused.Apply(a, ctx, event); err != nil { + return nil, err + } + } else { + if err := TransitionPauseRequested.Apply(a, ctx, event); err != nil { + return nil, err + } + } + return &activitypb.PauseActivityExecutionResponse{}, nil +} + +func (a *Activity) handleUnpauseRequested(ctx chasm.MutableContext, req *activitypb.UnpauseActivityExecutionRequest) ( + _ *activitypb.UnpauseActivityExecutionResponse, retErr error, +) { + frontendReq := req.GetFrontendRequest() + requestID := frontendReq.GetRequestId() + if requestID != "" && requestID == a.GetLastUnpauseRequestId() { + return &activitypb.UnpauseActivityExecutionResponse{}, nil + } + if requestID != "" { + defer func() { + if retErr == nil { + a.LastUnpauseRequestId = requestID + } + }() + } + + if a.isTerminal() { + return nil, serviceerror.NewFailedPreconditionf("activity is in terminal state %v", a.GetStatus()) + } + metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityUnpausedScope) + + event := unpauseEvent{req: frontendReq, metricsHandler: metricsHandler} + switch { + case TransitionUnpaused.Possible(a): + if err := TransitionUnpaused.Apply(a, ctx, event); err != nil { + return nil, err + } + case TransitionUnpausedWhilePauseRequested.Possible(a): + if err := TransitionUnpausedWhilePauseRequested.Apply(a, ctx, event); err != nil { + return nil, err + } + default: + return nil, serviceerror.NewFailedPreconditionf("activity is in non-unpausable state %v", a.GetStatus()) + } + a.emitOnUnpausedMetrics(metricsHandler) + return &activitypb.UnpauseActivityExecutionResponse{}, nil +} + +// isPaused reports whether the activity is currently paused (waiting) or has a pending pause request +// (worker still running). +func (a *Activity) isPaused() bool { + switch a.GetStatus() { + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, + activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: + return true + case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: + return a.GetResetShouldPause() + default: + return false + } +} + +// unpauseDispatchTime computes when an unpaused attempt should be dispatched +func (a *Activity) unpauseDispatchTime(ctx chasm.MutableContext, event unpauseEvent) time.Time { + unpauseTime := ctx.Now(a) + if jitter := event.req.GetJitter().AsDuration(); jitter > 0 { + unpauseTime = unpauseTime.Add(time.Duration(rand.Int63n(int64(jitter)))) //nolint:gosec + } + dispatchTime := a.dispatchTimeRespectingStartDelay(unpauseTime) + retryDispatchTime := dispatchTimeForRetry(a.LastAttempt.Get(ctx)) + if retryDispatchTime != nil && retryDispatchTime.AsTime().After(dispatchTime) { + return retryDispatchTime.AsTime() + } + return dispatchTime +} + +func (a *Activity) recordPauseState( + ctx chasm.MutableContext, + event pauseEvent, +) { + a.LastPauseState = &activitypb.ActivityPauseState{ + PauseTime: timestamppb.New(ctx.Now(a)), + Identity: event.req.GetIdentity(), + Reason: event.req.GetReason(), + RequestId: event.req.GetRequestId(), + } + a.emitOnPausedMetrics(event.metricsHandler) +} + +func (a *Activity) clearHeartbeatDetails(ctx chasm.MutableContext) { + if hb, ok := a.LastHeartbeat.TryGet(ctx); ok { + hb.Details = nil + hb.RecordedTime = nil + } +} + +// handleReset handles the activity execution reset. +// +// For SCHEDULED and PAUSED activities (no worker running): re-dispatches at attempt 1. Any pending +// retry backoff is discarded (reset clears CurrentRetryInterval), but a pending start_delay is +// honored so the re-dispatched attempt 1 does not fire before its original requested start time. A +// PAUSED activity is unpaused first — unless keepPaused is set, in which case the counter is reset +// to 1 but the activity stays PAUSED until a later unpause. +// +// For STARTED activities: transitions to RESET_REQUESTED. The worker is notified via +// ActivityReset=true on its next heartbeat response and continues to use its existing task token. +// If the attempt fails, the activity transitions back to SCHEDULED at attempt 1 via +// TransitionResetAttemptFailedToScheduled. +// +// RestoreOriginalOptions follows the same split: applied immediately for a non-running activity, +// and deferred for a running one, so the in-flight attempt is not disturbed — every restored option +// takes effect on the new attempt 1. +// +// For CANCEL_REQUESTED activities: rejected with FailedPrecondition; cancel takes precedence. +func (a *Activity) handleReset( + ctx chasm.MutableContext, + req *activitypb.ResetActivityExecutionRequest, +) (_ *activitypb.ResetActivityExecutionResponse, retErr error) { + frontendReq := req.GetFrontendRequest() + requestID := frontendReq.GetRequestId() + if requestID != "" && requestID == a.GetLastResetRequestId() { + return &activitypb.ResetActivityExecutionResponse{}, nil + } + if requestID != "" { + defer func() { + if retErr == nil { + a.LastResetRequestId = requestID + } + }() + } + + if frontendReq.GetRestoreOriginalOptions() { + if err := validateOriginalOptionsRestorable(a.GetOriginalOptions()); err != nil { + return nil, err + } + } + + metricsHandler := a.enrichedMetricsHandler(ctx, metrics.ActivityResetScope) + + switch a.Status { + case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: + return nil, serviceerror.NewFailedPrecondition("cannot reset an activity with a pending cancellation") + case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: + // A reset is already pending on this running attempt. For now we reject a second reset + // request clearly. + // TODO (dan): define desired behavior and implement + return nil, serviceerror.NewFailedPrecondition("cannot reset an activity with a pending reset") + case activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: + return a.deferResetWhileRunning(ctx, frontendReq, metricsHandler) + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: + // No worker is running; restore takes effect immediately. + if frontendReq.GetRestoreOriginalOptions() { + a.restoreOriginalOptions(ctx) + } + if frontendReq.GetKeepPaused() { + return a.resetKeepPaused(ctx, frontendReq, metricsHandler) + } + // No keepPaused: perform an immediate reset. restoreOriginalOptions (if requested) already + // ran above, so skip it in resetImmediately to avoid restoring twice. + return a.resetImmediately(ctx, frontendReq, metricsHandler, false) + case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED: + return a.resetImmediately(ctx, frontendReq, metricsHandler, frontendReq.GetRestoreOriginalOptions()) + default: + // Terminal or unspecified state. + return nil, serviceerror.NewFailedPrecondition("activity execution is not running") + } +} + +// deferResetWhileRunning defers reset mutations (option restore, heartbeat details clear, +// attempt-count rewind) while a worker is still executing (STARTED or PAUSE_REQUESTED), so that the +// in-flight attempt is undisturbed. The deferred reset applies when the worker yields. +func (a *Activity) deferResetWhileRunning( + ctx chasm.MutableContext, + frontendReq *workflowservice.ResetActivityExecutionRequest, + metricsHandler metrics.Handler, +) (*activitypb.ResetActivityExecutionResponse, error) { + keepPaused := frontendReq.GetKeepPaused() + pauseRequested := a.Status == activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED + + if pauseRequested && !keepPaused { + // Unpause; the deferred reset will apply on the next retry via STARTED->SCHEDULED. + if err := TransitionUnpausedWhilePauseRequested.Apply(a, ctx, unpauseEvent{ + req: &workflowservice.UnpauseActivityExecutionRequest{}, + metricsHandler: metricsHandler, + }); err != nil { + return nil, err + } + } + if frontendReq.GetRestoreOriginalOptions() { + a.ResetRestoreOptions = true + } + if frontendReq.GetResetHeartbeat() { + a.ResetShouldClearHeartbeat = true + } + // keepPaused on a paused (PAUSE_REQUESTED) activity preserves the pause: when the worker + // yields the activity lands back in PAUSED rather than SCHEDULED. + a.ResetShouldPause = keepPaused && pauseRequested + if err := TransitionResetRequested.Apply(a, ctx, nil); err != nil { + return nil, err + } + a.emitOnResetMetrics(metricsHandler) + return &activitypb.ResetActivityExecutionResponse{}, nil +} + +func (a *Activity) resetKeepPaused( + ctx chasm.MutableContext, + frontendReq *workflowservice.ResetActivityExecutionRequest, + metricsHandler metrics.Handler, +) (*activitypb.ResetActivityExecutionResponse, error) { + attempt := a.LastAttempt.Get(ctx) + attempt.Count = 1 + attempt.Stamp++ + attempt.CurrentRetryInterval = nil + attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + attempt.DispatchTime = nil + if frontendReq.GetResetHeartbeat() { + a.clearHeartbeatDetails(ctx) + } + a.emitOnResetMetrics(metricsHandler) + return &activitypb.ResetActivityExecutionResponse{}, nil +} + +func (a *Activity) resetImmediately( + ctx chasm.MutableContext, + frontendReq *workflowservice.ResetActivityExecutionRequest, + metricsHandler metrics.Handler, + restore bool, +) (*activitypb.ResetActivityExecutionResponse, error) { + if restore { + a.restoreOriginalOptions(ctx) + } + resetTime := ctx.Now(a) + if jitter := frontendReq.GetJitter().AsDuration(); jitter > 0 { + resetTime = resetTime.Add(time.Duration(rand.Int63n(int64(jitter)))) //nolint:gosec + } + if err := TransitionReset.Apply(a, ctx, resetEvent{ + req: frontendReq, + resetTime: resetTime, + metricsHandler: metricsHandler, + }); err != nil { + return nil, err + } + return &activitypb.ResetActivityExecutionResponse{}, nil +} + +// applyDeferredOptionRestore applies a Reset(RestoreOriginalOptions) that was deferred because a +// worker was running an attempt at reset time (see handleReset). +func (a *Activity) applyDeferredOptionRestore(ctx chasm.MutableContext) { + if !a.ResetRestoreOptions { + return + } + a.ResetRestoreOptions = false + a.restoreOriginalOptions(ctx) +} + +// applyDeferredHeartbeatClear applies a Reset(ResetHeartbeat) that was deferred because a worker was +// running an attempt at reset time (see handleReset). +func (a *Activity) applyDeferredHeartbeatClear(ctx chasm.MutableContext) { + if !a.ResetShouldClearHeartbeat { + return + } + a.ResetShouldClearHeartbeat = false + a.clearHeartbeatDetails(ctx) +} + +// restoreOriginalOptions resets the activity's options to the values it was originally scheduled +// with and reissues the ScheduleToClose timer at the resulting deadline. start_delay is restored +// only if the activity has never started. +func (a *Activity) restoreOriginalOptions(ctx chasm.MutableContext) { + og := a.GetOriginalOptions() + a.TaskQueue = common.CloneProto(og.GetTaskQueue()) + a.ScheduleToCloseTimeout = common.CloneProto(og.GetScheduleToCloseTimeout()) + a.ScheduleToStartTimeout = common.CloneProto(og.GetScheduleToStartTimeout()) + a.StartToCloseTimeout = common.CloneProto(og.GetStartToCloseTimeout()) + a.HeartbeatTimeout = common.CloneProto(og.GetHeartbeatTimeout()) + a.RetryPolicy = common.CloneProto(og.GetRetryPolicy()) + a.Priority = common.CloneProto(og.GetPriority()) + if a.GetFirstAttemptStartedTime() == nil { + a.StartDelay = common.CloneProto(og.GetStartDelay()) + } + a.reissueScheduleToClose(ctx) +} diff --git a/chasm/lib/activity/operator_commands_test.go b/chasm/lib/activity/operator_commands_test.go new file mode 100644 index 00000000000..e407807644a --- /dev/null +++ b/chasm/lib/activity/operator_commands_test.go @@ -0,0 +1,709 @@ +package activity + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/fieldmaskpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestRequestDeduplicationAfterTerminalState(t *testing.T) { + t.Run("CancellationAfterCanceled", func(t *testing.T) { + const requestID = "cancel-request-id" + + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, + CancelState: &activitypb.ActivityCancelState{ + RequestId: requestID, + }, + }, + } + + _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ + FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{ + RequestId: requestID, + }, + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) + }) + + t.Run("NewCancellationAfterCanceled", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, + CancelState: &activitypb.ActivityCancelState{ + RequestId: "original-request-id", + }, + }, + } + + _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ + FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{ + RequestId: "new-request-id", + }, + }) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.EqualError(t, err, "activity is in terminal state Canceled") + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) + }) + + t.Run("TerminationAfterTerminated", func(t *testing.T) { + const requestID = "terminate-request-id" + + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + TerminateState: &activitypb.ActivityTerminateState{ + RequestId: requestID, + }, + }, + } + + _, err := activity.Terminate(ctx, chasm.TerminateComponentRequest{ + RequestID: requestID, + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) + }) + + t.Run("NewTerminationAfterTerminated", func(t *testing.T) { + const requestID = "terminate-request-id" + + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + TerminateState: &activitypb.ActivityTerminateState{ + RequestId: requestID, + }, + }, + } + + _, err := activity.Terminate(ctx, chasm.TerminateComponentRequest{ + RequestID: "new-request-id", + }) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.EqualError(t, err, "already terminated with request ID "+requestID) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) + }) + + t.Run("PauseAfterTerminated", func(t *testing.T) { + const requestID = "pause-request-id" + + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + LastPauseState: &activitypb.ActivityPauseState{ + RequestId: requestID, + }, + }, + } + + _, err := activity.handlePauseRequested(ctx, &activitypb.PauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ + RequestId: requestID, + }, + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) + }) + + t.Run("NewPauseAfterTerminated", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + LastPauseState: &activitypb.ActivityPauseState{ + RequestId: "pause-request-id", + }, + }, + } + + _, err := activity.handlePauseRequested(ctx, &activitypb.PauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ + RequestId: "new-request-id", + }, + }) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.EqualError(t, err, "activity is in non-pausable state Terminated") + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, activity.Status) + }) +} + +// Check that we do not emit a StartToCloseLatency metric when cancelling an activity that has no +// attempt in progress. Cancelling a SCHEDULED or PAUSED activity transitions straight to Canceled, +// and the status captured before the CancelRequested transition must be forwarded so that the +// metric is not emitted. +func TestHandleCancellationRequestedDirectCancelMetrics(t *testing.T) { + testCases := []struct { + name string + status activitypb.ActivityExecutionStatus + }{ + { + name: "scheduled during retry backoff, stale StartedTime", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + }, + { + name: "paused during retry backoff, stale StartedTime", + status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return defaultTime }, + HandleMetricsHandler: func() metrics.Handler { return metricsHandler }, + HandleNamespaceEntry: testNamespaceEntry, + GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ + config: &Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + }, + }), + }, + } + + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + RetryPolicy: defaultRetryPolicy, + Status: tc.status, + ScheduleTime: timestamppb.New(defaultTime), + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + ScheduleToCloseTimeout: durationpb.New(defaultScheduleToCloseTimeout), + ScheduleToStartTimeout: durationpb.New(defaultScheduleToStartTimeout), + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + }, + // A stale StartedTime from a prior attempt: no attempt is currently running, so + // StartToCloseLatency must not be recorded on cancellation. + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + Count: 1, + StartedTime: timestamppb.New(defaultTime), + }), + Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), + } + + _, err := activity.handleCancellationRequested(ctx, &activitypb.RequestCancelActivityExecutionRequest{ + FrontendRequest: &workflowservice.RequestCancelActivityExecutionRequest{Reason: "test"}, + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, activity.Status) + + snapshot := capture.Snapshot() + require.NotEmpty(t, snapshot[metrics.ActivityCancel.Name()]) + require.NotEmpty(t, snapshot[metrics.ActivityScheduleToCloseLatency.Name()]) + require.Empty(t, snapshot[metrics.ActivityStartToCloseLatency.Name()], + "no attempt was running; StartToCloseLatency must not be recorded") + }) + } +} + +func TestUpdateStartedActivityExecutionOptionsDoesNotBumpStartedStamp(t *testing.T) { + testTime := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC) + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return testTime }, + HandleNamespaceEntry: testNamespaceEntry, + GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ + config: &Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + }, + }), + }, + } + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + ScheduleToCloseTimeout: durationpb.New(10 * time.Minute), + ScheduleToStartTimeout: durationpb.New(2 * time.Minute), + StartToCloseTimeout: durationpb.New(3 * time.Minute), + HeartbeatTimeout: durationpb.New(time.Minute), + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + Count: 1, + Stamp: 7, + StartedStamp: 7, + }), + Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), + } + attempt := activity.LastAttempt.Get(ctx) + originalStamp := attempt.GetStamp() + originalStartedStamp := attempt.GetStartedStamp() + + _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ + FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ + ActivityId: "test-activity-id", + RequestId: "update-request-id", + ActivityOptions: &apiactivitypb.ActivityOptions{ + HeartbeatTimeout: durationpb.New(2 * time.Minute), + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, + }, + }) + require.NoError(t, err) + + require.Equal(t, originalStamp+1, attempt.GetStamp()) + require.Equal(t, originalStartedStamp, attempt.GetStartedStamp()) +} + +func TestHandlePauseRequestedDedupBeforeValidation(t *testing.T) { + for _, status := range []activitypb.ActivityExecutionStatus{ + activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + } { + t.Run(status.String(), func(t *testing.T) { + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: status, + LastPauseState: &activitypb.ActivityPauseState{ + RequestId: "pause-request-id", + }, + }, + } + + _, err := activity.handlePauseRequested( + &chasm.MockMutableContext{}, + &activitypb.PauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.PauseActivityExecutionRequest{ + RequestId: "pause-request-id", + }, + }, + ) + require.NoError(t, err) + require.Equal(t, status, activity.GetStatus()) + }) + } +} + +func TestHandleUnpauseRequestedRequestID(t *testing.T) { + t.Run("deduplicates latest request ID", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + LastUnpauseRequestId: "unpause-request-id", + ScheduleToStartTimeout: durationpb.New(time.Minute), + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), + } + + _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ + RequestId: "unpause-request-id", + }, + }) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, activity.GetStatus()) + require.Equal(t, int32(3), activity.LastAttempt.Get(ctx).GetCount()) + }) + + t.Run("does not persist request ID when unpause fails", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + }, + } + + _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ + RequestId: "unpause-request-id", + }, + }) + var failedPreconditionErr *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPreconditionErr) + require.Empty(t, activity.GetLastUnpauseRequestId(), "a failed unpause must not be recorded for de-dup") + }) + + t.Run("does not record failed request ID", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + LastUnpauseRequestId: "previous-unpause-request-id", + }, + } + + _, err := activity.handleUnpauseRequested(ctx, &activitypb.UnpauseActivityExecutionRequest{ + FrontendRequest: &workflowservice.UnpauseActivityExecutionRequest{ + RequestId: "failed-unpause-request-id", + }, + }) + require.Error(t, err) + require.Equal(t, "previous-unpause-request-id", activity.GetLastUnpauseRequestId()) + }) +} + +func TestHandleResetRequestID(t *testing.T) { + t.Run("deduplicates latest request ID", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + LastResetRequestId: "reset-request-id", + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), + } + + _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ + FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ + RequestId: "reset-request-id", + }, + }) + require.NoError(t, err) + require.Equal(t, int32(3), activity.LastAttempt.Get(ctx).GetCount()) + }) + + t.Run("records successful request ID", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Count: 3}), + } + + _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ + FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ + RequestId: "reset-request-id", + }, + }) + require.NoError(t, err) + require.Equal(t, "reset-request-id", activity.GetLastResetRequestId()) + }) + + t.Run("does not record failed request ID", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + LastResetRequestId: "previous-reset-request-id", + }, + } + + _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ + FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ + RequestId: "failed-reset-request-id", + }, + }) + require.Error(t, err) + require.Equal(t, "previous-reset-request-id", activity.GetLastResetRequestId()) + }) +} + +func TestUpdateActivityExecutionOptionsRequestID(t *testing.T) { + t.Run("deduplicates latest request ID", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, + LastUpdateOptionsRequestId: "update-request-id", + }, + } + + resp, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ + FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ + RequestId: "update-request-id", + }, + }) + require.NoError(t, err) + require.Equal(t, "current-task-queue", resp.GetFrontendResponse().GetActivityOptions().GetTaskQueue().GetName()) + }) + + t.Run("records successful request ID", func(t *testing.T) { + ctx := newOperatorCommandTestContext(t) + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + Status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + ScheduleToCloseTimeout: durationpb.New(10 * time.Minute), + ScheduleToStartTimeout: durationpb.New(2 * time.Minute), + StartToCloseTimeout: durationpb.New(3 * time.Minute), + HeartbeatTimeout: durationpb.New(time.Minute), + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + Count: 1, + Stamp: 7, + StartedStamp: 7, + }), + Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), + } + + _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ + FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ + RequestId: "update-request-id", + ActivityOptions: &apiactivitypb.ActivityOptions{ + HeartbeatTimeout: durationpb.New(2 * time.Minute), + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, + }, + }) + require.NoError(t, err) + require.Equal(t, "update-request-id", activity.GetLastUpdateOptionsRequestId()) + }) + + t.Run("does not record failed request ID", func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + LastUpdateOptionsRequestId: "previous-update-request-id", + }, + } + + _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ + FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ + RequestId: "failed-update-request-id", + }, + }) + require.Error(t, err) + require.Equal(t, "previous-update-request-id", activity.GetLastUpdateOptionsRequestId()) + }) +} + +func newOperatorCommandTestContext(t *testing.T) *chasm.MockMutableContext { + t.Helper() + return &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, + HandleNamespaceEntry: testNamespaceEntry, + GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ + config: &Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + }, + }), + }, + } +} + +func TestMergeActivityOptionsRejectsInvalidMergedRetryPolicy(t *testing.T) { + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "T"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "Q"}, + ScheduleToCloseTimeout: durationpb.New(30 * time.Second), + ScheduleToStartTimeout: durationpb.New(20 * time.Second), + StartToCloseTimeout: durationpb.New(10 * time.Second), + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: durationpb.New(10 * time.Second), + BackoffCoefficient: 2, + MaximumInterval: durationpb.New(30 * time.Second), + MaximumAttempts: 5, + }, + }, + } + + err := activity.mergeActivityOptions(&workflowservice.UpdateActivityExecutionOptionsRequest{ + ActivityId: "act", + ActivityOptions: &apiactivitypb.ActivityOptions{ + RetryPolicy: &commonpb.RetryPolicy{ + InitialInterval: durationpb.New(60 * time.Second), + }, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"retry_policy.initial_interval"}}, + }) + require.ErrorContains(t, err, "MaximumInterval cannot be less than InitialInterval") +} + +func TestShouldRecalculateCurrentRetryInterval(t *testing.T) { + retryInterval := 2 * time.Second + + testCases := []struct { + name string + status activitypb.ActivityExecutionStatus + restoreOriginal bool + updateFields map[string]struct{} + currentRetryInterval *durationpb.Duration + retryIntervalSource activitypb.ActivityRetryIntervalSource + expectRecalculate bool + }{ + { + name: "retry policy subfield update with policy-derived interval", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, + expectRecalculate: true, + }, + { + name: "retry policy replacement with policy-derived interval", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"retryPolicy": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, + expectRecalculate: true, + }, + { + name: "restore original with policy-derived interval", + status: activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, + restoreOriginal: true, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, + expectRecalculate: true, + }, + { + name: "unrelated update preserves retry interval", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"heartbeatTimeout": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, + }, + { + name: "worker override is preserved regardless of value", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_WORKER_OVERRIDE, + }, + { + name: "unspecified source (pre-existing attempt state) is preserved", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, + }, + { + name: "started activity is not in retry backoff", + status: activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, + currentRetryInterval: durationpb.New(retryInterval), + retryIntervalSource: activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_RETRY_POLICY, + }, + { + name: "missing retry interval", + status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + updateFields: map[string]struct{}{"retryPolicy.initialInterval": {}}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: tc.status, + }, + } + attempt := &activitypb.ActivityAttemptState{ + CurrentRetryInterval: tc.currentRetryInterval, + CurrentRetryIntervalSource: tc.retryIntervalSource, + } + + got := activity.shouldRecalculateCurrentRetryInterval( + attempt, + tc.restoreOriginal, + tc.updateFields, + ) + + require.Equal(t, tc.expectRecalculate, got) + }) + } +} + +// TestUpdateActivityExecutionOptions_RestoreOriginal_RejectsMissingOriginalOptions mimics an +// activity persisted before original_options existed (e.g. created by a binary predating this +// field's introduction). Restoring such an activity's options has nothing valid to fall back to +// and must be rejected rather than silently wiping TaskQueue and both close/start timeouts. +func TestUpdateActivityExecutionOptions_RestoreOriginal_RejectsMissingOriginalOptions(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, + }, + } + + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + ActivityType: &commonpb.ActivityType{Name: "T"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, + ScheduleToCloseTimeout: durationpb.New(30 * time.Second), + StartToCloseTimeout: durationpb.New(10 * time.Second), + OriginalOptions: nil, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{}), + } + + _, err := activity.UpdateActivityExecutionOptions(ctx, &activitypb.UpdateActivityExecutionOptionsRequest{ + FrontendRequest: &workflowservice.UpdateActivityExecutionOptionsRequest{ + ActivityId: "act", + RestoreOriginal: true, + RequestId: "failed-update-request-id", + }, + }) + + require.Error(t, err) + require.Empty(t, activity.GetLastUpdateOptionsRequestId()) + require.Equal(t, "current-task-queue", activity.GetTaskQueue().GetName()) + require.Equal(t, 30*time.Second, activity.GetScheduleToCloseTimeout().AsDuration()) + require.Equal(t, 10*time.Second, activity.GetStartToCloseTimeout().AsDuration()) +} + +// TestHandleReset_RestoreOriginalOptions_RejectsMissingOriginalOptions covers the equivalent gap +// on the Reset(RestoreOriginalOptions=true) path, for an activity with no original_options +// snapshot (see the Update test above for the scenario this mimics). +func TestHandleReset_RestoreOriginalOptions_RejectsMissingOriginalOptions(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, + }, + } + + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + ActivityType: &commonpb.ActivityType{Name: "T"}, + TaskQueue: &taskqueuepb.TaskQueue{Name: "current-task-queue"}, + ScheduleToCloseTimeout: durationpb.New(30 * time.Second), + StartToCloseTimeout: durationpb.New(10 * time.Second), + OriginalOptions: nil, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{}), + } + + _, err := activity.handleReset(ctx, &activitypb.ResetActivityExecutionRequest{ + FrontendRequest: &workflowservice.ResetActivityExecutionRequest{ + ActivityId: "act", + RestoreOriginalOptions: true, + RequestId: "failed-reset-request-id", + }, + }) + + require.Error(t, err) + require.Empty(t, activity.GetLastResetRequestId()) + require.Equal(t, "current-task-queue", activity.GetTaskQueue().GetName()) +} diff --git a/chasm/lib/activity/responses.go b/chasm/lib/activity/responses.go new file mode 100644 index 00000000000..8aaa11103fb --- /dev/null +++ b/chasm/lib/activity/responses.go @@ -0,0 +1,292 @@ +package activity + +import ( + "fmt" + + apiactivitypb "go.temporal.io/api/activity/v1" //nolint:importas + callbackpb "go.temporal.io/api/callback/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Projection of activity state onto the API response protos. + +// InternalStatusToAPIStatus converts internal activity execution status to API status. +func InternalStatusToAPIStatus(status activitypb.ActivityExecutionStatus) enumspb.ActivityExecutionStatus { + switch status { + case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, + activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, + activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED, + activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: + return enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: + return enumspb.ACTIVITY_EXECUTION_STATUS_PAUSED + case activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED: + return enumspb.ACTIVITY_EXECUTION_STATUS_COMPLETED + case activitypb.ACTIVITY_EXECUTION_STATUS_FAILED: + return enumspb.ACTIVITY_EXECUTION_STATUS_FAILED + case activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED: + return enumspb.ACTIVITY_EXECUTION_STATUS_CANCELED + case activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED: + return enumspb.ACTIVITY_EXECUTION_STATUS_TERMINATED + case activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT: + return enumspb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT + case activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: + return enumspb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED + default: + panic(fmt.Sprintf("unknown activity execution status: %v", status)) //nolint:forbidigo + } +} + +func (a *Activity) runState() enumspb.PendingActivityState { + status := a.GetStatus() + switch status { + case activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED: + return enumspb.PENDING_ACTIVITY_STATE_SCHEDULED + case activitypb.ACTIVITY_EXECUTION_STATUS_STARTED: + return enumspb.PENDING_ACTIVITY_STATE_STARTED + case activitypb.ACTIVITY_EXECUTION_STATUS_RESET_REQUESTED: + // The worker is still executing under its existing task token; the public PendingActivityState + // enum does not have a RESET_REQUESTED variant. The reset is surfaced to the worker via + // ActivityReset=true on its next heartbeat response. + if a.isPaused() { + return enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED + } + return enumspb.PENDING_ACTIVITY_STATE_STARTED + case activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED: + return enumspb.PENDING_ACTIVITY_STATE_CANCEL_REQUESTED + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED: + return enumspb.PENDING_ACTIVITY_STATE_PAUSED + case activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED: + return enumspb.PENDING_ACTIVITY_STATE_PAUSE_REQUESTED + case activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, + activitypb.ACTIVITY_EXECUTION_STATUS_FAILED, + activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, + activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, + activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, + activitypb.ACTIVITY_EXECUTION_STATUS_UNSPECIFIED: + return enumspb.PENDING_ACTIVITY_STATE_UNSPECIFIED + default: + panic(fmt.Sprintf("unknown activity execution status: %v", status)) //nolint:forbidigo + } +} + +func (a *Activity) buildActivityExecutionInfo( + ctx chasm.Context, + request *workflowservice.DescribeActivityExecutionRequest, +) *apiactivitypb.ActivityExecutionInfo { + status := InternalStatusToAPIStatus(a.GetStatus()) + runState := a.runState() + + requestData := a.RequestData.Get(ctx) + attempt := a.LastAttempt.Get(ctx) + heartbeat, _ := a.LastHeartbeat.TryGet(ctx) + key := ctx.ExecutionKey() + executionInfo := ctx.ExecutionInfo() + var closeTime *timestamppb.Timestamp + var executionDuration *durationpb.Duration + if a.LifecycleState(ctx) != chasm.LifecycleStateRunning { + executionDuration = durationpb.New(executionInfo.CloseTime.Sub(a.GetScheduleTime().AsTime())) + closeTime = timestamppb.New(executionInfo.CloseTime) + } + + var expirationTime *timestamppb.Timestamp + if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { + expirationTime = timestamppb.New(deadline) + } + + sa := &commonpb.SearchAttributes{ + IndexedFields: a.Visibility.Get(ctx).CustomSearchAttributes(ctx), + } + + info := &apiactivitypb.ActivityExecutionInfo{ + ActivityId: key.BusinessID, + ActivityType: a.GetActivityType(), + Attempt: attempt.GetCount(), + CanceledReason: a.CancelState.GetReason(), + CloseTime: closeTime, + CurrentRetryInterval: a.currentRetryInterval(ctx, attempt), + ExecutionDuration: executionDuration, + ExecutionTime: timestamppb.New(a.firstDispatchTime()), + ExpirationTime: expirationTime, + Header: requestData.GetHeader(), + HeartbeatTimeout: a.GetHeartbeatTimeout(), + Links: ctx.Links(a), + TotalHeartbeatCount: heartbeat.GetTotalHeartbeatCount(), + LastAttemptCompleteTime: attempt.GetCompleteTime(), + LastHeartbeatTime: heartbeat.GetRecordedTime(), + LastStartedTime: attempt.GetStartedTime(), + LastWorkerIdentity: attempt.GetLastWorkerIdentity(), + LastDeploymentVersion: attempt.GetLastDeploymentVersion(), + SdkName: attempt.GetSdkName(), + SdkVersion: attempt.GetSdkVersion(), + NextAttemptScheduleTime: a.nextAttemptDispatchTime(ctx, attempt), + Priority: a.GetPriority(), + RetryPolicy: a.GetRetryPolicy(), + RunId: key.RunID, + RunState: runState, + ScheduleTime: a.GetScheduleTime(), + ScheduleToCloseTimeout: a.GetScheduleToCloseTimeout(), + ScheduleToStartTimeout: a.GetScheduleToStartTimeout(), + StartDelay: a.GetStartDelay(), + StartToCloseTimeout: a.GetStartToCloseTimeout(), + StateSizeBytes: int64(executionInfo.ApproximateStateSize), + StateTransitionCount: executionInfo.StateTransitionCount, + SearchAttributes: sa, + Status: status, + TaskQueue: a.GetTaskQueue().GetName(), + UserMetadata: a.effectiveUserMetadata(ctx), + } + if request.GetIncludeHeartbeatDetails() { + info.HeartbeatDetails = heartbeat.GetDetails() + } + if request.GetIncludeLastFailure() { + info.LastFailure = attempt.GetLastFailureDetails().GetFailure() + } + + return info +} + +func (a *Activity) buildDescribeActivityExecutionResponse( + ctx chasm.Context, + req *activitypb.DescribeActivityExecutionRequest, +) (*activitypb.DescribeActivityExecutionResponse, error) { + request := req.GetFrontendRequest() + + token, err := ctx.Ref(a) + if err != nil { + return nil, err + } + + info := a.buildActivityExecutionInfo(ctx, request) + + var input *commonpb.Payloads + if request.GetIncludeInput() { + input = a.RequestData.Get(ctx).GetInput() + } + + callbackInfos, err := a.buildCallbackInfos(ctx) + if err != nil { + return nil, err + } + + response := &workflowservice.DescribeActivityExecutionResponse{ + Info: info, + RunId: ctx.ExecutionKey().RunID, + Input: input, + LongPollToken: token, + Callbacks: callbackInfos, + } + + if request.GetIncludeOutcome() { + response.Outcome = a.outcome(ctx) + } + + return &activitypb.DescribeActivityExecutionResponse{ + FrontendResponse: response, + }, nil +} + +func (a *Activity) buildCallbackInfos(ctx chasm.Context) ([]*apiactivitypb.CallbackInfo, error) { + if len(a.Callbacks) == 0 { + return nil, nil + } + + cbInfos := make([]*apiactivitypb.CallbackInfo, 0, len(a.Callbacks)) + for _, field := range a.Callbacks { + cb := field.Get(ctx) + + cbSpec, err := cb.ToAPICallback() + if err != nil { + return nil, err + } + + var state enumspb.CallbackState + switch cb.Status { + case callbackspb.CALLBACK_STATUS_UNSPECIFIED: + return nil, serviceerror.NewInternal("callback with UNSPECIFIED state") + case callbackspb.CALLBACK_STATUS_STANDBY: + state = enumspb.CALLBACK_STATE_STANDBY + case callbackspb.CALLBACK_STATUS_SCHEDULED: + state = enumspb.CALLBACK_STATE_SCHEDULED + case callbackspb.CALLBACK_STATUS_BACKING_OFF: + state = enumspb.CALLBACK_STATE_BACKING_OFF + case callbackspb.CALLBACK_STATUS_FAILED: + state = enumspb.CALLBACK_STATE_FAILED + case callbackspb.CALLBACK_STATUS_SUCCEEDED: + state = enumspb.CALLBACK_STATE_SUCCEEDED + default: + return nil, serviceerror.NewInternalf("unknown callback state: %v", cb.Status) + } + + cbInfos = append(cbInfos, &apiactivitypb.CallbackInfo{ + Trigger: &apiactivitypb.CallbackInfo_Trigger{ + Variant: &apiactivitypb.CallbackInfo_Trigger_ActivityClosed{}, + }, + Info: &callbackpb.CallbackInfo{ + Callback: cbSpec, + RegistrationTime: cb.RegistrationTime, + State: state, + Attempt: cb.Attempt, + LastAttemptCompleteTime: cb.LastAttemptCompleteTime, + LastAttemptFailure: cb.LastAttemptFailure, + NextAttemptScheduleTime: cb.NextAttemptScheduleTime, + }, + }) + } + return cbInfos, nil +} + +func (a *Activity) buildPollActivityExecutionResponse( + ctx chasm.Context, +) *activitypb.PollActivityExecutionResponse { + return &activitypb.PollActivityExecutionResponse{ + FrontendResponse: &workflowservice.PollActivityExecutionResponse{ + RunId: ctx.ExecutionKey().RunID, + Outcome: a.outcome(ctx), + }, + } +} + +// outcome retrieves the activity outcome (result or failure) if the activity has completed. +// Returns nil if the activity has not completed. +func (a *Activity) outcome(ctx chasm.Context) *apiactivitypb.ActivityExecutionOutcome { + if !a.LifecycleState(ctx).IsClosed() { + return nil + } + activityOutcome := a.Outcome.Get(ctx) + if successful := activityOutcome.GetSuccessful(); successful != nil { + return &apiactivitypb.ActivityExecutionOutcome{ + Value: &apiactivitypb.ActivityExecutionOutcome_Result{Result: successful.GetOutput()}, + } + } + if failure := a.terminalFailure(ctx); failure != nil { + return &apiactivitypb.ActivityExecutionOutcome{ + Value: &apiactivitypb.ActivityExecutionOutcome_Failure{Failure: failure}, + RetryState: activityOutcome.GetRetryState(), + } + } + return nil +} + +// terminalFailure returns the failure for a closed activity. The failure may be stored in Outcome.Failed +// (terminated, canceled, timed out) or in LastAttempt.LastFailureDetails (failed after exhausting retries). +// Returns nil if no failure is found. +func (a *Activity) terminalFailure(ctx chasm.Context) *failurepb.Failure { + if f := a.Outcome.Get(ctx).GetFailed(); f != nil { + return f.GetFailure() + } + if details := a.LastAttempt.Get(ctx).GetLastFailureDetails(); details != nil { + return details.GetFailure() + } + return nil +} diff --git a/chasm/lib/activity/responses_test.go b/chasm/lib/activity/responses_test.go new file mode 100644 index 00000000000..106b3e69d27 --- /dev/null +++ b/chasm/lib/activity/responses_test.go @@ -0,0 +1,45 @@ +package activity + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + deploymentpb "go.temporal.io/api/deployment/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common/testing/protorequire" +) + +func TestBuildActivityExecutionInfo_IncludeLastDeploymentVersion(t *testing.T) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return time.Unix(0, 0) }, + }, + } + + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + LastDeploymentVersion: &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "test-deployment", + BuildId: "test-build-1", + }, + }), + RequestData: chasm.NewDataField(ctx, &activitypb.ActivityRequestData{}), + Visibility: chasm.NewComponentField(ctx, &chasm.Visibility{}), + } + + resp, err := activity.buildDescribeActivityExecutionResponse(ctx, &activitypb.DescribeActivityExecutionRequest{ + FrontendRequest: &workflowservice.DescribeActivityExecutionRequest{}, + }) + + require.NoError(t, err) + protorequire.ProtoEqual(t, &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "test-deployment", + BuildId: "test-build-1", + }, resp.FrontendResponse.GetInfo().GetLastDeploymentVersion()) +} diff --git a/chasm/lib/activity/statemachine.go b/chasm/lib/activity/statemachine.go index a1d342a9b37..a9cd9be2d07 100644 --- a/chasm/lib/activity/statemachine.go +++ b/chasm/lib/activity/statemachine.go @@ -34,6 +34,62 @@ func (a *Activity) SetStateMachineState(state activitypb.ActivityExecutionStatus a.Status = state } +type ( + rescheduleEvent struct { + retryInterval time.Duration + retryIntervalSource activitypb.ActivityRetryIntervalSource + failure *failurepb.Failure + timeoutType enumspb.TimeoutType + } + + completeEvent struct { + req *historyservice.RespondActivityTaskCompletedRequest + baseHandler metrics.Handler + enrichedHandler metrics.Handler + } + + failedEvent struct { + req *historyservice.RespondActivityTaskFailedRequest + retryState enumspb.RetryState + baseHandler metrics.Handler + enrichedHandler metrics.Handler + } + + terminateEvent struct { + request chasm.TerminateComponentRequest + metricsHandler metrics.Handler + fromStatus activitypb.ActivityExecutionStatus + } + + cancelEvent struct { + details *commonpb.Payloads + metricsHandler metrics.Handler + fromStatus activitypb.ActivityExecutionStatus + } + + timeoutEvent struct { + metricsHandler metrics.Handler + timeoutType enumspb.TimeoutType + retryState enumspb.RetryState + } + + pauseEvent struct { + req *workflowservice.PauseActivityExecutionRequest + metricsHandler metrics.Handler + } + + unpauseEvent struct { + req *workflowservice.UnpauseActivityExecutionRequest + metricsHandler metrics.Handler + } + + resetEvent struct { + req *workflowservice.ResetActivityExecutionRequest + resetTime time.Time + metricsHandler metrics.Handler + } +) + // TransitionScheduled transitions to Scheduled status. This is only called on the initial // scheduling of the activity. var TransitionScheduled = chasm.NewTransition( @@ -42,57 +98,10 @@ var TransitionScheduled = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, _ any) error { - attempt := a.LastAttempt.Get(ctx) - - attempt.Count++ - attempt.Stamp++ - - // Start delay defers the dispatch and extends ScheduleToClose and ScheduleToStart timeouts. StartToClose and - // Heartbeat timeouts are unaffected as they only start when a worker picks up the task. - dispatchTime := a.firstDispatchTime() - attempt.DispatchTime = timestamppb.New(dispatchTime) - - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: dispatchTime.Add(timeout), - }, - &activitypb.ScheduleToStartTimeoutTask{ - Stamp: attempt.GetStamp(), - }) - } - - if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { - a.ScheduleToCloseStamp++ - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: deadline, - }, - &activitypb.ScheduleToCloseTimeoutTask{Stamp: a.GetScheduleToCloseStamp()}) - } - - dispatchAttrs := chasm.TaskAttributes{} - if dispatchTime.After(a.ScheduleTime.AsTime()) { - dispatchAttrs.ScheduledTime = dispatchTime - } - ctx.AddTask( - a, - dispatchAttrs, - a.newActivityDispatchTask(ctx)) - - return nil + return a.applyScheduled(ctx) }, ) -type rescheduleEvent struct { - retryInterval time.Duration - retryIntervalSource activitypb.ActivityRetryIntervalSource - failure *failurepb.Failure - timeoutType enumspb.TimeoutType -} - // TransitionRescheduled transitions to Scheduled from Started, which happens on retries. The event // to pass in is the failure to be recorded from the previously failed attempt. var TransitionRescheduled = chasm.NewTransition( @@ -101,33 +110,7 @@ var TransitionRescheduled = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, event rescheduleEvent) error { - if err := a.applyFailedAttempt(ctx, event); err != nil { - return err - } - - attempt := a.LastAttempt.Get(ctx) - retryScheduledTime := dispatchTimeForRetry(attempt).AsTime() - attempt.DispatchTime = timestamppb.New(retryScheduledTime) - - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: retryScheduledTime.Add(timeout), - }, - &activitypb.ScheduleToStartTimeoutTask{ - Stamp: attempt.GetStamp(), - }) - } - - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: retryScheduledTime, - }, - a.newActivityDispatchTask(ctx)) - - return nil + return a.applyRescheduled(ctx, event) }, ) @@ -138,54 +121,10 @@ var TransitionStarted = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_STARTED, func(a *Activity, ctx chasm.MutableContext, request *historyservice.RecordActivityTaskStartedRequest) error { - attempt := a.LastAttempt.Get(ctx) - attempt.StartedTime = timestamppb.New(ctx.Now(a)) - attempt.StartedStamp = request.GetStamp() - // Record the first-ever worker pickup time once and never update on retries or resets. - if a.FirstAttemptStartedTime == nil { - a.FirstAttemptStartedTime = attempt.GetStartedTime() - } - attempt.StartRequestId = request.GetRequestId() - attempt.LastWorkerIdentity = request.GetPollRequest().GetIdentity() - attempt.SdkName = ctx.RequestHeader(headers.ClientNameHeaderName) - attempt.SdkVersion = ctx.RequestHeader(headers.ClientVersionHeaderName) - if versionDirective := request.GetVersionDirective().GetDeploymentVersion(); versionDirective != nil { - attempt.LastDeploymentVersion = &deploymentpb.WorkerDeploymentVersion{ - BuildId: versionDirective.GetBuildId(), - DeploymentName: versionDirective.GetDeploymentName(), - } - } - startTime := attempt.GetStartedTime().AsTime() - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: startTime.Add(a.GetStartToCloseTimeout().AsDuration()), - }, - &activitypb.StartToCloseTimeoutTask{ - Stamp: a.LastAttempt.Get(ctx).GetStamp(), - }) - - if heartbeatTimeout := a.GetHeartbeatTimeout().AsDuration(); heartbeatTimeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: startTime.Add(heartbeatTimeout), - }, - &activitypb.HeartbeatTimeoutTask{ - Stamp: attempt.GetStamp(), - }) - } - - return nil + return a.applyStarted(ctx, request) }, ) -type completeEvent struct { - req *historyservice.RespondActivityTaskCompletedRequest - baseHandler metrics.Handler - enrichedHandler metrics.Handler -} - // TransitionCompleted transitions to Completed status. SCHEDULED and PAUSED are included because // RespondActivityTaskCompletedById can force-complete an activity that has no attempt in // progress, mirroring workflow-activity behavior. @@ -200,42 +139,10 @@ var TransitionCompleted = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_COMPLETED, func(a *Activity, ctx chasm.MutableContext, event completeEvent) error { - return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { - req := event.req.GetCompleteRequest() - - attemptWasStarted := a.hasAttemptInProgress() - attempt := a.LastAttempt.Get(ctx) - if !attemptWasStarted { - // RespondActivityTaskCompletedById can complete an activity when no attempt is in - // progress. - attempt.StartedTime = timestamppb.New(ctx.Now(a)) - if a.FirstAttemptStartedTime == nil { - a.FirstAttemptStartedTime = attempt.StartedTime - } - } - attempt.CompleteTime = timestamppb.New(ctx.Now(a)) - attempt.LastWorkerIdentity = req.GetIdentity() - outcome := a.Outcome.Get(ctx) - outcome.Variant = &activitypb.ActivityOutcome_Successful_{ - Successful: &activitypb.ActivityOutcome_Successful{ - Output: req.GetResult(), - }, - } - - a.emitOnCompletedMetrics(ctx, event.baseHandler, event.enrichedHandler, req.GetResult(), attemptWasStarted) - - return nil - }) + return a.applyCompleted(ctx, event) }, ) -type failedEvent struct { - req *historyservice.RespondActivityTaskFailedRequest - retryState enumspb.RetryState - baseHandler metrics.Handler - enrichedHandler metrics.Handler -} - // TransitionFailed transitions to Failed status. var TransitionFailed = chasm.NewTransition( []activitypb.ActivityExecutionStatus{ @@ -246,41 +153,10 @@ var TransitionFailed = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_FAILED, func(a *Activity, ctx chasm.MutableContext, event failedEvent) error { - return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { - req := event.req.GetFailedRequest() - a.Outcome.Get(ctx).RetryState = event.retryState - - attempt := a.LastAttempt.Get(ctx) - attempt.LastWorkerIdentity = req.GetIdentity() - - // A worker may respond failed without a Failure. Synthesize a generic terminal failure so - // the closed activity still exposes a consumable outcome; otherwise PollActivityExecution - // returns a nil outcome and a client cannot tell the closed activity apart from one that - // simply has no result yet, and polls forever. (Workflow activities tolerate a nil failure - // because the SDK wraps the failed history event in an ActivityError; a standalone activity - // returns the raw outcome with no such wrapper.) - failure := req.GetFailure() - if failure == nil { - failure = &failurepb.Failure{Message: "activity task failed without failure details"} - } - - if err := a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true); err != nil { - return err - } - - a.emitOnFailedMetrics(ctx, event.baseHandler, event.enrichedHandler, req.GetFailure()) - - return nil - }) + return a.applyFailed(ctx, event) }, ) -type terminateEvent struct { - request chasm.TerminateComponentRequest - metricsHandler metrics.Handler - fromStatus activitypb.ActivityExecutionStatus -} - // TransitionTerminated transitions to Terminated status. var TransitionTerminated = chasm.NewTransition( []activitypb.ActivityExecutionStatus{ @@ -293,29 +169,7 @@ var TransitionTerminated = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_TERMINATED, func(a *Activity, ctx chasm.MutableContext, event terminateEvent) error { - return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { - a.TerminateState = &activitypb.ActivityTerminateState{ - RequestId: event.request.RequestID, - } - outcome := a.Outcome.Get(ctx) - failure := &failurepb.Failure{ - Message: event.request.Reason, - FailureInfo: &failurepb.Failure_TerminatedFailureInfo{ - TerminatedFailureInfo: &failurepb.TerminatedFailureInfo{ - Identity: event.request.Identity, - }, - }, - } - outcome.Variant = &activitypb.ActivityOutcome_Failed_{ - Failed: &activitypb.ActivityOutcome_Failed{ - Failure: failure, - }, - } - - a.emitOnTerminatedMetrics(event.metricsHandler) - - return nil - }) + return a.applyTerminated(ctx, event) }, ) @@ -331,26 +185,10 @@ var TransitionCancelRequested = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED, func(a *Activity, ctx chasm.MutableContext, req *workflowservice.RequestCancelActivityExecutionRequest) error { - a.CancelState = &activitypb.ActivityCancelState{ - Identity: req.GetIdentity(), - RequestId: req.GetRequestId(), - Reason: req.GetReason(), - RequestTime: timestamppb.New(ctx.Now(a)), - } - // Cancel takes precedence over a pending reset so clear its deferred intents - a.ResetRestoreOptions = false - a.ResetShouldClearHeartbeat = false - - return nil + return a.applyCancelRequested(ctx, req) }, ) -type cancelEvent struct { - details *commonpb.Payloads - metricsHandler metrics.Handler - fromStatus activitypb.ActivityExecutionStatus -} - // TransitionCanceled transitions to Canceled status. var TransitionCanceled = chasm.NewTransition( []activitypb.ActivityExecutionStatus{ @@ -358,36 +196,10 @@ var TransitionCanceled = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_CANCELED, func(a *Activity, ctx chasm.MutableContext, event cancelEvent) error { - return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { - outcome := a.Outcome.Get(ctx) - failure := &failurepb.Failure{ - Message: "Activity canceled", - FailureInfo: &failurepb.Failure_CanceledFailureInfo{ - CanceledFailureInfo: &failurepb.CanceledFailureInfo{ - Details: event.details, - Identity: a.GetCancelState().GetIdentity(), - }, - }, - } - outcome.Variant = &activitypb.ActivityOutcome_Failed_{ - Failed: &activitypb.ActivityOutcome_Failed{ - Failure: failure, - }, - } - - a.emitOnCanceledMetrics(ctx, event.metricsHandler, event.fromStatus) - - return nil - }) + return a.applyCanceled(ctx, event) }, ) -type timeoutEvent struct { - metricsHandler metrics.Handler - timeoutType enumspb.TimeoutType - retryState enumspb.RetryState -} - // TransitionTimedOut transitions to TimedOut status. var TransitionTimedOut = chasm.NewTransition( []activitypb.ActivityExecutionStatus{ @@ -400,64 +212,10 @@ var TransitionTimedOut = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_TIMED_OUT, func(a *Activity, ctx chasm.MutableContext, event timeoutEvent) error { - timeoutType := event.timeoutType - - return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { - a.Outcome.Get(ctx).RetryState = event.retryState - priorAttemptFailure := a.LastAttempt.Get(ctx).GetLastFailureDetails().GetFailure() - var err error - switch timeoutType { - case enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, - enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE: - err = a.recordScheduleToStartOrCloseTimeoutFailure( - ctx, - timeoutType, - fmt.Sprintf(common.FailureReasonActivityTimeout, timeoutType.String()), - priorAttemptFailure, - ) - case enumspb.TIMEOUT_TYPE_START_TO_CLOSE: - failure := createStartToCloseTimeoutFailure() - failure.GetTimeoutFailureInfo().LastHeartbeatDetails = a.lastHeartbeatDetails(ctx) - failure.Cause = priorAttemptFailure - err = a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true) - case enumspb.TIMEOUT_TYPE_HEARTBEAT: - failure := createHeartbeatTimeoutFailure() - failure.GetTimeoutFailureInfo().LastHeartbeatDetails = a.lastHeartbeatDetails(ctx) - failure.Cause = priorAttemptFailure - err = a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true) - default: - err = fmt.Errorf("unhandled activity timeout: %v", timeoutType) - } - if err != nil { - return err - } - - retryPreventedByScheduleToClose := event.retryState == enumspb.RETRY_STATE_TIMEOUT && - (timeoutType == enumspb.TIMEOUT_TYPE_START_TO_CLOSE || - timeoutType == enumspb.TIMEOUT_TYPE_HEARTBEAT) - if retryPreventedByScheduleToClose { - if err := a.recordScheduleToStartOrCloseTimeoutFailure( - ctx, - enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, - common.FailureReasonActivityRetryScheduleToCloseTimeout, - priorAttemptFailure, - ); err != nil { - return err - } - } - - a.emitOnTimedOutMetrics(event.metricsHandler, timeoutType) - - return nil - }) + return a.applyTimedOut(ctx, event) }, ) -type pauseEvent struct { - req *workflowservice.PauseActivityExecutionRequest - metricsHandler metrics.Handler -} - // TransitionPaused transitions a SCHEDULED activity to PAUSED status. The stamp is bumped to // invalidate any pending dispatch task so the activity is not dispatched while paused. var TransitionPaused = chasm.NewTransition( @@ -466,10 +224,7 @@ var TransitionPaused = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, func(a *Activity, ctx chasm.MutableContext, event pauseEvent) error { - a.recordPauseState(ctx, event) - attempt := a.LastAttempt.Get(ctx) - attempt.Stamp++ - return nil + return a.applyPaused(ctx, event) }, ) @@ -483,16 +238,10 @@ var TransitionPauseRequested = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSE_REQUESTED, func(a *Activity, ctx chasm.MutableContext, event pauseEvent) error { - a.recordPauseState(ctx, event) - return nil + return a.applyPauseRequested(ctx, event) }, ) -type unpauseEvent struct { - req *workflowservice.UnpauseActivityExecutionRequest - metricsHandler metrics.Handler -} - // TransitionUnpaused transitions PAUSED → SCHEDULED, triggering a dispatch task that will lead to // another activity attempt. var TransitionUnpaused = chasm.NewTransition( @@ -501,8 +250,7 @@ var TransitionUnpaused = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, event unpauseEvent) error { - a.unpause(ctx, event) - return nil + return a.applyUnpaused(ctx, event) }, ) @@ -532,12 +280,6 @@ var TransitionAttemptFailedWhilePauseRequested = chasm.NewTransition( }, ) -type resetEvent struct { - req *workflowservice.ResetActivityExecutionRequest - resetTime time.Time - metricsHandler metrics.Handler -} - // TransitionReset resets a SCHEDULED or PAUSED activity back to attempt 1. The stamp is bumped to // invalidate any pending dispatch task, then a new dispatch task is added at the given schedule time. // For STARTED activities the reset is deferred — the activity transitions to RESET_REQUESTED via @@ -550,8 +292,7 @@ var TransitionReset = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, event resetEvent) error { - a.reset(ctx, event) - return nil + return a.applyReset(ctx, event) }, ) @@ -584,20 +325,7 @@ var TransitionResetAttemptFailedToPaused = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, func(a *Activity, ctx chasm.MutableContext, event rescheduleEvent) error { - attempt := a.LastAttempt.Get(ctx) - a.ResetShouldPause = false - a.applyDeferredOptionRestore(ctx) - a.applyDeferredHeartbeatClear(ctx) - attempt.Count = 1 - attempt.Stamp++ - if err := a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, ctx.Now(a), false); err != nil { - return err - } - // Reset discards the retry backoff - attempt.CurrentRetryInterval = nil - attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - attempt.DispatchTime = nil - return nil + return a.applyResetAttemptFailedToPaused(ctx, event) }, ) @@ -611,42 +339,433 @@ var TransitionResetAttemptFailedToScheduled = chasm.NewTransition( }, activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, func(a *Activity, ctx chasm.MutableContext, event rescheduleEvent) error { + return a.applyResetAttemptFailedToScheduled(ctx, event) + }, +) + +// The methods below are transition bodies, invoked from the transitions above once the +// source state has been validated. Do not call them from outside this file. + +func (a *Activity) applyScheduled(ctx chasm.MutableContext) error { + attempt := a.LastAttempt.Get(ctx) + + attempt.Count++ + attempt.Stamp++ + + // Start delay defers the dispatch and extends ScheduleToClose and ScheduleToStart timeouts. StartToClose and + // Heartbeat timeouts are unaffected as they only start when a worker picks up the task. + dispatchTime := a.firstDispatchTime() + attempt.DispatchTime = timestamppb.New(dispatchTime) + + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: dispatchTime.Add(timeout), + }, + &activitypb.ScheduleToStartTimeoutTask{ + Stamp: attempt.GetStamp(), + }) + } + + if deadline := a.scheduleToCloseDeadline(); !deadline.IsZero() { + a.ScheduleToCloseStamp++ + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: deadline, + }, + &activitypb.ScheduleToCloseTimeoutTask{Stamp: a.GetScheduleToCloseStamp()}) + } + + dispatchAttrs := chasm.TaskAttributes{} + if dispatchTime.After(a.ScheduleTime.AsTime()) { + dispatchAttrs.ScheduledTime = dispatchTime + } + ctx.AddTask( + a, + dispatchAttrs, + a.newActivityDispatchTask(ctx)) + + return nil +} + +func (a *Activity) applyRescheduled(ctx chasm.MutableContext, event rescheduleEvent) error { + if err := a.applyFailedAttempt(ctx, event); err != nil { + return err + } + + attempt := a.LastAttempt.Get(ctx) + retryScheduledTime := dispatchTimeForRetry(attempt).AsTime() + attempt.DispatchTime = timestamppb.New(retryScheduledTime) + + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: retryScheduledTime.Add(timeout), + }, + &activitypb.ScheduleToStartTimeoutTask{ + Stamp: attempt.GetStamp(), + }) + } + + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: retryScheduledTime, + }, + a.newActivityDispatchTask(ctx)) + + return nil +} + +func (a *Activity) applyStarted(ctx chasm.MutableContext, request *historyservice.RecordActivityTaskStartedRequest) error { + attempt := a.LastAttempt.Get(ctx) + attempt.StartedTime = timestamppb.New(ctx.Now(a)) + attempt.StartedStamp = request.GetStamp() + // Record the first-ever worker pickup time once and never update on retries or resets. + if a.FirstAttemptStartedTime == nil { + a.FirstAttemptStartedTime = attempt.GetStartedTime() + } + attempt.StartRequestId = request.GetRequestId() + attempt.LastWorkerIdentity = request.GetPollRequest().GetIdentity() + attempt.SdkName = ctx.RequestHeader(headers.ClientNameHeaderName) + attempt.SdkVersion = ctx.RequestHeader(headers.ClientVersionHeaderName) + if versionDirective := request.GetVersionDirective().GetDeploymentVersion(); versionDirective != nil { + attempt.LastDeploymentVersion = &deploymentpb.WorkerDeploymentVersion{ + BuildId: versionDirective.GetBuildId(), + DeploymentName: versionDirective.GetDeploymentName(), + } + } + startTime := attempt.GetStartedTime().AsTime() + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: startTime.Add(a.GetStartToCloseTimeout().AsDuration()), + }, + &activitypb.StartToCloseTimeoutTask{ + Stamp: a.LastAttempt.Get(ctx).GetStamp(), + }) + + if heartbeatTimeout := a.GetHeartbeatTimeout().AsDuration(); heartbeatTimeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: startTime.Add(heartbeatTimeout), + }, + &activitypb.HeartbeatTimeoutTask{ + Stamp: attempt.GetStamp(), + }) + } + + return nil +} + +func (a *Activity) applyCompleted(ctx chasm.MutableContext, event completeEvent) error { + return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { + req := event.req.GetCompleteRequest() + + attemptWasStarted := a.hasAttemptInProgress() attempt := a.LastAttempt.Get(ctx) - currentTime := ctx.Now(a) + if !attemptWasStarted { + // RespondActivityTaskCompletedById can complete an activity when no attempt is in + // progress. + attempt.StartedTime = timestamppb.New(ctx.Now(a)) + if a.FirstAttemptStartedTime == nil { + a.FirstAttemptStartedTime = attempt.StartedTime + } + } + attempt.CompleteTime = timestamppb.New(ctx.Now(a)) + attempt.LastWorkerIdentity = req.GetIdentity() + outcome := a.Outcome.Get(ctx) + outcome.Variant = &activitypb.ActivityOutcome_Successful_{ + Successful: &activitypb.ActivityOutcome_Successful{ + Output: req.GetResult(), + }, + } + + a.emitOnCompletedMetrics(ctx, event.baseHandler, event.enrichedHandler, req.GetResult(), attemptWasStarted) + + return nil + }) +} - a.ResetShouldPause = false - a.applyDeferredOptionRestore(ctx) - a.applyDeferredHeartbeatClear(ctx) +// applyFailed is the body of TransitionFailed: it records a terminal failure outcome. +// Not to be confused with applyFailedAttempt/recordFailedAttempt, which record the +// failure of a single attempt that may still be retried. +func (a *Activity) applyFailed(ctx chasm.MutableContext, event failedEvent) error { + return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { + req := event.req.GetFailedRequest() + a.Outcome.Get(ctx).RetryState = event.retryState - attempt.Count = 1 - attempt.Stamp++ + attempt := a.LastAttempt.Get(ctx) + attempt.LastWorkerIdentity = req.GetIdentity() + + // A worker may respond failed without a Failure. Synthesize a generic terminal failure so + // the closed activity still exposes a consumable outcome; otherwise PollActivityExecution + // returns a nil outcome and a client cannot tell the closed activity apart from one that + // simply has no result yet, and polls forever. (Workflow activities tolerate a nil failure + // because the SDK wraps the failed history event in an ActivityError; a standalone activity + // returns the raw outcome with no such wrapper.) + failure := req.GetFailure() + if failure == nil { + failure = &failurepb.Failure{Message: "activity task failed without failure details"} + } - if err := a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, currentTime, false); err != nil { + if err := a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true); err != nil { return err } - // Reset discards the retry backoff - attempt.CurrentRetryInterval = nil - attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED - - dispatchTime := a.dispatchTimeRespectingStartDelay(currentTime) - attempt.DispatchTime = timestamppb.New(dispatchTime) - if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { - ctx.AddTask( - a, - chasm.TaskAttributes{ - ScheduledTime: dispatchTime.Add(timeout), + + a.emitOnFailedMetrics(ctx, event.baseHandler, event.enrichedHandler, req.GetFailure()) + + return nil + }) +} + +func (a *Activity) applyTerminated(ctx chasm.MutableContext, event terminateEvent) error { + return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { + a.TerminateState = &activitypb.ActivityTerminateState{ + RequestId: event.request.RequestID, + } + outcome := a.Outcome.Get(ctx) + failure := &failurepb.Failure{ + Message: event.request.Reason, + FailureInfo: &failurepb.Failure_TerminatedFailureInfo{ + TerminatedFailureInfo: &failurepb.TerminatedFailureInfo{ + Identity: event.request.Identity, }, - &activitypb.ScheduleToStartTimeoutTask{ - Stamp: attempt.GetStamp(), - }) + }, } + outcome.Variant = &activitypb.ActivityOutcome_Failed_{ + Failed: &activitypb.ActivityOutcome_Failed{ + Failure: failure, + }, + } + + a.emitOnTerminatedMetrics(event.metricsHandler) + + return nil + }) +} + +func (a *Activity) applyCancelRequested(ctx chasm.MutableContext, req *workflowservice.RequestCancelActivityExecutionRequest) error { + a.CancelState = &activitypb.ActivityCancelState{ + Identity: req.GetIdentity(), + RequestId: req.GetRequestId(), + Reason: req.GetReason(), + RequestTime: timestamppb.New(ctx.Now(a)), + } + // Cancel takes precedence over a pending reset so clear its deferred intents + a.ResetRestoreOptions = false + a.ResetShouldClearHeartbeat = false + + return nil +} + +func (a *Activity) applyCanceled(ctx chasm.MutableContext, event cancelEvent) error { + return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { + outcome := a.Outcome.Get(ctx) + failure := &failurepb.Failure{ + Message: "Activity canceled", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{ + Details: event.details, + Identity: a.GetCancelState().GetIdentity(), + }, + }, + } + outcome.Variant = &activitypb.ActivityOutcome_Failed_{ + Failed: &activitypb.ActivityOutcome_Failed{ + Failure: failure, + }, + } + + a.emitOnCanceledMetrics(ctx, event.metricsHandler, event.fromStatus) + + return nil + }) +} + +func (a *Activity) applyTimedOut(ctx chasm.MutableContext, event timeoutEvent) error { + timeoutType := event.timeoutType + + return a.StoreOrSelf(ctx).RecordCompleted(ctx, func(ctx chasm.MutableContext) error { + a.Outcome.Get(ctx).RetryState = event.retryState + priorAttemptFailure := a.LastAttempt.Get(ctx).GetLastFailureDetails().GetFailure() + var err error + switch timeoutType { + case enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, + enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE: + err = a.recordScheduleToStartOrCloseTimeoutFailure( + ctx, + timeoutType, + fmt.Sprintf(common.FailureReasonActivityTimeout, timeoutType.String()), + priorAttemptFailure, + ) + case enumspb.TIMEOUT_TYPE_START_TO_CLOSE: + failure := createStartToCloseTimeoutFailure() + failure.GetTimeoutFailureInfo().LastHeartbeatDetails = a.lastHeartbeatDetails(ctx) + failure.Cause = priorAttemptFailure + err = a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true) + case enumspb.TIMEOUT_TYPE_HEARTBEAT: + failure := createHeartbeatTimeoutFailure() + failure.GetTimeoutFailureInfo().LastHeartbeatDetails = a.lastHeartbeatDetails(ctx) + failure.Cause = priorAttemptFailure + err = a.recordFailedAttempt(ctx, 0, activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED, failure, ctx.Now(a), true) + default: + err = fmt.Errorf("unhandled activity timeout: %v", timeoutType) + } + if err != nil { + return err + } + + retryPreventedByScheduleToClose := event.retryState == enumspb.RETRY_STATE_TIMEOUT && + (timeoutType == enumspb.TIMEOUT_TYPE_START_TO_CLOSE || + timeoutType == enumspb.TIMEOUT_TYPE_HEARTBEAT) + if retryPreventedByScheduleToClose { + if err := a.recordScheduleToStartOrCloseTimeoutFailure( + ctx, + enumspb.TIMEOUT_TYPE_SCHEDULE_TO_CLOSE, + common.FailureReasonActivityRetryScheduleToCloseTimeout, + priorAttemptFailure, + ); err != nil { + return err + } + } + + a.emitOnTimedOutMetrics(event.metricsHandler, timeoutType) + + return nil + }) +} + +func (a *Activity) applyPaused(ctx chasm.MutableContext, event pauseEvent) error { + a.recordPauseState(ctx, event) + attempt := a.LastAttempt.Get(ctx) + attempt.Stamp++ + return nil +} + +// applyPauseRequested is the body of TransitionPauseRequested: it records the pause +// request. Unlike applyPaused it does not bump the attempt stamp, because the worker +// remains in charge of the attempt and its StartToClose and Heartbeat tasks must stay +// valid. +func (a *Activity) applyPauseRequested(ctx chasm.MutableContext, event pauseEvent) error { + a.recordPauseState(ctx, event) + return nil +} + +func (a *Activity) applyUnpaused( + ctx chasm.MutableContext, + event unpauseEvent, +) error { + attempt := a.LastAttempt.Get(ctx) + dispatchTime := a.unpauseDispatchTime(ctx, event) + attempt.DispatchTime = timestamppb.New(dispatchTime) + + attempt.Stamp++ + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, + &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}) + } + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime}, + a.newActivityDispatchTask(ctx)) + return nil +} + +// applyFailedAttempt mutates activity state when a worker yields with retries remaining. +func (a *Activity) applyFailedAttempt(ctx chasm.MutableContext, event rescheduleEvent) error { + attempt := a.LastAttempt.Get(ctx) + attempt.Count++ + attempt.Stamp++ + return a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, ctx.Now(a), false) +} + +func (a *Activity) applyReset(ctx chasm.MutableContext, event resetEvent) error { + attempt := a.LastAttempt.Get(ctx) + attempt.Count = 1 + attempt.Stamp++ + attempt.CurrentRetryInterval = nil + attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + if event.req.GetResetHeartbeat() { + a.clearHeartbeatDetails(ctx) + } + dispatchTime := a.dispatchTimeRespectingStartDelay(event.resetTime) + attempt.DispatchTime = timestamppb.New(dispatchTime) + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime.Add(timeout)}, + &activitypb.ScheduleToStartTimeoutTask{Stamp: attempt.GetStamp()}, + ) + } + ctx.AddTask( + a, + chasm.TaskAttributes{ScheduledTime: dispatchTime}, + a.newActivityDispatchTask(ctx), + ) + a.emitOnResetMetrics(event.metricsHandler) + return nil +} + +func (a *Activity) applyResetAttemptFailedToPaused(ctx chasm.MutableContext, event rescheduleEvent) error { + attempt := a.LastAttempt.Get(ctx) + a.ResetShouldPause = false + a.applyDeferredOptionRestore(ctx) + a.applyDeferredHeartbeatClear(ctx) + attempt.Count = 1 + attempt.Stamp++ + if err := a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, ctx.Now(a), false); err != nil { + return err + } + // Reset discards the retry backoff + attempt.CurrentRetryInterval = nil + attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + attempt.DispatchTime = nil + return nil +} + +func (a *Activity) applyResetAttemptFailedToScheduled(ctx chasm.MutableContext, event rescheduleEvent) error { + attempt := a.LastAttempt.Get(ctx) + currentTime := ctx.Now(a) + + a.ResetShouldPause = false + a.applyDeferredOptionRestore(ctx) + a.applyDeferredHeartbeatClear(ctx) + + attempt.Count = 1 + attempt.Stamp++ + + if err := a.recordFailedAttempt(ctx, event.retryInterval, event.retryIntervalSource, event.failure, currentTime, false); err != nil { + return err + } + // Reset discards the retry backoff + attempt.CurrentRetryInterval = nil + attempt.CurrentRetryIntervalSource = activitypb.ACTIVITY_RETRY_INTERVAL_SOURCE_UNSPECIFIED + + dispatchTime := a.dispatchTimeRespectingStartDelay(currentTime) + attempt.DispatchTime = timestamppb.New(dispatchTime) + if timeout := a.GetScheduleToStartTimeout().AsDuration(); timeout > 0 { ctx.AddTask( a, chasm.TaskAttributes{ - ScheduledTime: dispatchTime, + ScheduledTime: dispatchTime.Add(timeout), }, - a.newActivityDispatchTask(ctx)) - - return nil - }, -) + &activitypb.ScheduleToStartTimeoutTask{ + Stamp: attempt.GetStamp(), + }) + } + ctx.AddTask( + a, + chasm.TaskAttributes{ + ScheduledTime: dispatchTime, + }, + a.newActivityDispatchTask(ctx)) + + return nil +} diff --git a/chasm/lib/activity/activity_tasks.go b/chasm/lib/activity/tasks.go similarity index 100% rename from chasm/lib/activity/activity_tasks.go rename to chasm/lib/activity/tasks.go diff --git a/chasm/lib/activity/activity_tasks_test.go b/chasm/lib/activity/tasks_test.go similarity index 100% rename from chasm/lib/activity/activity_tasks_test.go rename to chasm/lib/activity/tasks_test.go From adf694f823cfedb05d78fb955df92f5ac9c5a48c Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 08:35:43 -0700 Subject: [PATCH 046/139] Reuse test context for namespace setup (#11623) ## What changed? Pass one test-owned context through namespace creation, namespace cache polling, and search attribute registration during functional test setup. ## Why? Reusing the test context avoids creating independent timeout contexts for each setup RPC and ties their resources to the test lifecycle. --- tests/archival_test.go | 2 ++ tests/testcore/functional_test_base.go | 15 +++++++++------ tests/testcore/test_env.go | 3 ++- tests/timeskipping_test.go | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/archival_test.go b/tests/archival_test.go index 8e71780d33c..4be1a1519d7 100644 --- a/tests/archival_test.go +++ b/tests/archival_test.go @@ -174,6 +174,7 @@ func (s *ArchivalSuite) newTestEnv() *archivalTestEnv { // Register namespace using built-in filestore archiver ae.archivalNamespace = namespace.Name(testcore.RandomizeStr("archival-enabled-namespace")) ae.archivalNamespaceID, err = ae.RegisterNamespace( + s.Context(), ae.archivalNamespace, 0, // Archive right away. enumspb.ARCHIVAL_STATE_ENABLED, @@ -187,6 +188,7 @@ func (s *ArchivalSuite) newTestEnv() *archivalTestEnv { customHistoryURI := customArchiverScheme + "://custom-history-archiver" customVisibilityURI := customArchiverScheme + "://custom-visibility-archiver" ae.customArchiverNamespaceID, err = ae.RegisterNamespace( + s.Context(), ae.customArchiverNamespace, 0, // Archive right away. enumspb.ARCHIVAL_STATE_ENABLED, diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index 7af807f6dc3..20dc013812f 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -40,6 +40,7 @@ import ( "go.temporal.io/server/common/testing/historyrequire" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/taskpoller" + "go.temporal.io/server/common/testing/testcontext" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testlogger" "go.temporal.io/server/common/testing/testtelemetry" @@ -342,12 +343,13 @@ func (s *FunctionalTestBase) setupCluster(options ...TestClusterOption) { s.Require().NoError(err) // Setup test cluster namespaces. + ctx := testcontext.For(s.T()) s.namespace = namespace.Name(RandomizeStr("namespace")) - s.namespaceID, err = s.RegisterNamespace(s.Namespace(), 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + s.namespaceID, err = s.RegisterNamespace(ctx, s.Namespace(), 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") s.Require().NoError(err) s.externalNamespace = namespace.Name(RandomizeStr("external-namespace")) - _, err = s.RegisterNamespace(s.ExternalNamespace(), 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + _, err = s.RegisterNamespace(ctx, s.ExternalNamespace(), 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") s.Require().NoError(err) } @@ -504,6 +506,7 @@ func (s *FunctionalTestBase) tearDownSdk() { // 2. Update search attributes would require an extra API call, // 3. One more extra API call would be necessary to get namespace.ID. func (s *FunctionalTestBase) RegisterNamespace( + ctx context.Context, nsName namespace.Name, retentionDays int32, archivalState enumspb.ArchivalState, @@ -540,7 +543,7 @@ func (s *FunctionalTestBase) RegisterNamespace( }, IsGlobalNamespace: false, } - _, err := s.testCluster.testBase.MetadataManager.CreateNamespace(context.Background(), namespaceRequest) + _, err := s.testCluster.testBase.MetadataManager.CreateNamespace(ctx, namespaceRequest) if err != nil { return namespace.EmptyID, err @@ -550,7 +553,7 @@ func (s *FunctionalTestBase) RegisterNamespace( ticker := time.NewTicker(NamespaceCacheRefreshInterval / 2) defer ticker.Stop() for { - _, describeErr := s.FrontendClient().DescribeNamespace(NewContext(), &workflowservice.DescribeNamespaceRequest{ + _, describeErr := s.FrontendClient().DescribeNamespace(ctx, &workflowservice.DescribeNamespaceRequest{ Namespace: nsName.String(), }) if describeErr == nil { @@ -562,7 +565,7 @@ func (s *FunctionalTestBase) RegisterNamespace( <-ticker.C } - _, err = s.OperatorClient().AddSearchAttributes(NewContext(), &operatorservice.AddSearchAttributesRequest{ + _, err = s.OperatorClient().AddSearchAttributes(ctx, &operatorservice.AddSearchAttributesRequest{ Namespace: nsName.String(), SearchAttributes: expectedSearchAttributes, }) @@ -572,7 +575,7 @@ func (s *FunctionalTestBase) RegisterNamespace( namespaceCacheDeadline = time.Now().Add(5 * NamespaceCacheRefreshInterval) for { - listResp, listErr := s.OperatorClient().ListSearchAttributes(NewContext(), &operatorservice.ListSearchAttributesRequest{ + listResp, listErr := s.OperatorClient().ListSearchAttributes(ctx, &operatorservice.ListSearchAttributesRequest{ Namespace: nsName.String(), }) if listErr == nil { diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index 3e0126835e9..81642f463f7 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -259,7 +259,7 @@ func NewEnv(t *testing.T, opts ...TestOption) *TestEnv { // Create the test context before any expensive setup, so that the deadline // extension below can compensate for the time setup takes. - testcontext.For(t) + ctx := testcontext.For(t) var options testOptions for _, opt := range opts { @@ -296,6 +296,7 @@ func NewEnv(t *testing.T, opts ...TestOption) *TestEnv { baseName := strings.ReplaceAll(t.Name(), "/", "-") ns := namespace.Name(RandomizeStr(baseName)) nsID, err := base.RegisterNamespace( + ctx, ns, 1, // 1 day retention enumspb.ARCHIVAL_STATE_DISABLED, diff --git a/tests/timeskipping_test.go b/tests/timeskipping_test.go index 5b18eaeb09a..51fb362c7a8 100644 --- a/tests/timeskipping_test.go +++ b/tests/timeskipping_test.go @@ -516,7 +516,7 @@ func (s *TimeSkippingTestSuite) TestTimeSkipping_RetentionClearsSkippedWorkflowI // NewEnv's namespace has 1 day retention, which cannot be lowered through the frontend. ns := namespace.Name(testcore.RandomizeStr("ts-retention")) - _, err := env.RegisterNamespace(ns, 0, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + _, err := env.RegisterNamespace(s.Context(), ns, 0, enumspb.ARCHIVAL_STATE_DISABLED, "", "") s.NoError(err) tv := testvars.New(s.T()) From 33e99171eb83053c3cd98a9802a583bd4601421d Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 24 Aug 2026 08:36:33 -0700 Subject: [PATCH 047/139] Clarify functionality; cleanup testcases (#11693) ## What changed? The `commonpb.Callback`-variant of `commonpb.Callback_Internal` is unused and should be removed entirely. This PR removes the remaining unnecessary instances of that type. (My actual motivation is really to avoid a larger diff later, since introducing Worker-variant callbacks will start returning errors when you try to attach an `Internal`-variant callback.) However, changing `Callback_Internal` to `Callback_Nexus` changed the behavior of `TestDedupLinksFromCallbacks`. After scratching my head for a while, I add a doc comment to clarify exactly what the function does, and then updated the tests to be easier to read and understand. ## Why? The call to `dedupLinksFromCallbacks(...)` in the testcase did _not_ dedupe the links attached to `callbacks[0]` because it was the `commonpb.Callback_Internal` variant. (Relying on a quirk of the function only filtering callbacks from Nexus-variant callbacks.) I kept that behavior in, but added a couple more test scenarios to provide better coverage and clarify the expected behavior. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks None. --- service/frontend/workflow_handler.go | 5 +- service/frontend/workflow_handler_test.go | 157 +++++++++------------- 2 files changed, 65 insertions(+), 97 deletions(-) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 433272e5998..415ceb898e4 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -6568,6 +6568,8 @@ func (wh *WorkflowHandler) checkWorkerDeploymentReadRateLimit(ctx context.Contex return nil } +// Returns the provided links, but filtering out any which are also present on the supplied callbacks. +// Links from non-Nexus callbacks are not considered for deduping. func dedupLinksFromCallbacks( links []*commonpb.Link, callbacks []*commonpb.Callback, @@ -6575,7 +6577,6 @@ func dedupLinksFromCallbacks( if len(links) == 0 { return nil } - var res []*commonpb.Link callbacksLinks := make([]*commonpb.Link, 0, len(callbacks)) for _, cb := range callbacks { if cb.GetNexus() != nil { @@ -6583,6 +6584,8 @@ func dedupLinksFromCallbacks( callbacksLinks = append(callbacksLinks, cb.GetLinks()...) } } + + var res []*commonpb.Link for _, link := range links { isDup := false for _, cbLink := range callbacksLinks { diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index a0eea679ae3..2e08831c71f 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -66,7 +66,6 @@ import ( "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/tasktoken" - "go.temporal.io/server/common/testing/protoassert" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/wideevents" @@ -1143,9 +1142,7 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidCallback RequestId: uuid.NewString(), CompletionCallbacks: []*commonpb.Callback{ { - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }, + Variant: nexusCallbackVariant(), Links: []*commonpb.Link{ { Variant: &commonpb.Link_WorkflowEvent_{ @@ -1189,11 +1186,7 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidAggregat RequestId: uuid.NewString(), CompletionCallbacks: []*commonpb.Callback{ { - Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{ - Url: "http://localhost/test", - }, - }, + Variant: nexusCallbackVariant(), Links: []*commonpb.Link{ { Variant: &commonpb.Link_WorkflowEvent_{ @@ -4467,89 +4460,59 @@ func TestDedupLinksFromCallbacks(t *testing.T) { }, }, } - callbacks := []*commonpb.Callback{ - { - Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{}, - }, - Links: []*commonpb.Link{ - { - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: "test-ns", - WorkflowId: "test-workflow-id", - RunId: "test-run-id", - Reference: &commonpb.Link_WorkflowEvent_EventRef{ - EventRef: &commonpb.Link_WorkflowEvent_EventReference{ - EventId: 3, - EventType: enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, - }, - }, - }, - }, - }, - { - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: "test-ns", - WorkflowId: "test-workflow-id", - RunId: "test-run-id", - Reference: &commonpb.Link_WorkflowEvent_EventRef{ - EventRef: &commonpb.Link_WorkflowEvent_EventReference{ - EventId: 5, - EventType: enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, - }, - }, - }, - }, + + // Returns a new pair of callbacks to be mutated in tests. + getCallbacks := func() []*commonpb.Callback { + return []*commonpb.Callback{ + { + Variant: nexusCallbackVariant(), + Links: []*commonpb.Link{ + common.CloneProto(links[0]), + common.CloneProto(links[1]), }, }, - }, - { - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }, - Links: []*commonpb.Link{ - { - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: "test-ns", - WorkflowId: "test-workflow-id", - RunId: "test-run-id", - Reference: &commonpb.Link_WorkflowEvent_RequestIdRef{ - RequestIdRef: &commonpb.Link_WorkflowEvent_RequestIdReference{ - RequestId: "test-request-id", - EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, - }, - }, - }, - }, + { + Variant: &commonpb.Callback_Internal_{ + Internal: &commonpb.Callback_Internal{}, + }, + Links: []*commonpb.Link{ + common.CloneProto(links[2]), }, }, - }, + } } - dedupedLinks := dedupLinksFromCallbacks(links, callbacks) - assert.Len(t, dedupedLinks, 1) - protoassert.ProtoEqual( - t, - &commonpb.Link{ - Variant: &commonpb.Link_WorkflowEvent_{ - WorkflowEvent: &commonpb.Link_WorkflowEvent{ - Namespace: "test-ns", - WorkflowId: "test-workflow-id", - RunId: "test-run-id", - Reference: &commonpb.Link_WorkflowEvent_RequestIdRef{ - RequestIdRef: &commonpb.Link_WorkflowEvent_RequestIdReference{ - RequestId: "test-request-id", - EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, - }, - }, - }, - }, - }, - dedupedLinks[0], - ) + // Because the second Callback is the Internal-variant, the link + // it carries will not be considered for deduping. + t.Run("IgnoreNonNexusCallbackLinks", func(t *testing.T) { + callbacks := getCallbacks() + dedupedLinks := dedupLinksFromCallbacks(links, callbacks) + require.Len(t, dedupedLinks, 1) + protorequire.ProtoEqual(t, links[2], dedupedLinks[0]) + }) + + // Remove links[1] from callbacks[0].Links. links[1] should then not + // be deduped, and be in the resulting slice. + t.Run("UniqueNexusLink", func(t *testing.T) { + callbacks := getCallbacks() + callbacks[0].Links = callbacks[0].Links[:1] + + dedupedLinks := dedupLinksFromCallbacks(links, callbacks) + require.Len(t, dedupedLinks, 2) + protorequire.ProtoEqual(t, links[1], dedupedLinks[0]) + protorequire.ProtoEqual(t, links[2], dedupedLinks[1]) // Same as before. + }) + + // Change the type of the second callback to be a Nexus-variant. + // Now all callback links will be considered for deduping, filtering + // everything out. + t.Run("AllDupes", func(t *testing.T) { + callbacks := getCallbacks() + callbacks[1].Variant = nexusCallbackVariant() + + dedupedLinks := dedupLinksFromCallbacks(links, callbacks) + require.Empty(t, dedupedLinks) + }) } func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution() { @@ -5856,10 +5819,8 @@ func (s *WorkflowHandlerSuite) TestPrepareUpdateWorkflowRequest_ValidatesComplet }, } request := newRequest([]*commonpb.Callback{{ - Variant: &commonpb.Callback_Nexus_{ - Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/callback"}, - }, - Links: []*commonpb.Link{callbackLink}, + Variant: nexusCallbackVariant(), + Links: []*commonpb.Link{callbackLink}, }}) request.Request.Links = []*commonpb.Link{common.CloneProto(callbackLink)} @@ -5874,9 +5835,7 @@ func (s *WorkflowHandlerSuite) TestPrepareUpdateWorkflowRequest_ValidatesComplet context.Background(), s.testNamespace, newRequest([]*commonpb.Callback{{ - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }, + Variant: nexusCallbackVariant(), Links: []*commonpb.Link{{ Variant: &commonpb.Link_BatchJob_{ BatchJob: &commonpb.Link_BatchJob{}, @@ -5892,9 +5851,7 @@ func (s *WorkflowHandlerSuite) TestPrepareUpdateWorkflowRequest_ValidatesComplet s.Run("rejects too many combined links", func() { request := newRequest([]*commonpb.Callback{{ - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }, + Variant: nexusCallbackVariant(), Links: []*commonpb.Link{{ Variant: &commonpb.Link_BatchJob_{ BatchJob: &commonpb.Link_BatchJob{JobId: "callback-job"}, @@ -5951,3 +5908,11 @@ type routingMatchingClient struct { func (r *routingMatchingClient) Route(p tqid.Partition) (string, error) { return r.routeFn(p) } + +func nexusCallbackVariant() *commonpb.Callback_Nexus_ { + return &commonpb.Callback_Nexus_{ + Nexus: &commonpb.Callback_Nexus{ + Url: "https://localhost/nexus-callback", + }, + } +} From fc7b395f6358a13e8899323ceb173aaf431a238e Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 08:42:14 -0700 Subject: [PATCH 048/139] Make Nexus link warnings aggregatable (#11685) ## What changed All three warnings in `ConvertNexusLinksToProtoLinks` interpolated the link type into the message, and two also embedded the link URL. Moved both into tags and kept the message static. --------- Co-authored-by: Claude Opus 5 (1M context) --- common/nexus/links.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/common/nexus/links.go b/common/nexus/links.go index 83f253bb3d2..0577978dbd3 100644 --- a/common/nexus/links.go +++ b/common/nexus/links.go @@ -1,8 +1,6 @@ package nexus import ( - "fmt" - "github.com/nexus-rpc/sdk-go/nexus" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/temporalnexus" @@ -21,8 +19,10 @@ func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) [ link, err := ConvertNexusLinkToLinkWorkflowEvent(nexusLink) if err != nil { logger.Warn( - fmt.Sprintf("failed to parse link to %q: %s", nexusLink.Type, nexusLink.URL), + "failed to parse Nexus link", tag.Error(err), + tag.NewStringTag("nexus-link-type", nexusLink.Type), + tag.URL(nexusLink.URL.String()), ) continue } @@ -33,8 +33,10 @@ func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) [ link, err := ConvertNexusLinkToLinkActivity(nexusLink) if err != nil { logger.Warn( - fmt.Sprintf("failed to parse link to %q: %s", nexusLink.Type, nexusLink.URL), + "failed to parse Nexus link", tag.Error(err), + tag.NewStringTag("nexus-link-type", nexusLink.Type), + tag.URL(nexusLink.URL.String()), ) continue } @@ -54,7 +56,9 @@ func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) [ Variant: &commonpb.Link_Workflow_{Workflow: link}, }) default: - logger.Warn(fmt.Sprintf("invalid link data type: %q", nexusLink.Type)) + logger.Warn("invalid Nexus link data type", + tag.NewStringTag("nexus-link-type", nexusLink.Type), + ) } } return out From 4fc665f120a95256df2c55ac591a177dde05904b Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 08:52:40 -0700 Subject: [PATCH 049/139] Tag HSM outbound Nexus call failure logs (#11662) ## What changed The HSM Nexus executor logged outbound call failure logs were missing tags. ## Why `chasm/lib/nexusoperation` already logs exactly these fields via `invocationTraceContext.tags()`, --------- Co-authored-by: Claude Opus 5 (1M context) --- components/nexusoperations/executors.go | 115 ++++++++++++++++-------- 1 file changed, 77 insertions(+), 38 deletions(-) diff --git a/components/nexusoperations/executors.go b/components/nexusoperations/executors.go index cfc4acd514d..184d5ad77bb 100644 --- a/components/nexusoperations/executors.go +++ b/components/nexusoperations/executors.go @@ -270,6 +270,19 @@ func (e taskExecutor) executeInvocationTask(ctx context.Context, env hsm.Environ Links: []nexus.Link{args.nexusLink}, } + traceCtx := invocationTraceContext{ + operationTag: "StartOperation", + namespaceName: ns.Name().String(), + targetNamespaceID: endpoint.GetEndpoint().GetSpec().GetTarget().GetWorker().GetNamespaceId(), + requestID: args.requestID, + operation: args.operation, + endpointName: args.endpointName, + workflowID: ref.WorkflowKey.WorkflowID, + runID: ref.WorkflowKey.RunID, + attemptStart: time.Now().UTC(), + attempt: task.Attempt, + } + var result *nexusrpc.ClientStartOperationResponse[*commonpb.Payload] var callErr error var startTime time.Time @@ -291,18 +304,8 @@ func (e taskExecutor) executeInvocationTask(ctx context.Context, env hsm.Environ } if e.HTTPTraceProvider != nil { - traceLogger := log.With(e.Logger, - tag.Operation("StartOperation"), - tag.WorkflowNamespace(ns.Name().String()), - tag.RequestID(args.requestID), - tag.NexusOperation(args.operation), - tag.Endpoint(args.endpointName), - tag.WorkflowID(ref.WorkflowKey.WorkflowID), - tag.WorkflowRunID(ref.WorkflowKey.RunID), - tag.AttemptStart(time.Now().UTC()), - tag.Attempt(task.Attempt), - ) - if trace := e.HTTPTraceProvider.NewTrace(task.Attempt, traceLogger); trace != nil { + traceLogger := log.With(e.Logger, traceCtx.tags()...) + if trace := e.HTTPTraceProvider.NewTrace(traceCtx.attempt, traceLogger); trace != nil { callCtx = httptrace.WithClientTrace(callCtx, trace) } } @@ -333,13 +336,7 @@ func (e taskExecutor) executeInvocationTask(ctx context.Context, env hsm.Environ chasmnexus.OutboundRequestCounter.With(e.MetricsHandler).Record(1, namespaceTag, destTag, methodTag, outcomeTag, failureSourceTag) chasmnexus.OutboundRequestLatency.With(e.MetricsHandler).Record(time.Since(startTime), namespaceTag, destTag, methodTag, outcomeTag, failureSourceTag) - if callErr != nil { - if failureSource == commonnexus.FailureSourceWorker || errors.As(callErr, new(*operationTimeoutBelowMinError)) { - e.Logger.Debug("Nexus StartOperation request failed", tag.Error(callErr)) - } else { - e.Logger.Error("Nexus StartOperation request failed", tag.Error(callErr)) - } - } + e.logCallFailure(traceCtx, callErr, failureSource) err = e.saveResult(ctx, env, ref, result, callErr) @@ -734,6 +731,19 @@ func (e taskExecutor) executeCancelationTask(ctx context.Context, env hsm.Enviro // Set this value on the parent context so that our custom HTTP caller can mutate it since we cannot access response headers directly. callCtx = context.WithValue(callCtx, commonnexus.FailureSourceContextKey, &atomic.Value{}) + traceCtx := invocationTraceContext{ + operationTag: "CancelOperation", + namespaceName: ns.Name().String(), + targetNamespaceID: endpoint.GetEndpoint().GetSpec().GetTarget().GetWorker().GetNamespaceId(), + requestID: args.requestID, + operation: args.operation, + endpointName: args.endpointName, + workflowID: ref.WorkflowKey.WorkflowID, + runID: ref.WorkflowKey.RunID, + attemptStart: time.Now().UTC(), + attempt: task.Attempt, + } + var callErr error var startTime time.Time if callTimeout < e.Config.MinRequestTimeout(ns.Name().String()) { @@ -758,18 +768,8 @@ func (e taskExecutor) executeCancelationTask(ctx context.Context, env hsm.Enviro } if e.HTTPTraceProvider != nil { - traceLogger := log.With(e.Logger, - tag.Operation("CancelOperation"), - tag.WorkflowNamespace(ns.Name().String()), - tag.RequestID(args.requestID), - tag.NexusOperation(args.operation), - tag.Endpoint(args.endpointName), - tag.WorkflowID(ref.WorkflowKey.WorkflowID), - tag.WorkflowRunID(ref.WorkflowKey.RunID), - tag.AttemptStart(time.Now().UTC()), - tag.Attempt(task.Attempt), - ) - if trace := e.HTTPTraceProvider.NewTrace(task.Attempt, traceLogger); trace != nil { + traceLogger := log.With(e.Logger, traceCtx.tags()...) + if trace := e.HTTPTraceProvider.NewTrace(traceCtx.attempt, traceLogger); trace != nil { callCtx = httptrace.WithClientTrace(callCtx, trace) } } @@ -791,13 +791,7 @@ func (e taskExecutor) executeCancelationTask(ctx context.Context, env hsm.Enviro chasmnexus.OutboundRequestCounter.With(e.MetricsHandler).Record(1, namespaceTag, destTag, methodTag, statusCodeTag, failureSourceTag) chasmnexus.OutboundRequestLatency.With(e.MetricsHandler).Record(time.Since(startTime), namespaceTag, destTag, methodTag, statusCodeTag, failureSourceTag) - if callErr != nil { - if failureSource == commonnexus.FailureSourceWorker || errors.As(callErr, new(*operationTimeoutBelowMinError)) { - e.Logger.Debug("Nexus CancelOperation request failed", tag.Error(callErr)) - } else { - e.Logger.Error("Nexus CancelOperation request failed", tag.Error(callErr)) - } - } + e.logCallFailure(traceCtx, callErr, failureSource) err = e.saveCancelationResult(ctx, env, ref, callErr, args.scheduledEventID) @@ -1025,6 +1019,51 @@ func cancelCallOutcomeTag(callCtx context.Context, callErr error) string { return "successful" } +// invocationTraceContext captures per-call contextual information used for HTTP tracing and failure logging. +type invocationTraceContext struct { + operationTag string // "StartOperation" or "CancelOperation" + namespaceName string // source (caller) namespace + targetNamespaceID string + requestID string + operation string + endpointName string + workflowID string + runID string + attemptStart time.Time + attempt int32 +} + +// tags returns the structured log tags describing the call. +func (c invocationTraceContext) tags() []tag.Tag { + return []tag.Tag{ + tag.Operation(c.operationTag), + tag.WorkflowNamespace(c.namespaceName), + tag.NexusEndpointTargetNamespaceID(c.targetNamespaceID), + tag.RequestID(c.requestID), + tag.NexusOperation(c.operation), + tag.Endpoint(c.endpointName), + tag.WorkflowID(c.workflowID), + tag.WorkflowRunID(c.runID), + tag.AttemptStart(c.attemptStart), + tag.Attempt(c.attempt), + } +} + +// logCallFailure logs a failed outbound Nexus call. +func (e taskExecutor) logCallFailure(traceCtx invocationTraceContext, callErr error, failureSource string) { + if callErr == nil { + return + } + tags := append(traceCtx.tags(), tag.Error(callErr)) + msg := fmt.Sprintf("Nexus %s request failed", traceCtx.operationTag) + _, isTimeoutBelowMin := errors.AsType[*operationTimeoutBelowMinError](callErr) + if failureSource == commonnexus.FailureSourceWorker || isTimeoutBelowMin { + e.Logger.Debug(msg, tags...) + } else { + e.Logger.Error(msg, tags...) + } +} + func isDestinationDown(err error) bool { var serviceErr serviceerror.ServiceError // For the system endpoint, we don't even consider the destination down since it's internal. From baa033892586df799bb48ceb03d947ac87c35a13 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 24 Aug 2026 10:46:17 -0700 Subject: [PATCH 050/139] Fix build break (#11747) ## What changed? Fixes a build error, seemingly introduced when multiple changes were merged automatically after approval. ## Why? Because a broken build stops the flow of spice. And the spice must flow. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks None. But there might be other issues, I'm curious to see if there are any other issues flagged by CI/CD. --- common/nexus/links.go | 4 +++- tests/gethistory_test.go | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/common/nexus/links.go b/common/nexus/links.go index 0577978dbd3..56dc9834d2a 100644 --- a/common/nexus/links.go +++ b/common/nexus/links.go @@ -47,7 +47,9 @@ func ConvertNexusLinksToProtoLinks(nexusLinks []nexus.Link, logger log.Logger) [ link, err := temporalnexus.ConvertNexusLinkToLinkWorkflow(nexusLink) if err != nil { logger.Warn( - fmt.Sprintf("failed to parse link to %q: %s", nexusLink.Type, nexusLink.URL), + "failed to parse link", + tag.NewStringTag("nexus-link-type", nexusLink.Type), + tag.URL(nexusLink.URL.String()), tag.Error(err), ) continue diff --git a/tests/gethistory_test.go b/tests/gethistory_test.go index e4428d2c621..c4ac2ba2a17 100644 --- a/tests/gethistory_test.go +++ b/tests/gethistory_test.go @@ -789,6 +789,7 @@ func startMultiBatchWorkflow( func (s *GetHistorySuite) TestGetWorkflowExecutionHistory_ContinuationFromAnotherNamespace( enableTransitionHistory bool, ) { + ctx := s.Context() // A single-shard cluster so both namespaces map to the same history shard. On SQL backends // shard_id is part of the history_node primary key, so without this the replay would be // rejected for the wrong reason; on Cassandra it is not part of the key at all. @@ -800,7 +801,7 @@ func (s *GetHistorySuite) TestGetWorkflowExecutionHistory_ContinuationFromAnothe ) otherNamespace := namespace.Name(testcore.RandomizeStr("other-namespace")) - _, err := env.RegisterNamespace(otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + _, err := env.RegisterNamespace(ctx, otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") s.Require().NoError(err) startMultiBatchWorkflow(s.Context(), s.Require(), env) @@ -841,6 +842,7 @@ func (s *GetHistorySuite) TestGetWorkflowExecutionHistory_ContinuationFromAnothe } func (s *RawHistorySuite) TestGetWorkflowExecutionHistoryReverse_ContinuationFromAnotherNamespace() { + ctx := s.Context() env := s.newTestEnv( testcore.WithHistoryShardCount(1), // Shadow mode reports the mismatch but still serves the page. @@ -848,7 +850,7 @@ func (s *RawHistorySuite) TestGetWorkflowExecutionHistoryReverse_ContinuationFro ) otherNamespace := namespace.Name(testcore.RandomizeStr("other-namespace")) - _, err := env.RegisterNamespace(otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") + _, err := env.RegisterNamespace(ctx, otherNamespace, 1, enumspb.ARCHIVAL_STATE_DISABLED, "", "") s.Require().NoError(err) startMultiBatchWorkflow(s.Context(), s.Require(), env) From c1a0869365c25a83a96163441d8d61bd5f233a08 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Mon, 24 Aug 2026 13:17:48 -0700 Subject: [PATCH 051/139] Reconstruct missing current execution record on replication apply (#11257) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? When `dispatchForExistingWorkflow` finds `currentRunID == ""`, it treats the missing current execution record as a deletion issued by the user, and handles the target run accordingly: - **Running target:** return an internal invariant error. Replication tasks are ordered, so a deletion cannot overtake the close event — a running run with no current record is conceptually impossible. The task retries and is eventually sent to the DLQ. - **Closed target:** apply the target as a zombie with `UpdateWorkflowModeBypassCurrent`, leaving the current record missing. The current record is never re-established here, because that would resurrect a user-deleted workflow. If the task carries a new run (continue-as-new / cron / retry successor), it is **not dropped**: when it is not already present locally it is persisted as a zombie via bypass-current, so **no history is lost** — but it never becomes the current run. A zombie is not a dead end: when the successor's own close event later replicates, `ZOMBIE -> COMPLETED` is a valid transition and it converges like any other run (intermediate events keep it a zombie, so they never hit the running-invariant error). While open, a zombie is invisible to visibility, so it never shows up as a stray running workflow. For example, given `r1 -> r2 -> r3` (continued-as-new chain) followed by deletion of the current run `r3`, a later replication update of `r1` may carry `r2` or `r3` as its new run. Either way the successor is persisted as a zombie, never as current, so the user's deletion intent is preserved while its history is retained. The current record is (re)established only by a separately replicated new/reset run through its own new-workflow path. ## Why? Cross-cluster deletion removes the passive cluster's current execution record while closed run rows can survive until retention. Re-establishing a current record from a later replication task would resurrect a deliberately deleted workflow, so the passive cluster preserves the deletion intent: closed historical runs converge as zombies, and only a separately replicated new/reset run may establish a new current record. Carried successors are still persisted (as zombies) so their history is never lost, and they close normally once their own close events replicate. ## How did you test it? - [x] built - [x] run locally and tested manually — reproduced the original duplicate-run failure with `r1 -> r2 -> r3`, deleted `r3` on both clusters, then reset `r1` - [x] covered by existing tests - [x] added unit tests — missing-current dispatch for the closed, rebuilt, and impossible running cases, plus a carried new run in both the persist-if-absent and skip-if-already-present paths - [ ] added new functional test(s) --------- Co-authored-by: Claude Opus 4.8 --- service/history/ndc/transaction_manager.go | 1 - .../transaction_manager_existing_workflow.go | 104 +++++++--- ...nsaction_manager_existing_workflow_test.go | 195 +++++++++++++++++- 3 files changed, 259 insertions(+), 41 deletions(-) diff --git a/service/history/ndc/transaction_manager.go b/service/history/ndc/transaction_manager.go index 629ea149cee..20f434128f8 100644 --- a/service/history/ndc/transaction_manager.go +++ b/service/history/ndc/transaction_manager.go @@ -69,7 +69,6 @@ import ( // conflict resolve path + suppress current // 1. update from zombie to current & suppress current -> nDCTransactionPolicySuppressCurrentAndUpdateAsCurrent // 2. update from zombie to current & new created as current & suppress current -> nDCTransactionPolicySuppressCurrentAndUpdateAsCurrent - type nDCTransactionPolicy int const ( diff --git a/service/history/ndc/transaction_manager_existing_workflow.go b/service/history/ndc/transaction_manager_existing_workflow.go index 0663109cc4f..b93e476bf9a 100644 --- a/service/history/ndc/transaction_manager_existing_workflow.go +++ b/service/history/ndc/transaction_manager_existing_workflow.go @@ -5,7 +5,9 @@ package ndc import ( "context" + enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" @@ -96,28 +98,41 @@ func (r *nDCTransactionMgrForExistingWorkflowImpl) dispatchForExistingWorkflow( return err } if currentRunID == "" { - // currentRunID == "" means the current execution record is gone — the current run was - // deleted, leaving this closed run an orphan. With no new run to create, persist it via the - // bypass-current path (which tolerates a missing current record). Any other shape (a running - // run, or a closed run carrying a new run) still needs a current run we don't have, so - // surface it as an inconsistency. - if !mutableState.IsWorkflowExecutionRunning() && newWorkflow == nil { - r.shardContext.GetThrottledLogger().Warn( - "Applying replication update as zombie (bypass-current) for closed run with no current execution; workflow appears deleted", - tag.WorkflowNamespaceID(namespaceID.String()), - tag.WorkflowID(workflowID), - tag.WorkflowRunID(targetRunID), - ) - return r.dispatchWorkflowUpdateAsZombie( - ctx, - isWorkflowRebuilt, - nil, // no current workflow: the current run was deleted - targetWorkflow, - nil, // no new workflow (guarded above) - archetypeID, + // The current execution record is missing. It is written atomically with the run, so this only + // happens when delete replication removed it (the workflow is being deleted). We must not + // re-establish the current record here - that would resurrect a deleted workflow - so the target + // is persisted bypass-current: + // - running target: a non-current run is never running, so this is an anomaly -> error. + // - not-running target (closed, or already a zombie - IsWorkflowExecutionRunning() is false + // for both): bypass-current, leaving the current record missing. + // A carried newWorkflow (continue-as-new/cron/retry successor) still holds real history we must + // not drop, so it is passed through to the zombie path: it is persisted bypass-current (as a + // zombie) when not already present locally, and skipped when it is. It never becomes the current + // run; if it is truly the live head of the lineage, it is reconciled by its own replication. + if mutableState.IsWorkflowExecutionRunning() { + if newWorkflow != nil { + newWorkflow.GetReleaseFn()(nil) + } + return serviceerror.NewInternalf( + "dispatchForExistingWorkflow: run %v is running but its current execution record is missing (workflow %v)", + targetRunID, + workflowID, ) } - return serviceerror.NewInternal("transactionMgr: unable to locate current workflow during update") + r.shardContext.GetThrottledLogger().Warn( + "Applying replication update as zombie (bypass-current) for closed run with no current execution; workflow appears deleted", + tag.WorkflowNamespaceID(namespaceID.String()), + tag.WorkflowID(workflowID), + tag.WorkflowRunID(targetRunID), + ) + return r.dispatchWorkflowUpdateAsZombie( + ctx, + isWorkflowRebuilt, + nil, // no current workflow: the current run was deleted + targetWorkflow, + newWorkflow, // persisted bypass-current if not already present; never becomes current + archetypeID, + ) } if currentRunID == targetRunID { @@ -254,8 +269,9 @@ func (r *nDCTransactionMgrForExistingWorkflowImpl) updateAsCurrent( // // It exists solely for the deleted-current-run (orphan) path in this file, where currentWorkflow is // nil because the current execution record is gone. In that case there is nothing to suppress -// against and the target is guaranteed closed (dispatchForExistingWorkflow only reaches the zombie -// path for a non-running target), so it stays passive. Do NOT reuse this elsewhere or call it with a +// against and the target is guaranteed not running - closed or already a zombie +// (dispatchForExistingWorkflow only reaches the zombie path for a non-running target), so it stays +// passive. Do NOT reuse this elsewhere or call it with a // nil current for a running target: it would report the target as suppressed without zombifying it. // Everywhere else, call targetWorkflow.SuppressBy(currentWorkflow) directly. func suppressTargetPolicy(targetWorkflow Workflow, currentWorkflow Workflow) (historyi.TransactionPolicy, error) { @@ -265,6 +281,34 @@ func suppressTargetPolicy(targetWorkflow Workflow, currentWorkflow Workflow) (hi return targetWorkflow.SuppressBy(currentWorkflow) } +// suppressNewWorkflowPolicy suppresses the carried new run (continue-as-new/cron/retry successor) by +// the current workflow and returns the resulting transaction policy. It mirrors suppressTargetPolicy +// for the deleted-current-run (orphan) path, where currentWorkflow is nil because the current +// execution record is gone. +// +// With a real current workflow it defers to newWorkflow.SuppressBy(currentWorkflow). With no current +// to compare against, the new run cannot be the current run, so a still-running successor is forced +// into the zombie state (a closed one is left as is) before the caller persists it bypass-current. On +// the passive apply path the successor is always remote-active, so zombie (never terminate) is the +// correct suppression, matching SuppressBy's remote-active branch. +func suppressNewWorkflowPolicy(newWorkflow Workflow, currentWorkflow Workflow) (historyi.TransactionPolicy, error) { + if currentWorkflow != nil { + return newWorkflow.SuppressBy(currentWorkflow) + } + newMutableState := newWorkflow.GetMutableState() + if newMutableState.IsWorkflowExecutionRunning() { + if _, err := newMutableState.UpdateWorkflowStateStatus( + enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + ); err != nil { + // The policy is unused on error (callers return immediately); return the conservative + // passive policy rather than active so a leaked value can never be mistaken for current. + return historyi.TransactionPolicyPassive, err + } + } + return historyi.TransactionPolicyPassive, nil +} + func (r *nDCTransactionMgrForExistingWorkflowImpl) updateAsZombie( ctx context.Context, currentWorkflow Workflow, @@ -285,11 +329,9 @@ func (r *nDCTransactionMgrForExistingWorkflowImpl) updateAsZombie( var newMutableState historyi.MutableState var newTransactionPolicy *historyi.TransactionPolicy if newWorkflow != nil { - // newWorkflow is only set when a current workflow exists (dispatchForExistingWorkflow skips - // the new run when the current record is missing), so suppressing against it is safe. - newWorkflowPolicy, err := newWorkflow.SuppressBy( - currentWorkflow, - ) + // currentWorkflow is nil on the deleted-current-run (orphan) path; suppressNewWorkflowPolicy + // handles that by parking the successor as a zombie instead of suppressing against a current. + newWorkflowPolicy, err := suppressNewWorkflowPolicy(newWorkflow, currentWorkflow) if err != nil { return err } @@ -443,11 +485,9 @@ func (r *nDCTransactionMgrForExistingWorkflowImpl) conflictResolveAsZombie( var newContext historyi.WorkflowContext var newMutableState historyi.MutableState if newWorkflow != nil { - // newWorkflow is only set when a current workflow exists (dispatchForExistingWorkflow skips - // the new run when the current record is missing), so suppressing against it is safe. - newWorkflowPolicy, err = newWorkflow.SuppressBy( - currentWorkflow, - ) + // currentWorkflow is nil on the deleted-current-run (orphan) path; suppressNewWorkflowPolicy + // handles that by parking the successor as a zombie instead of suppressing against a current. + newWorkflowPolicy, err = suppressNewWorkflowPolicy(newWorkflow, currentWorkflow) if err != nil { return err } diff --git a/service/history/ndc/transaction_manager_existing_workflow_test.go b/service/history/ndc/transaction_manager_existing_workflow_test.go index 30e3d96d135..ce2c97bd506 100644 --- a/service/history/ndc/transaction_manager_existing_workflow_test.go +++ b/service/history/ndc/transaction_manager_existing_workflow_test.go @@ -6,12 +6,13 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + enumspb "go.temporal.io/api/enums/v1" + enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/log" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" - "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" @@ -352,20 +353,55 @@ func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow s.True(targetReleaseCalled) } -func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoCurrentRecord_RunningTargetRun_Errors() { - // Running run with no current execution record: a real inconsistency that - // must still surface as an error. +func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoCurrentRecord_RunningTargetRun_ReturnsError() { + // A running run with no current execution record is not an expected state - current records are + // only removed by deletion, which targets closed chains - so it is surfaced as an error. ctx := context.Background() namespaceID := namespace.ID("some random namespace ID") workflowID := "some random workflow ID" targetRunID := "some random run ID" - isWorkflowRebuilt := false + targetWorkflow := NewMockWorkflow(s.controller) + targetMutableState := historyi.NewMockMutableState(s.controller) + targetWorkflow.EXPECT().GetMutableState().Return(targetMutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(historyi.ReleaseWorkflowContextFunc(func(error) {})).AnyTimes() + + targetMutableState.EXPECT().IsCurrentWorkflowGuaranteed().Return(false).AnyTimes() + targetMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + }).AnyTimes() + targetMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: targetRunID, + }).AnyTimes() + s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID(ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID).Return("", nil) + targetMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true).AnyTimes() + + err := s.updateMgr.dispatchForExistingWorkflow(ctx, false, chasm.WorkflowArchetypeID, targetWorkflow, nil) + s.Error(err) +} + +func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoCurrentRecord_RunningTargetRun_WithNewRun_ReturnsError() { + // Defensive: a running target should never carry a new run (running means it has not continued-as-new), + // but if one is present it is released (never persisted) and the running-without-current invariant error + // is still returned. + ctx := context.Background() + + namespaceID := namespace.ID("some random namespace ID") + workflowID := "some random workflow ID" + targetRunID := "some random run ID" + + newReleaseCalled := false targetWorkflow := NewMockWorkflow(s.controller) targetMutableState := historyi.NewMockMutableState(s.controller) targetWorkflow.EXPECT().GetMutableState().Return(targetMutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(historyi.ReleaseWorkflowContextFunc(func(error) {})).AnyTimes() + + newWorkflow := NewMockWorkflow(s.controller) + var newReleaseFn historyi.ReleaseWorkflowContextFunc = func(error) { newReleaseCalled = true } + newWorkflow.EXPECT().GetReleaseFn().Return(newReleaseFn).AnyTimes() targetMutableState.EXPECT().IsCurrentWorkflowGuaranteed().Return(false).AnyTimes() targetMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ @@ -378,10 +414,153 @@ func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID(ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID).Return("", nil) targetMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true).AnyTimes() - err := s.updateMgr.dispatchForExistingWorkflow(ctx, isWorkflowRebuilt, chasm.WorkflowArchetypeID, targetWorkflow, nil) + err := s.updateMgr.dispatchForExistingWorkflow(ctx, false, chasm.WorkflowArchetypeID, targetWorkflow, newWorkflow) s.Error(err) - s.NotErrorIs(err, consts.ErrDuplicate) - s.Contains(err.Error(), "unable to locate current workflow") + s.True(newReleaseCalled) +} + +func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoCurrentRecord_ClosedTargetRunWithNewRun_NewRunDoesNotExist_PersistsNewRun() { + // Closed target carrying a continue-as-new successor with no current execution record: the current + // record was deleted, so the target is persisted via bypass-current and never becomes current. The + // successor still holds real history, so - because it is not already present locally - it is forced + // into the zombie state and persisted bypass-current (never as the current run). + ctx := context.Background() + + namespaceID := namespace.ID("some random namespace ID") + workflowID := "some random workflow ID" + targetRunID := "some random run ID" + newRunID := "some random new run ID" + + targetReleaseCalled := false + newReleaseCalled := false + + targetWorkflow := NewMockWorkflow(s.controller) + targetContext := historyi.NewMockWorkflowContext(s.controller) + targetMutableState := historyi.NewMockMutableState(s.controller) + var targetReleaseFn historyi.ReleaseWorkflowContextFunc = func(error) { targetReleaseCalled = true } + targetWorkflow.EXPECT().GetContext().Return(targetContext).AnyTimes() + targetWorkflow.EXPECT().GetMutableState().Return(targetMutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(targetReleaseFn).AnyTimes() + + newWorkflow := NewMockWorkflow(s.controller) + newContext := historyi.NewMockWorkflowContext(s.controller) + newMutableState := historyi.NewMockMutableState(s.controller) + var newReleaseFn historyi.ReleaseWorkflowContextFunc = func(error) { newReleaseCalled = true } + newWorkflow.EXPECT().GetContext().Return(newContext).AnyTimes() + newWorkflow.EXPECT().GetMutableState().Return(newMutableState).AnyTimes() + newWorkflow.EXPECT().GetReleaseFn().Return(newReleaseFn).AnyTimes() + + targetMutableState.EXPECT().IsCurrentWorkflowGuaranteed().Return(false).AnyTimes() + targetMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + }).AnyTimes() + targetMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: targetRunID, + }).AnyTimes() + newMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + }).AnyTimes() + newMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: newRunID, + }).AnyTimes() + + s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID(ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID).Return("", nil) + // target run is closed -> takes the bypass-current path + targetMutableState.EXPECT().IsWorkflowExecutionRunning().Return(false).AnyTimes() + // with no current to compare against, a still-running successor is forced into the zombie state + newMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true).AnyTimes() + newMutableState.EXPECT().UpdateWorkflowStateStatus( + enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + ).Return(true, nil) + // the successor is not already present locally, so it is persisted alongside the bypass-current update + s.mockTransactionMgr.EXPECT().CheckWorkflowExists(ctx, namespaceID, workflowID, newRunID, chasm.WorkflowArchetypeID).Return(false, nil) + + targetContext.EXPECT().UpdateWorkflowExecutionWithNew( + gomock.Any(), + s.mockShard, + persistence.UpdateWorkflowModeBypassCurrent, + newContext, + newMutableState, + historyi.TransactionPolicyPassive, + historyi.TransactionPolicyPassive.Ptr(), + ).Return(nil) + + err := s.updateMgr.dispatchForExistingWorkflow(ctx, false, chasm.WorkflowArchetypeID, targetWorkflow, newWorkflow) + s.NoError(err) + s.True(targetReleaseCalled) + s.True(newReleaseCalled) +} + +func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoCurrentRecord_ClosedTargetRunWithNewRun_NewRunExists_SkipsNewRun() { + // Same as above, but the successor is already present locally (e.g. created by workflow resend), so + // it is not re-created; only the closed target is persisted via bypass-current. + ctx := context.Background() + + namespaceID := namespace.ID("some random namespace ID") + workflowID := "some random workflow ID" + targetRunID := "some random run ID" + newRunID := "some random new run ID" + + targetReleaseCalled := false + newReleaseCalled := false + + targetWorkflow := NewMockWorkflow(s.controller) + targetContext := historyi.NewMockWorkflowContext(s.controller) + targetMutableState := historyi.NewMockMutableState(s.controller) + var targetReleaseFn historyi.ReleaseWorkflowContextFunc = func(error) { targetReleaseCalled = true } + targetWorkflow.EXPECT().GetContext().Return(targetContext).AnyTimes() + targetWorkflow.EXPECT().GetMutableState().Return(targetMutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(targetReleaseFn).AnyTimes() + + newWorkflow := NewMockWorkflow(s.controller) + newMutableState := historyi.NewMockMutableState(s.controller) + var newReleaseFn historyi.ReleaseWorkflowContextFunc = func(error) { newReleaseCalled = true } + newWorkflow.EXPECT().GetMutableState().Return(newMutableState).AnyTimes() + newWorkflow.EXPECT().GetReleaseFn().Return(newReleaseFn).AnyTimes() + + targetMutableState.EXPECT().IsCurrentWorkflowGuaranteed().Return(false).AnyTimes() + targetMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + }).AnyTimes() + targetMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: targetRunID, + }).AnyTimes() + newMutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + }).AnyTimes() + newMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: newRunID, + }).AnyTimes() + + s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID(ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID).Return("", nil) + targetMutableState.EXPECT().IsWorkflowExecutionRunning().Return(false).AnyTimes() + newMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true).AnyTimes() + newMutableState.EXPECT().UpdateWorkflowStateStatus( + enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + ).Return(true, nil) + // the successor already exists locally, so it is skipped (not re-created) + s.mockTransactionMgr.EXPECT().CheckWorkflowExists(ctx, namespaceID, workflowID, newRunID, chasm.WorkflowArchetypeID).Return(true, nil) + + targetContext.EXPECT().UpdateWorkflowExecutionWithNew( + gomock.Any(), + s.mockShard, + persistence.UpdateWorkflowModeBypassCurrent, + (historyi.WorkflowContext)(nil), + (historyi.MutableState)(nil), + historyi.TransactionPolicyPassive, + (*historyi.TransactionPolicy)(nil), + ).Return(nil) + + err := s.updateMgr.dispatchForExistingWorkflow(ctx, false, chasm.WorkflowArchetypeID, targetWorkflow, newWorkflow) + s.NoError(err) + s.True(targetReleaseCalled) + s.True(newReleaseCalled) } func (s *transactionMgrForExistingWorkflowSuite) TestDispatchForExistingWorkflow_NoRebuild_CurrentWorkflowNotGuaranteed_NotCurrent_UpdateAsZombie_NewRunDoesNotExists() { From 3c5aaed9a11dd452724fbce9e221662a29714930 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Mon, 24 Aug 2026 14:47:21 -0700 Subject: [PATCH 052/139] Move CHASM Link and Callback validators into common (#11697) ## What changed? - Moves `activity.linkValidator` into `common/links`. - Moves `callback.Validator` into `common/callbacks` In addition, this PR performs some minor refactorings for consistency and clarity. - Moved some `links.Validator`-specific tests from `chasm/lib/activity/validator_test.go` elsewhere. - Introduced a `callbacks.ValidatorConfig` to bundle all of the specific settings. (Since we'll need to wire 3+ more parameters when updating the `callbacks.Validator` to support worker callbacks.) > The singular package names `common/link` or `common/callback` would be more consistent. But `common/links` already existed, there are other pluralized ones like `common/enums` or `common/headers`. And IMHO, the plural seems a little more applicable since the validations are only on groupings of links or callbacks. ## Why? The `activity.linkValidator` and `callback.Validator` types are great, but they aren't able to be used as across other CHASM components as easily. Moreover, `callback.Validator` uses types that are exposed from the CHASM `callback` package, it will lead to circular dependencies in the future. (I'm hitting this now in PRs for landing worker callbacks.) Moving the `commonpb` protobuf validation into `common/` means we can better separate the the distinction between validation logic and the CHASM executions that rely on it. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks This should just be a standard refactoring. There should not be any new validation checks enabled on codepaths where they weren't already present. (Or in test cases, we initialize fields of `callback.ValidatorConfig` that weren't used before.) --- chasm/lib/activity/activity.go | 2 +- chasm/lib/activity/frontend.go | 8 +- chasm/lib/activity/link_validator.go | 53 +----- chasm/lib/activity/validator_test.go | 97 +++++----- chasm/lib/callback/config.go | 113 +---------- common/callbacks/address.go | 114 ++++++++++++ .../callbacks/address_test.go | 2 +- .../callbacks}/validator.go | 70 ++++--- .../callbacks}/validator_test.go | 94 ++++++---- common/links/validator.go | 176 ++++++++++++------ common/links/validator_test.go | 97 ++++++++++ service/frontend/fx.go | 21 ++- service/frontend/service.go | 7 +- service/frontend/workflow_handler.go | 10 +- service/frontend/workflow_handler_test.go | 25 +-- tests/activity_standalone_test.go | 2 +- 16 files changed, 537 insertions(+), 354 deletions(-) create mode 100644 common/callbacks/address.go rename chasm/lib/callback/config_test.go => common/callbacks/address_test.go (99%) rename {chasm/lib/callback => common/callbacks}/validator.go (50%) rename {chasm/lib/callback => common/callbacks}/validator_test.go (67%) diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index e13784b1311..04a65799714 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -387,7 +387,7 @@ func (a *Activity) attachLinks(ctx chasm.MutableContext, links []*commonpb.Link, if err := validator.ValidateRequest(namespaceName, links); err != nil { return err } - if err := validator.ValidateComponentTotal(namespaceName, len(ctx.Links(a)), len(links)); err != nil { + if err := validator.ValidateTotal(namespaceName, len(ctx.Links(a)), len(links)); err != nil { return err } return ctx.SetRequestLinks(a, requestID, links) diff --git a/chasm/lib/activity/frontend.go b/chasm/lib/activity/frontend.go index f2641bafa4f..26596b08946 100644 --- a/chasm/lib/activity/frontend.go +++ b/chasm/lib/activity/frontend.go @@ -11,8 +11,8 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" - "go.temporal.io/server/chasm/lib/callback" "go.temporal.io/server/common" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -44,7 +44,7 @@ var ErrStandaloneActivityOperatorCommandsDisabled = serviceerror.NewUnimplemente type frontendHandler struct { FrontendHandler - callbackValidator callback.Validator + callbackValidator callbacks.Validator linkValidator *linkValidator client activitypb.ActivityServiceClient config *Config @@ -57,7 +57,7 @@ type frontendHandler struct { // NewFrontendHandler creates a new FrontendHandler instance for processing activity frontend requests. func NewFrontendHandler( - callbackValidator callback.Validator, + callbackValidator callbacks.Validator, linkValidator *linkValidator, client activitypb.ActivityServiceClient, config *Config, @@ -423,7 +423,7 @@ func (h *frontendHandler) validateAndPopulateStartRequest( } } - if err := h.linkValidator.ValidateStartRequest(req.GetNamespace(), req.GetLinks(), req.GetCompletionCallbacks()); err != nil { + if err := h.linkValidator.ValidateRequestWithCallbacks(req.GetNamespace(), req.GetLinks(), req.GetCompletionCallbacks()); err != nil { return nil, err } diff --git a/chasm/lib/activity/link_validator.go b/chasm/lib/activity/link_validator.go index 1c7bd98a626..b17cec1d48c 100644 --- a/chasm/lib/activity/link_validator.go +++ b/chasm/lib/activity/link_validator.go @@ -1,19 +1,15 @@ package activity import ( - commonpb "go.temporal.io/api/common/v1" - "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" commonlinks "go.temporal.io/server/common/links" ) // linkValidator validates links attached to standalone activity executions. -// It enforces both per-request limits (count + size + variant shape) and a -// per-component cumulative cap across start/attach calls. type linkValidator struct { - maxLinksPerRequest dynamicconfig.IntPropertyFnWithNamespaceFilter - maxLinksPerComponent dynamicconfig.IntPropertyFnWithNamespaceFilter - linkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter + // Distinct per-component type: all CHASM lib fx modules provide into the same + // container, so *commonlinks.Validator cannot be injected directly. + *commonlinks.Validator } func newLinkValidator( @@ -22,42 +18,11 @@ func newLinkValidator( linkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter, ) *linkValidator { return &linkValidator{ - maxLinksPerRequest: maxLinksPerRequest, - maxLinksPerComponent: maxLinksPerComponent, - linkMaxSize: linkMaxSize, + commonlinks.NewValidator( + "an activity", + maxLinksPerRequest, + maxLinksPerComponent, + linkMaxSize, + ), } } - -// ValidateRequest checks count, per-link size, and variant shape for the links -// on a single incoming request. -func (v *linkValidator) ValidateRequest(namespaceName string, links []*commonpb.Link) error { - return commonlinks.Validate(links, v.maxLinksPerRequest(namespaceName), v.linkMaxSize(namespaceName)) -} - -// ValidateStartRequest also includes links embedded in completion callbacks. -func (v *linkValidator) ValidateStartRequest( - namespaceName string, - links []*commonpb.Link, - callbacks []*commonpb.Callback, -) error { - return commonlinks.ValidateRequest( - links, - callbacks, - v.maxLinksPerRequest(namespaceName), - v.linkMaxSize(namespaceName), - ) -} - -// ValidateComponentTotal checks that adding addingCount links to a component -// already holding existingCount links would not exceed the per-component cap. -func (v *linkValidator) ValidateComponentTotal(namespaceName string, existingCount, addingCount int) error { - maxLinks := v.maxLinksPerComponent(namespaceName) - if existingCount+addingCount > maxLinks { - return serviceerror.NewFailedPreconditionf( - "cannot attach more than %d links to an activity (%d links already attached)", - maxLinks, - existingCount, - ) - } - return nil -} diff --git a/chasm/lib/activity/validator_test.go b/chasm/lib/activity/validator_test.go index f9f44468e36..95a725ef37f 100644 --- a/chasm/lib/activity/validator_test.go +++ b/chasm/lib/activity/validator_test.go @@ -2,6 +2,7 @@ package activity import ( "fmt" + "regexp" "testing" "time" @@ -14,7 +15,7 @@ import ( "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" - "go.temporal.io/server/chasm/lib/callback" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/namespace" @@ -529,7 +530,24 @@ func TestRequestIDGeneratedWhenMissing(t *testing.T) { }) } -func TestValidateAndPopulateStartRequest_ValidatesCompletionCallbackLinks(t *testing.T) { +// TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks verifies that both +// the request's own links and those embedded in its completion callbacks reach the link +// validator. Link shape and limit semantics are covered in common/links. +func TestValidateAndPopulateStartRequest_CombinesRequestAndCallbackLinks(t *testing.T) { + callbackValidator, err := callbacks.NewValidator(callbacks.ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 2000 }, + URLMaxLength: func(string) int { return 1000 }, + HeaderMaxSize: func(string) int { return 2000 }, + EndpointRules: func(string) callbacks.AddressMatchRules { + return callbacks.AddressMatchRules{ + Rules: []callbacks.AddressMatchRule{ + {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, + }, + } + }, + }) + require.NoError(t, err) + h := &frontendHandler{ config: &Config{ BlobSizeLimitError: defaultBlobSizeLimitError, @@ -540,12 +558,7 @@ func TestValidateAndPopulateStartRequest_ValidatesCompletionCallbackLinks(t *tes MaxUserMetadataDetailsSize: defaultMaxUserMetadataDetailsSize, MaxUserMetadataSummarySize: defaultMaxUserMetadataSummarySize, }, - callbackValidator: callback.NewValidator( - func(string) int { return 10 }, - func(string) int { return 1000 }, - func(string) int { return 4096 }, - func(string) callback.AddressMatchRules { return callback.AddressMatchRules{} }, - ), + callbackValidator: callbackValidator, linkValidator: newLinkValidator( func(string) int { return 1 }, func(string) int { return 2000 }, @@ -553,54 +566,34 @@ func TestValidateAndPopulateStartRequest_ValidatesCompletionCallbackLinks(t *tes ), logger: log.NewNoopLogger(), } - newRequest := func(callbackLinks []*commonpb.Link) *workflowservice.StartActivityExecutionRequest { - return &workflowservice.StartActivityExecutionRequest{ - Namespace: defaultNamespaceID, - ActivityId: defaultActivityID, - ActivityType: &commonpb.ActivityType{Name: defaultActivityType}, - TaskQueue: &taskqueuepb.TaskQueue{Name: defaultTaskQueue}, - StartToCloseTimeout: durationpb.New(10 * time.Second), - CompletionCallbacks: []*commonpb.Callback{{ - Variant: &commonpb.Callback_Internal_{ - Internal: &commonpb.Callback_Internal{}, - }, - Links: callbackLinks, - }}, - } - } - - t.Run("invalid callback link", func(t *testing.T) { - req := newRequest([]*commonpb.Link{{ - Variant: &commonpb.Link_BatchJob_{ - BatchJob: &commonpb.Link_BatchJob{}, - }, - }}) - - _, err := h.validateAndPopulateStartRequest(t.Context(), req, namespace.ID(defaultNamespaceID)) - - var invalidArgument *serviceerror.InvalidArgument - require.ErrorAs(t, err, &invalidArgument) - require.ErrorContains(t, err, "batch job link must not have an empty job ID") - }) - - t.Run("too many combined links", func(t *testing.T) { - req := newRequest([]*commonpb.Link{{ - Variant: &commonpb.Link_BatchJob_{ - BatchJob: &commonpb.Link_BatchJob{JobId: "callback-job"}, - }, - }}) - req.Links = []*commonpb.Link{{ + // The per-request limit is 1, so a single link from each source only trips the + // validator if both sources are forwarded. + req := &workflowservice.StartActivityExecutionRequest{ + Namespace: defaultNamespaceID, + ActivityId: defaultActivityID, + ActivityType: &commonpb.ActivityType{Name: defaultActivityType}, + TaskQueue: &taskqueuepb.TaskQueue{Name: defaultTaskQueue}, + StartToCloseTimeout: durationpb.New(10 * time.Second), + Links: []*commonpb.Link{{ Variant: &commonpb.Link_BatchJob_{ BatchJob: &commonpb.Link_BatchJob{JobId: "request-job"}, }, - }} - - _, err := h.validateAndPopulateStartRequest(t.Context(), req, namespace.ID(defaultNamespaceID)) + }}, + CompletionCallbacks: []*commonpb.Callback{{ + Variant: &commonpb.Callback_Internal_{ + Internal: &commonpb.Callback_Internal{}, + }, + Links: []*commonpb.Link{{ + Variant: &commonpb.Link_BatchJob_{ + BatchJob: &commonpb.Link_BatchJob{JobId: "callback-job"}, + }, + }}, + }}, + } - var invalidArgument *serviceerror.InvalidArgument - require.ErrorAs(t, err, &invalidArgument) - require.ErrorContains(t, err, "cannot attach more than 1 links per request, got 2") - }) + _, err = h.validateAndPopulateStartRequest(t.Context(), req, namespace.ID(defaultNamespaceID)) + require.ErrorAs(t, err, new(*serviceerror.InvalidArgument)) + require.ErrorContains(t, err, "cannot attach more than 1 links per request, got 2") } func validateUUID(s string) error { diff --git a/chasm/lib/callback/config.go b/chasm/lib/callback/config.go index 102bfaae1fb..7149c423871 100644 --- a/chasm/lib/callback/config.go +++ b/chasm/lib/callback/config.go @@ -1,17 +1,11 @@ package callback import ( - "net/url" - "regexp" - "strings" "time" - "go.temporal.io/server/chasm" "go.temporal.io/server/common/backoff" + commoncallbacks "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" - "go.temporal.io/server/common/nexus" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" ) var MaxPerExecution = dynamicconfig.NewNamespaceIntSetting( @@ -70,8 +64,8 @@ per-namespace.`, var AllowedAddresses = dynamicconfig.NewNamespaceTypedSettingWithConverter( "callback.allowedAddresses", - allowedAddressConverter, - AddressMatchRules{}, + commoncallbacks.AllowedAddressConverter, + commoncallbacks.AddressMatchRules{}, `The per-namespace list of addresses that are allowed for callbacks and whether secure connections (https) are required. URLs: "temporal://system" and "temporal://internal" are always allowed. The default is no address rules. URLs are checked against each in order when starting a workflow or activitiy with attached callbacks or a standalone @@ -81,104 +75,3 @@ any invalid entries are ignored. Each entry is a map with possible values: Wildcards, '*', are supported and can match any number of characters (e.g. '*' matches everything, 'prefix.*.domain' matches 'prefix.a.domain' as well as 'prefix.a.b.domain'). - "AllowInsecure":bool (optional, default=false) indicates whether https is required`) - -type AddressMatchRules struct { - Rules []AddressMatchRule -} - -func (a AddressMatchRules) Validate(rawURL string) error { - // Exact match only; no path, query, or fragment allowed for system URL - if rawURL == nexus.SystemCallbackURL || rawURL == chasm.NexusCompletionHandlerURL { - return nil - } - u, err := url.Parse(rawURL) - if err != nil { - return status.Errorf(codes.InvalidArgument, "invalid callback url: %v", err) - } - if u.Scheme != "http" && u.Scheme != "https" { - return status.Errorf(codes.InvalidArgument, "invalid url: unknown scheme: %v", u) - } - if u.Host == "" { - return status.Errorf(codes.InvalidArgument, "invalid url: missing host") - } - for _, rule := range a.Rules { - allow, err := rule.Allow(u) - if err != nil { - return err - } - if allow { - return nil - } - } - return status.Errorf(codes.InvalidArgument, "invalid url: url does not match any configured callback address: %v", u) -} - -type AddressMatchRule struct { - Regexp *regexp.Regexp - AllowInsecure bool -} - -// Allow validates the URL by: -// 1. true, nil if the provided url matches the rule and passed validation -// for the given rule. -// 2. false, nil if the URL does not match the rule. -// 3. It false, error if there is a match and the URL fails validation -func (a AddressMatchRule) Allow(u *url.URL) (bool, error) { - if !a.Regexp.MatchString(u.Host) { - return false, nil - } - if a.AllowInsecure { - return true, nil - } - if u.Scheme != "https" { - return false, - status.Errorf(codes.InvalidArgument, - "invalid url: callback address does not allow insecure connections: %v", u) - } - return true, nil -} - -func allowedAddressConverter(val any) (AddressMatchRules, error) { - type entry struct { - Pattern string - AllowInsecure bool - } - intermediate, err := dynamicconfig.ConvertStructure[[]entry](nil)(val) - if err != nil { - return AddressMatchRules{}, err - } - - configs := []AddressMatchRule{} - for _, e := range intermediate { - if e.Pattern == "" { - // Skip configs with missing / unparsable Pattern - continue - } - re, err := regexp.Compile(addressPatternToRegexp(e.Pattern)) - if err != nil { - // Skip configs with malformed Pattern - continue - } - configs = append(configs, AddressMatchRule{ - Regexp: re, - AllowInsecure: e.AllowInsecure, - }) - } - return AddressMatchRules{Rules: configs}, nil -} - -func addressPatternToRegexp(pattern string) string { - var result strings.Builder - result.WriteString("^") - first := true - for literal := range strings.SplitSeq(pattern, "*") { - if !first { - // Replace * with .* - result.WriteString(".*") - } - result.WriteString(regexp.QuoteMeta(literal)) - first = false - } - result.WriteString("$") - return result.String() -} diff --git a/common/callbacks/address.go b/common/callbacks/address.go new file mode 100644 index 00000000000..d27a347ba51 --- /dev/null +++ b/common/callbacks/address.go @@ -0,0 +1,114 @@ +package callbacks + +import ( + "net/url" + "regexp" + "strings" + + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/nexus" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type AddressMatchRules struct { + Rules []AddressMatchRule +} + +func (a AddressMatchRules) Validate(rawURL string) error { + // Exact match only; no path, query, or fragment allowed for system URL + if rawURL == nexus.SystemCallbackURL || rawURL == chasm.NexusCompletionHandlerURL { + return nil + } + u, err := url.Parse(rawURL) + if err != nil { + return status.Errorf(codes.InvalidArgument, "invalid callback url: %v", err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return status.Errorf(codes.InvalidArgument, "invalid url: unknown scheme: %v", u) + } + if u.Host == "" { + return status.Errorf(codes.InvalidArgument, "invalid url: missing host") + } + for _, rule := range a.Rules { + allow, err := rule.Allow(u) + if err != nil { + return err + } + if allow { + return nil + } + } + return status.Errorf(codes.InvalidArgument, "invalid url: url does not match any configured callback address: %v", u) +} + +type AddressMatchRule struct { + Regexp *regexp.Regexp + AllowInsecure bool +} + +// Allow validates the URL by: +// 1. true, nil if the provided url matches the rule and passed validation +// for the given rule. +// 2. false, nil if the URL does not match the rule. +// 3. It false, error if there is a match and the URL fails validation +func (a AddressMatchRule) Allow(u *url.URL) (bool, error) { + if !a.Regexp.MatchString(u.Host) { + return false, nil + } + if a.AllowInsecure { + return true, nil + } + if u.Scheme != "https" { + return false, + status.Errorf(codes.InvalidArgument, + "invalid url: callback address does not allow insecure connections: %v", u) + } + return true, nil +} + +func AllowedAddressConverter(val any) (AddressMatchRules, error) { + type entry struct { + Pattern string + AllowInsecure bool + } + intermediate, err := dynamicconfig.ConvertStructure[[]entry](nil)(val) + if err != nil { + return AddressMatchRules{}, err + } + + configs := []AddressMatchRule{} + for _, e := range intermediate { + if e.Pattern == "" { + // Skip configs with missing / unparsable Pattern + continue + } + re, err := regexp.Compile(addressPatternToRegexp(e.Pattern)) + if err != nil { + // Skip configs with malformed Pattern + continue + } + configs = append(configs, AddressMatchRule{ + Regexp: re, + AllowInsecure: e.AllowInsecure, + }) + } + return AddressMatchRules{Rules: configs}, nil +} + +func addressPatternToRegexp(pattern string) string { + var result strings.Builder + result.WriteString("^") + first := true + for literal := range strings.SplitSeq(pattern, "*") { + if !first { + // Replace * with .* + result.WriteString(".*") + } + result.WriteString(regexp.QuoteMeta(literal)) + first = false + } + result.WriteString("$") + return result.String() +} diff --git a/chasm/lib/callback/config_test.go b/common/callbacks/address_test.go similarity index 99% rename from chasm/lib/callback/config_test.go rename to common/callbacks/address_test.go index 2d4f9fe36a3..c12a2372f2f 100644 --- a/chasm/lib/callback/config_test.go +++ b/common/callbacks/address_test.go @@ -1,4 +1,4 @@ -package callback +package callbacks import ( "regexp" diff --git a/chasm/lib/callback/validator.go b/common/callbacks/validator.go similarity index 50% rename from chasm/lib/callback/validator.go rename to common/callbacks/validator.go index d9de4ea607b..e55c9c0c0fd 100644 --- a/chasm/lib/callback/validator.go +++ b/common/callbacks/validator.go @@ -1,4 +1,4 @@ -package callback +package callbacks import ( "context" @@ -11,38 +11,60 @@ import ( "google.golang.org/grpc/status" ) -// Validator validates completion callbacks attached to executions (workflows and standalone activities). +// Validator validates completion callbacks attached to executions (e.g. workflows and standalone activities). type Validator interface { Validate(ctx context.Context, namespaceName string, cbs []*commonpb.Callback) error } +// ValidatorConfig holds the limits a [Validator] enforces. +type ValidatorConfig struct { + MaxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter + + // Nexus-variant limits. + URLMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter + HeaderMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter + EndpointRules dynamicconfig.TypedPropertyFnWithNamespaceFilter[AddressMatchRules] +} + +func (vc *ValidatorConfig) Validate() error { + var missingFields []string + if vc.MaxCallbacksPerExecution == nil { + missingFields = append(missingFields, "MaxCallbacksPerExecution") + } + if vc.URLMaxLength == nil { + missingFields = append(missingFields, "URLMaxLength") + } + if vc.HeaderMaxSize == nil { + missingFields = append(missingFields, "HeaderMaxSize") + } + if vc.EndpointRules == nil { + missingFields = append(missingFields, "EndpointRules") + } + + if len(missingFields) != 0 { + return fmt.Errorf("missing required fields: %v", missingFields) + } + return nil +} + type validator struct { - maxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter - urlMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter - headerMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter - endpointRules dynamicconfig.TypedPropertyFnWithNamespaceFilter[AddressMatchRules] + config ValidatorConfig } -func NewValidator( - maxCallbacksPerExecution dynamicconfig.IntPropertyFnWithNamespaceFilter, - urlMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter, - headerMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter, - endpointRules dynamicconfig.TypedPropertyFnWithNamespaceFilter[AddressMatchRules], -) Validator { - return &validator{ - maxCallbacksPerExecution: maxCallbacksPerExecution, - urlMaxLength: urlMaxLength, - headerMaxSize: headerMaxSize, - endpointRules: endpointRules, +// NewValidator returns a new Validator. +func NewValidator(config ValidatorConfig) (Validator, error) { + if err := config.Validate(); err != nil { + return nil, err } + return &validator{config: config}, nil } // Validate validates completion callbacks: count, URL length, endpoint allowlist, header size, and normalizes header // keys to lowercase. func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*commonpb.Callback) error { - if len(cbs) > v.maxCallbacksPerExecution(namespaceName) { + if len(cbs) > v.config.MaxCallbacksPerExecution(namespaceName) { return serviceerror.NewInvalidArgumentf( - "cannot attach more than %d callbacks to an execution", v.maxCallbacksPerExecution(namespaceName), + "cannot attach more than %d callbacks to an execution", v.config.MaxCallbacksPerExecution(namespaceName), ) } @@ -50,12 +72,12 @@ func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*com switch variant := cb.GetVariant().(type) { case *commonpb.Callback_Nexus_: rawURL := variant.Nexus.GetUrl() - if len(rawURL) > v.urlMaxLength(namespaceName) { + if len(rawURL) > v.config.URLMaxLength(namespaceName) { return serviceerror.NewInvalidArgumentf( - "invalid url: url length longer than max length allowed of %d", v.urlMaxLength(namespaceName), + "invalid url: url length longer than max length allowed of %d", v.config.URLMaxLength(namespaceName), ) } - if err := v.endpointRules(namespaceName).Validate(rawURL); err != nil { + if err := v.config.EndpointRules(namespaceName).Validate(rawURL); err != nil { if s, ok := status.FromError(err); ok { return serviceerror.NewInvalidArgument(s.Message()) } @@ -68,9 +90,9 @@ func (v *validator) Validate(_ context.Context, namespaceName string, cbs []*com headerSize += len(k) + len(val) lowerCaseHeaders[strings.ToLower(k)] = val } - if headerSize > v.headerMaxSize(namespaceName) { + if headerSize > v.config.HeaderMaxSize(namespaceName) { return serviceerror.NewInvalidArgumentf( - "invalid header: header size longer than max allowed size of %d", v.headerMaxSize(namespaceName), + "invalid header: header size longer than max allowed size of %d", v.config.HeaderMaxSize(namespaceName), ) } variant.Nexus.Header = lowerCaseHeaders diff --git a/chasm/lib/callback/validator_test.go b/common/callbacks/validator_test.go similarity index 67% rename from chasm/lib/callback/validator_test.go rename to common/callbacks/validator_test.go index 63d95d87d96..a27f19830f4 100644 --- a/chasm/lib/callback/validator_test.go +++ b/common/callbacks/validator_test.go @@ -1,4 +1,4 @@ -package callback +package callbacks import ( "context" @@ -10,18 +10,40 @@ import ( "go.temporal.io/api/serviceerror" ) +func mustNewValidator(t *testing.T, cfg ValidatorConfig) Validator { + t.Helper() + v, err := NewValidator(cfg) + require.NoError(t, err) + return v +} + +func TestValidatorConfigValidate(t *testing.T) { + cfg := ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 10 }, + HeaderMaxSize: func(string) int { return 4096 }, + } + + _, err := NewValidator(cfg) + require.EqualError(t, err, "missing required fields: [URLMaxLength EndpointRules]") +} + func TestValidateCallbacks(t *testing.T) { - allowAll := AddressMatchRules{ + ctx := context.Background() + + allowAllAddresses := AddressMatchRules{ Rules: []AddressMatchRule{ {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, }, } - v := NewValidator( - func(string) int { return 10 }, - func(string) int { return 1000 }, - func(string) int { return 4096 }, - func(string) AddressMatchRules { return allowAll }, - ) + getStandardConfig := func() ValidatorConfig { + return ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 10 }, + URLMaxLength: func(string) int { return 1000 }, + HeaderMaxSize: func(string) int { return 4096 }, + EndpointRules: func(string) AddressMatchRules { return allowAllAddresses }, + } + } + v := mustNewValidator(t, getStandardConfig()) t.Run("ValidNexusCallback", func(t *testing.T) { cbs := []*commonpb.Callback{ @@ -32,34 +54,28 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + err := v.Validate(ctx, "ns", cbs) require.NoError(t, err) }) t.Run("TooManyCallbacks", func(t *testing.T) { - v := NewValidator( - func(string) int { return 1 }, - func(string) int { return 1000 }, - func(string) int { return 4096 }, - func(string) AddressMatchRules { return allowAll }, - ) cbs := []*commonpb.Callback{ {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/cb1"}}}, {Variant: &commonpb.Callback_Nexus_{Nexus: &commonpb.Callback_Nexus{Url: "http://localhost/cb2"}}}, } - err := v.Validate(context.Background(), "ns", cbs) + + cfg := getStandardConfig() + cfg.MaxCallbacksPerExecution = func(string) int { return 1 } + v := mustNewValidator(t, cfg) + + err := v.Validate(ctx, "ns", cbs) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) require.Contains(t, err.Error(), "cannot attach more than 1 callbacks") }) t.Run("URLTooLong", func(t *testing.T) { - v := NewValidator( - func(string) int { return 10 }, - func(string) int { return 50 }, - func(string) int { return 4096 }, - func(string) AddressMatchRules { return allowAll }, - ) cbs := []*commonpb.Callback{ {Variant: &commonpb.Callback_Nexus_{ Nexus: &commonpb.Callback_Nexus{ @@ -67,7 +83,12 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + cfg := getStandardConfig() + cfg.URLMaxLength = func(string) int { return 50 } + v := mustNewValidator(t, cfg) + + err := v.Validate(ctx, "ns", cbs) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) require.Contains(t, err.Error(), "url length longer than max length allowed") @@ -82,7 +103,8 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + err := v.Validate(ctx, "ns", cbs) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) require.Contains(t, err.Error(), "header size longer than max allowed size") @@ -97,7 +119,8 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + err := v.Validate(ctx, "ns", cbs) require.NoError(t, err) nexus := cbs[0].GetNexus() require.Equal(t, "application/json", nexus.Header["content-type"]) @@ -107,12 +130,6 @@ func TestValidateCallbacks(t *testing.T) { }) t.Run("URLNotInAllowlist", func(t *testing.T) { - v := NewValidator( - func(string) int { return 10 }, - func(string) int { return 1000 }, - func(string) int { return 4096 }, - func(string) AddressMatchRules { return AddressMatchRules{} }, - ) cbs := []*commonpb.Callback{ {Variant: &commonpb.Callback_Nexus_{ Nexus: &commonpb.Callback_Nexus{ @@ -120,7 +137,12 @@ func TestValidateCallbacks(t *testing.T) { }, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + cfg := getStandardConfig() + cfg.EndpointRules = func(string) AddressMatchRules { return AddressMatchRules{} } + v := mustNewValidator(t, cfg) + + err := v.Validate(ctx, "ns", cbs) var invalidArgErr *serviceerror.InvalidArgument require.ErrorAs(t, err, &invalidArgErr) require.Contains(t, err.Error(), "does not match any configured callback address") @@ -130,14 +152,15 @@ func TestValidateCallbacks(t *testing.T) { cbs := []*commonpb.Callback{ {Variant: nil}, } - err := v.Validate(context.Background(), "ns", cbs) + + err := v.Validate(ctx, "ns", cbs) var unimplementedErr *serviceerror.Unimplemented require.ErrorAs(t, err, &unimplementedErr) require.Contains(t, err.Error(), "unknown callback variant") }) t.Run("EmptyCallbacksNoError", func(t *testing.T) { - err := v.Validate(context.Background(), "ns", nil) + err := v.Validate(ctx, "ns", nil) require.NoError(t, err) }) @@ -147,7 +170,8 @@ func TestValidateCallbacks(t *testing.T) { Internal: &commonpb.Callback_Internal{}, }}, } - err := v.Validate(context.Background(), "ns", cbs) + + err := v.Validate(ctx, "ns", cbs) require.NoError(t, err) }) } diff --git a/common/links/validator.go b/common/links/validator.go index e05c25b4d8d..b6499102c66 100644 --- a/common/links/validator.go +++ b/common/links/validator.go @@ -5,10 +5,11 @@ import ( commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/dynamicconfig" ) -// ValidateRequest validates all links attached directly to a request or embedded in its callbacks. -func ValidateRequest( +// ValidateWithCallbacks validates all links attached directly to a request or embedded in its callbacks. +func ValidateWithCallbacks( links []*commonpb.Link, callbacks []*commonpb.Callback, maxAllowedLinks int, @@ -25,7 +26,6 @@ func ValidateRequest( // Validate checks that the given links do not exceed the configured count and // per-link size limits, and that each link's variant has its required fields // populated. -// nolint:revive // cognitive-complexity is high but justified to keep each case together for readability. func Validate(links []*commonpb.Link, maxAllowedLinks, maxSize int) error { if len(links) > maxAllowedLinks { return serviceerror.NewInvalidArgumentf("cannot attach more than %d links per request, got %d", maxAllowedLinks, len(links)) @@ -34,57 +34,127 @@ func Validate(links []*commonpb.Link, maxAllowedLinks, maxSize int) error { if l.Size() > maxSize { return serviceerror.NewInvalidArgumentf("link exceeds allowed size of %d, got %d", maxSize, l.Size()) } - switch t := l.Variant.(type) { - case *commonpb.Link_WorkflowEvent_: - if t.WorkflowEvent.GetNamespace() == "" { - return serviceerror.NewInvalidArgument("workflow event link must not have an empty namespace field") - } - if t.WorkflowEvent.GetWorkflowId() == "" { - return serviceerror.NewInvalidArgument("workflow event link must not have an empty workflow ID field") - } - if t.WorkflowEvent.GetRunId() == "" { - return serviceerror.NewInvalidArgument("workflow event link must not have an empty run ID field") - } - if t.WorkflowEvent.GetEventRef().GetEventType() == enumspb.EVENT_TYPE_UNSPECIFIED && t.WorkflowEvent.GetEventRef().GetEventId() != 0 { - return serviceerror.NewInvalidArgument("workflow event link ref cannot have an unspecified event type and a non-zero event ID") - } - case *commonpb.Link_BatchJob_: - if t.BatchJob.GetJobId() == "" { - return serviceerror.NewInvalidArgument("batch job link must not have an empty job ID") - } - case *commonpb.Link_NexusOperation_: - if t.NexusOperation.GetNamespace() == "" { - return serviceerror.NewInvalidArgument("nexus operation link must not have an empty namespace field") - } - if t.NexusOperation.GetOperationId() == "" { - return serviceerror.NewInvalidArgument("nexus operation link must not have an empty operation ID field") - } - if t.NexusOperation.GetRunId() == "" { - return serviceerror.NewInvalidArgument("nexus operation link must not have an empty run ID field") - } - case *commonpb.Link_Activity_: - if t.Activity.GetNamespace() == "" { - return serviceerror.NewInvalidArgument("activity link must not have an empty namespace field") - } - if t.Activity.GetActivityId() == "" { - return serviceerror.NewInvalidArgument("activity link must not have an empty activity ID field") - } - if t.Activity.GetRunId() == "" { - return serviceerror.NewInvalidArgument("activity link must not have an empty run ID field") - } - case *commonpb.Link_Workflow_: - if t.Workflow.GetNamespace() == "" { - return serviceerror.NewInvalidArgument("workflow link must not have an empty namespace field") - } - if t.Workflow.GetWorkflowId() == "" { - return serviceerror.NewInvalidArgument("workflow link must not have an empty workflow ID field") - } - if t.Workflow.GetRunId() == "" { - return serviceerror.NewInvalidArgument("workflow link must not have an empty run ID field") - } - default: - return serviceerror.NewInvalidArgument("unsupported link variant") + if err := validateFields(l); err != nil { + return err + } + } + return nil +} + +// validateFields confirms that Link has all the required fields. +// nolint:revive // cognitive-complexity is high but justified to keep each case together for readability. +func validateFields(l *commonpb.Link) error { + switch t := l.Variant.(type) { + case *commonpb.Link_WorkflowEvent_: + if t.WorkflowEvent.GetNamespace() == "" { + return serviceerror.NewInvalidArgument("workflow event link must not have an empty namespace field") + } + if t.WorkflowEvent.GetWorkflowId() == "" { + return serviceerror.NewInvalidArgument("workflow event link must not have an empty workflow ID field") + } + if t.WorkflowEvent.GetRunId() == "" { + return serviceerror.NewInvalidArgument("workflow event link must not have an empty run ID field") + } + if t.WorkflowEvent.GetEventRef().GetEventType() == enumspb.EVENT_TYPE_UNSPECIFIED && t.WorkflowEvent.GetEventRef().GetEventId() != 0 { + return serviceerror.NewInvalidArgument("workflow event link ref cannot have an unspecified event type and a non-zero event ID") + } + case *commonpb.Link_BatchJob_: + if t.BatchJob.GetJobId() == "" { + return serviceerror.NewInvalidArgument("batch job link must not have an empty job ID") + } + case *commonpb.Link_NexusOperation_: + if t.NexusOperation.GetNamespace() == "" { + return serviceerror.NewInvalidArgument("nexus operation link must not have an empty namespace field") + } + if t.NexusOperation.GetOperationId() == "" { + return serviceerror.NewInvalidArgument("nexus operation link must not have an empty operation ID field") + } + if t.NexusOperation.GetRunId() == "" { + return serviceerror.NewInvalidArgument("nexus operation link must not have an empty run ID field") + } + case *commonpb.Link_Activity_: + if t.Activity.GetNamespace() == "" { + return serviceerror.NewInvalidArgument("activity link must not have an empty namespace field") + } + if t.Activity.GetActivityId() == "" { + return serviceerror.NewInvalidArgument("activity link must not have an empty activity ID field") + } + if t.Activity.GetRunId() == "" { + return serviceerror.NewInvalidArgument("activity link must not have an empty run ID field") + } + case *commonpb.Link_Workflow_: + if t.Workflow.GetNamespace() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty namespace field") + } + if t.Workflow.GetWorkflowId() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty workflow ID field") } + if t.Workflow.GetRunId() == "" { + return serviceerror.NewInvalidArgument("workflow link must not have an empty run ID field") + } + default: + return serviceerror.NewInvalidArgument("unsupported link variant") + } + + return nil +} + +// Validator validates links attached to executions. It enforces both +// per-request limits (count + size + variant shape) and a cumulative cap. +type Validator struct { + // resourceName describes the resource the link is attached to in an error message, as a noun + // phrase including its article. e.g. "an activity", "a nexus operation". + resourceName string + + maxLinksPerRequest dynamicconfig.IntPropertyFnWithNamespaceFilter + maxLinksPerResource dynamicconfig.IntPropertyFnWithNamespaceFilter + linkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter +} + +func NewValidator( + resourceName string, + maxLinksPerRequest dynamicconfig.IntPropertyFnWithNamespaceFilter, + maxLinksPerResource dynamicconfig.IntPropertyFnWithNamespaceFilter, + linkMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter, +) *Validator { + return &Validator{ + resourceName: resourceName, + maxLinksPerRequest: maxLinksPerRequest, + maxLinksPerResource: maxLinksPerResource, + linkMaxSize: linkMaxSize, + } +} + +// ValidateRequest checks count, per-link size, and variant shape for the links +// on a single incoming request. +func (v *Validator) ValidateRequest(namespaceName string, links []*commonpb.Link) error { + return Validate(links, v.maxLinksPerRequest(namespaceName), v.linkMaxSize(namespaceName)) +} + +// ValidateRequestWithCallbacks also counts and validates links embedded in completion callbacks. +func (v *Validator) ValidateRequestWithCallbacks( + namespaceName string, + links []*commonpb.Link, + callbacks []*commonpb.Callback, +) error { + return ValidateWithCallbacks( + links, + callbacks, + v.maxLinksPerRequest(namespaceName), + v.linkMaxSize(namespaceName), + ) +} + +// ValidateTotal checks that adding addingCount links to a resource will not exceed the limit. +func (v *Validator) ValidateTotal(namespaceName string, existingCount, addingCount int) error { + maxLinks := v.maxLinksPerResource(namespaceName) + if existingCount+addingCount > maxLinks { + return serviceerror.NewFailedPreconditionf( + "cannot attach more than %d links to %s (%d links already attached)", + maxLinks, + v.resourceName, + existingCount, + ) } return nil } diff --git a/common/links/validator_test.go b/common/links/validator_test.go index 816de48f597..ef64cae5eac 100644 --- a/common/links/validator_test.go +++ b/common/links/validator_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/links" "google.golang.org/protobuf/proto" ) @@ -187,3 +188,99 @@ func TestValidate(t *testing.T) { require.EqualError(t, err, "workflow link must not have an empty run ID field") }) } + +func batchJobLink(jobID string) *commonpb.Link { + return &commonpb.Link{ + Variant: &commonpb.Link_BatchJob_{ + BatchJob: &commonpb.Link_BatchJob{JobId: jobID}, + }, + } +} + +func TestValidateWithCallbacks(t *testing.T) { + const maxSize = 1024 + + t.Run("HappyPath", func(t *testing.T) { + err := links.ValidateWithCallbacks( + []*commonpb.Link{batchJobLink("request-job")}, + []*commonpb.Callback{{Links: []*commonpb.Link{batchJobLink("callback-job")}}}, + 2, + maxSize, + ) + require.NoError(t, err) + }) + + t.Run("CountsRequestAndCallbackLinksTogether", func(t *testing.T) { + err := links.ValidateWithCallbacks( + []*commonpb.Link{batchJobLink("request-job")}, + []*commonpb.Callback{{Links: []*commonpb.Link{batchJobLink("callback-job")}}}, + 1, + maxSize, + ) + require.ErrorContains(t, err, "cannot attach more than 1 links per request, got 2") + }) + + t.Run("InvalidCallbackLink", func(t *testing.T) { + err := links.ValidateWithCallbacks( + nil, + []*commonpb.Callback{{Links: []*commonpb.Link{batchJobLink("")}}}, + 2, + maxSize, + ) + require.ErrorContains(t, err, "batch job link must not have an empty job ID") + }) +} + +func TestValidator(t *testing.T) { + limit := func(n int) func(string) int { + return func(string) int { return n } + } + + t.Run("ValidateRequest", func(t *testing.T) { + v := links.NewValidator("an activity", limit(1), limit(10), limit(1024)) + require.NoError(t, v.ValidateRequest("ns", []*commonpb.Link{batchJobLink("a")})) + require.ErrorContains(t, + v.ValidateRequest("ns", []*commonpb.Link{batchJobLink("a"), batchJobLink("b")}), + "cannot attach more than 1 links per request, got 2") + }) + + t.Run("ValidateRequestWithCallbacks", func(t *testing.T) { + v := links.NewValidator("an activity", limit(1), limit(10), limit(1024)) + err := v.ValidateRequestWithCallbacks( + "ns", + []*commonpb.Link{batchJobLink("request-job")}, + []*commonpb.Callback{{Links: []*commonpb.Link{batchJobLink("callback-job")}}}, + ) + require.ErrorContains(t, err, "cannot attach more than 1 links per request, got 2") + }) + + t.Run("ValidateTotal", func(t *testing.T) { + v := links.NewValidator("an activity", limit(10), limit(3), limit(1024)) + require.NoError(t, v.ValidateTotal("ns", 1, 2)) + + err := v.ValidateTotal("ns", 1, 3) + require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.EqualError(t, err, "cannot attach more than 3 links to an activity (1 links already attached)") + }) + + t.Run("LimitsAreNamespaceScoped", func(t *testing.T) { + perNamespace := func(limits map[string]int) func(string) int { + return func(ns string) int { return limits[ns] } + } + v := links.NewValidator( + "an activity", + perNamespace(map[string]int{"strict": 1, "loose": 5}), + perNamespace(map[string]int{"strict": 1, "loose": 5}), + limit(1024), + ) + twoLinks := []*commonpb.Link{batchJobLink("a"), batchJobLink("b")} + + require.NoError(t, v.ValidateRequest("loose", twoLinks)) + require.ErrorContains(t, v.ValidateRequest("strict", twoLinks), + "cannot attach more than 1 links per request, got 2") + + require.NoError(t, v.ValidateTotal("loose", 2, 2)) + require.ErrorContains(t, v.ValidateTotal("strict", 2, 2), + "cannot attach more than 1 links to an activity") + }) +} diff --git a/service/frontend/fx.go b/service/frontend/fx.go index b160db42856..15b12f1575a 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -9,7 +9,7 @@ import ( "go.temporal.io/server/api/adminservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity" - "go.temporal.io/server/chasm/lib/callback" + chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" nexusoperationpb "go.temporal.io/server/chasm/lib/nexusoperation/gen/nexusoperationpb/v1" chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" @@ -21,6 +21,7 @@ import ( "go.temporal.io/server/common/archiver" "go.temporal.io/server/common/archiver/provider" "go.temporal.io/server/common/authorization" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/config" @@ -143,7 +144,7 @@ var Module = fx.Options( chasmnexus.Module, chasmscheduler.Module, chasmworkflow.Module, - callback.Module, + chasmcallback.Module, activity.FrontendModule, fx.Provide(visibility.ChasmVisibilityManagerProvider), fx.Provide(chasm.ChasmVisibilityInterceptorProvider), @@ -903,13 +904,13 @@ func OperatorHandlerProvider( // callbackValidatorProvider creates a callback Validator using the production dynamic config keys // so that existing operator configurations (callback.allowedAddresses) are honored. -func callbackValidatorProvider(dc *dynamicconfig.Collection) callback.Validator { - return callback.NewValidator( - callback.MaxPerExecution.Get(dc), - dynamicconfig.FrontendCallbackURLMaxLength.Get(dc), - dynamicconfig.FrontendCallbackHeaderMaxSize.Get(dc), - callback.AllowedAddresses.Get(dc), - ) +func callbackValidatorProvider(dc *dynamicconfig.Collection) (callbacks.Validator, error) { + return callbacks.NewValidator(callbacks.ValidatorConfig{ + MaxCallbacksPerExecution: chasmcallback.MaxPerExecution.Get(dc), + URLMaxLength: dynamicconfig.FrontendCallbackURLMaxLength.Get(dc), + HeaderMaxSize: dynamicconfig.FrontendCallbackHeaderMaxSize.Get(dc), + EndpointRules: chasmcallback.AllowedAddresses.Get(dc), + }) } func HandlerProvider( @@ -948,7 +949,7 @@ func HandlerProvider( healthInterceptor *interceptor.HealthInterceptor, scheduleSpecBuilder *scheduler.SpecBuilder, activityHandler activity.FrontendHandler, - callbackValidator callback.Validator, + callbackValidator callbacks.Validator, nexusOperationHandler chasmnexus.FrontendHandler, registry *chasm.Registry, frontendServiceResolver membership.ServiceResolver, diff --git a/service/frontend/service.go b/service/frontend/service.go index ef40b03c330..f48d86c33bc 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -11,8 +11,9 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" "go.temporal.io/server/chasm/lib/activity" - "go.temporal.io/server/chasm/lib/callback" + chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -214,7 +215,7 @@ type Config struct { CallbackURLMaxLength dynamicconfig.IntPropertyFnWithNamespaceFilter CallbackHeaderMaxSize dynamicconfig.IntPropertyFnWithNamespaceFilter MaxCallbacksPerWorkflow dynamicconfig.IntPropertyFnWithNamespaceFilter - CallbackEndpointConfigs dynamicconfig.TypedPropertyFnWithNamespaceFilter[callback.AddressMatchRules] + CallbackEndpointConfigs dynamicconfig.TypedPropertyFnWithNamespaceFilter[callbacks.AddressMatchRules] MaxNexusOperationTokenLength dynamicconfig.IntPropertyFnWithNamespaceFilter NexusRequestHeadersBlacklist dynamicconfig.TypedPropertyFn[*regexp.Regexp] @@ -418,7 +419,7 @@ func NewConfig( LinkMaxSize: dynamicconfig.FrontendLinkMaxSize.Get(dc), MaxLinksPerRequest: dynamicconfig.FrontendMaxLinksPerRequest.Get(dc), - CallbackEndpointConfigs: callback.AllowedAddresses.Get(dc), + CallbackEndpointConfigs: chasmcallback.AllowedAddresses.Get(dc), AdminEnableListHistoryTasks: dynamicconfig.AdminEnableListHistoryTasks.Get(dc), MaskInternalErrorDetails: dynamicconfig.FrontendMaskInternalErrorDetails.Get(dc), diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 415ceb898e4..9bed3410555 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -38,7 +38,6 @@ import ( taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity" - "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" @@ -49,6 +48,7 @@ import ( "go.temporal.io/server/common/archiver" "go.temporal.io/server/common/archiver/provider" "go.temporal.io/server/common/backoff" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/collection" @@ -129,7 +129,7 @@ type ( status int32 - callbackValidator callback.Validator + callbackValidator callbacks.Validator tokenSerializer *tasktoken.Serializer config *Config versionChecker headers.VersionChecker @@ -308,7 +308,7 @@ func (wh *WorkflowHandler) ValidateWorkerDeploymentVersionComputeConfig( // NewWorkflowHandler creates a gRPC handler for workflowservice func NewWorkflowHandler( - callbackValidator callback.Validator, + callbackValidator callbacks.Validator, config *Config, namespaceReplicationQueue persistence.NamespaceReplicationQueue, visibilityMgr manager.VisibilityManager, @@ -695,7 +695,7 @@ func (wh *WorkflowHandler) prepareStartWorkflowRequest( request.Links = dedupLinksFromCallbacks(request.GetLinks(), request.GetCompletionCallbacks()) - if err := commonlinks.ValidateRequest( + if err := commonlinks.ValidateWithCallbacks( request.GetLinks(), request.GetCompletionCallbacks(), wh.config.MaxLinksPerRequest(namespaceName.String()), @@ -5510,7 +5510,7 @@ func (wh *WorkflowHandler) prepareUpdateWorkflowRequest( request.GetRequest().GetCompletionCallbacks(), ) - return commonlinks.ValidateRequest( + return commonlinks.ValidateWithCallbacks( request.GetRequest().GetLinks(), request.GetRequest().GetCompletionCallbacks(), wh.config.MaxLinksPerRequest(namespaceName.String()), diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index 2e08831c71f..b58abcc4a81 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -39,12 +39,12 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" tokenspb "go.temporal.io/server/api/token/v1" - "go.temporal.io/server/chasm/lib/callback" "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common" "go.temporal.io/server/common/archiver" "go.temporal.io/server/common/archiver/provider" + "go.temporal.io/server/common/callbacks" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" dc "go.temporal.io/server/common/dynamicconfig" @@ -175,18 +175,21 @@ func (s *WorkflowHandlerSuite) getWorkflowHandler(config *Config) *WorkflowHandl s.mockVisibilityMgr.EXPECT().GetIndexName().Return(esIndexName).AnyTimes() healthInterceptor := interceptor.NewHealthInterceptor() healthInterceptor.SetHealthy(true) - cbValidator := callback.NewValidator( - func(string) int { return 2000 }, - config.CallbackURLMaxLength, - config.CallbackHeaderMaxSize, - func(string) callback.AddressMatchRules { - return callback.AddressMatchRules{ - Rules: []callback.AddressMatchRule{ + + cbValidator, err := callbacks.NewValidator(callbacks.ValidatorConfig{ + MaxCallbacksPerExecution: func(string) int { return 2000 }, + URLMaxLength: config.CallbackURLMaxLength, + HeaderMaxSize: config.CallbackHeaderMaxSize, + EndpointRules: func(string) callbacks.AddressMatchRules { + return callbacks.AddressMatchRules{ + Rules: []callbacks.AddressMatchRule{ {Regexp: regexp.MustCompile(`.*`), AllowInsecure: true}, }, } }, - ) + }) + s.NoError(err) + saValidator := searchattribute.NewValidator( s.mockResource.GetSearchAttributesProvider(), s.mockResource.GetSearchAttributesMapperProvider(), @@ -1164,8 +1167,8 @@ func (s *WorkflowHandlerSuite) TestStartWorkflowExecution_Failed_InvalidAggregat s.mockSearchAttributesMapperProvider.EXPECT().GetMapper(gomock.Any()).AnyTimes().Return(nil, nil) config := s.newConfig() config.MaxLinksPerRequest = dc.GetIntPropertyFnFilteredByNamespace(10) - config.CallbackEndpointConfigs = dc.GetTypedPropertyFnFilteredByNamespace(callback.AddressMatchRules{ - Rules: []callback.AddressMatchRule{ + config.CallbackEndpointConfigs = dc.GetTypedPropertyFnFilteredByNamespace(callbacks.AddressMatchRules{ + Rules: []callbacks.AddressMatchRule{ { Regexp: regexp.MustCompile(`.*`), AllowInsecure: true, diff --git a/tests/activity_standalone_test.go b/tests/activity_standalone_test.go index 3192614b533..9407c3db49c 100644 --- a/tests/activity_standalone_test.go +++ b/tests/activity_standalone_test.go @@ -1128,8 +1128,8 @@ func (s *standaloneActivityTestSuite) TestStart() { RequestId: env.Tv().Any().String(), Links: links, }) - require.Error(t, err) require.ErrorAs(t, err, new(*serviceerror.FailedPrecondition)) + require.ErrorContains(t, err, "cannot attach more than 2 links to an activity") }) t.Run("PerExecutionCapNotEnforcedWhenLinksWillBeDropped", func(t *testing.T) { From 6d0d534e0222e8d4306e4346e45baa622e8715a4 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 18:48:05 -0700 Subject: [PATCH 053/139] Add missing Nexus CODEOWNERS rules (#11754) WISOTT --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .github/CODEOWNERS | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 954ab831479..e890a020ffb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,8 @@ /chasm/lib/nexusoperation/ @temporalio/act @temporalio/nexus /chasm/lib/callback/ @temporalio/act @temporalio/nexus +/chasm/lib/workflow/nexus_* @temporalio/act @temporalio/nexus +/chasm/nexus_* @temporalio/oss-foundations @temporalio/act @temporalio/nexus # Service Clients @@ -31,6 +33,7 @@ /common/tasks/ @temporalio/oss-foundations @temporalio/cgs /common/persistence/ @temporalio/oss-foundations @temporalio/oss-matching @temporalio/cgs +/common/persistence/**/nexus_endpoint* @temporalio/oss-foundations @temporalio/oss-matching @temporalio/nexus # Components @@ -51,6 +54,10 @@ # DB Schema /schema/**/visibility/ @temporalio/oss-foundations +# Frontend Service + +/service/frontend/nexus_* @temporalio/nexus + # History Service /service/history/archival/ @temporalio/oss-foundations @@ -82,7 +89,7 @@ /service/worker/dlq/ @temporalio/oss-foundations /service/worker/scanner/ @temporalio/oss-foundations @temporalio/oss-matching -/service/worker/scanner/executions/ @temporalio/oss-foundations +/service/worker/scanner/executions/ @temporalio/oss-foundations /service/worker/scanner/history/ @temporalio/oss-foundations /service/worker/scanner/build_ids/ @temporalio/oss-matching /service/worker/scanner/taskqueue/ @temporalio/oss-matching From 3ba31f2ac021411f9ec275df99f036ef05ab862a Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 24 Aug 2026 19:30:41 -0700 Subject: [PATCH 054/139] Add context-aware channel test helpers (#11700) Adds generic `await.Rcv` and `await.Snd` helpers that bound blocking channel operations by the test context and report closure or cancellation clearly. Replaces both previous helpers and existing unsafe channel rcv/snd across tests. --- .github/copilot-instructions.md | 2 +- common/membership/grpc_resolver_test.go | 3 +- common/testing/await/channel.go | 43 +++++++ common/testing/await/channel_test.go | 95 ++++++++++++++++ common/testing/await/doc.go | 8 +- common/testing/nettest/rpc_factory_test.go | 3 +- docs/development/testing.md | 12 ++ tests/activity_api_pause_test.go | 12 +- tests/activity_api_reset_test.go | 16 +-- tests/activity_api_rules_test.go | 21 ++-- tests/activity_api_update_test.go | 8 +- tests/activity_test.go | 8 +- tests/add_tasks_test.go | 8 +- tests/client_misc_test.go | 3 +- tests/nexus_api_test.go | 15 +-- tests/nexus_otel_test.go | 24 +--- tests/nexus_standalone_test.go | 81 ++++---------- tests/nexus_workflow_test.go | 5 +- tests/pause_workflow_execution_test.go | 11 +- tests/sizelimit_test.go | 7 +- tests/testcore/functional_test_base.go | 20 ---- tests/testcore/test_env.go | 20 ---- tests/versioning_3_query_test.go | 13 ++- tests/versioning_3_test.go | 79 ++++++------- tests/versioning_test.go | 123 +++++++++++---------- tests/versioning_test_env.go | 16 +-- tests/worker_deployment_version_test.go | 3 +- 27 files changed, 356 insertions(+), 303 deletions(-) create mode 100644 common/testing/await/channel.go create mode 100644 common/testing/await/channel_test.go diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7bd7b32ce8c..1a9b5239556 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -32,7 +32,7 @@ Apply these patterns when reviewing PRs or suggesting code changes. - Do not use single-value type assertions on errors (`err.(*T)`); this panics instead of failing the test when the type doesn't match. Use `errors.As` with a guarded return. - When launching a goroutine to maintain a precondition for later assertions (e.g., keeping pollers active so a deployment version gets registered), loop until context cancellation rather than running once. A single attempt that times out exits silently, leaving downstream Eventually/propagation waits to hang until their own deadline. - Never call testify assertions (`s.NoError`, `s.Equal`, `require.NoError`, even `assert.NoError`) inside a `go func()` — if the goroutine outlives the test, the assertion panics the binary with `panic: Fail in goroutine after TestXxx has completed`. Move assertions to the test goroutine or use a buffered error channel. -- Any `<-ch` that isn't inside a `select` with `ctx.Done()` will hang indefinitely if the sender never sends. Always provide a context cancellation fallback. +- Prefer `await.Rcv` and `await.Snd` for blocking channel operations so tests fail on timeout instead of hanging indefinitely. - Never write to package-level or global variables in tests — parallel tests share the same process; thread values through function parameters instead. - Never use `time.Sleep` or `time.Since(start) > threshold` to enforce ordering — use channels, `sync.WaitGroup`, or `EventuallyWithT` instead. - When using `EventuallyWithT` (or similar) to wait for a condition driven by a background goroutine, ensure the goroutine's timeout is longer than the `EventuallyWithT` deadline — if the background op times out first, the condition will never be satisfied and the wait will hang until its own deadline. diff --git a/common/membership/grpc_resolver_test.go b/common/membership/grpc_resolver_test.go index 973b7d106cc..5461f12b646 100644 --- a/common/membership/grpc_resolver_test.go +++ b/common/membership/grpc_resolver_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.temporal.io/server/common/primitives" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/nettest" "go.uber.org/mock/gomock" "google.golang.org/grpc" @@ -77,7 +78,7 @@ func TestGRPCBuilder(t *testing.T) { ) require.NoError(t, err) conn.Connect() - require.NoError(t, <-serverErrs) + require.NoError(t, await.Rcv(t, serverErrs)) // The gRPC library calls [resolver.Resolver.Close] when the connection is closed in a background goroutine, so we // can't synchronously assert that [ServiceResolver.RemoveListener] was called right after the connection is closed. diff --git a/common/testing/await/channel.go b/common/testing/await/channel.go new file mode 100644 index 00000000000..675005ea33a --- /dev/null +++ b/common/testing/await/channel.go @@ -0,0 +1,43 @@ +package await + +import ( + "testing" + + "go.temporal.io/server/common/testing/testcontext" +) + +// Rcv waits for and returns the next value from ch. +// A closed ch yields the zero value, as with a plain receive. +// It fails the test if the context ends before ch produces a value. +func Rcv[T any](tb testing.TB, ch <-chan T) T { + tb.Helper() + + ctx := testcontext.For(tb) + select { + case value := <-ch: + return value + case <-ctx.Done(): + tb.Fatalf("context ended while waiting to receive from channel: %v", ctx.Err()) + var zero T + return zero + } +} + +// Snd waits to send value to ch. +// It fails the test if the context ends or ch closes before accepting the value. +func Snd[T any](tb testing.TB, ch chan<- T, value T) { + tb.Helper() + + ctx := testcontext.For(tb) + defer func() { + if recover() != nil { + tb.Fatal("channel closed before sending a value") + } + }() + + select { + case ch <- value: + case <-ctx.Done(): + tb.Fatalf("context ended while waiting to send to channel: %v", ctx.Err()) + } +} diff --git a/common/testing/await/channel_test.go b/common/testing/await/channel_test.go new file mode 100644 index 00000000000..f534fdc6426 --- /dev/null +++ b/common/testing/await/channel_test.go @@ -0,0 +1,95 @@ +package await_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/testcontext" +) + +func TestRcv(t *testing.T) { + t.Parallel() + + t.Run("receives value", func(t *testing.T) { + t.Parallel() + + ch := make(chan string, 1) + ch <- "value" + + require.Equal(t, "value", await.Rcv(t, ch)) + }) + + t.Run("returns zero value when channel closes", func(t *testing.T) { + t.Parallel() + + ch := make(chan string) + close(ch) + + require.Empty(t, await.Rcv(t, ch)) + }) + + t.Run("fails when context ends", func(t *testing.T) { + t.Parallel() + + ch := make(chan string) + tb := newRecordingTB() + + tb.run(func() { + cancelTestContext(tb) + await.Rcv(tb, ch) + }) + + require.Contains(t, tb.fatals(), "context canceled") + }) +} + +func TestSnd(t *testing.T) { + t.Parallel() + + t.Run("sends value", func(t *testing.T) { + t.Parallel() + + ch := make(chan string, 1) + await.Snd(t, ch, "value") + + require.Equal(t, "value", <-ch) + }) + + t.Run("fails instead of panicking when channel closes", func(t *testing.T) { + t.Parallel() + + ch := make(chan string) + close(ch) + tb := newRecordingTB() + + tb.run(func() { + await.Snd(tb, ch, "value") + }) + + require.Contains(t, tb.fatals(), "channel closed before sending a value") + }) + + t.Run("fails when context ends", func(t *testing.T) { + t.Parallel() + + ch := make(chan string) + tb := newRecordingTB() + + tb.run(func() { + cancelTestContext(tb) + await.Snd(tb, ch, "value") + }) + + require.Contains(t, tb.fatals(), "context canceled") + }) +} + +func cancelTestContext(tb testing.TB) { + testcontext.AttachDecorator(tb, "cancel", func(ctx context.Context) context.Context { + ctx, cancel := context.WithCancel(ctx) + cancel() + return ctx + }) +} diff --git a/common/testing/await/doc.go b/common/testing/await/doc.go index 83de1733274..b8d2c0f10fe 100644 --- a/common/testing/await/doc.go +++ b/common/testing/await/doc.go @@ -1,7 +1,9 @@ -// Package await provides polling-based test assertions as a replacement -// for testify's Eventually, EventuallyWithT, and their formatted variants. +// Package await provides context-bounded test waits, including polling +// assertions and blocking channel operations. // -// By default, it enforces a 10s timeout for each await attempt. +// Its polling assertions replace testify's Eventually, EventuallyWithT, and +// their formatted variants. By default, they enforce a 10s timeout for each +// await attempt. // // Improvements over testify's eventually functions: // diff --git a/common/testing/nettest/rpc_factory_test.go b/common/testing/nettest/rpc_factory_test.go index 9ad3dabf9f8..ef7fedd28da 100644 --- a/common/testing/nettest/rpc_factory_test.go +++ b/common/testing/nettest/rpc_factory_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/nettest" "google.golang.org/grpc" ) @@ -75,7 +76,7 @@ func testDialer(t *testing.T, target string, dial func(rpcFactory *nettest.RPCFa conn := dial(rpcFactory) conn.Connect() - require.NoError(t, <-errs) + require.NoError(t, await.Rcv(t, errs)) assert.Equal(t, target, conn.Target()) assert.NoError(t, conn.Close()) }) diff --git a/docs/development/testing.md b/docs/development/testing.md index 92a412e681e..b8857dc6655 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -68,6 +68,18 @@ await.RequireTrue(t, func() bool { `RequireTrue` is the wrong tool when dealing with errors or assertions; use `Require` instead. +### Blocking on channels + +Use `await.Rcv` and `await.Snd` for blocking channel operations. +They fail the test if its context ends or the channel closes. + +```go +headers := await.Rcv(t, requestHeaders) +await.Snd(t, responses, response) +``` + +Keep an explicit `select` when testing non-blocking behavior or waiting on multiple channels. + ### Parallelization All tests (and subtests!) should use `t.Parallel()` to be run concurrently; diff --git a/tests/activity_api_pause_test.go b/tests/activity_api_pause_test.go index 2e89dd8c6d7..a0c4580e136 100644 --- a/tests/activity_api_pause_test.go +++ b/tests/activity_api_pause_test.go @@ -145,7 +145,7 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { activityFunction := func() (string, error) { startedActivityCount.Add(1) if startedActivityCount.Load() == 1 { - s.WaitForChannel(activityPausedCn) + await.Rcv(t, activityPausedCn) return "", activityErr } return "done!", nil @@ -272,7 +272,7 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { activityFunction := func() (string, error) { startedActivityCount.Add(1) if startedActivityCount.Load() == 1 { - s.WaitForChannel(activityPausedCn) + await.Rcv(t, activityPausedCn) return "", activityErr } if shouldSucceed.Load() { @@ -571,7 +571,7 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { activityErr := errors.New("bad-luck-please-retry") return "", activityErr } - s.WaitForChannel(activityCompleteCn) + await.Rcv(t, activityCompleteCn) return "done!", nil } @@ -669,7 +669,7 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { activityFunction := func() (string, error) { startedActivityCount.Add(1) if startedActivityCount.Load() == 1 { - s.WaitForChannel(activityPausedCn) + await.Rcv(t, activityPausedCn) return "", activityErr } return "done!", nil @@ -860,7 +860,7 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { if !activityWasReset.Load() { return "", errors.New("bad-luck-please-retry") } - s.WaitForChannel(activityCompleteCh) + await.Rcv(t, activityCompleteCh) return "done!", nil } @@ -1006,7 +1006,7 @@ func TestActivityApiPause_AttributesToActivityInContextMetadata(t *testing.T) { }, workflowFn) require.NoError(t, err) - s.WaitForChannel(activityStartedCn) + await.Rcv(t, activityStartedCn) var trailer metadata.MD _, err = s.FrontendClient().PauseActivity(ctx, &workflowservice.PauseActivityRequest{ diff --git a/tests/activity_api_reset_test.go b/tests/activity_api_reset_test.go index 7678508f808..bf558143000 100644 --- a/tests/activity_api_reset_test.go +++ b/tests/activity_api_reset_test.go @@ -121,7 +121,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_AfterRetry() { return "", activityErr } - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -180,7 +180,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_WhileRunning() { var startedActivityCount atomic.Int32 activityFunction := func() (string, error) { startedActivityCount.Add(1) - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -240,7 +240,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_UnpausesRunningAc var startedActivityCount atomic.Int32 activityFunction := func() (string, error) { startedActivityCount.Add(1) - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -392,7 +392,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_InRetry() { return "", activityErr } - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -464,7 +464,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_KeepPaused() { return "", activityErr } - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -592,7 +592,7 @@ func (s *ActivityApiResetClientTestSuite) runResetHeartbeatDetails(resetHeartbea return "", errors.New("bad-luck-please-retry") } // not the first iteration - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) for activityShouldFinish.Load() == false { activity.RecordHeartbeat(ctx, "second") time.Sleep(time.Second) //nolint:forbidigo @@ -704,7 +704,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_WhilePaused() { if !activityWasReset.Load() { return "", errors.New("bad-luck-please-retry") } - s.WaitForChannel(ctx, activityCompleteCh) //nolint:staticcheck + await.Rcv(s.T(), activityCompleteCh) return "done!", nil } @@ -783,7 +783,7 @@ func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_TerminateWhileDef activityFunction := func() (string, error) { startedActivityCount.Add(1) - s.WaitForChannel(ctx, activityBlockCh) //nolint:staticcheck + await.Rcv(s.T(), activityBlockCh) return "done!", nil } diff --git a/tests/activity_api_rules_test.go b/tests/activity_api_rules_test.go index 9dbd5914d81..512455873f2 100644 --- a/tests/activity_api_rules_test.go +++ b/tests/activity_api_rules_test.go @@ -17,6 +17,7 @@ import ( "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/util" "go.temporal.io/server/tests/testcore" @@ -42,17 +43,17 @@ type internalRulesTestWorkflow struct { activityCompleteCn chan struct{} activityFailedCn chan struct{} - env *testcore.TestEnv + tb testing.TB } -func newInternalRulesTestWorkflow(env *testcore.TestEnv) *internalRulesTestWorkflow { +func newInternalRulesTestWorkflow(tb testing.TB) *internalRulesTestWorkflow { wf := &internalRulesTestWorkflow{ initialRetryInterval: 1 * time.Second, scheduleToCloseTimeout: 30 * time.Minute, startToCloseTimeout: 15 * time.Minute, activityCompleteCn: make(chan struct{}), activityFailedCn: make(chan struct{}), - env: env, + tb: tb, } wf.activityRetryPolicy = &temporal.RetryPolicy{ InitialInterval: wf.initialRetryInterval, @@ -89,11 +90,11 @@ func (w *internalRulesTestWorkflow) ActivityFuncForRetryActivity() (string, erro w.startedActivityCount.Add(1) if !w.letActivitySucceed.Load() { - w.env.WaitForChannel(w.activityFailedCn) + await.Rcv(w.tb, w.activityFailedCn) activityErr := errors.New("bad-luck-please-retry") return "", activityErr } - w.env.WaitForChannel(w.activityCompleteCn) + await.Rcv(w.tb, w.activityCompleteCn) return "done!", nil } @@ -104,13 +105,13 @@ func (w *internalRulesTestWorkflow) ActivityFuncForRetryTask() (string, error) { activityErr := errors.New("bad-luck-please-retry") return "", activityErr } - w.env.WaitForChannel(w.activityCompleteCn) + await.Rcv(w.tb, w.activityCompleteCn) return "done!", nil } func (w *internalRulesTestWorkflow) ActivityFuncForPrePause() (string, error) { w.startedActivityCount.Add(1) - w.env.WaitForChannel(w.activityCompleteCn) + await.Rcv(w.tb, w.activityCompleteCn) return "done!", nil } @@ -252,7 +253,7 @@ func (s *ActivityApiRulesClientTestSuite) TestActivityRulesApi_CRUD() { func (s *ActivityApiRulesClientTestSuite) TestActivityRulesApi_RetryActivity() { env := s.newTestEnv() - testWorkflow := newInternalRulesTestWorkflow(env) + testWorkflow := newInternalRulesTestWorkflow(s.T()) env.SdkWorker().RegisterWorkflow(testWorkflow.WorkflowFuncForRetryActivity) env.SdkWorker().RegisterActivity(testWorkflow.ActivityFuncForRetryActivity) @@ -380,7 +381,7 @@ func (s *ActivityApiRulesClientTestSuite) TestActivityRulesApi_RetryTask() { // 8. Let activity complete // 9. Wait for workflow to finish - testRetryTaskWorkflow := newInternalRulesTestWorkflow(env) + testRetryTaskWorkflow := newInternalRulesTestWorkflow(s.T()) // set much longer retry interval to make sure that activity is retried at least once testRetryTaskWorkflow.activityRetryPolicy.InitialInterval = 4 * time.Second @@ -514,7 +515,7 @@ func (s *ActivityApiRulesClientTestSuite) TestActivityRulesApi_PrePause() { // 11. Let activity complete // 12. Wait for workflow to finish - testRetryTaskWorkflow := newInternalRulesTestWorkflow(env) + testRetryTaskWorkflow := newInternalRulesTestWorkflow(s.T()) env.SdkWorker().RegisterWorkflow(testRetryTaskWorkflow.WorkflowFuncForPrePause) env.SdkWorker().RegisterActivity(testRetryTaskWorkflow.ActivityFuncForPrePause) diff --git a/tests/activity_api_update_test.go b/tests/activity_api_update_test.go index f447522d65c..6c3506190ba 100644 --- a/tests/activity_api_update_test.go +++ b/tests/activity_api_update_test.go @@ -126,7 +126,7 @@ func TestActivityApiUpdateClientTestSuite(t *testing.T) { return "", activityErr } - s.WaitForChannel(activityUpdated) + await.Rcv(t, activityUpdated) return "done!", nil } @@ -356,7 +356,7 @@ func TestActivityApiUpdateClientTestSuite(t *testing.T) { return "", activityErr } - s.WaitForChannel(activityUpdated) + await.Rcv(t, activityUpdated) return "done!", nil } @@ -477,7 +477,7 @@ func TestActivityUpdateExecutionOptionsApi(t *testing.T) { if startedActivityCount.Load() == 1 { return "", errors.New("bad-luck-please-retry") } - s.WaitForChannel(activityUpdated) + await.Rcv(t, activityUpdated) return "done!", nil } @@ -669,7 +669,7 @@ func TestActivityUpdateExecutionOptionsApi(t *testing.T) { if startedActivityCount.Load() == 1 { return "", errors.New("bad-luck-please-retry") } - s.WaitForChannel(activityUpdated) + await.Rcv(t, activityUpdated) return "done!", nil } diff --git a/tests/activity_test.go b/tests/activity_test.go index b643df0c881..f0fb08d9a83 100644 --- a/tests/activity_test.go +++ b/tests/activity_test.go @@ -31,6 +31,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/tests/testcore" @@ -1045,12 +1046,7 @@ func (s *ActivityTestSuite) TestTryActivityCancellationFromWorkflow() { s.True(err == nil || errors.Is(err, testcore.ErrNoTasks)) env.Logger.Info("Waiting for cancel to complete.", tag.WorkflowRunID(we.RunId)) - select { - case <-cancelCh: - case <-s.Context().Done(): - s.Fail("Test timed out for activity cancellation", s.Context().Err()) - return - } + await.Rcv(s.T(), cancelCh) s.True(activityCanceled, "Activity was not cancelled.") env.Logger.Info("Activity cancelled.", tag.WorkflowRunID(we.RunId)) } diff --git a/tests/add_tasks_test.go b/tests/add_tasks_test.go index e94a4b89bc4..ce8d17f2d5f 100644 --- a/tests/add_tasks_test.go +++ b/tests/add_tasks_test.go @@ -16,6 +16,7 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/tasks" @@ -87,12 +88,7 @@ func (s *AddTasksSuite) TestAddTasks_Ok() { s.NoError(err) // Get the task that we skipped, and add it back - var task tasks.Task - select { - case task = <-skippedTasks: - case <-s.Context().Done(): - s.FailNow("timed out waiting for skipped task") - } + task := await.Rcv(s.T(), skippedTasks) shouldSkip.Store(false) blob, err := serialization.NewSerializer().SerializeTask(task) diff --git a/tests/client_misc_test.go b/tests/client_misc_test.go index 461515427b6..c16a8b80245 100644 --- a/tests/client_misc_test.go +++ b/tests/client_misc_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/searchattribute/sadefs" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/workflow/update" @@ -837,7 +838,7 @@ func (s *ClientMiscTestSuite) Test_BufferedQuery() { err = workflowRun.Get(s.Context(), nil) s.NoError(err) - s.NoError(<-describeErrCh) // assert on test goroutine after workflow completes + s.NoError(await.Rcv(s.T(), describeErrCh)) // assert on test goroutine after workflow completes } func (s *ClientMiscTestSuite) assertHistory(env *testcore.TestEnv, wid, rid string, expected []enumspb.EventType) { diff --git a/tests/nexus_api_test.go b/tests/nexus_api_test.go index 0559abd7c95..a68d2d1a5ce 100644 --- a/tests/nexus_api_test.go +++ b/tests/nexus_api_test.go @@ -23,6 +23,7 @@ import ( "go.temporal.io/server/common/metrics/metricstest" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/frontend/configs" @@ -301,7 +302,7 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Outcomes(useTemporalFailures }) tc.assertion(s, result, err, headerCapture.lastHeaders) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) requests := capture.Metric("nexus_requests") s.Len(requests, 1) @@ -441,7 +442,7 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Claims(useTemporalFailures b tc.assertion(s, result, err, preprocessErrors) if pollerErrCh != nil { - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) } } @@ -560,7 +561,7 @@ func (s *NexusApiTestSuite) TestNexusCancelOperation_Outcomes(useTemporalFailure err = handle.Cancel(s.Context(), nexus.CancelOperationOptions{Header: header}) tc.assertion(s, err, headerCapture.lastHeaders) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) requests := capture.Metric("nexus_requests") s.Len(requests, 1) @@ -639,7 +640,7 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_WithNamespaceAndTaskQueue_Su result, err := nexusrpc.StartOperation(ctx, client, op, "input", nexus.StartOperationOptions{}) s.NoError(err) s.Equal("input", result.Successful) - s.NoError(<-pollerErrCh1) + s.NoError(await.Rcv(s.T(), pollerErrCh1)) // Unversioned poller doesn't get a task pollerErrCh2 := env.nexusTaskPoller(ctx, s.T(), taskQueue, nexusEchoHandler) @@ -657,8 +658,8 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_WithNamespaceAndTaskQueue_Su } // Cancel the parent context to unblock the pollers that didn't receive a task. cancel() - s.NoError(<-pollerErrCh2) - s.NoError(<-pollerErrCh3) + s.NoError(await.Rcv(s.T(), pollerErrCh2)) + s.NoError(await.Rcv(s.T(), pollerErrCh3)) } // TestNexusClientNameMetricPropagation verifies that when an SDK worker polls for Nexus tasks @@ -693,7 +694,7 @@ func (s *NexusApiTestSuite) TestNexusClientNameMetricPropagation(useTemporalFail _, err = nexusrpc.StartOperation(s.Context(), client, op, "input", nexus.StartOperationOptions{}) s.NoError(err) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) // Verify that the matching service emitted nexus_task_requests with client_name tag. var found bool diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go index af5ba6790e5..85c61265dad 100644 --- a/tests/nexus_otel_test.go +++ b/tests/nexus_otel_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/testtelemetry" "go.temporal.io/server/tests/testcore" @@ -111,12 +112,7 @@ func (s *NexusOTELSuite) TestCallback() { s.NoError(err) s.NoError(env.SdkClient().GetWorkflow(s.Context(), env.Tv().WorkflowID(), startResponse.RunId).Get(s.Context(), nil)) - var headers http.Header - select { - case headers = <-requestHeaders: - case <-s.Context().Done(): - s.FailNow("timed out waiting for Nexus callback", s.Context().Err().Error()) - } + headers := await.Rcv(s.T(), requestHeaders) s.Equal(callbackHeaderValue, headers.Get("X-Callback-Header")) httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ TraceID: 1, @@ -181,12 +177,7 @@ func (s *NexusOTELSuite) TestOperation() { ScheduleToCloseTimeout: durationpb.New(time.Minute), }) s.NoError(err) - var nexusRequestID string - select { - case nexusRequestID = <-requestIDs: - case <-s.Context().Done(): - s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) - } + nexusRequestID := await.Rcv(s.T(), requestIDs) pollResponse, err := callerEnv.FrontendClient().PollNexusOperationExecution(s.Context(), &workflowservice.PollNexusOperationExecutionRequest{ Namespace: callerEnv.Namespace().String(), @@ -293,12 +284,7 @@ func (s *NexusOTELSuite) TestWorkerOperation() { }) s.NoError(err) - var request nexusRequest - select { - case request = <-requests: - case <-s.Context().Done(): - s.FailNow("timed out waiting for Nexus operation", s.Context().Err().Error()) - } + request := await.Rcv(s.T(), requests) operationURLPath := "/nexus/endpoints/" + endpoint.Id + "/services/" + service.Name + "/" + operation.Name() httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{ { @@ -369,7 +355,7 @@ func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { }) var handlerErr *nexus.HandlerError s.Require().ErrorAs(err, &handlerErr) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) httpSpans := s.requireNexusHTTPSpans(exporter, []nexusHTTPSpan{{ TraceID: 1, diff --git a/tests/nexus_standalone_test.go b/tests/nexus_standalone_test.go index 4daea03d738..ab44d3efc47 100644 --- a/tests/nexus_standalone_test.go +++ b/tests/nexus_standalone_test.go @@ -28,6 +28,7 @@ import ( "go.temporal.io/server/common/nexus/nexustest" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/searchattribute/sadefs" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/tests/testcore" @@ -295,22 +296,12 @@ func (s *NexusStandaloneTestSuite) TestDescribeStandaloneNexusOperation() { terminateErrCh <- err }() - select { - case err := <-terminateErrCh: - s.NoError(err) - case <-s.Context().Done(): - s.T().Fatal("TerminateNexusOperationExecution timed out") - } + s.NoError(await.Rcv(s.T(), terminateErrCh)) // Verify the longpoll result. - var longPollResp *workflowservice.DescribeNexusOperationExecutionResponse - select { - case result := <-describeResultCh: - s.NoError(result.err) - longPollResp = result.resp - case <-s.Context().Done(): - s.T().Fatal("DescribeNexusOperationExecution timed out") - } + result := await.Rcv(s.T(), describeResultCh) + s.NoError(result.err) + longPollResp := result.resp s.Equal(startResp.RunId, longPollResp.GetRunId()) s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_TERMINATED, longPollResp.GetInfo().GetStatus()) @@ -506,7 +497,7 @@ func (s *NexusStandaloneTestSuite) TestDescribeStandaloneNexusOperation() { protorequire.ProtoEqual(t, expectedResult, pollResp.GetResult()) }, 10*time.Second, 100*time.Millisecond) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) }) s.Run("IncludeOutcome_Failure", func(s *NexusStandaloneTestSuite) { @@ -822,7 +813,7 @@ func (s *NexusStandaloneTestSuite) TestDescribeStandaloneNexusOperation() { tc.assertOutcome(t, descResp, pollResp) }, 10*time.Second, 100*time.Millisecond) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) }) } }) @@ -1893,11 +1884,7 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationPoll() { pollResultCh <- pollResult{resp: resp, err: err} }() - select { - case <-pollStartedCh: - case <-s.Context().Done(): - s.T().Fatal("PollNexusOperationExecution did not start before timeout") - } + await.Rcv(s.T(), pollStartedCh) // PollNexusOperationExecution should not resolve before the operation is started. select { @@ -1949,17 +1936,13 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationPoll() { s.NoError(err) // Verify the poll result. - select { - case result := <-pollResultCh: - s.NoError(result.err) - protorequire.ProtoEqual(s.T(), &workflowservice.PollNexusOperationExecutionResponse{ - RunId: startResp.RunId, - WaitStage: enumspb.NEXUS_OPERATION_WAIT_STAGE_STARTED, - OperationToken: "test-operation-token", - }, result.resp) - case <-s.Context().Done(): - s.T().Fatal("PollNexusOperationExecution did not resolve before timeout") - } + result := await.Rcv(s.T(), pollResultCh) + s.NoError(result.err) + protorequire.ProtoEqual(s.T(), &workflowservice.PollNexusOperationExecutionResponse{ + RunId: startResp.RunId, + WaitStage: enumspb.NEXUS_OPERATION_WAIT_STAGE_STARTED, + OperationToken: "test-operation-token", + }, result.resp) descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), @@ -2018,11 +2001,7 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationPoll() { pollResultCh <- pollResult{resp: resp, err: err} }() - select { - case <-pollStartedCh: - case <-s.Context().Done(): - s.T().Fatal("PollNexusOperationExecution did not start before timeout") - } + await.Rcv(s.T(), pollStartedCh) // PollNexusOperationExecution should not resolve before the operation is closed. select { @@ -2044,20 +2023,10 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationPoll() { terminateErrCh <- err }() - select { - case err := <-terminateErrCh: - s.NoError(err) - case <-s.Context().Done(): - s.T().Fatal("TerminateNexusOperationExecution timed out") - } + s.NoError(await.Rcv(s.T(), terminateErrCh)) // Verify the poll result. - var result pollResult - select { - case result = <-pollResultCh: - case <-s.Context().Done(): - s.T().Fatal("PollNexusOperationExecution did not resolve before timeout") - } + result := await.Rcv(s.T(), pollResultCh) s.NoError(result.err) pollResp := result.resp @@ -2171,7 +2140,7 @@ func (s *NexusStandaloneTestSuite) TestStandaloneNexusOperationPoll() { }, pollResp.GetFailure()) }, 10*time.Second, 100*time.Millisecond) - s.NoError(<-pollerErrCh) + s.NoError(await.Rcv(s.T(), pollerErrCh)) }) s.Run("NamespaceNotFound", func(s *NexusStandaloneTestSuite) { @@ -2274,15 +2243,9 @@ func (s *NexusStandaloneTestSuite) TestAsyncCompletionIgnoresExecutionTransition Endpoint: endpointName, }) s.NoError(err) - var callbackToken string - var callbackURL string - select { - case callback := <-callbackCh: - callbackToken = callback.token - callbackURL = callback.url - case <-s.Context().Done(): - s.FailNow("timed out waiting for Nexus callback details", s.Context().Err().Error()) - } + callback := await.Rcv(s.T(), callbackCh) + callbackToken := callback.token + callbackURL := callback.url gen := &commonnexus.CallbackTokenGenerator{} decodedToken, err := commonnexus.DecodeCallbackToken(callbackToken) diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index 39f6e266f28..db4cadd0f48 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -44,6 +44,7 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/nexus/nexustest" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/historyrequire" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protorequire" @@ -1938,8 +1939,8 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationCancelBeforeStarted_Cancelati require.NotNil(t, desc.PendingNexusOperations[0].CancellationInfo) }, time.Second*10, time.Millisecond*100) - env.SendToChannel(canStartCh) - env.WaitForChannel(cancelSentCh) + await.Snd(s.T(), canStartCh, struct{}{}) + await.Rcv(s.T(), cancelSentCh) // Terminate the workflow for good measure. err = env.SdkClient().TerminateWorkflow(ctx, run.GetID(), run.GetRunID(), "test") diff --git a/tests/pause_workflow_execution_test.go b/tests/pause_workflow_execution_test.go index 07dae285b09..0d7b7b29121 100644 --- a/tests/pause_workflow_execution_test.go +++ b/tests/pause_workflow_execution_test.go @@ -25,6 +25,7 @@ import ( "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/payload" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/util" "go.temporal.io/server/tests/testcore" @@ -247,7 +248,7 @@ func (s *PauseWorkflowExecutionSuite) TestPauseUnpauseWorkflowExecution() { }, 5*time.Second, 200*time.Millisecond) // Unblock the activity to complete the workflow. - env.SendToChannel(env.activityCompletedCh) + await.Snd(s.T(), env.activityCompletedCh, struct{}{}) // assert that the workflow completes now. s.EventuallyWithT(func(t *assert.CollectT) { @@ -1054,7 +1055,7 @@ func (s *PauseWorkflowExecutionSuite) TestQueryWorkflowWhenPaused() { s.NotNil(unpauseResp) // Unblock the activity and send the signal to complete the workflow. - env.SendToChannel(env.activityCompletedCh) + await.Snd(s.T(), env.activityCompletedCh, struct{}{}) err = env.SdkClient().SignalWorkflow(s.Context(), workflowID, runID, env.testEndSignal, "test end signal") s.NoError(err) @@ -1278,7 +1279,7 @@ func (s *PauseWorkflowExecutionSuite) TestPauseWorkflowExecutionAlreadyPaused() }, 5*time.Second, 200*time.Millisecond) // Unblock the activity and send the signal to complete the workflow. - env.SendToChannel(env.activityCompletedCh) + await.Snd(s.T(), env.activityCompletedCh, struct{}{}) err = env.SdkClient().SignalWorkflow(s.Context(), workflowID, runID, env.testEndSignal, "test end signal") s.NoError(err) @@ -1597,7 +1598,7 @@ func (s *PauseWorkflowExecutionSuite) TestSignalWithStartWhilePaused() { RequestId: uuid.NewString(), }) s.NoError(err) - env.SendToChannel(env.activityCompletedCh) + await.Snd(s.T(), env.activityCompletedCh, struct{}{}) // The buffered signal is delivered: the workflow completes and its result // includes the signal payload. @@ -1785,7 +1786,7 @@ func (s *PauseWorkflowExecutionSuite) TestPauseIdempotentSameRequestId() { RequestId: uuid.NewString(), }) s.NoError(err) - env.SendToChannel(env.activityCompletedCh) + await.Snd(s.T(), env.activityCompletedCh, struct{}{}) err = env.SdkClient().SignalWorkflow(s.Context(), workflowID, runID, env.testEndSignal, "done") s.NoError(err) s.Await(func(s *PauseWorkflowExecutionSuite) { diff --git a/tests/sizelimit_test.go b/tests/sizelimit_test.go index 85a9f67ae54..d9f4d0ca869 100644 --- a/tests/sizelimit_test.go +++ b/tests/sizelimit_test.go @@ -20,6 +20,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/taskpoller" "go.temporal.io/server/common/testing/testvars" @@ -285,11 +286,7 @@ func (s *SizeLimitSuite) TestWorkflowFailed_PayloadSizeTooLarge() { env.Logger.Info("PollAndHandleWorkflowTask", tag.Error(err)) }() - select { - case <-sigReadyToSendChan: - case <-s.Context().Done(): - s.FailNow("timed out waiting for workflow task handler to be ready") - } + await.Rcv(s.T(), sigReadyToSendChan) _, err = env.FrontendClient().SignalWorkflowExecution(s.Context(), &workflowservice.SignalWorkflowExecutionRequest{ Namespace: env.Namespace().String(), diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index 20dc013812f..d9f5b325c06 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -710,26 +710,6 @@ func (s *FunctionalTestBase) RunTestWithMatchingBehavior(subtest func()) { } } -// Deprecated: use (*TestEnv).WaitForChannel instead. -func (s *FunctionalTestBase) WaitForChannel(ctx context.Context, ch chan struct{}) { - s.T().Helper() - select { - case <-ch: - case <-ctx.Done(): - s.FailNow("context timeout while waiting for channel") - } -} - -// Deprecated: use (*TestEnv).SendToChannel instead. -func (s *FunctionalTestBase) SendToChannel(ctx context.Context, ch chan struct{}) { - s.T().Helper() - select { - case ch <- struct{}{}: - case <-ctx.Done(): - s.FailNow("context timeout while sending to channel") - } -} - // TODO (alex): change to nsName namespace.Name func (s *FunctionalTestBase) SendSignal(nsName string, execution *commonpb.WorkflowExecution, signalName string, input *commonpb.Payloads, identity string) error { diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index 81642f463f7..61f1371a35d 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -474,26 +474,6 @@ func (e *TestEnv) Context() context.Context { return testcontext.For(e.t) } -// WaitForChannel waits for ch to receive using the TestEnv context. -func (e *TestEnv) WaitForChannel(ch <-chan struct{}) { - e.t.Helper() - select { - case <-ch: - case <-e.Context().Done(): - e.FailNow("context timeout while waiting for channel") - } -} - -// SendToChannel sends to ch using the TestEnv context. -func (e *TestEnv) SendToChannel(ch chan<- struct{}) { - e.t.Helper() - select { - case ch <- struct{}{}: - case <-e.Context().Done(): - e.FailNow("context timeout while sending to channel") - } -} - // SdkClient returns the SDK client. It is lazily initialized on the first call. func (e *TestEnv) SdkClient() sdkclient.Client { e.sdkClientOnce.Do(func() { diff --git a/tests/versioning_3_query_test.go b/tests/versioning_3_query_test.go index 23adb1b9d89..58ffa95474f 100644 --- a/tests/versioning_3_query_test.go +++ b/tests/versioning_3_query_test.go @@ -9,6 +9,7 @@ import ( "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/matching" @@ -75,7 +76,7 @@ func (s *Versioning3QuerySuite) testPinnedQueryDrainedVersion(env *VersioningTes close(idlePollerDone) }() env.setCurrentDeployment(s, tv) - env.WaitForChannel(idlePollerDone) + await.Rcv(s.T(), idlePollerDone) wftCompleted := make(chan struct{}) env.pollWftAndHandle(s, tv, false, wftCompleted, @@ -85,7 +86,7 @@ func (s *Versioning3QuerySuite) testPinnedQueryDrainedVersion(env *VersioningTes }) env.startWorkflow(s, tv, tv.VersioningOverridePinned()) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbPinned, tv.Deployment(), tv.VersioningOverridePinned(), nil) // create version v2 and make it current which shall make v1 go from current -> draining/drained @@ -96,7 +97,7 @@ func (s *Versioning3QuerySuite) testPinnedQueryDrainedVersion(env *VersioningTes close(idlePollerDone) }() env.setCurrentDeployment(s, tv2) - env.WaitForChannel(idlePollerDone) + await.Rcv(s.T(), idlePollerDone) // wait for v1 to become drained s.Await(func(s *Versioning3QuerySuite) { @@ -175,7 +176,7 @@ func (s *Versioning3QuerySuite) testQueryWithPinnedOverride(env *VersioningTestE runID := env.startWorkflow(s, tv, tv.VersioningOverridePinned()) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), tv.VersioningOverridePinned(), nil) if sticky { env.verifyWorkflowStickyQueue(s, tv.WithRunID(runID)) @@ -216,7 +217,7 @@ func (s *Versioning3QuerySuite) testUnpinnedQuery(env *VersioningTestEnv, sticky runID := env.startWorkflow(s, tv, nil) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), nil, nil) if sticky { env.verifyWorkflowStickyQueue(s, tv.WithRunID(runID)) @@ -228,7 +229,7 @@ func (s *Versioning3QuerySuite) testUnpinnedQuery(env *VersioningTestEnv, sticky close(pollerDone) }() env.pollAndQueryWorkflow(s, tv, sticky) - env.WaitForChannel(pollerDone) // wait for the idle poller to complete to not interfere with the next poller + await.Rcv(s.T(), pollerDone) // wait for the idle poller to complete to not interfere with the next poller env.setCurrentDeployment(s, tv2) env.waitForDeploymentDataPropagation(s, tv2, versionStatusCurrent, false, tqTypeWf) diff --git a/tests/versioning_3_test.go b/tests/versioning_3_test.go index 40a7b7ed9eb..8a3f5c8affb 100644 --- a/tests/versioning_3_test.go +++ b/tests/versioning_3_test.go @@ -34,6 +34,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/primitives/timestamp" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protoutils" "go.temporal.io/server/common/testing/taskpoller" @@ -262,14 +263,14 @@ func (s *Versioning3Suite) testWorkflowWithPinnedOverride(env *VersioningTestEnv runID := env.startWorkflow(s, tv, tv.VersioningOverridePinned()) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), tv.VersioningOverridePinned(), nil) env.verifyVersioningSAs(s, tv, vbPinned, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, tv) if sticky { env.verifyWorkflowStickyQueue(s, tv.WithRunID(runID)) } - env.WaitForChannel(actCompleted) + await.Rcv(s.T(), actCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), tv.VersioningOverridePinned(), nil) env.pollWftAndHandle(s, tv, sticky, nil, @@ -307,7 +308,7 @@ func (s *Versioning3Suite) testPinnedWorkflowWithLateActivityPoller(env *Version override := tv.VersioningOverridePinned() env.startWorkflow(s, tv, override) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), override, nil) // Wait long enough to make sure the activity is backlogged. env.validateBacklogCount(s, tv, tqTypeAct, 1) @@ -368,14 +369,14 @@ func (s *Versioning3Suite) testUnpinnedWorkflow(env *VersioningTestEnv, sticky b runID := env.startWorkflow(s, tv, nil) - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), nil, nil) env.verifyVersioningSAs(s, tv, vbUnpinned, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, tv) if sticky { env.verifyWorkflowStickyQueue(s, tv.WithRunID(runID)) } - env.WaitForChannel(actCompleted) + await.Rcv(s.T(), actCompleted) env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), nil, nil) env.pollWftAndHandle(s, tv, sticky, nil, @@ -1319,7 +1320,7 @@ func (s *Versioning3Suite) testTransitionFromActivity(sticky bool) { return env.respondActivity(), nil }) - env.WaitForChannel(act1Started) + await.Rcv(s.T(), act1Started) env.pollActivityAndHandle(s, tv1, act2Failed, func(task *workflowservice.PollActivityTaskQueueResponse) (*workflowservice.RespondActivityTaskCompletedRequest, error) { s.NotNil(task) @@ -1336,7 +1337,7 @@ func (s *Versioning3Suite) testTransitionFromActivity(sticky bool) { return nil, errors.New("intentional activity failure") }) - env.WaitForChannel(act2Started) + await.Rcv(s.T(), act2Started) env.verifyWorkflowVersioning(s, tv1, vbUnpinned, tv1.Deployment(), nil, nil) // 2. Set d2 as the current deployment @@ -1392,16 +1393,8 @@ func (s *Versioning3Suite) testTransitionFromActivity(sticky bool) { close(transitionStarted) env.Logger.Info("Transition wft started") // 8. Complete the transition after act1 completes and act2's first attempt fails. - select { - case <-act1Completed: - case <-s.Context().Done(): - s.FailNow("context expired waiting for act1 to complete") - } - select { - case <-act2Failed: - case <-s.Context().Done(): - s.FailNow("context expired waiting for act2 to fail") - } + await.Rcv(s.T(), act1Completed) + await.Rcv(s.T(), act2Failed) transitionCompleted.Store(true) env.Logger.Info("Transition wft completed") return env.respondEmptyWft(tv2, sticky, vbUnpinned), nil @@ -1412,7 +1405,7 @@ func (s *Versioning3Suite) testTransitionFromActivity(sticky bool) { } // 9. Now all activities should complete. - env.WaitForChannel(act2To4Completed) + await.Rcv(s.T(), act2To4Completed) select { case err := <-act2To4Err: s.NoError(err) @@ -1620,7 +1613,7 @@ func (s *Versioning3Suite) testChildWorkflowInheritanceExpectInherit(crossTq boo run, err := env.SdkClient().ExecuteWorkflow(s.Context(), startOpts, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(wfStarted) + await.Rcv(s.T(), wfStarted) close(wfStarted) // force panic if replayed // make v2 current for both parent and child and unblock the wf to start the child @@ -2167,7 +2160,7 @@ func (s *Versioning3Suite) testChildWorkflowInheritanceExpectNoInherit(crossTq b s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(wfStarted) + await.Rcv(s.T(), wfStarted) close(wfStarted) // make v2 current for both parent and child and unblock the wf to start the child @@ -2441,7 +2434,7 @@ func (s *Versioning3Suite) testPinnedCaNUpgradeOnCaN(normalTask, speculativeTask }) // Wait for the new run's first workflow task - env.WaitForChannel(wftNewRunDone) + await.Rcv(s.T(), wftNewRunDone) // Verify the new workflow run is on v2 (not v1) because of AUTO_UPGRADE initial behavior // But if testing with Pinned V1 override, it should be on v1 @@ -2574,7 +2567,7 @@ func (s *Versioning3Suite) testPinnedCaNUseRampingVersionOnCaN(pinnedOverride, n s.NotNil(task) return env.respondCompleteWorkflow(postCaNTV, vbPinned), nil }) - env.WaitForChannel(wftNewRunDone) + await.Rcv(s.T(), wftNewRunDone) env.verifyWorkflowVersioning(s, postCaNTV, vbPinned, postCaNTV.Deployment(), override, nil) env.verifyVersioningSAs(s, postCaNTV, vbPinned, enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, postCaNTV) @@ -2628,7 +2621,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_SubsequentWFTGoe s.NotNil(task) return env.respondEmptyWft(tv2, false, vbUnpinned), nil }) - env.WaitForChannel(firstWFTDone) + await.Rcv(s.T(), firstWFTDone) // Signal the UseRampingVersion run (omit run ID to target the current run). // The resulting WFT should go to v1 (Target/current), not v2 (ramping) — (1). @@ -2663,7 +2656,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_SubsequentWFTGoe DeploymentOptions: tv2.WorkerDeploymentOptions(true), }, nil }) - env.WaitForChannel(secondWFTDone) + await.Rcv(s.T(), secondWFTDone) // The CaN run's first WFT must land on v1 (current), NOT v2 (ramping) — (2). // UseRampingVersion is not inherited by a CaN issued from the resulting workflow. @@ -2673,7 +2666,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_SubsequentWFTGoe s.NotNil(task) return env.respondCompleteWorkflow(tv1, vbUnpinned), nil }) - env.WaitForChannel(thirdRunDone) + await.Rcv(s.T(), thirdRunDone) env.verifyWorkflowVersioning(s, tv1, vbUnpinned, tv1.Deployment(), nil, nil) env.verifyVersioningSAs(s, tv1, vbUnpinned, enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, tv1) @@ -2738,7 +2731,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_RetryInheritsIni DeploymentOptions: tv2.WorkerDeploymentOptions(true), }, nil }) - env.WaitForChannel(firstAttemptDone) + await.Rcv(s.T(), firstAttemptDone) // Retry run: ContinueAsNewInitialVersioningBehavior=USE_RAMPING_VERSION is propagated, // so the retry's first WFT must also land on v2 (ramping), not v1 (current). @@ -2748,7 +2741,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_RetryInheritsIni s.NotNil(task) return env.respondCompleteWorkflow(tv2, vbUnpinned), nil }) - env.WaitForChannel(retryRunDone) + await.Rcv(s.T(), retryRunDone) env.verifyWorkflowVersioning(s, tv2, vbUnpinned, tv2.Deployment(), nil, nil) env.verifyVersioningSAs(s, tv2, vbUnpinned, enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, tv2) @@ -2816,7 +2809,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_ChildDoesNotInhe DeploymentOptions: tv2.WorkerDeploymentOptions(true), }, nil }) - env.WaitForChannel(parentWFT1Done) + await.Rcv(s.T(), parentWFT1Done) // Child workflows do NOT inherit UseRampingVersion. childWFTDone := make(chan struct{}) @@ -2825,7 +2818,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UseRampingVersionOnCaN_ChildDoesNotInhe s.NotNil(task) return env.respondCompleteWorkflow(tv1, vbUnpinned), nil }) - env.WaitForChannel(childWFTDone) + await.Rcv(s.T(), childWFTDone) // Verify the child did not inherit UseRampingVersion. s.Await(func(s *Versioning3Suite) { @@ -2973,7 +2966,7 @@ func (s *Versioning3Suite) TestAutoUpgradeCaN_UpgradeOnCaN() { }) // Wait for the new run's first workflow task on v2 - env.WaitForChannel(wftNewRunDone) + await.Rcv(s.T(), wftNewRunDone) // Verify the new workflow run is on v2 (not v1) because of AUTO_UPGRADE initial behavior // The new workflow has the worker-sent AutoUpgrade behavior from the first WFT completion after CaN. @@ -3121,7 +3114,7 @@ func (s *Versioning3Suite) testCan(crossTq bool, behavior enumspb.VersioningBeha s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(wfStarted) + await.Rcv(s.T(), wfStarted) close(wfStarted) // force panic if replayed // make v2 current @@ -3636,7 +3629,7 @@ func (s *Versioning3Suite) TestActivityTQLags_DependentActivityCompletesOnTheNew return env.respondActivity(), nil }) - env.WaitForChannel(activityTaskCh) + await.Rcv(s.T(), activityTaskCh) // Verify that the workflow is still running on v1. s.Await(func(s *Versioning3Suite) { @@ -4530,7 +4523,7 @@ func (s *Versioning3Suite) TestVersionedQueueUnload() { }() // Wait for 10 seconds - env.WaitForChannel(keepAliveDone) + await.Rcv(s.T(), keepAliveDone) // Send a poll from v1 and ensure the poller gets the task taskReceived := false @@ -4723,7 +4716,7 @@ func (s *Versioning3Suite) TestPinnedCaN_NoAUOnCaN_NoInfiniteLoop() { // Start workflow — first WFT is handled by the async poller above (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) env.pollUntilRegistered(s, tv2) @@ -5091,7 +5084,7 @@ func (s *Versioning3Suite) TestOverride_SuppressesTargetVersionChangedSignal() { // Start workflow — first WFT is handled by the async poller above (pinned on v1) env.startWorkflow(s, tv1, nil) execution := tv1.WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) // Apply pinned override to v1 via UpdateWorkflowExecutionOptions @@ -5149,7 +5142,7 @@ func (s *Versioning3Suite) TestAutoUpgrade_SuppressesTargetVersionChangedSignal( // Start workflow — first WFT is handled by the async poller above (AutoUpgrade on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbUnpinned, tv1.Deployment(), nil, nil) env.pollUntilRegistered(s, tv2) @@ -5200,7 +5193,7 @@ func (s *Versioning3Suite) TestPinnedCaN_TargetChangesAgain_SignalsTrue() { // Start workflow — first WFT is handled by the async poller above (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) env.pollUntilRegistered(s, tv2) @@ -5301,7 +5294,7 @@ func (s *Versioning3Suite) TestRemoveOverride_ClearsDeclinedState() { // Start workflow — first WFT is handled by the async poller above (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) // Set v2 as current, trigger signal @@ -5454,7 +5447,7 @@ func (s *Versioning3Suite) TestStalePartition_RevisionSuppressesTrampolining() { // Start workflow — first WFT handled by async poller (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) // Register v2, set v2 as current (revision increments) @@ -5591,7 +5584,7 @@ func (s *Versioning3Suite) TestInlinePath_StableRouting_NoSpuriousFlag() { runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) // Trigger a regular WFT via a first signal. @@ -5825,7 +5818,7 @@ func (s *Versioning3Suite) TestPinnedCaN_RollbackResetsDeclined() { // Start workflow — first WFT is handled by the async poller above (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) env.pollUntilRegistered(s, tv2) @@ -5969,7 +5962,7 @@ func (s *Versioning3Suite) TestPinnedCaN_NeverSignaled_NewRunGetsSignalForUnvers // Start workflow — first WFT is handled by the async poller above (pinned on v1) runID := env.startWorkflow(s, tv1, nil) execution := tv1.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.T(), wftCompleted) env.verifyWorkflowVersioning(s, tv1, vbPinned, tv1.Deployment(), nil, nil) // Trigger a WFT and CaN without any target change — no signal fires. @@ -6106,7 +6099,7 @@ func (s *Versioning3Suite) TestPinnedCaN_UpgradeToUnversioned() { "CaN run should have a different run ID") return env.respondCompleteWorkflowUnversioned(tv1), nil }) - env.WaitForChannel(wftNewRunDone) + await.Rcv(s.T(), wftNewRunDone) }) } diff --git a/tests/versioning_test.go b/tests/versioning_test.go index 6a3e51da9f8..3f5cf42938c 100644 --- a/tests/versioning_test.go +++ b/tests/versioning_test.go @@ -30,6 +30,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/searchattribute/sadefs" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/worker_versioning" @@ -595,7 +596,7 @@ func (s *VersioningIntegSuite) workflowStaysInBuildID(env *testcore.TestEnv) { } act2 := func() (string, error) { - env.WaitForChannel(rulesUpdated) + await.Rcv(s.T(), rulesUpdated) return "act2 done!", nil } @@ -636,7 +637,7 @@ func (s *VersioningIntegSuite) workflowStaysInBuildID(env *testcore.TestEnv) { run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(act1Done) + await.Rcv(s.T(), act1Done) s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v1, true, v1, "", nil) // update rules with v2 as the default build @@ -679,7 +680,7 @@ func (s *VersioningIntegSuite) unversionedWorkflowStaysUnversioned(env *testcore } act2 := func() (string, error) { - env.WaitForChannel(rulesUpdated) + await.Rcv(s.T(), rulesUpdated) return "act2 done!", nil } @@ -715,7 +716,7 @@ func (s *VersioningIntegSuite) unversionedWorkflowStaysUnversioned(env *testcore run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(act1Done) + await.Rcv(s.T(), act1Done) s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), "", true, "binary-checksum", "", nil) // update rules with v1 as the default build @@ -781,7 +782,7 @@ func (s *VersioningIntegSuite) firstWorkflowTaskAssignmentSpooled(env *testcore. s.NoError(w1.Start()) defer w1.Stop() - env.WaitForChannel(failedTask) + await.Rcv(s.T(), failedTask) // After scheduling the second time, now MS should be assigned to v2 s.waitForWorkflowBuildID(env, run.GetID(), run.GetRunID(), v2) @@ -812,7 +813,7 @@ func (s *VersioningIntegSuite) firstWorkflowTaskAssignmentSpooled(env *testcore. s.NoError(w2.Start()) defer w2.Stop() - env.WaitForChannel(timedoutTask) + await.Rcv(s.T(), timedoutTask) // After scheduling the third time, now MS should be assigned to v3 s.waitForWorkflowBuildID(env, run.GetID(), run.GetRunID(), v3) @@ -883,8 +884,8 @@ func (s *VersioningIntegSuite) firstWorkflowTaskAssignmentSyncMatch(env *testcor s.NoError(err) // wait for two failures to make sure more attempts does not generate more history tasks - env.WaitForChannel(failedTask) - env.WaitForChannel(failedTask) + await.Rcv(s.T(), failedTask) + await.Rcv(s.T(), failedTask) // MS should have the correct build ID s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v1, true, "", "", nil) @@ -915,9 +916,9 @@ func (s *VersioningIntegSuite) firstWorkflowTaskAssignmentSyncMatch(env *testcor s.waitForAssignmentRulePropagation(env, tq, rule) // wait for multiple timeouts to make sure more attempts do not generate more history events - env.WaitForChannel(timedoutTask) - env.WaitForChannel(timedoutTask) - env.WaitForChannel(timedoutTask) + await.Rcv(s.T(), timedoutTask) + await.Rcv(s.T(), timedoutTask) + await.Rcv(s.T(), timedoutTask) // After scheduling the second time, now MS should be assigned to v2 s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v2, true, "", "", []string{v1}) @@ -1058,7 +1059,7 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSpooled( s.NoError(w1.Start()) defer w1.Stop() - env.WaitForChannel(failedTask) + await.Rcv(s.T(), failedTask) // After scheduling the second time, now pending activity should be assigned to v2 s.Eventually( @@ -1097,7 +1098,7 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSpooled( s.NoError(w2.Start()) defer w2.Stop() - env.WaitForChannel(timedoutTask) + await.Rcv(s.T(), timedoutTask) // After scheduling the third time, now pending activity should be assigned to v3 s.Eventually( @@ -1226,7 +1227,7 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSyncMatch( }, wf) s.NoError(err) - env.WaitForChannel(failedTask) + await.Rcv(s.T(), failedTask) // MS should have the correct build ID after finishing the first WFT s.Eventually( @@ -1274,7 +1275,7 @@ func (s *VersioningIntegSuite) independentActivityTaskAssignmentSyncMatch( rule = s.addAssignmentRule(env, actTq, v2) s.waitForAssignmentRulePropagation(env, actTq, rule) - env.WaitForChannel(timedoutTask) + await.Rcv(s.T(), timedoutTask) // After scheduling the second time, now pending activity should be assigned to v2 s.Eventually( @@ -1396,9 +1397,9 @@ func (s *VersioningIntegSuite) testWorkflowTaskRedirectInRetry( s.NoError(err) // wait for multiple failures to make sure more attempts does not generate more history tasks - env.WaitForChannel(failedTask) - env.WaitForChannel(failedTask) - env.WaitForChannel(failedTask) + await.Rcv(s.T(), failedTask) + await.Rcv(s.T(), failedTask) + await.Rcv(s.T(), failedTask) expectedStampBuildId := "" if !firstTask { @@ -1440,9 +1441,9 @@ func (s *VersioningIntegSuite) testWorkflowTaskRedirectInRetry( s.waitForRedirectRulePropagation(env, tq, rule2) // wait for multiple timeouts to make sure more attempts does not generate more history tasks - env.WaitForChannel(timedoutTask) - env.WaitForChannel(timedoutTask) - env.WaitForChannel(timedoutTask) + await.Rcv(s.T(), timedoutTask) + await.Rcv(s.T(), timedoutTask) + await.Rcv(s.T(), timedoutTask) // After scheduling the second time, now MS should be assigned to v2 s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v11, true, expectedStampBuildId, "", []string{v1}) @@ -1615,7 +1616,7 @@ func (s *VersioningIntegSuite) dispatchUnversionedRemainsUnversioned(env *testco run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(started) + await.Rcv(s.T(), started) s.addNewDefaultBuildID(env, tq, v1) s.waitForVersionSetPropagation(env, tq, v1) @@ -1685,7 +1686,7 @@ func (s *VersioningIntegSuite) dispatchUpgrade( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) - env.WaitForChannel(started) + await.Rcv(s.T(), started) // now add v11 as compatible so the next workflow task runs there if newVersioning { @@ -1900,7 +1901,7 @@ func (s *VersioningIntegSuite) dispatchActivity( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started) + await.Rcv(s.T(), started) close(started) // force panic if replayed // now register v2 as default @@ -1971,12 +1972,12 @@ func (s *VersioningIntegSuite) TestDispatchActivityUpgrade() { } act11 := func() (string, error) { started11 <- struct{}{} - env.WaitForChannel(proceed11) + await.Rcv(s.T(), proceed11) return "v1.1", nil } act12 := func() (string, error) { started12 <- struct{}{} - env.WaitForChannel(proceed12) + await.Rcv(s.T(), proceed12) return "v1.2", nil } wf := func(ctx workflow.Context) (string, error) { @@ -2051,37 +2052,37 @@ func (s *VersioningIntegSuite) TestDispatchActivityUpgrade() { s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(startedWf) + await.Rcv(s.T(), startedWf) rule2 := s.addRedirectRule(env, tq, v1, v11) s.waitForRedirectRulePropagation(env, tq, rule2) - env.SendToChannel(proceedWf) + await.Snd(s.T(), proceedWf, struct{}{}) - env.WaitForChannel(started11) + await.Rcv(s.T(), started11) // wf assigned build ID should be updated by activity redirect s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v11, true, v1, "", []string{v1}) // let activity finish - env.SendToChannel(proceed11) + await.Snd(s.T(), proceed11, struct{}{}) // wf replays on 1.1 so need to unblock it an extra time - env.WaitForChannel(startedWf) - env.SendToChannel(proceedWf) + await.Rcv(s.T(), startedWf) + await.Snd(s.T(), proceedWf, struct{}{}) - env.WaitForChannel(startedWf) + await.Rcv(s.T(), startedWf) rule2 = s.addRedirectRule(env, tq, v11, v12) s.waitForRedirectRulePropagation(env, tq, rule2) - env.SendToChannel(proceedWf) + await.Snd(s.T(), proceedWf, struct{}{}) - env.WaitForChannel(started12) + await.Rcv(s.T(), started12) // wf assigned build ID should not be updated by independent activity redirect s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v11, true, v11, "", []string{v1}) // let activity finish - env.SendToChannel(proceed12) + await.Snd(s.T(), proceed12, struct{}{}) // wf replays on 1.2 so need to unblock it two extra times - env.WaitForChannel(startedWf) - env.SendToChannel(proceedWf) - env.WaitForChannel(startedWf) - env.SendToChannel(proceedWf) + await.Rcv(s.T(), startedWf) + await.Snd(s.T(), proceedWf, struct{}{}) + await.Rcv(s.T(), startedWf) + await.Snd(s.T(), proceedWf, struct{}{}) var out string s.NoError(run.Get(s.Context(), &out)) @@ -2362,7 +2363,7 @@ func (s *VersioningIntegSuite) dispatchActivityCompatible(env *testcore.TestEnv) run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started) + await.Rcv(s.T(), started) // now register v1.1 as compatible s.addCompatibleBuildID(env, tq, v11, v1, false) @@ -2640,7 +2641,7 @@ func (s *VersioningIntegSuite) dispatchChildWorkflow( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started) + await.Rcv(s.T(), started) close(started) // force panic if replayed // now register v2 as default @@ -2770,7 +2771,7 @@ func (s *VersioningIntegSuite) dispatchChildWorkflowUpgrade( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started) + await.Rcv(s.T(), started) // now register v1.1 as compatible if newVersioning { @@ -2937,7 +2938,7 @@ func (s *VersioningIntegSuite) dispatchQuery( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started) + await.Rcv(s.T(), started) if newVersioning { rule := s.addAssignmentRule(env, tq, v2) @@ -3118,7 +3119,7 @@ func (s *VersioningIntegSuite) dispatchContinueAsNew( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) // now make v2 as a new default if newVersioning { @@ -3161,7 +3162,7 @@ func (s *VersioningIntegSuite) dispatchContinueAsNew( // unblock the workflow. it should get kicked off the sticky queue and replay on v1 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) var out string s.NoError(run.Get(s.Context(), &out)) @@ -3248,7 +3249,7 @@ func (s *VersioningIntegSuite) dispatchContinueAsNewUpgrade( run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) // now register v11 as newer compatible with v1 AND v2 as a new default if newVersioning { @@ -3290,10 +3291,10 @@ func (s *VersioningIntegSuite) dispatchContinueAsNewUpgrade( // unblock the workflow. it should get kicked off the sticky queue and replay on v11 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) - env.WaitForChannel(started11) + await.Rcv(s.T(), started11) // then continue-as-new onto v11 - env.WaitForChannel(started11) + await.Rcv(s.T(), started11) // initial run s.validateWorkflowBuildIds(env, run.GetID(), run.GetRunID(), v11, newVersioning, v11, "", []string{v1}) @@ -3379,7 +3380,7 @@ func (s *VersioningIntegSuite) dispatchRetryOld(env *testcore.TestEnv) { }, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) // now register v11 as newer compatible with v1 AND v2 as a new default s.addCompatibleBuildID(env, tq, v11, v1, false) @@ -3412,14 +3413,14 @@ func (s *VersioningIntegSuite) dispatchRetryOld(env *testcore.TestEnv) { // unblock the workflow. it should replay on v11 and then retry (on v11). s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) - env.WaitForChannel(started11) // replay - env.WaitForChannel(started11) // attempt 2 + await.Rcv(s.T(), started11) // replay + await.Rcv(s.T(), started11) // attempt 2 // now it's blocked in attempt 2. unblock it. s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) // wait for attempt 3. unblock that and it should return. - env.WaitForChannel(started11) // attempt 3 + await.Rcv(s.T(), started11) // attempt 3 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) var out string @@ -3482,7 +3483,7 @@ func (s *VersioningIntegSuite) dispatchRetry(env *testcore.TestEnv) { }, "wf") s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) // now register v2 as a new default rule = s.addAssignmentRule(env, tq, v2) @@ -3502,12 +3503,12 @@ func (s *VersioningIntegSuite) dispatchRetry(env *testcore.TestEnv) { // unblock the workflow on v1 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) - env.WaitForChannel(started2) // attempt 2 + await.Rcv(s.T(), started2) // attempt 2 // now it's blocked in attempt 2. unblock it. s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) // wait for attempt 3. unblock that and it should return. - env.WaitForChannel(started2) // attempt 3 + await.Rcv(s.T(), started2) // attempt 3 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) var out string @@ -3880,7 +3881,7 @@ func (s *VersioningIntegSuite) resetWorkflowAssignsToCorrectBuildIDChildWf( _, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(childStarted) + await.Rcv(s.T(), childStarted) s.validateBuildIDAfterReset(env, childWfId, "", inheritBuildID) } @@ -3981,7 +3982,7 @@ func (s *VersioningIntegSuite) TestDescribeTaskQueueEnhanced_Versioned_Reachabil defer w.Stop() run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(started) + await.Rcv(s.T(), started) // 2. Wait for visibility to show A as running with BuildId SearchAttribute 'assigned:A' s.Await(func(s *VersioningIntegSuite) { @@ -4048,7 +4049,7 @@ func (s *VersioningIntegSuite) TestDescribeTaskQueueEnhanced_Versioned_BasicReac defer w.Stop() run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) - env.WaitForChannel(started) + await.Rcv(s.T(), started) // wait for visibility to show A as running with BuildId SearchAttribute 'assigned:A' s.Await(func(s *VersioningIntegSuite) { @@ -4355,7 +4356,7 @@ func (s *VersioningIntegSuite) TestDescribeWorkflowExecution() { run, err := env.SdkClient().ExecuteWorkflow(s.Context(), sdkclient.StartWorkflowOptions{TaskQueue: tq}, wf) s.NoError(err) // wait for it to start on v1 - env.WaitForChannel(started1) + await.Rcv(s.T(), started1) // describe and check build ID s.Await(func(s *VersioningIntegSuite) { @@ -4385,7 +4386,7 @@ func (s *VersioningIntegSuite) TestDescribeWorkflowExecution() { // unblock the workflow. it should get kicked off the sticky queue and replay on v11 s.NoError(env.SdkClient().SignalWorkflow(s.Context(), run.GetID(), "", "wait", nil)) - env.WaitForChannel(started11) + await.Rcv(s.T(), started11) s.Await(func(s *VersioningIntegSuite) { resp, err := env.SdkClient().DescribeWorkflowExecution(s.Context(), run.GetID(), "") diff --git a/tests/versioning_test_env.go b/tests/versioning_test_env.go index 27bdfa214c4..ade8d9466de 100644 --- a/tests/versioning_test_env.go +++ b/tests/versioning_test_env.go @@ -124,7 +124,7 @@ func (env *VersioningTestEnv) pollAndQueryWorkflow( _, err := env.queryWorkflow(s.Context(), tv) s.Require().NoError(err) - <-queryResultCh + await.Rcv(s.TB(), queryResultCh) } // drainWorkflowTaskAfterSetCurrent is a helper that sets the current deployment version, @@ -152,7 +152,7 @@ func (env *VersioningTestEnv) drainWorkflowTaskAfterSetCurrentWithOverride( runID := env.startWorkflow(s, tv, override) execution := tv.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.TB(), wftCompleted) return execution, runID } @@ -177,7 +177,7 @@ func (env *VersioningTestEnv) drainWorkflowTaskAfterSetCurrent( runID := env.startWorkflow(s, tv, nil) execution := tv.WithRunID(runID).WorkflowExecution() - env.WaitForChannel(wftCompleted) + await.Rcv(s.TB(), wftCompleted) return execution, runID } @@ -198,7 +198,7 @@ func (env *VersioningTestEnv) pollAndDispatchNexusTask( _, err := matchingClient.DispatchNexusTask(s.Context(), nexusRequest) s.Require().NoError(err) - <-nexusCompleted + await.Rcv(s.TB(), nexusCompleted) } func (env *VersioningTestEnv) describeVersioningInfo( @@ -885,7 +885,7 @@ func (env *VersioningTestEnv) doPollWftAndHandle( return poller, resp } go func() { - _, _ = f() // errors are surfaced via test context timeout on WaitForChannel + _, _ = f() // errors are surfaced via the test context timeout in Rcv close(async) }() return nil, nil @@ -917,7 +917,7 @@ func (env *VersioningTestEnv) pollWftAndHandleQueries( return poller, resp } go func() { - _, _ = f() // errors are surfaced via test context timeout on WaitForChannel + _, _ = f() // errors are surfaced via the test context timeout in Rcv close(async) }() return nil, nil @@ -949,7 +949,7 @@ func (env *VersioningTestEnv) pollNexusTaskAndHandle( return poller, resp } go func() { - _, _ = f() // errors are surfaced via test context timeout on WaitForChannel + _, _ = f() // errors are surfaced via the test context timeout in Rcv close(async) }() return nil, nil @@ -995,7 +995,7 @@ func (env *VersioningTestEnv) doPollActivityAndHandle( s.Require().NoError(f()) } else { go func() { - _ = f() // errors are surfaced via test context timeout on WaitForChannel + _ = f() // errors are surfaced via the test context timeout in Rcv close(async) }() } diff --git a/tests/worker_deployment_version_test.go b/tests/worker_deployment_version_test.go index cd90e0c1e85..d154aec94de 100644 --- a/tests/worker_deployment_version_test.go +++ b/tests/worker_deployment_version_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testvars" @@ -504,7 +505,7 @@ func (s *DeploymentVersionSuite) startVersionedWorkflow(ctx context.Context, env defer w.Stop() run, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{TaskQueue: tv.TaskQueue().String()}, wf) s.NoError(err) - env.WaitForChannel(started) + await.Rcv(s.T(), started) return run } From 9e63eac46a0448c58a03080918cc89857c2f205d Mon Sep 17 00:00:00 2001 From: Prathyush PV Date: Mon, 24 Aug 2026 20:49:38 -0700 Subject: [PATCH 055/139] Fix SequentialScheduler test deadlock from bounded test task queue (#11772) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Back `testSequentialTaskQueue` with an unbounded slice instead of a 3000-capacity channel, so `Add` no longer blocks. ## Why? `common/tasks` deadlocked and timed out after 15m on main ([run 32773080605](https://github.com/temporalio/temporal/actions/runs/32773080605)). `SequentialScheduler.Submit` calls `Add` from inside `PutOrDo`'s callback, which runs under the shard write lock. Once the test queue's channel filled, the submitter blocked in `Add` while holding that lock, and the only worker was blocked on the same lock in `RemoveIf`. Holding the lock across the add is deliberate — it keeps the add atomic against `RemoveIf`'s empty-check — so the invariant is that `Add` must not block, and only the test queue violated it. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests --- common/tasks/sequential_scheduler_test.go | 4 +-- common/tasks/sequential_task_queue_test.go | 35 ++++++++++++++-------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/common/tasks/sequential_scheduler_test.go b/common/tasks/sequential_scheduler_test.go index 7a410d905d4..0e44f05ad5e 100644 --- a/common/tasks/sequential_scheduler_test.go +++ b/common/tasks/sequential_scheduler_test.go @@ -385,7 +385,7 @@ func (s *sequentialSchedulerSuite) newTestProcessor() *SequentialScheduler[*Mock return 1 } factory := func(task *MockTask) SequentialTaskQueue[*MockTask] { - return newTestSequentialTaskQueue[*MockTask](1, 3000) + return newTestSequentialTaskQueue[*MockTask](1) } return NewSequentialScheduler[*MockTask]( &SequentialSchedulerOptions{ @@ -405,7 +405,7 @@ func (s *sequentialSchedulerSuite) newTestProcessorWithQueueSize(queueSize int) return 1 } factory := func(task *MockTask) SequentialTaskQueue[*MockTask] { - return newTestSequentialTaskQueue[*MockTask](1, 3000) + return newTestSequentialTaskQueue[*MockTask](1) } return NewSequentialScheduler[*MockTask]( &SequentialSchedulerOptions{ diff --git a/common/tasks/sequential_task_queue_test.go b/common/tasks/sequential_task_queue_test.go index 72c37cbcc0a..c833cda3ca3 100644 --- a/common/tasks/sequential_task_queue_test.go +++ b/common/tasks/sequential_task_queue_test.go @@ -1,13 +1,15 @@ package tasks +import "sync" + type testSequentialTaskQueue[T Task] struct { - q chan T - id int + sync.Mutex + tasks []T + id int } -func newTestSequentialTaskQueue[T Task](id, capacity int) SequentialTaskQueue[T] { +func newTestSequentialTaskQueue[T Task](id int) SequentialTaskQueue[T] { return &testSequentialTaskQueue[T]{ - q: make(chan T, capacity), id: id, } } @@ -17,23 +19,30 @@ func (s *testSequentialTaskQueue[T]) ID() any { } func (s *testSequentialTaskQueue[T]) Add(task T) { - s.q <- task + s.Lock() + defer s.Unlock() + s.tasks = append(s.tasks, task) } func (s *testSequentialTaskQueue[T]) Remove() T { - select { - case t := <-s.q: - return t - default: - var emptyT T - return emptyT + s.Lock() + defer s.Unlock() + var task T + if len(s.tasks) == 0 { + return task } + task, s.tasks = s.tasks[0], s.tasks[1:] + return task } func (s *testSequentialTaskQueue[T]) IsEmpty() bool { - return len(s.q) == 0 + s.Lock() + defer s.Unlock() + return len(s.tasks) == 0 } func (s *testSequentialTaskQueue[T]) Len() int { - return len(s.q) + s.Lock() + defer s.Unlock() + return len(s.tasks) } From 5406711d685f4b203c39b9a49a04e5aed3ad6af1 Mon Sep 17 00:00:00 2001 From: Rob Holland Date: Tue, 25 Aug 2026 16:53:45 +0100 Subject: [PATCH 056/139] Add replication stream lane wire protocol and receiver-side lane routing (#11303) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > **Part 3 of a 5-PR series** building to replication stream namespace isolation (a restructuring of #10147): read buffer → reader group → lane protocol → isolation manager → sender isolation. > #11302 (reader group) has merged, so this PR's diff is now standalone against `main`. · **Next in series: #11304** (isolation manager). ## What changed? The wire-level building blocks for per-namespace lane isolation, receiver side only: - **Proto**: `SyncReplicationState` gains `throttle_high_namespace_ids` (namespaces the receiver reports as overwhelming the HIGH lane — the priority lives in the field name, so a future LOW extension adds its own field rather than widening this one), `isolated_lane_states` (per-lane applied watermarks keyed by namespace ID — with the documented caveat that a missing key is ambiguous between "not tracked yet" and "retired and drained", so a sender must never treat absence alone as drain proof), and `supports_namespace_isolation` (capability advertisement, so a sender never emits lane-tagged traffic to a receiver that would misroute it). `WorkflowReplicationMessages` gains `isolated_namespace_id` — when set, the batch belongs to that namespace's dedicated lane — and `retire_isolated_lane`, marking a lane's final message. - **Receiver**: lane-tagged batches route to lazily-created per-namespace task trackers. Each lane is its own monotonic stream for the life of the connection — there is no rewind or rotation machinery, because the sender-side design (later in the series) gives every lane a single owner cursor that never goes backwards. Member-lane watermarks fold into the overall ack minimum (cleanup safety) and are reported per lane; member-lane backlogs count toward HIGH flow control. Lane lifecycle is defensive about ordering: a batch's tasks are tracked BEFORE its retire flag is applied (so the concurrent ack loop can never delete a lane whose final batch is mid-track), a retiring lane is only dropped once it is drained AND has tracked at least one batch, and non-retire traffic arriving on a retiring lane revives it (the sender re-isolated the namespace before the lane drained). Lane-tagged traffic at any priority other than HIGH is a protocol violation and fails the stream rather than silently mis-acking (isolation splits the HIGH lane only). Lanes created concurrently with `Stop()` are pre-cancelled so no tasks run after shutdown. - **`NamespaceThrottler`** interface (default: noop, via fx) observes per-namespace HIGH-priority task load and decides which namespaces to report. The sender does not tag lanes yet, so this is inert until the sender-side isolation lands. ## Why? Isolation needs a wire contract before the sender can use it: capability advertisement, per-lane routing and progress reporting, and the throttled-namespace feedback channel. Landing the receiver first makes mixed-version clusters safe by construction. Compared to #10147, lanes are per-namespace rather than shared per severity tier — which is what eliminates that design's cursor rewinds and the watermark-regression/tracker-rotation protocol this PR previously needed to compensate for them. ## How did you test it? - [x] built - [x] covered by existing tests - [x] added new unit test(s) — lane routing (priority routing when unset, per-namespace tracker identity, non-HIGH rejection), retirement lifecycle (drop once drained, never-tracked retiring lane survives the ack snapshot, revive on re-isolation traffic, fresh lane after drop), and post-Stop lane creation being pre-cancelled - [x] added new functional test(s) — exercised end-to-end by the xdc test in the final PR of the series ## Potential risks Inert until a sender emits `isolated_namespace_id`, which is gated behind both a config flag and the capability advertisement. Receiver-side lane state is bounded by the sender's isolation cap (final PR). 🤖 Generated with [Claude Code](https://claude.com/claude-code) --- > [!NOTE] > **Medium Risk** > Touches replication ack watermarks and stream failure paths on the passive cluster; lane mis-handling could stall cleanup or mis-ack, though lane-tagged traffic is not sent until follow-up sender work. > > **Overview** > Adds the **wire contract and receiver behavior** for per-namespace HIGH-lane isolation before the sender starts tagging traffic. > > **Proto:** `SyncReplicationState` now carries `throttle_high_namespace_ids`, per-namespace `isolated_lane_states`, and `supports_namespace_isolation`. `WorkflowReplicationMessages` adds `isolated_namespace_id` and `retire_isolated_lane` so batches can be routed and retired on dedicated lanes. > > **Receiver:** Lane-tagged HIGH batches use lazily created per-namespace task trackers (monotonic per connection). Member-lane watermarks fold into the overall ack minimum and are reported per namespace; member backlog counts toward HIGH flow control. Acks include shard-scoped throttled namespace IDs via a new **`NamespaceThrottler`** (noop by default). Lane lifecycle handles retire/drain/revive and rejects non-HIGH lane traffic. > > Sender emission of lane tags is not in this PR, so behavior stays inert until a later change gates on capability advertisement. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 688a5173f00116fb5214d8b088e482d799995ed5. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --------- Co-authored-by: Claude Co-authored-by: Cursor --- api/replication/v1/message.pb.go | 260 +++++++++++------ .../server/api/replication/v1/message.proto | 24 ++ .../replication/executable_task_tool_box.go | 1 + service/history/replication/fx.go | 3 + .../replication/namespace_throttler.go | 27 ++ .../history/replication/stream_receiver.go | 214 +++++++++++++- .../replication/stream_receiver_test.go | 273 ++++++++++++++++++ 7 files changed, 695 insertions(+), 107 deletions(-) create mode 100644 service/history/replication/namespace_throttler.go diff --git a/api/replication/v1/message.pb.go b/api/replication/v1/message.pb.go index 1fdcca398b9..dd070f38583 100644 --- a/api/replication/v1/message.pb.go +++ b/api/replication/v1/message.pb.go @@ -424,8 +424,23 @@ type SyncReplicationState struct { InclusiveLowWatermarkTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=inclusive_low_watermark_time,json=inclusiveLowWatermarkTime,proto3" json:"inclusive_low_watermark_time,omitempty"` HighPriorityState *ReplicationState `protobuf:"bytes,3,opt,name=high_priority_state,json=highPriorityState,proto3" json:"high_priority_state,omitempty"` LowPriorityState *ReplicationState `protobuf:"bytes,4,opt,name=low_priority_state,json=lowPriorityState,proto3" json:"low_priority_state,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Namespace IDs the receiver reports as overwhelming the HIGH (live) lane. The + // sender isolates these namespaces' HIGH traffic onto throttled severity-tier + // lanes. The priority lives in the field name: a future LOW-lane extension adds + // its own throttle_low_namespace_ids rather than widening this one. + ThrottleHighNamespaceIds []string `protobuf:"bytes,5,rep,name=throttle_high_namespace_ids,json=throttleHighNamespaceIds,proto3" json:"throttle_high_namespace_ids,omitempty"` + // Low watermarks of the isolated per-namespace lanes, keyed by namespace ID. + // Absence of a key is ambiguous: it means either "the receiver has not tracked + // this lane yet" or "the lane retired and drained". A sender must therefore never + // treat absence alone as proof a lane drained — it must correlate with its own + // lane state (e.g. only conclude drain for lanes it has retired). + IsolatedLaneStates map[string]*ReplicationState `protobuf:"bytes,6,rep,name=isolated_lane_states,json=isolatedLaneStates,proto3" json:"isolated_lane_states,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // True when the receiver understands per-namespace lane isolation (routing of + // isolated_namespace_id-tagged messages). The sender requires this before it emits + // lane-tagged traffic, so an old receiver is never sent lanes it would misroute. + SupportsNamespaceIsolation bool `protobuf:"varint,7,opt,name=supports_namespace_isolation,json=supportsNamespaceIsolation,proto3" json:"supports_namespace_isolation,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SyncReplicationState) Reset() { @@ -486,6 +501,27 @@ func (x *SyncReplicationState) GetLowPriorityState() *ReplicationState { return nil } +func (x *SyncReplicationState) GetThrottleHighNamespaceIds() []string { + if x != nil { + return x.ThrottleHighNamespaceIds + } + return nil +} + +func (x *SyncReplicationState) GetIsolatedLaneStates() map[string]*ReplicationState { + if x != nil { + return x.IsolatedLaneStates + } + return nil +} + +func (x *SyncReplicationState) GetSupportsNamespaceIsolation() bool { + if x != nil { + return x.SupportsNamespaceIsolation + } + return false +} + type ReplicationState struct { state protoimpl.MessageState `protogen:"open.v1"` InclusiveLowWatermark int64 `protobuf:"varint,1,opt,name=inclusive_low_watermark,json=inclusiveLowWatermark,proto3" json:"inclusive_low_watermark,omitempty"` @@ -623,8 +659,17 @@ type WorkflowReplicationMessages struct { ExclusiveHighWatermark int64 `protobuf:"varint,2,opt,name=exclusive_high_watermark,json=exclusiveHighWatermark,proto3" json:"exclusive_high_watermark,omitempty"` ExclusiveHighWatermarkTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=exclusive_high_watermark_time,json=exclusiveHighWatermarkTime,proto3" json:"exclusive_high_watermark_time,omitempty"` Priority v1.TaskPriority `protobuf:"varint,4,opt,name=priority,proto3,enum=temporal.server.api.enums.v1.TaskPriority" json:"priority,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // When set, this batch belongs to the named namespace's isolated lane rather + // than the shared priority lane. Used by the receiver only for tracker routing; + // scheduler priority still comes from `priority`. Each isolated lane is a + // monotonic stream for the life of the stream connection. + IsolatedNamespaceId string `protobuf:"bytes,5,opt,name=isolated_namespace_id,json=isolatedNamespaceId,proto3" json:"isolated_namespace_id,omitempty"` + // True on the final message of an isolated lane: the sender has merged the + // namespace back to the shared lane and will send no further traffic on this + // lane. The receiver drops the lane's tracking once its pending work drains. + RetireIsolatedLane bool `protobuf:"varint,6,opt,name=retire_isolated_lane,json=retireIsolatedLane,proto3" json:"retire_isolated_lane,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkflowReplicationMessages) Reset() { @@ -685,6 +730,20 @@ func (x *WorkflowReplicationMessages) GetPriority() v1.TaskPriority { return v1.TaskPriority(0) } +func (x *WorkflowReplicationMessages) GetIsolatedNamespaceId() string { + if x != nil { + return x.IsolatedNamespaceId + } + return "" +} + +func (x *WorkflowReplicationMessages) GetRetireIsolatedLane() bool { + if x != nil { + return x.RetireIsolatedLane + } + return false +} + // TODO: Deprecate this definition, it only used by the deprecated replication DLQ v1 logic type ReplicationTaskInfo struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -2146,12 +2205,18 @@ const file_temporal_server_api_replication_v1_message_proto_rawDesc = "" + "\x1elast_processed_visibility_time\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x1blastProcessedVisibilityTime\"N\n" + "\x0fSyncShardStatus\x12;\n" + "\vstatus_time\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\n" + - "statusTime\"\xf5\x02\n" + + "statusTime\"\xf8\x05\n" + "\x14SyncReplicationState\x126\n" + "\x17inclusive_low_watermark\x18\x01 \x01(\x03R\x15inclusiveLowWatermark\x12[\n" + "\x1cinclusive_low_watermark_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x19inclusiveLowWatermarkTime\x12d\n" + "\x13high_priority_state\x18\x03 \x01(\v24.temporal.server.api.replication.v1.ReplicationStateR\x11highPriorityState\x12b\n" + - "\x12low_priority_state\x18\x04 \x01(\v24.temporal.server.api.replication.v1.ReplicationStateR\x10lowPriorityState\"\x96\x02\n" + + "\x12low_priority_state\x18\x04 \x01(\v24.temporal.server.api.replication.v1.ReplicationStateR\x10lowPriorityState\x12=\n" + + "\x1bthrottle_high_namespace_ids\x18\x05 \x03(\tR\x18throttleHighNamespaceIds\x12\x82\x01\n" + + "\x14isolated_lane_states\x18\x06 \x03(\v2P.temporal.server.api.replication.v1.SyncReplicationState.IsolatedLaneStatesEntryR\x12isolatedLaneStates\x12@\n" + + "\x1csupports_namespace_isolation\x18\a \x01(\bR\x1asupportsNamespaceIsolation\x1a{\n" + + "\x17IsolatedLaneStatesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12J\n" + + "\x05value\x18\x02 \x01(\v24.temporal.server.api.replication.v1.ReplicationStateR\x05value:\x028\x01\"\x96\x02\n" + "\x10ReplicationState\x126\n" + "\x17inclusive_low_watermark\x18\x01 \x01(\x03R\x15inclusiveLowWatermark\x12[\n" + "\x1cinclusive_low_watermark_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x19inclusiveLowWatermarkTime\x12m\n" + @@ -2160,12 +2225,14 @@ const file_temporal_server_api_replication_v1_message_proto_rawDesc = "" + "\x11replication_tasks\x18\x01 \x03(\v23.temporal.server.api.replication.v1.ReplicationTaskR\x10replicationTasks\x129\n" + "\x19last_retrieved_message_id\x18\x02 \x01(\x03R\x16lastRetrievedMessageId\x12\x19\n" + "\bhas_more\x18\x03 \x01(\bR\ahasMore\x12_\n" + - "\x11sync_shard_status\x18\x04 \x01(\v23.temporal.server.api.replication.v1.SyncShardStatusR\x0fsyncShardStatus\"\xe0\x02\n" + + "\x11sync_shard_status\x18\x04 \x01(\v23.temporal.server.api.replication.v1.SyncShardStatusR\x0fsyncShardStatus\"\xc6\x03\n" + "\x1bWorkflowReplicationMessages\x12`\n" + "\x11replication_tasks\x18\x01 \x03(\v23.temporal.server.api.replication.v1.ReplicationTaskR\x10replicationTasks\x128\n" + "\x18exclusive_high_watermark\x18\x02 \x01(\x03R\x16exclusiveHighWatermark\x12]\n" + "\x1dexclusive_high_watermark_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x1aexclusiveHighWatermarkTime\x12F\n" + - "\bpriority\x18\x04 \x01(\x0e2*.temporal.server.api.enums.v1.TaskPriorityR\bpriority\"\xa8\x03\n" + + "\bpriority\x18\x04 \x01(\x0e2*.temporal.server.api.enums.v1.TaskPriorityR\bpriority\x122\n" + + "\x15isolated_namespace_id\x18\x05 \x01(\tR\x13isolatedNamespaceId\x120\n" + + "\x14retire_isolated_lane\x18\x06 \x01(\bR\x12retireIsolatedLane\"\xa8\x03\n" + "\x13ReplicationTaskInfo\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12\x1f\n" + "\vworkflow_id\x18\x02 \x01(\tR\n" + @@ -2314,7 +2381,7 @@ func file_temporal_server_api_replication_v1_message_proto_rawDescGZIP() []byte return file_temporal_server_api_replication_v1_message_proto_rawDescData } -var file_temporal_server_api_replication_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 23) +var file_temporal_server_api_replication_v1_message_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_temporal_server_api_replication_v1_message_proto_goTypes = []any{ (*ReplicationTask)(nil), // 0: temporal.server.api.replication.v1.ReplicationTask (*ReplicationToken)(nil), // 1: temporal.server.api.replication.v1.ReplicationToken @@ -2339,32 +2406,33 @@ var file_temporal_server_api_replication_v1_message_proto_goTypes = []any{ (*SyncVersionedTransitionTaskAttributes)(nil), // 20: temporal.server.api.replication.v1.SyncVersionedTransitionTaskAttributes (*VersionedTransitionArtifact)(nil), // 21: temporal.server.api.replication.v1.VersionedTransitionArtifact (*MigrationExecutionInfo)(nil), // 22: temporal.server.api.replication.v1.MigrationExecutionInfo - (v1.ReplicationTaskType)(0), // 23: temporal.server.api.enums.v1.ReplicationTaskType - (*v11.DataBlob)(nil), // 24: temporal.api.common.v1.DataBlob - (*timestamppb.Timestamp)(nil), // 25: google.protobuf.Timestamp - (v1.TaskPriority)(0), // 26: temporal.server.api.enums.v1.TaskPriority - (*v12.VersionedTransition)(nil), // 27: temporal.server.api.persistence.v1.VersionedTransition - (*v12.ReplicationTaskInfo)(nil), // 28: temporal.server.api.persistence.v1.ReplicationTaskInfo - (v1.ReplicationFlowControlCommand)(0), // 29: temporal.server.api.enums.v1.ReplicationFlowControlCommand - (v1.TaskType)(0), // 30: temporal.server.api.enums.v1.TaskType - (v1.NamespaceOperation)(0), // 31: temporal.server.api.enums.v1.NamespaceOperation - (*v13.NamespaceInfo)(nil), // 32: temporal.api.namespace.v1.NamespaceInfo - (*v13.NamespaceConfig)(nil), // 33: temporal.api.namespace.v1.NamespaceConfig - (*v14.NamespaceReplicationConfig)(nil), // 34: temporal.api.replication.v1.NamespaceReplicationConfig - (*v14.FailoverStatus)(nil), // 35: temporal.api.replication.v1.FailoverStatus - (*v11.Payloads)(nil), // 36: temporal.api.common.v1.Payloads - (*v15.Failure)(nil), // 37: temporal.api.failure.v1.Failure - (*v16.VersionHistory)(nil), // 38: temporal.server.api.history.v1.VersionHistory - (*v17.BaseExecutionInfo)(nil), // 39: temporal.server.api.workflow.v1.BaseExecutionInfo - (*durationpb.Duration)(nil), // 40: google.protobuf.Duration - (*v16.VersionHistoryItem)(nil), // 41: temporal.server.api.history.v1.VersionHistoryItem - (*v12.WorkflowMutableState)(nil), // 42: temporal.server.api.persistence.v1.WorkflowMutableState - (*v12.TaskQueueUserData)(nil), // 43: temporal.server.api.persistence.v1.TaskQueueUserData - (*v12.StateMachineNode)(nil), // 44: temporal.server.api.persistence.v1.StateMachineNode - (*v12.WorkflowMutableStateMutation)(nil), // 45: temporal.server.api.persistence.v1.WorkflowMutableStateMutation + nil, // 23: temporal.server.api.replication.v1.SyncReplicationState.IsolatedLaneStatesEntry + (v1.ReplicationTaskType)(0), // 24: temporal.server.api.enums.v1.ReplicationTaskType + (*v11.DataBlob)(nil), // 25: temporal.api.common.v1.DataBlob + (*timestamppb.Timestamp)(nil), // 26: google.protobuf.Timestamp + (v1.TaskPriority)(0), // 27: temporal.server.api.enums.v1.TaskPriority + (*v12.VersionedTransition)(nil), // 28: temporal.server.api.persistence.v1.VersionedTransition + (*v12.ReplicationTaskInfo)(nil), // 29: temporal.server.api.persistence.v1.ReplicationTaskInfo + (v1.ReplicationFlowControlCommand)(0), // 30: temporal.server.api.enums.v1.ReplicationFlowControlCommand + (v1.TaskType)(0), // 31: temporal.server.api.enums.v1.TaskType + (v1.NamespaceOperation)(0), // 32: temporal.server.api.enums.v1.NamespaceOperation + (*v13.NamespaceInfo)(nil), // 33: temporal.api.namespace.v1.NamespaceInfo + (*v13.NamespaceConfig)(nil), // 34: temporal.api.namespace.v1.NamespaceConfig + (*v14.NamespaceReplicationConfig)(nil), // 35: temporal.api.replication.v1.NamespaceReplicationConfig + (*v14.FailoverStatus)(nil), // 36: temporal.api.replication.v1.FailoverStatus + (*v11.Payloads)(nil), // 37: temporal.api.common.v1.Payloads + (*v15.Failure)(nil), // 38: temporal.api.failure.v1.Failure + (*v16.VersionHistory)(nil), // 39: temporal.server.api.history.v1.VersionHistory + (*v17.BaseExecutionInfo)(nil), // 40: temporal.server.api.workflow.v1.BaseExecutionInfo + (*durationpb.Duration)(nil), // 41: google.protobuf.Duration + (*v16.VersionHistoryItem)(nil), // 42: temporal.server.api.history.v1.VersionHistoryItem + (*v12.WorkflowMutableState)(nil), // 43: temporal.server.api.persistence.v1.WorkflowMutableState + (*v12.TaskQueueUserData)(nil), // 44: temporal.server.api.persistence.v1.TaskQueueUserData + (*v12.StateMachineNode)(nil), // 45: temporal.server.api.persistence.v1.StateMachineNode + (*v12.WorkflowMutableStateMutation)(nil), // 46: temporal.server.api.persistence.v1.WorkflowMutableStateMutation } var file_temporal_server_api_replication_v1_message_proto_depIdxs = []int32{ - 23, // 0: temporal.server.api.replication.v1.ReplicationTask.task_type:type_name -> temporal.server.api.enums.v1.ReplicationTaskType + 24, // 0: temporal.server.api.replication.v1.ReplicationTask.task_type:type_name -> temporal.server.api.enums.v1.ReplicationTaskType 8, // 1: temporal.server.api.replication.v1.ReplicationTask.namespace_task_attributes:type_name -> temporal.server.api.replication.v1.NamespaceTaskAttributes 9, // 2: temporal.server.api.replication.v1.ReplicationTask.sync_shard_status_task_attributes:type_name -> temporal.server.api.replication.v1.SyncShardStatusTaskAttributes 10, // 3: temporal.server.api.replication.v1.ReplicationTask.sync_activity_task_attributes:type_name -> temporal.server.api.replication.v1.SyncActivityTaskAttributes @@ -2375,69 +2443,71 @@ var file_temporal_server_api_replication_v1_message_proto_depIdxs = []int32{ 15, // 8: temporal.server.api.replication.v1.ReplicationTask.backfill_history_task_attributes:type_name -> temporal.server.api.replication.v1.BackfillHistoryTaskAttributes 19, // 9: temporal.server.api.replication.v1.ReplicationTask.verify_versioned_transition_task_attributes:type_name -> temporal.server.api.replication.v1.VerifyVersionedTransitionTaskAttributes 20, // 10: temporal.server.api.replication.v1.ReplicationTask.sync_versioned_transition_task_attributes:type_name -> temporal.server.api.replication.v1.SyncVersionedTransitionTaskAttributes - 24, // 11: temporal.server.api.replication.v1.ReplicationTask.data:type_name -> temporal.api.common.v1.DataBlob - 25, // 12: temporal.server.api.replication.v1.ReplicationTask.visibility_time:type_name -> google.protobuf.Timestamp - 26, // 13: temporal.server.api.replication.v1.ReplicationTask.priority:type_name -> temporal.server.api.enums.v1.TaskPriority - 27, // 14: temporal.server.api.replication.v1.ReplicationTask.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 28, // 15: temporal.server.api.replication.v1.ReplicationTask.raw_task_info:type_name -> temporal.server.api.persistence.v1.ReplicationTaskInfo - 25, // 16: temporal.server.api.replication.v1.ReplicationToken.last_processed_visibility_time:type_name -> google.protobuf.Timestamp - 25, // 17: temporal.server.api.replication.v1.SyncShardStatus.status_time:type_name -> google.protobuf.Timestamp - 25, // 18: temporal.server.api.replication.v1.SyncReplicationState.inclusive_low_watermark_time:type_name -> google.protobuf.Timestamp + 25, // 11: temporal.server.api.replication.v1.ReplicationTask.data:type_name -> temporal.api.common.v1.DataBlob + 26, // 12: temporal.server.api.replication.v1.ReplicationTask.visibility_time:type_name -> google.protobuf.Timestamp + 27, // 13: temporal.server.api.replication.v1.ReplicationTask.priority:type_name -> temporal.server.api.enums.v1.TaskPriority + 28, // 14: temporal.server.api.replication.v1.ReplicationTask.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 29, // 15: temporal.server.api.replication.v1.ReplicationTask.raw_task_info:type_name -> temporal.server.api.persistence.v1.ReplicationTaskInfo + 26, // 16: temporal.server.api.replication.v1.ReplicationToken.last_processed_visibility_time:type_name -> google.protobuf.Timestamp + 26, // 17: temporal.server.api.replication.v1.SyncShardStatus.status_time:type_name -> google.protobuf.Timestamp + 26, // 18: temporal.server.api.replication.v1.SyncReplicationState.inclusive_low_watermark_time:type_name -> google.protobuf.Timestamp 4, // 19: temporal.server.api.replication.v1.SyncReplicationState.high_priority_state:type_name -> temporal.server.api.replication.v1.ReplicationState 4, // 20: temporal.server.api.replication.v1.SyncReplicationState.low_priority_state:type_name -> temporal.server.api.replication.v1.ReplicationState - 25, // 21: temporal.server.api.replication.v1.ReplicationState.inclusive_low_watermark_time:type_name -> google.protobuf.Timestamp - 29, // 22: temporal.server.api.replication.v1.ReplicationState.flow_control_command:type_name -> temporal.server.api.enums.v1.ReplicationFlowControlCommand - 0, // 23: temporal.server.api.replication.v1.ReplicationMessages.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 2, // 24: temporal.server.api.replication.v1.ReplicationMessages.sync_shard_status:type_name -> temporal.server.api.replication.v1.SyncShardStatus - 0, // 25: temporal.server.api.replication.v1.WorkflowReplicationMessages.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 25, // 26: temporal.server.api.replication.v1.WorkflowReplicationMessages.exclusive_high_watermark_time:type_name -> google.protobuf.Timestamp - 26, // 27: temporal.server.api.replication.v1.WorkflowReplicationMessages.priority:type_name -> temporal.server.api.enums.v1.TaskPriority - 30, // 28: temporal.server.api.replication.v1.ReplicationTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 26, // 29: temporal.server.api.replication.v1.ReplicationTaskInfo.priority:type_name -> temporal.server.api.enums.v1.TaskPriority - 31, // 30: temporal.server.api.replication.v1.NamespaceTaskAttributes.namespace_operation:type_name -> temporal.server.api.enums.v1.NamespaceOperation - 32, // 31: temporal.server.api.replication.v1.NamespaceTaskAttributes.info:type_name -> temporal.api.namespace.v1.NamespaceInfo - 33, // 32: temporal.server.api.replication.v1.NamespaceTaskAttributes.config:type_name -> temporal.api.namespace.v1.NamespaceConfig - 34, // 33: temporal.server.api.replication.v1.NamespaceTaskAttributes.replication_config:type_name -> temporal.api.replication.v1.NamespaceReplicationConfig - 35, // 34: temporal.server.api.replication.v1.NamespaceTaskAttributes.failover_history:type_name -> temporal.api.replication.v1.FailoverStatus - 25, // 35: temporal.server.api.replication.v1.SyncShardStatusTaskAttributes.status_time:type_name -> google.protobuf.Timestamp - 25, // 36: temporal.server.api.replication.v1.SyncActivityTaskAttributes.scheduled_time:type_name -> google.protobuf.Timestamp - 25, // 37: temporal.server.api.replication.v1.SyncActivityTaskAttributes.started_time:type_name -> google.protobuf.Timestamp - 25, // 38: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_heartbeat_time:type_name -> google.protobuf.Timestamp - 36, // 39: temporal.server.api.replication.v1.SyncActivityTaskAttributes.details:type_name -> temporal.api.common.v1.Payloads - 37, // 40: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_failure:type_name -> temporal.api.failure.v1.Failure - 38, // 41: temporal.server.api.replication.v1.SyncActivityTaskAttributes.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 39, // 42: temporal.server.api.replication.v1.SyncActivityTaskAttributes.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 25, // 43: temporal.server.api.replication.v1.SyncActivityTaskAttributes.first_scheduled_time:type_name -> google.protobuf.Timestamp - 25, // 44: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 40, // 45: temporal.server.api.replication.v1.SyncActivityTaskAttributes.retry_initial_interval:type_name -> google.protobuf.Duration - 40, // 46: temporal.server.api.replication.v1.SyncActivityTaskAttributes.retry_maximum_interval:type_name -> google.protobuf.Duration - 41, // 47: temporal.server.api.replication.v1.HistoryTaskAttributes.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 24, // 48: temporal.server.api.replication.v1.HistoryTaskAttributes.events:type_name -> temporal.api.common.v1.DataBlob - 24, // 49: temporal.server.api.replication.v1.HistoryTaskAttributes.new_run_events:type_name -> temporal.api.common.v1.DataBlob - 39, // 50: temporal.server.api.replication.v1.HistoryTaskAttributes.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 24, // 51: temporal.server.api.replication.v1.HistoryTaskAttributes.events_batches:type_name -> temporal.api.common.v1.DataBlob - 42, // 52: temporal.server.api.replication.v1.SyncWorkflowStateTaskAttributes.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 43, // 53: temporal.server.api.replication.v1.TaskQueueUserDataAttributes.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData - 38, // 54: temporal.server.api.replication.v1.SyncHSMAttributes.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 44, // 55: temporal.server.api.replication.v1.SyncHSMAttributes.state_machine_node:type_name -> temporal.server.api.persistence.v1.StateMachineNode - 41, // 56: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.event_version_history:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 24, // 57: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.event_batches:type_name -> temporal.api.common.v1.DataBlob - 16, // 58: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.new_run_info:type_name -> temporal.server.api.replication.v1.NewRunInfo - 24, // 59: temporal.server.api.replication.v1.NewRunInfo.event_batch:type_name -> temporal.api.common.v1.DataBlob - 27, // 60: temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes.exclusive_start_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 45, // 61: temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes.state_mutation:type_name -> temporal.server.api.persistence.v1.WorkflowMutableStateMutation - 42, // 62: temporal.server.api.replication.v1.SyncWorkflowStateSnapshotAttributes.state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 41, // 63: temporal.server.api.replication.v1.VerifyVersionedTransitionTaskAttributes.event_version_history:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 21, // 64: temporal.server.api.replication.v1.SyncVersionedTransitionTaskAttributes.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact - 17, // 65: temporal.server.api.replication.v1.VersionedTransitionArtifact.sync_workflow_state_mutation_attributes:type_name -> temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes - 18, // 66: temporal.server.api.replication.v1.VersionedTransitionArtifact.sync_workflow_state_snapshot_attributes:type_name -> temporal.server.api.replication.v1.SyncWorkflowStateSnapshotAttributes - 24, // 67: temporal.server.api.replication.v1.VersionedTransitionArtifact.event_batches:type_name -> temporal.api.common.v1.DataBlob - 16, // 68: temporal.server.api.replication.v1.VersionedTransitionArtifact.new_run_info:type_name -> temporal.server.api.replication.v1.NewRunInfo - 69, // [69:69] is the sub-list for method output_type - 69, // [69:69] is the sub-list for method input_type - 69, // [69:69] is the sub-list for extension type_name - 69, // [69:69] is the sub-list for extension extendee - 0, // [0:69] is the sub-list for field type_name + 23, // 21: temporal.server.api.replication.v1.SyncReplicationState.isolated_lane_states:type_name -> temporal.server.api.replication.v1.SyncReplicationState.IsolatedLaneStatesEntry + 26, // 22: temporal.server.api.replication.v1.ReplicationState.inclusive_low_watermark_time:type_name -> google.protobuf.Timestamp + 30, // 23: temporal.server.api.replication.v1.ReplicationState.flow_control_command:type_name -> temporal.server.api.enums.v1.ReplicationFlowControlCommand + 0, // 24: temporal.server.api.replication.v1.ReplicationMessages.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 2, // 25: temporal.server.api.replication.v1.ReplicationMessages.sync_shard_status:type_name -> temporal.server.api.replication.v1.SyncShardStatus + 0, // 26: temporal.server.api.replication.v1.WorkflowReplicationMessages.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 26, // 27: temporal.server.api.replication.v1.WorkflowReplicationMessages.exclusive_high_watermark_time:type_name -> google.protobuf.Timestamp + 27, // 28: temporal.server.api.replication.v1.WorkflowReplicationMessages.priority:type_name -> temporal.server.api.enums.v1.TaskPriority + 31, // 29: temporal.server.api.replication.v1.ReplicationTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 27, // 30: temporal.server.api.replication.v1.ReplicationTaskInfo.priority:type_name -> temporal.server.api.enums.v1.TaskPriority + 32, // 31: temporal.server.api.replication.v1.NamespaceTaskAttributes.namespace_operation:type_name -> temporal.server.api.enums.v1.NamespaceOperation + 33, // 32: temporal.server.api.replication.v1.NamespaceTaskAttributes.info:type_name -> temporal.api.namespace.v1.NamespaceInfo + 34, // 33: temporal.server.api.replication.v1.NamespaceTaskAttributes.config:type_name -> temporal.api.namespace.v1.NamespaceConfig + 35, // 34: temporal.server.api.replication.v1.NamespaceTaskAttributes.replication_config:type_name -> temporal.api.replication.v1.NamespaceReplicationConfig + 36, // 35: temporal.server.api.replication.v1.NamespaceTaskAttributes.failover_history:type_name -> temporal.api.replication.v1.FailoverStatus + 26, // 36: temporal.server.api.replication.v1.SyncShardStatusTaskAttributes.status_time:type_name -> google.protobuf.Timestamp + 26, // 37: temporal.server.api.replication.v1.SyncActivityTaskAttributes.scheduled_time:type_name -> google.protobuf.Timestamp + 26, // 38: temporal.server.api.replication.v1.SyncActivityTaskAttributes.started_time:type_name -> google.protobuf.Timestamp + 26, // 39: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_heartbeat_time:type_name -> google.protobuf.Timestamp + 37, // 40: temporal.server.api.replication.v1.SyncActivityTaskAttributes.details:type_name -> temporal.api.common.v1.Payloads + 38, // 41: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_failure:type_name -> temporal.api.failure.v1.Failure + 39, // 42: temporal.server.api.replication.v1.SyncActivityTaskAttributes.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 40, // 43: temporal.server.api.replication.v1.SyncActivityTaskAttributes.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 26, // 44: temporal.server.api.replication.v1.SyncActivityTaskAttributes.first_scheduled_time:type_name -> google.protobuf.Timestamp + 26, // 45: temporal.server.api.replication.v1.SyncActivityTaskAttributes.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 41, // 46: temporal.server.api.replication.v1.SyncActivityTaskAttributes.retry_initial_interval:type_name -> google.protobuf.Duration + 41, // 47: temporal.server.api.replication.v1.SyncActivityTaskAttributes.retry_maximum_interval:type_name -> google.protobuf.Duration + 42, // 48: temporal.server.api.replication.v1.HistoryTaskAttributes.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 25, // 49: temporal.server.api.replication.v1.HistoryTaskAttributes.events:type_name -> temporal.api.common.v1.DataBlob + 25, // 50: temporal.server.api.replication.v1.HistoryTaskAttributes.new_run_events:type_name -> temporal.api.common.v1.DataBlob + 40, // 51: temporal.server.api.replication.v1.HistoryTaskAttributes.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 25, // 52: temporal.server.api.replication.v1.HistoryTaskAttributes.events_batches:type_name -> temporal.api.common.v1.DataBlob + 43, // 53: temporal.server.api.replication.v1.SyncWorkflowStateTaskAttributes.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 44, // 54: temporal.server.api.replication.v1.TaskQueueUserDataAttributes.user_data:type_name -> temporal.server.api.persistence.v1.TaskQueueUserData + 39, // 55: temporal.server.api.replication.v1.SyncHSMAttributes.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 45, // 56: temporal.server.api.replication.v1.SyncHSMAttributes.state_machine_node:type_name -> temporal.server.api.persistence.v1.StateMachineNode + 42, // 57: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.event_version_history:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 25, // 58: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.event_batches:type_name -> temporal.api.common.v1.DataBlob + 16, // 59: temporal.server.api.replication.v1.BackfillHistoryTaskAttributes.new_run_info:type_name -> temporal.server.api.replication.v1.NewRunInfo + 25, // 60: temporal.server.api.replication.v1.NewRunInfo.event_batch:type_name -> temporal.api.common.v1.DataBlob + 28, // 61: temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes.exclusive_start_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 46, // 62: temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes.state_mutation:type_name -> temporal.server.api.persistence.v1.WorkflowMutableStateMutation + 43, // 63: temporal.server.api.replication.v1.SyncWorkflowStateSnapshotAttributes.state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 42, // 64: temporal.server.api.replication.v1.VerifyVersionedTransitionTaskAttributes.event_version_history:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 21, // 65: temporal.server.api.replication.v1.SyncVersionedTransitionTaskAttributes.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact + 17, // 66: temporal.server.api.replication.v1.VersionedTransitionArtifact.sync_workflow_state_mutation_attributes:type_name -> temporal.server.api.replication.v1.SyncWorkflowStateMutationAttributes + 18, // 67: temporal.server.api.replication.v1.VersionedTransitionArtifact.sync_workflow_state_snapshot_attributes:type_name -> temporal.server.api.replication.v1.SyncWorkflowStateSnapshotAttributes + 25, // 68: temporal.server.api.replication.v1.VersionedTransitionArtifact.event_batches:type_name -> temporal.api.common.v1.DataBlob + 16, // 69: temporal.server.api.replication.v1.VersionedTransitionArtifact.new_run_info:type_name -> temporal.server.api.replication.v1.NewRunInfo + 4, // 70: temporal.server.api.replication.v1.SyncReplicationState.IsolatedLaneStatesEntry.value:type_name -> temporal.server.api.replication.v1.ReplicationState + 71, // [71:71] is the sub-list for method output_type + 71, // [71:71] is the sub-list for method input_type + 71, // [71:71] is the sub-list for extension type_name + 71, // [71:71] is the sub-list for extension extendee + 0, // [0:71] is the sub-list for field type_name } func init() { file_temporal_server_api_replication_v1_message_proto_init() } @@ -2467,7 +2537,7 @@ func file_temporal_server_api_replication_v1_message_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_api_replication_v1_message_proto_rawDesc), len(file_temporal_server_api_replication_v1_message_proto_rawDesc)), NumEnums: 0, - NumMessages: 23, + NumMessages: 24, NumExtensions: 0, NumServices: 0, }, diff --git a/proto/internal/temporal/server/api/replication/v1/message.proto b/proto/internal/temporal/server/api/replication/v1/message.proto index 83d4566c591..1e2925bf257 100644 --- a/proto/internal/temporal/server/api/replication/v1/message.proto +++ b/proto/internal/temporal/server/api/replication/v1/message.proto @@ -67,6 +67,21 @@ message SyncReplicationState { google.protobuf.Timestamp inclusive_low_watermark_time = 2; ReplicationState high_priority_state = 3; ReplicationState low_priority_state = 4; + // Namespace IDs the receiver reports as overwhelming the HIGH (live) lane. The + // sender isolates these namespaces' HIGH traffic onto throttled severity-tier + // lanes. The priority lives in the field name: a future LOW-lane extension adds + // its own throttle_low_namespace_ids rather than widening this one. + repeated string throttle_high_namespace_ids = 5; + // Low watermarks of the isolated per-namespace lanes, keyed by namespace ID. + // Absence of a key is ambiguous: it means either "the receiver has not tracked + // this lane yet" or "the lane retired and drained". A sender must therefore never + // treat absence alone as proof a lane drained — it must correlate with its own + // lane state (e.g. only conclude drain for lanes it has retired). + map isolated_lane_states = 6; + // True when the receiver understands per-namespace lane isolation (routing of + // isolated_namespace_id-tagged messages). The sender requires this before it emits + // lane-tagged traffic, so an old receiver is never sent lanes it would misroute. + bool supports_namespace_isolation = 7; } message ReplicationState { int64 inclusive_low_watermark = 1; @@ -89,6 +104,15 @@ message WorkflowReplicationMessages { int64 exclusive_high_watermark = 2; google.protobuf.Timestamp exclusive_high_watermark_time = 3; temporal.server.api.enums.v1.TaskPriority priority = 4; + // When set, this batch belongs to the named namespace's isolated lane rather + // than the shared priority lane. Used by the receiver only for tracker routing; + // scheduler priority still comes from `priority`. Each isolated lane is a + // monotonic stream for the life of the stream connection. + string isolated_namespace_id = 5; + // True on the final message of an isolated lane: the sender has merged the + // namespace back to the shared lane and will send no further traffic on this + // lane. The receiver drops the lane's tracking once its pending work drains. + bool retire_isolated_lane = 6; } // TODO: Deprecate this definition, it only used by the deprecated replication DLQ v1 logic diff --git a/service/history/replication/executable_task_tool_box.go b/service/history/replication/executable_task_tool_box.go index 86094703769..ffa45705afa 100644 --- a/service/history/replication/executable_task_tool_box.go +++ b/service/history/replication/executable_task_tool_box.go @@ -42,5 +42,6 @@ type ( HistoryEventsHandler eventhandler.HistoryEventsHandler WorkflowCache wcache.Cache RemoteHistoryFetcher eventhandler.HistoryPaginatedFetcher + NamespaceThrottler NamespaceThrottler } ) diff --git a/service/history/replication/fx.go b/service/history/replication/fx.go index d97e84ca4c9..5c426bf4761 100644 --- a/service/history/replication/fx.go +++ b/service/history/replication/fx.go @@ -41,6 +41,9 @@ type ( var Module = fx.Provide( NewTaskFetcherFactory, + func() NamespaceThrottler { + return NoopNamespaceThrottler{} + }, func(m persistence.ExecutionManager) ExecutionManager { return m }, diff --git a/service/history/replication/namespace_throttler.go b/service/history/replication/namespace_throttler.go new file mode 100644 index 00000000000..eedad0b1219 --- /dev/null +++ b/service/history/replication/namespace_throttler.go @@ -0,0 +1,27 @@ +package replication + +// NamespaceThrottler tracks per-namespace HIGH-priority (live) replication task load on +// the receiver and reports which namespaces are overwhelming the shared lane. The +// reported namespace IDs travel back to the sender in SyncReplicationState acks +// (throttle_high_namespace_ids), which isolates those namespaces' live replication onto +// dedicated throttled lanes so they cannot stall the default lane. +// +// Load and throttle decisions can be scoped to the receiver's local shard: a namespace +// overwhelming one shard's lane is not necessarily throttled on others, letting it +// keep full speed on lower-traffic shards. The throttler is shared across all stream +// receivers on this host, so implementations must be safe for concurrent use. +type NamespaceThrottler interface { + // RecordTask records an incoming HIGH-priority task for the given namespace on + // the given local shard. + RecordTask(shardID int32, namespaceID string) + // ThrottledNamespaceIDs returns the namespace IDs the throttler currently + // considers overwhelming on the given local shard. How that is decided + // (thresholds, windows, decay) is internal to the implementation. + ThrottledNamespaceIDs(shardID int32) []string +} + +// NoopNamespaceThrottler is the default implementation which never throttles. +type NoopNamespaceThrottler struct{} + +func (NoopNamespaceThrottler) RecordTask(_ int32, _ string) {} +func (NoopNamespaceThrottler) ThrottledNamespaceIDs(_ int32) []string { return nil } diff --git a/service/history/replication/stream_receiver.go b/service/history/replication/stream_receiver.go index 2cc6795c00c..4f7a47c2dc0 100644 --- a/service/history/replication/stream_receiver.go +++ b/service/history/replication/stream_receiver.go @@ -5,6 +5,7 @@ package replication import ( "context" "fmt" + "maps" "sync" "sync/atomic" "time" @@ -45,13 +46,21 @@ type ( serverShardKey ClusterShardKey highPriorityTaskTracker ExecutableTaskTracker lowPriorityTaskTracker ExecutableTaskTracker - shutdownChan channel.ShutdownOnce - logger log.Logger - stream Stream - taskConverter ExecutableTaskConverter - receiverMode ReceiverMode - flowController ReceiverFlowController - recvSignalChan chan struct{} + // memberLanes holds one lane per isolated namespace, created lazily as the + // sender tags messages with isolated_namespace_id. Each lane is its own + // monotonic stream for the life of the stream connection; a retired lane + // (the sender merged the namespace back) is dropped once its pending work + // drains. Tasks are still scheduled at the message's priority. + memberLaneMu sync.Mutex + memberLanes map[string]*memberLane + memberLanesClosed bool // set by Stop; late-created lanes are pre-cancelled + shutdownChan channel.ShutdownOnce + logger log.Logger + stream Stream + taskConverter ExecutableTaskConverter + receiverMode ReceiverMode + flowController ReceiverFlowController + recvSignalChan chan struct{} slowSubmissionMu sync.RWMutex slowSubmissionTimestamps map[enumsspb.TaskPriority]time.Time @@ -61,6 +70,18 @@ type ( TrackableExecutableTask execute func() error } + + // memberLane is the receiver-side state of one isolated namespace's lane. + memberLane struct { + tracker ExecutableTaskTracker + // retiring is set when the sender marks the lane retired; the lane is + // dropped once its tracker drains. + retiring bool + // tracking counts batches resolved to this lane whose TrackTasks call has + // not finished yet; the ack loop must not drop the lane while one is in + // flight. + tracking int + } ) func (t executeInterceptedTask) Execute() error { @@ -100,6 +121,7 @@ func NewStreamReceiver( serverShardKey: serverShardKey, highPriorityTaskTracker: highPriorityTaskTracker, lowPriorityTaskTracker: lowPriorityTaskTracker, + memberLanes: make(map[string]*memberLane), shutdownChan: channel.NewShutdownOnce(), logger: logger, stream: newStream( @@ -115,7 +137,10 @@ func NewStreamReceiver( taskTrackerMap := make(map[enumsspb.TaskPriority]FlowControlSignalProvider) taskTrackerMap[enumsspb.TASK_PRIORITY_HIGH] = func() *FlowControlSignal { return &FlowControlSignal{ - taskTrackingCount: highPriorityTaskTracker.Size(), + // Include the isolated per-namespace lane trackers: isolation splits the + // HIGH lane, so flow control protects against total HIGH backlog (default + // lane + member lanes). + taskTrackingCount: receiver.highFamilyTrackingCount(), lastSlowSubmission: receiver.getLastSlowSubmissionTimestamp(enumsspb.TASK_PRIORITY_HIGH), } } @@ -126,6 +151,9 @@ func NewStreamReceiver( } } receiver.flowController = NewReceiverFlowControl(taskTrackerMap, processToolBox.Config) + if receiver.NamespaceThrottler == nil { + receiver.NamespaceThrottler = NoopNamespaceThrottler{} + } return receiver } @@ -173,6 +201,12 @@ func (r *StreamReceiverImpl) Stop() { r.stream.Close() r.highPriorityTaskTracker.Cancel() r.lowPriorityTaskTracker.Cancel() + r.memberLaneMu.Lock() + r.memberLanesClosed = true + for _, lane := range r.memberLanes { + lane.tracker.Cancel() + } + r.memberLaneMu.Unlock() r.logger.Info("StreamReceiver shutting down.") } @@ -241,9 +275,10 @@ func (r *StreamReceiverImpl) ackMessage( ) (int64, error) { highPriorityWaterMarkInfo := r.highPriorityTaskTracker.LowWatermark() lowPriorityWaterMarkInfo := r.lowPriorityTaskTracker.LowWatermark() - size := r.highPriorityTaskTracker.Size() + r.lowPriorityTaskTracker.Size() + size := r.highFamilyTrackingCount() + r.lowPriorityTaskTracker.Size() var highPriorityWatermark, lowPriorityWatermark *replicationspb.ReplicationState + var isolatedLaneStates map[string]*replicationspb.ReplicationState inclusiveLowWaterMark := int64(-1) var inclusiveLowWaterMarkTime time.Time @@ -286,6 +321,24 @@ func (r *StreamReceiverImpl) ackMessage( inclusiveLowWaterMark = lowPriorityWaterMarkInfo.Watermark inclusiveLowWaterMarkTime = lowPriorityWaterMarkInfo.Timestamp } + // Fold isolated member lanes into the overall min so Scopes[0] on the sender + // (and thus cleanup) accounts for a lagging lane: the overall min must be the + // lowest point in flight across every lane. Report each lane's watermark keyed + // by namespace so the sender can anchor per-member state on applied points. + memberWms := r.memberLaneWatermarks() + if len(memberWms) > 0 { + isolatedLaneStates = make(map[string]*replicationspb.ReplicationState, len(memberWms)) + } + for ns, wm := range memberWms { + if wm.Watermark < inclusiveLowWaterMark { + inclusiveLowWaterMark = wm.Watermark + inclusiveLowWaterMarkTime = wm.Timestamp + } + isolatedLaneStates[ns] = &replicationspb.ReplicationState{ + InclusiveLowWatermark: wm.Watermark, + InclusiveLowWatermarkTime: timestamppb.New(wm.Timestamp), + } + } case ReceiverModeSingleStack: if highPriorityWaterMarkInfo == nil { // This should not happen, more for a safety check return 0, NewStreamError("Single stack mode. High priority tracker does not have low watermark info", serviceerror.NewInternal("Invalid tracker state")) @@ -300,6 +353,10 @@ func (r *StreamReceiverImpl) ackMessage( return 0, NewStreamError("InclusiveLowWaterMark is not set", serviceerror.NewInternal("Invalid inclusive low watermark")) } + var throttleHighNamespaceIDs []string + if receiverMode == ReceiverModeTieredStack { + throttleHighNamespaceIDs = r.NamespaceThrottler.ThrottledNamespaceIDs(r.clientShardKey.ShardID) + } if err := stream.Send(&adminservice.StreamWorkflowReplicationMessagesRequest{ Attributes: &adminservice.StreamWorkflowReplicationMessagesRequest_SyncReplicationState{ SyncReplicationState: &replicationspb.SyncReplicationState{ @@ -307,6 +364,12 @@ func (r *StreamReceiverImpl) ackMessage( InclusiveLowWatermarkTime: timestamppb.New(inclusiveLowWaterMarkTime), HighPriorityState: highPriorityWatermark, LowPriorityState: lowPriorityWatermark, + ThrottleHighNamespaceIds: throttleHighNamespaceIDs, + IsolatedLaneStates: isolatedLaneStates, + // Advertise that this receiver routes isolated_namespace_id-tagged + // messages, so the sender may safely emit lane traffic to it. + // Isolation only applies to the tiered stack. + SupportsNamespaceIsolation: receiverMode == ReceiverModeTieredStack, }, }, }); err != nil { @@ -370,18 +433,39 @@ func (r *StreamReceiverImpl) processMessages( ) exclusiveHighWatermark := messages.ExclusiveHighWatermark exclusiveHighWatermarkTime := timestamp.TimeValue(messages.ExclusiveHighWatermarkTime) - taskTracker, err := r.getTaskTracker(priority) + // Route to the per-lane tracker: an isolated namespace's lane gets its own + // monotonic tracker; unset routes by priority (HIGH / default-LOW). + taskTracker, err := r.getTaskTrackerForLane(priority, messages.GetIsolatedNamespaceId(), messages.GetRetireIsolatedLane()) if err != nil { // Todo: Change to write Tasks to DLQ. As resend task will not help here return NewStreamError("ReplicationTask wrong priority", err) } + if priority == enumsspb.TASK_PRIORITY_HIGH { + // Isolation acts on HIGH (live) traffic: feed the throttler every HIGH task, + // including isolated-lane tasks, so an isolated namespace stays observable + // and is released once its HIGH rate drops. Load is scoped to the local shard. + for _, task := range convertedTasks { + if nsID := task.ReplicationTask().GetRawTaskInfo().GetNamespaceId(); nsID != "" { + r.NamespaceThrottler.RecordTask(r.clientShardKey.ShardID, nsID) + } + } + } + submissionThreshold := r.Config.ReplicationReceiverSlowSubmissionLatencyThreshold() - for _, task := range taskTracker.TrackTasks(WatermarkInfo{ + trackedTasks := taskTracker.TrackTasks(WatermarkInfo{ Watermark: exclusiveHighWatermark, Timestamp: exclusiveHighWatermarkTime, - }, convertedTasks...) { + }, convertedTasks...) + if isolatedNamespaceID := messages.GetIsolatedNamespaceId(); isolatedNamespaceID != "" { + // Finish strictly after TrackTasks: the ack loop may only delete a + // retiring lane once no batch sits between lane resolution and tracking, + // or the batch would be tracked on an orphaned lane and dropped from the + // ack fold. + r.finishLaneBatch(isolatedNamespaceID, messages.GetRetireIsolatedLane()) + } + for _, task := range trackedTasks { schedulerPriority, err := r.getTaskSchedulerPriority(priority, task) if err != nil { return err @@ -439,6 +523,112 @@ func (r *StreamReceiverImpl) getTaskTracker(priority enumsspb.TaskPriority) (Exe } } +// getTaskTrackerForLane returns the tracker for a message's lane: an isolated +// namespace's lane gets its own lazily-created tracker so each lane stays a single +// monotonic stream; an empty namespace routes by priority (HIGH / default-LOW). +// Every lane resolution must be paired with finishLaneBatch after the batch's +// TrackTasks call, so the ack loop never deletes the lane mid-batch. +func (r *StreamReceiverImpl) getTaskTrackerForLane(priority enumsspb.TaskPriority, isolatedNamespaceID string, retire bool) (ExecutableTaskTracker, error) { + if isolatedNamespaceID == "" { + return r.getTaskTracker(priority) + } + if priority != enumsspb.TASK_PRIORITY_HIGH { + // Isolation splits the HIGH lane only. Lane-tagged traffic at any other + // priority (including single-stack UNSPECIFIED, whose ack path would never + // fold a member lane) is a sender bug — fail the stream rather than + // silently mis-acking. + return nil, serviceerror.NewInternalf("isolated lane traffic must be HIGH priority, got %v", priority) + } + r.memberLaneMu.Lock() + defer r.memberLaneMu.Unlock() + lane, ok := r.memberLanes[isolatedNamespaceID] + if !ok { + lane = &memberLane{tracker: NewExecutableTaskTracker(r.logger, r.MetricsHandler)} + if r.memberLanesClosed { + // Lost the race with Stop(): pre-cancel so tracked tasks are + // cancelled rather than executed after shutdown. + lane.tracker.Cancel() + } + r.memberLanes[isolatedNamespaceID] = lane + } + if !retire && lane.retiring { + // Non-retire traffic on a retired lane means the sender re-isolated the + // namespace before the lane drained: the lane is live again. Without this, + // the stale retiring flag would delete an active lane at the first + // transient drain, dropping its watermark from the ack fold. + lane.retiring = false + } + lane.tracking++ + return lane.tracker, nil +} + +// finishLaneBatch completes a batch that getTaskTrackerForLane resolved to a lane, +// called after the batch's TrackTasks: it releases the batch's hold on the lane and +// applies the message's retire flag. A retiring lane is only dropped once it is +// drained and no batch is mid-track — so a retired lane can never pin the overall +// ack minimum, and a batch can never be orphaned onto a deleted lane. +func (r *StreamReceiverImpl) finishLaneBatch(isolatedNamespaceID string, retire bool) { + r.memberLaneMu.Lock() + defer r.memberLaneMu.Unlock() + lane, ok := r.memberLanes[isolatedNamespaceID] + if !ok { + return + } + lane.tracking-- + if retire { + lane.retiring = true + } +} + +// highFamilyTrackingCount is the total tracked-task count across the default HIGH lane +// and all isolated member lanes (used for HIGH flow control — isolation splits the +// HIGH lane). +func (r *StreamReceiverImpl) highFamilyTrackingCount() int { + count := r.highPriorityTaskTracker.Size() + r.memberLaneMu.Lock() + for _, lane := range r.memberLanes { + count += lane.tracker.Size() + } + r.memberLaneMu.Unlock() + return count +} + +// memberLaneWatermarks snapshots the low watermark of every isolated member lane that +// has received at least one batch, keyed by namespace id. Retired lanes are dropped +// once drained. +func (r *StreamReceiverImpl) memberLaneWatermarks() map[string]WatermarkInfo { + r.memberLaneMu.Lock() + lanes := make(map[string]*memberLane, len(r.memberLanes)) + maps.Copy(lanes, r.memberLanes) + r.memberLaneMu.Unlock() + + out := make(map[string]WatermarkInfo, len(lanes)) + for ns, lane := range lanes { + wm := lane.tracker.LowWatermark() + + r.memberLaneMu.Lock() + currentLane, ok := r.memberLanes[ns] + if !ok || currentLane != lane { + r.memberLaneMu.Unlock() + continue + } + // Drop only lanes that are retiring, drained, have tracked at least one + // batch (wm != nil), and have no batch between lane resolution and + // tracking: deleting a lane mid-batch would orphan that batch outside the + // ack fold. + if lane.retiring && lane.tracking == 0 && lane.tracker.Size() == 0 && wm != nil { + delete(r.memberLanes, ns) + r.memberLaneMu.Unlock() + continue + } + if wm != nil { + out[ns] = *wm + } + r.memberLaneMu.Unlock() + } + return out +} + func (r *StreamReceiverImpl) getTaskSchedulerPriority(priority enumsspb.TaskPriority, task TrackableExecutableTask) (enumsspb.TaskPriority, error) { switch priority { case enumsspb.TASK_PRIORITY_UNSPECIFIED: diff --git a/service/history/replication/stream_receiver_test.go b/service/history/replication/stream_receiver_test.go index c195ab21132..cc7221de91d 100644 --- a/service/history/replication/stream_receiver_test.go +++ b/service/history/replication/stream_receiver_test.go @@ -16,6 +16,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/service/history/tests" "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/timestamppb" @@ -45,8 +46,21 @@ type ( mockScheduler struct { tasks []TrackableExecutableTask } + // fakeNamespaceThrottler returns configured throttled namespace IDs per shard + // and records the shard ID it was queried with. + fakeNamespaceThrottler struct { + throttled map[int32][]string + queriedShardID int32 + } ) +func (f *fakeNamespaceThrottler) RecordTask(_ int32, _ string) {} + +func (f *fakeNamespaceThrottler) ThrottledNamespaceIDs(shardID int32) []string { + f.queriedShardID = shardID + return f.throttled[shardID] +} + func TestStreamReceiverSuite(t *testing.T) { s := new(streamReceiverSuite) suite.Run(t, s) @@ -83,6 +97,7 @@ func (s *streamReceiverSuite) SetupTest() { MetricsHandler: metrics.NoopMetricsHandler, Logger: log.NewTestLogger(), DLQWriter: NoopDLQWriter{}, + NamespaceThrottler: NoopNamespaceThrottler{}, } processToolBox.Config.ReplicationStreamSyncStatusDuration = dynamicconfig.GetDurationPropertyFn(5 * time.Millisecond) s.clusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, gomock.Any()).Return("some-cluster-name").AnyTimes() @@ -196,6 +211,263 @@ func (s *streamReceiverSuite) TestAckMessage_SyncStatus_ReceiverModeSingleStack_ s.Empty(s.stream.requests) } +func (s *streamReceiverSuite) TestGetTaskTrackerForLane_UnsetRoutesByPriority() { + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "", false) + s.NoError(err) + s.Same(s.streamReceiver.highPriorityTaskTracker, tracker) + + tracker, err = s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_LOW, "", false) + s.NoError(err) + s.Same(s.streamReceiver.lowPriorityTaskTracker, tracker) +} + +func (s *streamReceiverSuite) TestGetTaskTrackerForLane_PerMemberLanes() { + trackerA, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + s.NotNil(trackerA) + + // Same namespace, same lane; different namespaces are independent lanes. + trackerA2, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + s.Same(trackerA, trackerA2) + trackerB, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-b", false) + s.NoError(err) + s.NotSame(trackerA, trackerB) +} + +func (s *streamReceiverSuite) TestGetTaskTrackerForLane_RejectsNonHighPriority() { + // Isolation splits the HIGH lane only: lane-tagged traffic at LOW or + // single-stack UNSPECIFIED priority is a protocol violation, not a routing case. + _, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_LOW, "ns-a", false) + s.Error(err) + _, err = s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_UNSPECIFIED, "ns-a", false) + s.Error(err) + s.Empty(s.streamReceiver.memberLanes) +} + +func (s *streamReceiverSuite) TestMemberLane_NonRetireTrafficRevivesRetiringLane() { + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + tracker.TrackTasks(WatermarkInfo{Watermark: 100, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", false) + + // A retire marker arrives: the lane is now retiring. + markerTracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", true) + s.NoError(err) + s.Same(tracker, markerTracker) + markerTracker.TrackTasks(WatermarkInfo{Watermark: 200, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", true) + + // The namespace is re-isolated before the lane drains: new traffic clears the + // stale retiring flag so a transient drain can't delete the active lane. + trackerAgain, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + s.Same(tracker, trackerAgain) + wms := s.streamReceiver.memberLaneWatermarks() + s.Equal(int64(200), wms["ns-a"].Watermark) + s.Contains(s.streamReceiver.memberLanes, "ns-a") + s.streamReceiver.finishLaneBatch("ns-a", false) +} + +func (s *streamReceiverSuite) TestMemberLane_RetireBatchMidTrackSurvivesAckSnapshot() { + // A lane whose retire batch has been resolved but not yet tracked must survive + // the ack loop's snapshot; once the (watermark-only) retire batch is tracked + // and finished, the drained lane is dropped. + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", true) + s.NoError(err) + + s.streamReceiver.memberLaneWatermarks() + s.Contains(s.streamReceiver.memberLanes, "ns-a") + + tracker.TrackTasks(WatermarkInfo{Watermark: 50, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", true) + s.streamReceiver.memberLaneWatermarks() + s.NotContains(s.streamReceiver.memberLanes, "ns-a") +} + +func (s *streamReceiverSuite) TestMemberLane_RetireBatchOnRetiringLaneNotOrphaned() { + // A lane that already retired and drained can receive a second retire-tagged + // batch (e.g. a duplicate retirement marker after the namespace re-isolated and + // merged back before its lane was dropped). The ack loop must not delete the + // lane between the batch's lane resolution and its TrackTasks call, or the + // batch would be tracked on an orphaned lane and never re-enter the ack fold. + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", true) + s.NoError(err) + tracker.TrackTasks(WatermarkInfo{Watermark: 100, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", true) + + // The lane is now retiring and drained. A second retire batch resolves to it... + trackerAgain, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", true) + s.NoError(err) + s.Same(tracker, trackerAgain) + + // ...and the concurrent ack snapshot must find it undeletable mid-track. + wms := s.streamReceiver.memberLaneWatermarks() + s.Contains(s.streamReceiver.memberLanes, "ns-a") + s.Equal(int64(100), wms["ns-a"].Watermark) + + // Once the batch is tracked and finished, the drained lane is dropped as before. + trackerAgain.TrackTasks(WatermarkInfo{Watermark: 200, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", true) + s.streamReceiver.memberLaneWatermarks() + s.NotContains(s.streamReceiver.memberLanes, "ns-a") +} + +func (s *streamReceiverSuite) TestMemberLane_CreatedAfterStopIsCancelled() { + s.streamReceiver.memberLaneMu.Lock() + s.streamReceiver.memberLanesClosed = true + s.streamReceiver.memberLaneMu.Unlock() + + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-late", false) + s.NoError(err) + // The tracker was pre-cancelled: tasks tracked into it are cancelled instead + // of running to completion after shutdown. + task := NewMockTrackableExecutableTask(s.controller) + task.EXPECT().TaskID().Return(int64(1)).AnyTimes() + task.EXPECT().Cancel() + tracker.TrackTasks(WatermarkInfo{Watermark: 2, Timestamp: time.Now()}, task) +} + +func (s *streamReceiverSuite) TestMemberLane_RetiredLaneDroppedOnceDrained() { + tracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + tracker.TrackTasks(WatermarkInfo{Watermark: 100, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", false) + + // Active lane reports its watermark. + wms := s.streamReceiver.memberLaneWatermarks() + s.Equal(int64(100), wms["ns-a"].Watermark) + + // A retire marker arrives and is tracked (watermark-only batch): retired and + // drained, the lane is dropped on the next snapshot, so it can never pin the + // overall ack minimum. + markerTracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", true) + s.NoError(err) + s.Same(tracker, markerTracker) + markerTracker.TrackTasks(WatermarkInfo{Watermark: 200, Timestamp: time.Now()}) + s.streamReceiver.finishLaneBatch("ns-a", true) + wms = s.streamReceiver.memberLaneWatermarks() + s.NotContains(wms, "ns-a") + + // A later message would lazily create a fresh lane. + trackerNew, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + s.NotSame(tracker, trackerNew) +} + +func (s *streamReceiverSuite) TestMemberLane_WatermarkDoesNotHoldLaneLock() { + tracker := NewMockExecutableTaskTracker(s.controller) + lowWatermarkStarted := make(chan struct{}) + releaseLowWatermark := make(chan struct{}) + defer func() { + select { + case <-releaseLowWatermark: + default: + close(releaseLowWatermark) + } + }() + tracker.EXPECT().LowWatermark().DoAndReturn(func() *WatermarkInfo { + close(lowWatermarkStarted) + <-releaseLowWatermark + return nil + }) + s.streamReceiver.memberLanes["ns-a"] = &memberLane{tracker: tracker} + + snapshotDone := make(chan struct{}) + go func() { + s.streamReceiver.memberLaneWatermarks() + close(snapshotDone) + }() + <-lowWatermarkStarted + + laneLockAcquired := make(chan struct{}) + go func() { + s.streamReceiver.memberLaneMu.Lock() + close(laneLockAcquired) + s.streamReceiver.memberLaneMu.Unlock() + }() + await.RequireTrue(s.T(), func() bool { + select { + case <-laneLockAcquired: + return true + default: + return false + } + }, time.Second, 10*time.Millisecond) + + close(releaseLowWatermark) + <-snapshotDone +} + +func (s *streamReceiverSuite) TestAckMessage_TieredStack_FoldsMemberLaneWatermarkIntoAck() { + s.streamReceiver.receiverMode = ReceiverModeTieredStack + highWatermarkInfo := &WatermarkInfo{Watermark: 200, Timestamp: time.Unix(0, 2000)} + lowWatermarkInfo := &WatermarkInfo{Watermark: 300, Timestamp: time.Unix(0, 3000)} + s.highPriorityTaskTracker.EXPECT().LowWatermark().Return(highWatermarkInfo) + s.lowPriorityTaskTracker.EXPECT().LowWatermark().Return(lowWatermarkInfo) + s.receiverFlowController.EXPECT().GetFlowControlInfo(enumsspb.TASK_PRIORITY_HIGH).Return(FlowControlInfo{Command: enumsspb.REPLICATION_FLOW_CONTROL_COMMAND_RESUME}) + s.receiverFlowController.EXPECT().GetFlowControlInfo(enumsspb.TASK_PRIORITY_LOW).Return(FlowControlInfo{Command: enumsspb.REPLICATION_FLOW_CONTROL_COMMAND_RESUME}) + s.highPriorityTaskTracker.EXPECT().Size().Return(0).AnyTimes() + s.lowPriorityTaskTracker.EXPECT().Size().Return(0).AnyTimes() + + // An isolated lane lagging below both shared-lane watermarks. + laneWatermark := WatermarkInfo{Watermark: 100, Timestamp: time.Unix(0, 1000)} + laneTracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + laneTracker.TrackTasks(laneWatermark) + + _, err = s.streamReceiver.ackMessage(s.stream) + s.NoError(err) + s.Len(s.stream.requests, 1) + state := s.stream.requests[0].GetSyncReplicationState() + // The lane drags the overall min below both shared-lane watermarks, so the + // sender's cleanup accounts for the lowest point in flight across every lane. + s.Equal(laneWatermark.Watermark, state.InclusiveLowWatermark) + s.Equal(timestamppb.New(laneWatermark.Timestamp), state.InclusiveLowWatermarkTime) + // The shared-lane states keep their own watermarks. + s.Equal(highWatermarkInfo.Watermark, state.HighPriorityState.InclusiveLowWatermark) + s.Equal(lowWatermarkInfo.Watermark, state.LowPriorityState.InclusiveLowWatermark) + // The lane reports its own applied watermark keyed by namespace. + s.Len(state.IsolatedLaneStates, 1) + s.Equal(laneWatermark.Watermark, state.IsolatedLaneStates["ns-a"].InclusiveLowWatermark) + s.Equal(timestamppb.New(laneWatermark.Timestamp), state.IsolatedLaneStates["ns-a"].InclusiveLowWatermarkTime) +} + +func (s *streamReceiverSuite) TestAckMessage_TieredStack_ReportsThrottledNamespaces() { + s.streamReceiver.receiverMode = ReceiverModeTieredStack + throttler := &fakeNamespaceThrottler{throttled: map[int32][]string{ + s.streamReceiver.clientShardKey.ShardID: {"ns-hot-a", "ns-hot-b"}, + s.streamReceiver.clientShardKey.ShardID + 1: {"ns-other"}, + }} + s.streamReceiver.NamespaceThrottler = throttler + watermarkInfo := &WatermarkInfo{Watermark: 10, Timestamp: time.Unix(0, 1000)} + s.highPriorityTaskTracker.EXPECT().LowWatermark().Return(watermarkInfo) + s.lowPriorityTaskTracker.EXPECT().LowWatermark().Return(watermarkInfo) + s.receiverFlowController.EXPECT().GetFlowControlInfo(enumsspb.TASK_PRIORITY_HIGH).Return(FlowControlInfo{Command: enumsspb.REPLICATION_FLOW_CONTROL_COMMAND_RESUME}) + s.receiverFlowController.EXPECT().GetFlowControlInfo(enumsspb.TASK_PRIORITY_LOW).Return(FlowControlInfo{Command: enumsspb.REPLICATION_FLOW_CONTROL_COMMAND_RESUME}) + s.highPriorityTaskTracker.EXPECT().Size().Return(0).AnyTimes() + s.lowPriorityTaskTracker.EXPECT().Size().Return(0).AnyTimes() + + _, err := s.streamReceiver.ackMessage(s.stream) + s.NoError(err) + s.Len(s.stream.requests, 1) + state := s.stream.requests[0].GetSyncReplicationState() + // Only the local shard's throttled set is reported. + s.Equal([]string{"ns-hot-a", "ns-hot-b"}, state.ThrottleHighNamespaceIds) + s.Equal(s.streamReceiver.clientShardKey.ShardID, throttler.queriedShardID) +} + +func (s *streamReceiverSuite) TestHighFamilyTrackingCount_IncludesMemberLanes() { + s.highPriorityTaskTracker.EXPECT().Size().Return(5) + laneTracker, err := s.streamReceiver.getTaskTrackerForLane(enumsspb.TASK_PRIORITY_HIGH, "ns-a", false) + s.NoError(err) + task := NewMockTrackableExecutableTask(s.controller) + task.EXPECT().TaskID().Return(int64(1)).AnyTimes() + laneTracker.TrackTasks(WatermarkInfo{Watermark: 10, Timestamp: time.Now()}, task) + + s.Equal(6, s.streamReceiver.highFamilyTrackingCount()) +} + func (s *streamReceiverSuite) TestAckMessage_SyncStatus_ReceiverModeTieredStack_NoHighPriorityWatermark() { s.streamReceiver.receiverMode = ReceiverModeTieredStack watermarkInfo := &WatermarkInfo{ @@ -259,6 +531,7 @@ func (s *streamReceiverSuite) TestAckMessage_SyncStatus_ReceiverModeTieredStack( InclusiveLowWatermarkTime: timestamppb.New(lowWatermarkInfo.Timestamp), FlowControlCommand: enumsspb.REPLICATION_FLOW_CONTROL_COMMAND_PAUSE, }, + SupportsNamespaceIsolation: true, }, }, }, From 5c210f4c73bfb17bf0dcee39be70c8438d48dccf Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 25 Aug 2026 09:04:00 -0700 Subject: [PATCH 057/139] Replace errors.As with errors.AsType (#11674) Go 1.27 prerequisite that applies the `errorsastype` Go fixer and its required error-interface updates. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- chasm/lib/activity/handler.go | 3 +- chasm/lib/callback/invocable_outbound.go | 6 +-- chasm/lib/callback/request.go | 3 +- chasm/lib/scheduler/handler.go | 9 ++--- chasm/lib/scheduler/invoker_tasks.go | 3 +- chasm/lib/scheduler/scheduler_migrate_task.go | 3 +- chasm/lib/scheduler/scheduler_tasks.go | 3 +- chasm/lib/workflow/nexus_commands.go | 3 +- client/history/caching_redirector.go | 3 +- common/nexus/nexusrpc/server.go | 6 +-- common/persistence/client/fx.go | 3 +- common/persistence/execution_manager.go | 6 +-- .../store_fault_generator_test.go | 5 +-- .../nosqlplugin/cassandra/gocql/errors.go | 6 +-- .../persistence/persistence_metric_clients.go | 3 +- .../sql/sqlplugin/sqlite/driver.go | 6 +-- .../store/elasticsearch/processor.go | 3 +- .../visibility/store/query/errors.go | 3 +- .../visibility/store/sql/visibility_store.go | 18 +++------ .../visibility/visibility_manager_impl.go | 3 +- .../rpc/interceptor/request_error_handler.go | 3 +- common/searchattribute/mapper.go | 3 +- common/util/error_type.go | 4 +- common/worker_versioning/worker_versioning.go | 3 +- common/workercommands/dispatcher.go | 3 +- components/callbacks/nexus_invocation.go | 6 +-- components/callbacks/request.go | 3 +- components/nexusoperations/executors.go | 39 +++++++------------ service/frontend/admin_handler.go | 3 +- .../frontend/nexus_completion_http_handler.go | 18 +++------ service/frontend/nexus_endpoint_client.go | 3 +- service/frontend/nexus_handler.go | 12 ++---- .../frontend/nexus_operation_http_handler.go | 3 +- service/frontend/workflow_handler.go | 30 +++++--------- service/history/api/get_history_util.go | 3 +- .../api/getworkflowexecutionhistory/api.go | 3 +- service/history/api/multioperation/api.go | 3 +- .../api/recordworkflowtaskstarted/api.go | 3 +- .../api/respondworkflowtaskcompleted/api.go | 3 +- .../workflow_task_completed_handler.go | 6 +-- .../api.go | 3 +- service/history/archival/archiver.go | 4 +- service/history/handler.go | 3 +- service/history/ndc/workflow_resetter.go | 3 +- service/history/queues/executable.go | 7 ++-- service/history/queues/executable_mock.go | 14 +++++++ .../history/replication/executable_task.go | 15 +++---- .../history/replication/replication_events.go | 3 +- service/history/replication/stream.go | 3 +- service/history/workflow/update/registry.go | 3 +- service/matching/backlog_manager.go | 3 +- service/matching/fair_backlog_manager.go | 3 +- service/matching/fair_task_reader.go | 3 +- service/matching/pri_backlog_manager.go | 3 +- service/matching/pri_task_reader.go | 3 +- .../matching/task_queue_partition_manager.go | 3 +- service/matching/user_data_manager.go | 6 +-- service/worker/batcher/activities.go | 9 ++--- service/worker/deletenamespace/activities.go | 3 +- .../worker/deletenamespace/errors/errors.go | 3 +- .../reclaimresources/workflow.go | 3 +- service/worker/scheduler/activities.go | 6 +-- service/worker/scheduler/query.go | 6 +-- service/worker/workerdeployment/activities.go | 3 +- service/worker/workerdeployment/client.go | 33 ++++++---------- service/worker/workerdeployment/util.go | 6 +-- .../workerdeployment/version_activities.go | 6 +-- service/worker/workerdeployment/workflow.go | 3 +- tests/activity_workflow_driver.go | 3 +- tests/chasm_test.go | 3 +- tests/namespace_test.go | 9 ++--- tests/nexus_workflow_test.go | 3 +- tests/testcore/taskpoller.go | 3 +- tests/versioning_test_env.go | 3 +- tools/elasticsearch/tasks.go | 3 +- 75 files changed, 157 insertions(+), 285 deletions(-) diff --git a/chasm/lib/activity/handler.go b/chasm/lib/activity/handler.go index 50c675da471..a89d7934d2a 100644 --- a/chasm/lib/activity/handler.go +++ b/chasm/lib/activity/handler.go @@ -112,8 +112,7 @@ func (h *handler) StartActivityExecution(ctx context.Context, req *activitypb.St ) if err != nil { - var alreadyStartedErr *chasm.ExecutionAlreadyStartedError - if errors.As(err, &alreadyStartedErr) { + if alreadyStartedErr, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { return nil, serviceerror.NewActivityExecutionAlreadyStarted("activity execution already started", alreadyStartedErr.CurrentRequestID, alreadyStartedErr.CurrentRunID) } diff --git a/chasm/lib/callback/invocable_outbound.go b/chasm/lib/callback/invocable_outbound.go index 41a02a55ece..194223253ea 100644 --- a/chasm/lib/callback/invocable_outbound.go +++ b/chasm/lib/callback/invocable_outbound.go @@ -88,8 +88,7 @@ func (n invocableOutbound) Invoke( } func isRetryableCallError(err error) bool { - var handlerError *nexus.HandlerError - if errors.As(err, &handlerError) { + if handlerError, ok := errors.AsType[*nexus.HandlerError](err); ok { return handlerError.Retryable() } return true @@ -100,8 +99,7 @@ func outcomeTag(callCtx context.Context, callErr error) string { if callCtx.Err() != nil { return "request-timeout" } - var handlerErr *nexus.HandlerError - if errors.As(callErr, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](callErr); ok { return "handler-error:" + string(handlerErr.Type) } return "unknown-error" diff --git a/chasm/lib/callback/request.go b/chasm/lib/callback/request.go index 88747a8d33e..cb297afe54b 100644 --- a/chasm/lib/callback/request.go +++ b/chasm/lib/callback/request.go @@ -70,8 +70,7 @@ func routeSystemCallbackRequest( ns, err := namespaceRegistry.GetNamespaceByID(namespace.ID(namespaceID)) if err != nil { logger.Error("failed to get namespace for nexus completion request", tag.WorkflowNamespaceID(namespaceID), tag.Error(err)) - var nfe *serviceerror.NamespaceNotFound - if errors.As(err, &nfe) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeNotFound, "namespace %q not found", namespaceID) } return nil, commonnexus.ConvertGRPCError(err, false) diff --git a/chasm/lib/scheduler/handler.go b/chasm/lib/scheduler/handler.go index 710cee1743a..202e877b6e1 100644 --- a/chasm/lib/scheduler/handler.go +++ b/chasm/lib/scheduler/handler.go @@ -42,8 +42,7 @@ func (h *handler) CreateSchedule(ctx context.Context, req *schedulerpb.CreateSch chasm.WithRequestID(req.FrontendRequest.RequestId), ) - var alreadyStartedErr *chasm.ExecutionAlreadyStartedError - if errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { // Check if the existing schedule is a sentinel. // // TODO lina@ - this can be removed (as well as all other sentinel business) @@ -93,8 +92,7 @@ func (h *handler) CreateFromMigrationState(ctx context.Context, req *schedulerpb req, ) - var alreadyStartedErr *chasm.ExecutionAlreadyStartedError - if errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { // Check if the existing schedule is a sentinel. Sentinels are // auto-deleted SentinelIdleTime after schedule creation; the // V1 schedule will keep retrying migration until it expires. @@ -143,8 +141,7 @@ func (h *handler) CreateSentinel(ctx context.Context, req *schedulerpb.CreateSen req, ) - var alreadyStartedErr *chasm.ExecutionAlreadyStartedError - if errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*chasm.ExecutionAlreadyStartedError](err); ok { // If a sentinel already exists, succeed idempotently. // If a real scheduler exists, fail. _, readErr := chasm.ReadComponent( diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index 6da78f9d735..e3f654c4361 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -779,8 +779,7 @@ func isAlreadyStartedError(err error) bool { } func isRateLimitedError(err error) (time.Duration, bool) { - var expectedErr *rateLimitedError - if errors.As(err, &expectedErr) { + if expectedErr, ok := errors.AsType[*rateLimitedError](err); ok { return expectedErr.delay, true } return 0, false diff --git a/chasm/lib/scheduler/scheduler_migrate_task.go b/chasm/lib/scheduler/scheduler_migrate_task.go index f2b5237e8e7..536b41b4fc8 100644 --- a/chasm/lib/scheduler/scheduler_migrate_task.go +++ b/chasm/lib/scheduler/scheduler_migrate_task.go @@ -225,8 +225,7 @@ func (h *SchedulerMigrateToWorkflowTaskHandler) Execute( ) if err != nil { // Treat already-started as success for idempotency. - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - if !errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err); !ok { return fmt.Errorf("failed to start V1 scheduler workflow: %w", err) } } diff --git a/chasm/lib/scheduler/scheduler_tasks.go b/chasm/lib/scheduler/scheduler_tasks.go index b3975d625fb..3991950b06f 100644 --- a/chasm/lib/scheduler/scheduler_tasks.go +++ b/chasm/lib/scheduler/scheduler_tasks.go @@ -258,8 +258,7 @@ func (r *SchedulerCallbacksTaskHandler) watchRunningStart( }, }) if err != nil { - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return &watchResult{ completed: &schedulespb.CompletedResult{ Status: enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, diff --git a/chasm/lib/workflow/nexus_commands.go b/chasm/lib/workflow/nexus_commands.go index d33d2514a0f..e8c7abcc619 100644 --- a/chasm/lib/workflow/nexus_commands.go +++ b/chasm/lib/workflow/nexus_commands.go @@ -69,8 +69,7 @@ func (ch *nexusCommandHandler) handleScheduleCommand( // Links are not needed for validation. }, attrs.Service, attrs.Operation, attrs.Input) if err != nil { - var handlerErr *nexus.HandlerError - if errors.As(err, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](err); ok { //nolint:exhaustive switch handlerErr.Type { case nexus.HandlerErrorTypeNotFound, nexus.HandlerErrorTypeBadRequest: diff --git a/client/history/caching_redirector.go b/client/history/caching_redirector.go index 12ba8a2dba9..9ff93473dde 100644 --- a/client/history/caching_redirector.go +++ b/client/history/caching_redirector.go @@ -225,8 +225,7 @@ func (r *CachingRedirector[C]) handleSolError(opEntry cacheEntry[C], solErr *ser } func maybeHostDownError(opErr error) bool { - var unavail *serviceerror.Unavailable - if errors.As(opErr, &unavail) { + if _, ok := errors.AsType[*serviceerror.Unavailable](opErr); ok { return true } return common.IsContextDeadlineExceededErr(opErr) diff --git a/common/nexus/nexusrpc/server.go b/common/nexus/nexusrpc/server.go index 7e438e586bb..ca4dddfb497 100644 --- a/common/nexus/nexusrpc/server.go +++ b/common/nexus/nexusrpc/server.go @@ -94,13 +94,11 @@ func (h *httpHandler) writeResult(writer http.ResponseWriter, request *http.Requ // nolint:revive // Keeping all of the logic together for readability, even if it means the function is long. func (h *BaseHTTPHandler) WriteFailure(writer http.ResponseWriter, r *http.Request, err error) { var failure nexus.Failure - var failureError *nexus.FailureError - var opError *nexus.OperationError var handlerError *nexus.HandlerError var operationState nexus.OperationState statusCode := http.StatusInternalServerError - if errors.As(err, &opError) { + if opError, ok := errors.AsType[*nexus.OperationError](err); ok { operationState = opError.State var convErr error failure, convErr = h.FailureConverter.ErrorToFailure(opError) @@ -160,7 +158,7 @@ func (h *BaseHTTPHandler) WriteFailure(writer http.ResponseWriter, r *http.Reque default: h.Logger.Error("unexpected handler error type", "type", handlerError.Type) } - } else if errors.As(err, &failureError) { + } else if failureError, ok := errors.AsType[*nexus.FailureError](err); ok { failure = failureError.Failure } else { failure = nexus.Failure{ diff --git a/common/persistence/client/fx.go b/common/persistence/client/fx.go index 1409621ccd2..6d9e5a37ad6 100644 --- a/common/persistence/client/fx.go +++ b/common/persistence/client/fx.go @@ -227,8 +227,7 @@ func managerProvider[T persistence.Closeable](newManagerFn func(Factory) (T, err return func(f Factory, lc fx.Lifecycle) (T, error) { manager, err := newManagerFn(f) // passing receiver (Factory) as first argument. if err != nil { - var unimpl *serviceerror.Unimplemented - if errors.As(err, &unimpl) { + if _, ok := errors.AsType[*serviceerror.Unimplemented](err); ok { // allow factories to return Unimplemented, and turn into nil so that fx init doesn't fail. err = nil } diff --git a/common/persistence/execution_manager.go b/common/persistence/execution_manager.go index 25689c5242e..327ae634135 100644 --- a/common/persistence/execution_manager.go +++ b/common/persistence/execution_manager.go @@ -473,8 +473,7 @@ func (m *executionManagerImpl) GetWorkflowExecution( } response, respErr := m.persistence.GetWorkflowExecution(ctx, request) - var notFound *serviceerror.NotFound - if errors.As(respErr, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](respErr); ok { // strip persistence-specific error message respErr = serviceerror.NewNotFoundf( "workflow execution not found for workflow ID %q and run ID %q", request.WorkflowID, request.RunID) @@ -894,8 +893,7 @@ func (m *executionManagerImpl) GetCurrentExecution( response, respErr := m.persistence.GetCurrentExecution(ctx, request) - var notFound *serviceerror.NotFound - if errors.As(respErr, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](respErr); ok { // strip persistence-specific error message respErr = serviceerror.NewNotFoundf("workflow not found for ID: %v", request.WorkflowID) } diff --git a/common/persistence/faultinjection/store_fault_generator_test.go b/common/persistence/faultinjection/store_fault_generator_test.go index 1d456a2048f..764bf9a7364 100644 --- a/common/persistence/faultinjection/store_fault_generator_test.go +++ b/common/persistence/faultinjection/store_fault_generator_test.go @@ -39,10 +39,7 @@ func TestFaultInjection_Inject(t *testing.T) { if expectErr { assert.Error(t, err) assert.Contains(t, err.Error(), "fault injection error") - var reErr *serviceerror.ResourceExhausted - if errors.As(err, &reErr) { - assert.ErrorAs(t, err, &reErr) - } else { + if _, ok := errors.AsType[*serviceerror.ResourceExhausted](err); !ok { var timeoutErr *persistence.TimeoutError assert.ErrorAs(t, err, &timeoutErr) } diff --git a/common/persistence/nosql/nosqlplugin/cassandra/gocql/errors.go b/common/persistence/nosql/nosqlplugin/cassandra/gocql/errors.go index d057f4c493f..2a23c18f20f 100644 --- a/common/persistence/nosql/nosqlplugin/cassandra/gocql/errors.go +++ b/common/persistence/nosql/nosqlplugin/cassandra/gocql/errors.go @@ -27,13 +27,11 @@ func ConvertError( return serviceerror.NewNotFoundf("operation %v encountered %v", operation, err.Error()) } - var cqlTimeoutErr *gocql.RequestErrWriteTimeout - if errors.As(err, &cqlTimeoutErr) { + if cqlTimeoutErr, ok := errors.AsType[*gocql.RequestErrWriteTimeout](err); ok { return &persistence.TimeoutError{Msg: fmt.Sprintf("operation %v encountered %v", operation, cqlTimeoutErr.Error())} } - var cqlRequestErr gocql.RequestError - if errors.As(err, &cqlRequestErr) { + if cqlRequestErr, ok := errors.AsType[gocql.RequestError](err); ok { if cqlRequestErr.Code() == gocql.ErrCodeOverloaded { return &serviceerror.ResourceExhausted{ Cause: enumspb.RESOURCE_EXHAUSTED_CAUSE_SYSTEM_OVERLOADED, diff --git a/common/persistence/persistence_metric_clients.go b/common/persistence/persistence_metric_clients.go index 751ff433cb1..b451295c0d4 100644 --- a/common/persistence/persistence_metric_clients.go +++ b/common/persistence/persistence_metric_clients.go @@ -1425,8 +1425,7 @@ func (p *metricEmitter) recordRequestMetrics(operation string, caller string, la func (p *metricEmitter) recordDataLossMetrics(operation string, caller string, err error, workflowID, runID string) { // Emit data loss metrics if enabled and error is DataLoss - var dataLoss *serviceerror.DataLoss - if errors.As(err, &dataLoss) { + if _, ok := errors.AsType[*serviceerror.DataLoss](err); ok { if p.enableDataLossMetrics() { EmitDataLossMetric(p.metricsHandler, caller, workflowID, runID, operation, err) } diff --git a/common/persistence/sql/sqlplugin/sqlite/driver.go b/common/persistence/sql/sqlplugin/sqlite/driver.go index b9610cbe8b7..4d358fd5df4 100644 --- a/common/persistence/sql/sqlplugin/sqlite/driver.go +++ b/common/persistence/sql/sqlplugin/sqlite/driver.go @@ -17,8 +17,7 @@ const ( var sqlTableExistsRegex = regexp.MustCompile(sqlTableExistsPattern) func (*db) IsDupEntryError(err error) bool { - var sqlErr *sqlite.Error - if errors.As(err, &sqlErr) { + if sqlErr, ok := errors.AsType[*sqlite.Error](err); ok { return sqlErr.Code()&sqlConstraintCodes != 0 } @@ -26,8 +25,7 @@ func (*db) IsDupEntryError(err error) bool { } func isTableExistsError(err error) bool { - var sqlErr *sqlite.Error - if errors.As(err, &sqlErr) { + if sqlErr, ok := errors.AsType[*sqlite.Error](err); ok { return sqlTableExistsRegex.MatchString(sqlErr.Error()) } diff --git a/common/persistence/visibility/store/elasticsearch/processor.go b/common/persistence/visibility/store/elasticsearch/processor.go index 88736955d88..34626dfdd29 100644 --- a/common/persistence/visibility/store/elasticsearch/processor.go +++ b/common/persistence/visibility/store/elasticsearch/processor.go @@ -209,8 +209,7 @@ func (p *processorImpl) bulkAfterAction(_ int64, requests []elastic.BulkableRequ if err != nil { const logFirstNRequests = 5 var httpStatus int - var esErr *elastic.Error - if errors.As(err, &esErr) { + if esErr, ok := errors.AsType[*elastic.Error](err); ok { httpStatus = esErr.Status } diff --git a/common/persistence/visibility/store/query/errors.go b/common/persistence/visibility/store/query/errors.go index f87ad34c33e..d662d0821f7 100644 --- a/common/persistence/visibility/store/query/errors.go +++ b/common/persistence/visibility/store/query/errors.go @@ -49,8 +49,7 @@ func NewOperatorNotSupportedError( } func wrapConverterError(message string, err error) error { - var converterErr *ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*ConverterError](err); ok { return NewConverterError("%s: %v", message, converterErr) } return err diff --git a/common/persistence/visibility/store/sql/visibility_store.go b/common/persistence/visibility/store/sql/visibility_store.go index 7f85323a3de..1a9a0d80ec1 100644 --- a/common/persistence/visibility/store/sql/visibility_store.go +++ b/common/persistence/visibility/store/sql/visibility_store.go @@ -277,8 +277,7 @@ func (s *VisibilityStore) countChasmExecutions( request.ArchetypeId, ) if err != nil { - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -326,8 +325,7 @@ func (s *VisibilityStore) countChasmExecutionsLegacy( ) selectFilter, err := converter.BuildCountStmt() if err != nil { - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -391,8 +389,7 @@ func (s *VisibilityStore) listExecutionsInternal( if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be // only mapper errors). - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -493,8 +490,7 @@ func (s *VisibilityStore) listExecutionsInternalLegacy( selectFilter, err := converter.BuildSelectStmt(request.PageSize, request.NextPageToken) if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be only mapper errors). - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -575,8 +571,7 @@ func (s *VisibilityStore) countWorkflowExecutionsLegacy( selectFilter, err := converter.BuildCountStmt() if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be only mapper errors). - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -626,8 +621,7 @@ func (s *VisibilityStore) countWorkflowExecutions( if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be // only mapper errors). - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err diff --git a/common/persistence/visibility/visibility_manager_impl.go b/common/persistence/visibility/visibility_manager_impl.go index 13d01995481..d59d570b8c7 100644 --- a/common/persistence/visibility/visibility_manager_impl.go +++ b/common/persistence/visibility/visibility_manager_impl.go @@ -605,8 +605,7 @@ func aliasChasmSearchAttributes( if err != nil { // Silently ignore serviceerror.InvalidArgument because it indicates unregistered field. // INFO: Chasm search attributes must be registered with the CHASM Registry using the WithSearchAttributes() option. - var invalidArgumentErr *serviceerror.InvalidArgument - if errors.As(err, &invalidArgumentErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { continue } return nil, err diff --git a/common/rpc/interceptor/request_error_handler.go b/common/rpc/interceptor/request_error_handler.go index d419b75919a..315577800af 100644 --- a/common/rpc/interceptor/request_error_handler.go +++ b/common/rpc/interceptor/request_error_handler.go @@ -99,8 +99,7 @@ func (eh *RequestErrorHandler) logError( func recordErrorMetrics(metricsHandler metrics.Handler, err error, isExpectedError bool) { metrics.ServiceErrorWithType.With(metricsHandler).Record(1, metrics.ServiceErrorTypeTag(err)) - var resourceExhaustedErr *serviceerror.ResourceExhausted - if errors.As(err, &resourceExhaustedErr) { + if resourceExhaustedErr, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { metrics.ServiceErrResourceExhaustedCounter.With(metricsHandler).Record( 1, metrics.ResourceExhaustedCauseTag(resourceExhaustedErr.Cause), diff --git a/common/searchattribute/mapper.go b/common/searchattribute/mapper.go index 7d77b3ad00e..53c155acb79 100644 --- a/common/searchattribute/mapper.go +++ b/common/searchattribute/mapper.go @@ -163,8 +163,7 @@ func AliasFields( // Silently ignore serviceerror.InvalidArgument because it indicates unmapped field (alias was deleted, for example). // IMPORTANT: AliasFields should never return serviceerror.InvalidArgument because it is used by Poll API and the error // goes through up to SDK, which shutdowns worker when it receives serviceerror.InvalidArgument as poll response. - var invalidArgumentErr *serviceerror.InvalidArgument - if errors.As(err, &invalidArgumentErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { continue } return nil, err diff --git a/common/util/error_type.go b/common/util/error_type.go index c0e920ae673..4760b25711f 100644 --- a/common/util/error_type.go +++ b/common/util/error_type.go @@ -11,6 +11,7 @@ import ( // It is useful for attaching low-cardinality type names to errors that are suitable for telemetry tags. // See ErrorType for more details. type typedError interface { + error ErrorTypeName() string } @@ -28,8 +29,7 @@ var wrapperErrorTypes = map[string]bool{ // We consider errors wrapped via [fmt.Errorf], [errors.Join] and some pkg/errors functions to be wrapper errors. func ErrorType(err error) string { // If any error in the tree has an explicit type name, use it, preferring the first one in the DFS traversal. - var typedErr typedError - if errors.As(err, &typedErr) { + if typedErr, ok := errors.AsType[typedError](err); ok { return typedErr.ErrorTypeName() } diff --git a/common/worker_versioning/worker_versioning.go b/common/worker_versioning/worker_versioning.go index 4903bba6100..7b7a190c0c6 100644 --- a/common/worker_versioning/worker_versioning.go +++ b/common/worker_versioning/worker_versioning.go @@ -349,8 +349,7 @@ func checkVersionMembershipAndReactivationEligibility( if err != nil { // If matching is on an older version that doesn't have this RPC, // fall back to fetching full user data and checking version membership based on the received information. - var unimplErr *serviceerror.Unimplemented - if errors.As(err, &unimplErr) { + if _, ok := errors.AsType[*serviceerror.Unimplemented](err); ok { return checkVersionMembershipViaUserData(ctx, matchingClient, namespaceID, tq, tqType, version) } return false, false, 0, err diff --git a/common/workercommands/dispatcher.go b/common/workercommands/dispatcher.go index c9ef5d8c0da..fcd80fde65c 100644 --- a/common/workercommands/dispatcher.go +++ b/common/workercommands/dispatcher.go @@ -159,8 +159,7 @@ func (d *Dispatcher) dispatchToWorker( } func (d *Dispatcher) handleError(nexusErr error, task *tasks.WorkerCommandsTask, namespaceName string) error { - var handlerErr *nexus.HandlerError - if errors.As(nexusErr, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](nexusErr); ok { // Handler-level error (transport, timeout, internal). These are constructed by // MatchingDispatchResponseToError for non-worker-returned failures. if handlerErr.Type == nexus.HandlerErrorTypeUpstreamTimeout { diff --git a/components/callbacks/nexus_invocation.go b/components/callbacks/nexus_invocation.go index 69feed61164..6900dd5a7e5 100644 --- a/components/callbacks/nexus_invocation.go +++ b/components/callbacks/nexus_invocation.go @@ -93,8 +93,7 @@ func outcomeTag(callCtx context.Context, callErr error) string { if callCtx.Err() != nil { return "request-timeout" } - var handlerErr *nexus.HandlerError - if errors.As(callErr, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](callErr); ok { return "handler-error:" + string(handlerErr.Type) } return "unknown-error" @@ -103,8 +102,7 @@ func outcomeTag(callCtx context.Context, callErr error) string { } func isRetryableCallError(err error) bool { - var handlerError *nexus.HandlerError - if errors.As(err, &handlerError) { + if handlerError, ok := errors.AsType[*nexus.HandlerError](err); ok { return handlerError.Retryable() } return true diff --git a/components/callbacks/request.go b/components/callbacks/request.go index 55983ab1b27..186c5bae05f 100644 --- a/components/callbacks/request.go +++ b/components/callbacks/request.go @@ -54,8 +54,7 @@ func routeSystemCallbackRequest( ns, err := namespaceRegistry.GetNamespaceByID(namespace.ID(namespaceID)) if err != nil { logger.Error("failed to get namespace for nexus completion request", tag.WorkflowNamespaceID(namespaceID), tag.Error(err)) - var nfe *serviceerror.NamespaceNotFound - if errors.As(err, &nfe) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeNotFound, "namespace %q not found", namespaceID) } return nil, commonnexus.ConvertGRPCError(err, false) diff --git a/components/nexusoperations/executors.go b/components/nexusoperations/executors.go index 184d5ad77bb..f62d495c1ae 100644 --- a/components/nexusoperations/executors.go +++ b/components/nexusoperations/executors.go @@ -503,8 +503,7 @@ func (e taskExecutor) deferredOperationMetric(op Operation, callErr error, names } case enumsspb.NEXUS_OPERATION_STATE_TIMED_OUT: timeoutType := enumspb.TIMEOUT_TYPE_UNSPECIFIED - var belowMin *operationTimeoutBelowMinError - if errors.As(callErr, &belowMin) { + if belowMin, ok := errors.AsType[*operationTimeoutBelowMinError](callErr); ok { timeoutType = belowMin.timeoutType } return func() { @@ -961,8 +960,7 @@ func createNexusOperationFailure(operation Operation, scheduledEventID int64, ca func startCallOutcomeTag(callCtx context.Context, result *nexusrpc.ClientStartOperationResponse[*commonpb.Payload], callErr error) string { if callErr != nil { - var opTimeoutBelowMinErr *operationTimeoutBelowMinError - if errors.As(callErr, &opTimeoutBelowMinErr) { + if _, ok := errors.AsType[*operationTimeoutBelowMinError](callErr); ok { return "operation-timeout" } if errors.Is(callErr, ErrInvalidOperationToken) { @@ -974,16 +972,13 @@ func startCallOutcomeTag(callCtx context.Context, result *nexusrpc.ClientStartOp if callCtx.Err() != nil { return "request-timeout" } - var serviceErr serviceerror.ServiceError - if errors.As(callErr, &serviceErr) { + if serviceErr, ok := errors.AsType[serviceerror.ServiceError](callErr); ok { return "service-error:" + strings.Replace(fmt.Sprintf("%T", serviceErr), "*serviceerror.", "", 1) } - var opFailedError *nexus.OperationError - if errors.As(callErr, &opFailedError) { + if opFailedError, ok := errors.AsType[*nexus.OperationError](callErr); ok { return "operation-unsuccessful:" + string(opFailedError.State) } - var handlerError *nexus.HandlerError - if errors.As(callErr, &handlerError) { + if handlerError, ok := errors.AsType[*nexus.HandlerError](callErr); ok { return "handler-error:" + string(handlerError.Type) } return "unknown-error" @@ -999,19 +994,16 @@ func cancelCallOutcomeTag(callCtx context.Context, callErr error) string { if errors.Is(callErr, errOpProcessorFailed) { return "operation-processor-failed" } - var opTimeoutBelowMinErr *operationTimeoutBelowMinError - if errors.As(callErr, &opTimeoutBelowMinErr) { + if _, ok := errors.AsType[*operationTimeoutBelowMinError](callErr); ok { return "operation-timeout" } if callCtx.Err() != nil { return "request-timeout" } - var handlerErr *nexus.HandlerError - if errors.As(callErr, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](callErr); ok { return "handler-error:" + string(handlerErr.Type) } - var serviceErr serviceerror.ServiceError - if errors.As(callErr, &serviceErr) { + if serviceErr, ok := errors.AsType[serviceerror.ServiceError](callErr); ok { return "service-error:" + strings.Replace(fmt.Sprintf("%T", serviceErr), "*serviceerror.", "", 1) } return "unknown-error" @@ -1065,17 +1057,14 @@ func (e taskExecutor) logCallFailure(traceCtx invocationTraceContext, callErr er } func isDestinationDown(err error) bool { - var serviceErr serviceerror.ServiceError // For the system endpoint, we don't even consider the destination down since it's internal. - if errors.As(err, &serviceErr) { + if _, ok := errors.AsType[serviceerror.ServiceError](err); ok { return false } - var opFailedErr *nexus.OperationError - if errors.As(err, &opFailedErr) { + if _, ok := errors.AsType[*nexus.OperationError](err); ok { return false } - var handlerError *nexus.HandlerError - if errors.As(err, &handlerError) { + if handlerError, ok := errors.AsType[*nexus.HandlerError](err); ok { return handlerError.Retryable() } if errors.Is(err, errOpProcessorFailed) { @@ -1092,8 +1081,7 @@ func isDestinationDown(err error) bool { } func callErrToFailure(callErr error, retryable bool) (*failurepb.Failure, error) { - var serviceErr serviceerror.ServiceError - if errors.As(callErr, &serviceErr) { + if serviceErr, ok := errors.AsType[serviceerror.ServiceError](callErr); ok { return &failurepb.Failure{ Message: fmt.Sprintf("%s: %s", strings.Replace(fmt.Sprintf("%T", serviceErr), "*serviceerror.", "", 1), serviceErr.Error()), FailureInfo: &failurepb.Failure_ServerFailureInfo{ @@ -1103,8 +1091,7 @@ func callErrToFailure(callErr error, retryable bool) (*failurepb.Failure, error) }, }, nil } - var handlerErr *nexus.HandlerError - if errors.As(callErr, &handlerErr) { + if handlerErr, ok := errors.AsType[*nexus.HandlerError](callErr); ok { var nf nexus.Failure if handlerErr.OriginalFailure != nil { nf = *handlerErr.OriginalFailure diff --git a/service/frontend/admin_handler.go b/service/frontend/admin_handler.go index 73b131e8749..8468bf356c0 100644 --- a/service/frontend/admin_handler.go +++ b/service/frontend/admin_handler.go @@ -402,8 +402,7 @@ func (adh *AdminHandler) unaliasAndValidateSearchAttributes(historyBatches []*co unaliasedSas, err := searchattribute.UnaliasFields(adh.saMapperProvider, sas, nsName.String()) if err != nil { - var invArgErr *serviceerror.InvalidArgument - if !errors.As(err, &invArgErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); !ok { return nil, err } // Mapper returns InvalidArgument if alias is not found. It means that history has field names, not aliases. diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index 1603998b978..c97a7bca6c3 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -149,8 +149,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus if err != nil { h.Logger.Error("failed to get namespace for nexus completion request", tag.WorkflowNamespaceID(targetNamespaceID), tag.Error(err)) h.preProcessErrorsCounter.Record(1) - var nfe *serviceerror.NamespaceNotFound - if errors.As(err, &nfe) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeNotFound, "namespace %q not found", targetNamespaceID) } return commonnexus.ConvertGRPCError(err, false) @@ -200,8 +199,7 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus } if err := rCtx.interceptRequest(ctx, r); err != nil { - var notActiveErr *serviceerror.NamespaceNotActive - if errors.As(err, ¬ActiveErr) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { return h.forwardCompleteOperation(ctx, r, rCtx) } return err @@ -239,12 +237,10 @@ func (h *nexusCompletionHandler) CompleteOperation(ctx context.Context, r *nexus return nil } logger.Error("failed to process nexus completion request", tag.Error(err)) - var namespaceInactiveErr *serviceerror.NamespaceNotActive - if errors.As(err, &namespaceInactiveErr) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUnavailable, "cluster inactive") } - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return commonnexus.ConvertGRPCError(err, true) } return commonnexus.ConvertGRPCError(err, false) @@ -551,8 +547,7 @@ func (c *requestContext) capturePanicAndRecordMetrics(ctxPtr *context.Context, e } else if c.outcomeTag.Key != "" { c.metricsHandler = c.metricsHandler.WithTags(c.outcomeTag) } else { - var he *nexus.HandlerError - if errors.As(*errPtr, &he) { + if he, ok := errors.AsType[*nexus.HandlerError](*errPtr); ok { c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("error_" + strings.ToLower(string(he.Type)))) } else { c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("error_internal")) @@ -606,8 +601,7 @@ func (c *requestContext) interceptRequest(ctx context.Context, request *nexusrpc // If frontend.exposeAuthorizerErrors is false, Authorize err is either an explicitly set reason, or a generic // "Request unauthorized." message. // Otherwise, expose the underlying error. - var permissionDeniedError *serviceerror.PermissionDenied - if errors.As(err, &permissionDeniedError) { + if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { c.outcomeTag = metrics.OutcomeTag("unauthorized") return commonnexus.AdaptAuthorizeError(permissionDeniedError) } diff --git a/service/frontend/nexus_endpoint_client.go b/service/frontend/nexus_endpoint_client.go index 52e7f08b321..268f4d25f58 100644 --- a/service/frontend/nexus_endpoint_client.go +++ b/service/frontend/nexus_endpoint_client.go @@ -416,8 +416,7 @@ func (c *NexusEndpointClient) transformServiceError(err error, message string) e if err == nil { return nil } - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return err } c.logger.Error(message, tag.Error(err)) diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 75e2ab133b9..17d60adad85 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -184,8 +184,7 @@ func (c *operationContext) interceptRequest( // If frontend.exposeAuthorizerErrors is false, Authorize err is either an explicitly set reason, or a generic // "Request unauthorized." message. // Otherwise, expose the underlying error. - var permissionDeniedError *serviceerror.PermissionDenied - if errors.As(err, &permissionDeniedError) { + if permissionDeniedError, ok := errors.AsType[*serviceerror.PermissionDenied](err); ok { c.metricsHandler = c.metricsHandler.WithTags(metrics.OutcomeTag("unauthorized")) return commonnexus.AdaptAuthorizeError(permissionDeniedError) } @@ -404,8 +403,7 @@ func (h *nexusHandler) getOperationContext(ctx context.Context, method string) ( metrics.OutcomeTag("namespace_not_found"), ) - var nfe *serviceerror.NamespaceNotFound - if errors.As(err, &nfe) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeNotFound, "namespace not found: %q", nc.namespaceName) } return nil, commonnexus.ConvertGRPCError(err, false) @@ -460,8 +458,7 @@ func (h *nexusHandler) StartOperation( }) if err := oc.interceptRequest(ctx, request, options.Header); err != nil { - var notActiveErr *serviceerror.NamespaceNotActive - if errors.As(err, ¬ActiveErr) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { return h.forwardStartOperation(ctx, service, operation, input, options, oc) } return nil, err @@ -691,8 +688,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, }, }) if err := oc.interceptRequest(ctx, request, options.Header); err != nil { - var notActiveErr *serviceerror.NamespaceNotActive - if errors.As(err, ¬ActiveErr) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotActive](err); ok { return h.forwardCancelOperation(ctx, service, operation, token, options, oc) } return err diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index fa5fd022364..8623dd202bb 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -275,8 +275,7 @@ func (h *NexusOperationHTTPHandler) nexusContextFromEndpoint( tag.Error(err), tag.NexusEndpointTargetNamespaceID(v.Worker.GetNamespaceId()), ) - var notFoundErr *serviceerror.NamespaceNotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { h.writeFailure(w, r, &nexus.HandlerError{ Type: nexus.HandlerErrorTypeNotFound, Message: "invalid endpoint target", diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 9bed3410555..4db0c5a4053 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -795,8 +795,7 @@ func (wh *WorkflowHandler) ExecuteMultiOperation( historyResp, err := wh.historyClient.ExecuteMultiOperation(ctx, historyReq) if err != nil { - var multiErr *serviceerror.MultiOperationExecution - if errors.As(err, &multiErr) { + if multiErr, ok := errors.AsType[*serviceerror.MultiOperationExecution](err); ok { // Tweak error message for end-users to match the feature name. // The per-operation errors are embedded inside the error and unpacked by the SDK. multiErr.Message = "Update-with-Start could not be executed." @@ -3587,8 +3586,7 @@ func (wh *WorkflowHandler) createScheduleCHASM( // concurrent V1 CreateSchedule from succeeding for the same schedule ID. if wh.scheduleSentinelsEnabled(request.Namespace) { if err := wh.writeSchedulerWorkflowSentinel(ctx, namespaceID.String(), request); err != nil { - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - if !errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err); !ok { return nil, err } // V1 key is occupied. Check if it's a sentinel (proceed) or real scheduler (fail). @@ -3619,8 +3617,7 @@ func (wh *WorkflowHandler) createScheduleCHASM( return nil, serviceerror.NewWorkflowExecutionAlreadyStarted( fmt.Sprintf("schedule %q: concurrent creation detected", request.ScheduleId), "", "") } - var alreadyExistsErr *serviceerror.AlreadyExists - if errors.As(err, &alreadyExistsErr) { + if alreadyExistsErr, ok := errors.AsType[*serviceerror.AlreadyExists](err); ok { return nil, serviceerror.NewWorkflowExecutionAlreadyStarted(alreadyExistsErr.Message, "", "") } return nil, err @@ -3646,16 +3643,14 @@ func (wh *WorkflowHandler) createScheduleWorkflow( if err := wh.writeSchedulerCHASMSentinel(ctx, namespaceID.String(), namespaceName.String(), request.ScheduleId); err != nil { // Translate AlreadyExists (from CHASM handler) to // WorkflowExecutionAlreadyStarted for SDK compatibility. - var alreadyExistsErr *serviceerror.AlreadyExists - if errors.As(err, &alreadyExistsErr) { + if alreadyExistsErr, ok := errors.AsType[*serviceerror.AlreadyExists](err); ok { return nil, serviceerror.NewWorkflowExecutionAlreadyStarted(alreadyExistsErr.Message, "", "") } // Ignore unimplemented to avoid issues with mixed brain testing. // // We wouldn't hit this condition in prod, as we wouldn't migrate with the fleet // halfway deployed to the target version. - var unimplErr *serviceerror.Unimplemented - if !errors.As(err, &unimplErr) { + if _, ok := errors.AsType[*serviceerror.Unimplemented](err); !ok { return nil, err } } @@ -3732,8 +3727,7 @@ func (wh *WorkflowHandler) createScheduleWorkflow( ) if err != nil { - var alreadyStartedErr *serviceerror.WorkflowExecutionAlreadyStarted - if errors.As(err, &alreadyStartedErr) { + if _, ok := errors.AsType[*serviceerror.WorkflowExecutionAlreadyStarted](err); ok { // V1 key is occupied. Check if it's a sentinel (race) or real scheduler. isReal, checkErr := wh.isRealSchedulerInV1KeySpace(ctx, namespaceID.String(), request.Namespace, request.ScheduleId) if checkErr != nil { @@ -3821,8 +3815,7 @@ func (wh *WorkflowHandler) isRealSchedulerInV1KeySpace( }, }) if err != nil { - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return false, nil } return false, err @@ -3939,12 +3932,10 @@ func (wh *WorkflowHandler) scheduleSentinelsEnabled(namespaceName string) bool { // - ErrClosed: the CHASM schedule was migrated to V1 and marked closed; the // request should be retried against the workflow-backed stack. func isSchedulerErrorLegacyRoutable(err error) bool { - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return true } - var failedPreconditionErr *serviceerror.FailedPrecondition - if errors.As(err, &failedPreconditionErr) { + if failedPreconditionErr, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { return failedPreconditionErr.Message == chasmscheduler.ErrClosed.(*serviceerror.FailedPrecondition).Message } return false @@ -5757,8 +5748,7 @@ func (wh *WorkflowHandler) GetWorkerTaskReachability(ctx context.Context, reques response, err := wh.getWorkerTaskReachabilityValidated(ctx, ns, request) if err != nil { - var invalidArgument *serviceerror.InvalidArgument - if errors.As(err, &invalidArgument) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return nil, err } // Intentionally treat all errors as internal errors diff --git a/service/history/api/get_history_util.go b/service/history/api/get_history_util.go index ffd5c81058f..023cd459f0a 100644 --- a/service/history/api/get_history_util.go +++ b/service/history/api/get_history_util.go @@ -42,8 +42,7 @@ func GetRawHistory( branchToken []byte, ) (_ []*commonpb.DataBlob, _ []byte, retError error) { defer func() { - var dataLossErr *serviceerror.DataLoss - if errors.As(retError, &dataLossErr) { + if _, ok := errors.AsType[*serviceerror.DataLoss](retError); ok { if shardContext.GetConfig().EnableDataLossMetrics() { persistence.EmitDataLossMetric( shardContext.GetMetricsHandler(), diff --git a/service/history/api/getworkflowexecutionhistory/api.go b/service/history/api/getworkflowexecutionhistory/api.go index dfeab72a241..63e7aae902c 100644 --- a/service/history/api/getworkflowexecutionhistory/api.go +++ b/service/history/api/getworkflowexecutionhistory/api.go @@ -294,8 +294,7 @@ func Invoke( // when data inconsistency occurs. Long term solution should check event // batch pointing backwards within history store. defer func() { - var dataLossErr *serviceerror.DataLoss - if errors.As(retError, &dataLossErr) { + if _, ok := errors.AsType[*serviceerror.DataLoss](retError); ok { api.TrimHistoryNode( ctx, shardContext, diff --git a/service/history/api/multioperation/api.go b/service/history/api/multioperation/api.go index ac88ec15390..c58a32081bd 100644 --- a/service/history/api/multioperation/api.go +++ b/service/history/api/multioperation/api.go @@ -293,8 +293,7 @@ func (uws *updateWithStart) getWorkflowLease(ctx context.Context) (api.WorkflowL definition.NewWorkflowKey(uws.namespaceId.String(), uws.startReq.StartRequest.WorkflowId, ""), locks.PriorityHigh, ) - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, nil } if err != nil { diff --git a/service/history/api/recordworkflowtaskstarted/api.go b/service/history/api/recordworkflowtaskstarted/api.go index 19dd8ac944b..baee830603c 100644 --- a/service/history/api/recordworkflowtaskstarted/api.go +++ b/service/history/api/recordworkflowtaskstarted/api.go @@ -281,8 +281,7 @@ func setHistoryForRecordWfTaskStartedResp( // when data inconsistency occurs // long term solution should check event batch pointing backwards within history store defer func() { - var dataLossErr *serviceerror.DataLoss - if errors.As(retError, &dataLossErr) { + if _, ok := errors.AsType[*serviceerror.DataLoss](retError); ok { api.TrimHistoryNode( ctx, shardContext, diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index 3a6e1f551a8..945867c1c7e 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -908,8 +908,7 @@ func (handler *WorkflowTaskCompletedHandler) createPollWorkflowTaskQueueResponse // when data inconsistency occurs // long term solution should check event batch pointing backwards within history store defer func() { - var dataLossErr *serviceerror.DataLoss - if errors.As(retError, &dataLossErr) { + if _, ok := errors.AsType[*serviceerror.DataLoss](retError); ok { api.TrimHistoryNode( ctx, handler.shardContext, diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index a4b5fd8c04a..f0925d013c3 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -365,8 +365,7 @@ func (handler *workflowTaskCompletedHandler) handleCommand( err = hsmHandler(ctx, handler.mutableState, validator, handlerOpts.WorkflowTaskCompletedEventID, command) } - var failWFTErr chasmworkflow.FailWorkflowTaskError - if errors.As(err, &failWFTErr) { + if failWFTErr, ok := errors.AsType[chasmworkflow.FailWorkflowTaskError](err); ok { if failWFTErr.TerminateWorkflow { return nil, handler.terminateWorkflow(failWFTErr.Cause, failWFTErr) } @@ -1563,8 +1562,7 @@ func (handler *workflowTaskCompletedHandler) failWorkflowTaskOnInvalidArgument( wtFailedCause enumspb.WorkflowTaskFailedCause, err error, ) error { - var invalidArgument *serviceerror.InvalidArgument - if errors.As(err, &invalidArgument) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return handler.failWorkflowTask(wtFailedCause, err) } return err diff --git a/service/history/api/verifychildworkflowcompletionrecorded/api.go b/service/history/api/verifychildworkflowcompletionrecorded/api.go index 0e2b73ffc41..65f01388e55 100644 --- a/service/history/api/verifychildworkflowcompletionrecorded/api.go +++ b/service/history/api/verifychildworkflowcompletionrecorded/api.go @@ -245,8 +245,7 @@ func resendParentAndVerify( // TODO: add parent workflow to workflowNotFoundCache return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil } - var failedPreconditionErr *serviceerror.FailedPrecondition - if errors.As(err, &failedPreconditionErr) { + if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { // Unable to perform sync state. Transition history maybe disabled in source cluster. return nil, errVerify } diff --git a/service/history/archival/archiver.go b/service/history/archival/archiver.go index 5c64711a6b0..7f90471952a 100644 --- a/service/history/archival/archiver.go +++ b/service/history/archival/archiver.go @@ -122,9 +122,7 @@ func (a *archiver) Archive(ctx context.Context, request *Request) (res *Response if err != nil { status = "err" - var rateLimitExceededErr *serviceerror.ResourceExhausted - - if errors.As(err, &rateLimitExceededErr) { + if _, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { status = "rate_limit_exceeded" } diff --git a/service/history/handler.go b/service/history/handler.go index 3d1f325d4ee..d25e52b32bc 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -2571,8 +2571,7 @@ func (h *Handler) StartNexusOperation( } result, err := h.nexusHandler.StartOperation(ctx, req.GetRequest().GetService(), req.GetRequest().GetOperation(), input, options) if err != nil { - var opErr *nexus.OperationError - if errors.As(err, &opErr) { + if opErr, ok := errors.AsType[*nexus.OperationError](err); ok { nexusFailure, convErr := nexusrpc.DefaultFailureConverter().ErrorToFailure(opErr) if convErr != nil { return nil, convErr diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go index 9698de784db..8266d05288b 100644 --- a/service/history/ndc/workflow_resetter.go +++ b/service/history/ndc/workflow_resetter.go @@ -811,8 +811,7 @@ func (r *workflowResetterImpl) reapplyContinueAsNewWorkflowEvents( if err != nil { // A deleted run truncates the chain; other errors must fail the reset so a retry // can reapply the full surviving chain, since reset is one-shot. - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { break } return "", err diff --git a/service/history/queues/executable.go b/service/history/queues/executable.go index 3fe2b39af31..a6820920298 100644 --- a/service/history/queues/executable.go +++ b/service/history/queues/executable.go @@ -73,6 +73,7 @@ type ( // MaybeTerminalTaskError are errors which (if IsTerminalTaskError returns true) cannot be retried and should // not be rescheduled. Tasks should be enqueued to the DLQ immediately if an error is marked as terminal. MaybeTerminalTaskError interface { + error IsTerminalTaskError() bool } ) @@ -476,8 +477,7 @@ func (e *executableImpl) isExpectedRetryableError(err error) (isRetryable bool, } }() - var resourceExhaustedErr *serviceerror.ResourceExhausted - if errors.As(err, &resourceExhaustedErr) { + if resourceExhaustedErr, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { switch resourceExhaustedErr.Cause { //nolint:exhaustive case enumspb.RESOURCE_EXHAUSTED_CAUSE_BUSY_WORKFLOW: err = consts.ErrResourceExhaustedBusyWorkflow @@ -520,8 +520,7 @@ func (e *executableImpl) isExpectedRetryableError(err error) (isRetryable bool, } func (e *executableImpl) isUnexpectedNonRetryableError(err error) bool { - var terr MaybeTerminalTaskError - if errors.As(err, &terr) { + if terr, ok := errors.AsType[MaybeTerminalTaskError](err); ok { return terr.IsTerminalTaskError() } diff --git a/service/history/queues/executable_mock.go b/service/history/queues/executable_mock.go index 7a0c163f2f0..ffbbfee05ba 100644 --- a/service/history/queues/executable_mock.go +++ b/service/history/queues/executable_mock.go @@ -479,6 +479,20 @@ func (m *MockMaybeTerminalTaskError) EXPECT() *MockMaybeTerminalTaskErrorMockRec return m.recorder } +// Error mocks base method. +func (m *MockMaybeTerminalTaskError) Error() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Error") + ret0, _ := ret[0].(string) + return ret0 +} + +// Error indicates an expected call of Error. +func (mr *MockMaybeTerminalTaskErrorMockRecorder) Error() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Error", reflect.TypeOf((*MockMaybeTerminalTaskError)(nil).Error)) +} + // IsTerminalTaskError mocks base method. func (m *MockMaybeTerminalTaskError) IsTerminalTaskError() bool { m.ctrl.T.Helper() diff --git a/service/history/replication/executable_task.go b/service/history/replication/executable_task.go index f8ee4ca7d67..4e4f9716097 100644 --- a/service/history/replication/executable_task.go +++ b/service/history/replication/executable_task.go @@ -734,8 +734,7 @@ func (e *ExecutableTaskImpl) SyncState( } resp, err := remoteAdminClient.SyncWorkflowState(ctx, req) if err != nil { - var resourceExhaustedError *serviceerror.ResourceExhausted - if errors.As(err, &resourceExhaustedError) { + if _, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Sync workflow state request exceeded resource limits", err, map[string]any{ "request_payload_size": req.Size(), }) @@ -748,16 +747,14 @@ func (e *ExecutableTaskImpl) SyncState( tag.ReplicationTask(e.replicationTask), ) - var workflowNotReady *serviceerror.WorkflowNotReady - if errors.As(err, &workflowNotReady) { + if _, ok := errors.AsType[*serviceerror.WorkflowNotReady](err); ok { logger.Info("Dropped replication task as source mutable state has buffered events.", tag.Error(err)) e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Dropped replication task because source mutable state has buffered events", err, map[string]any{ "disposition": wideevents.ReplDispositionDiscarded, }) return false, nil } - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { logger.Error( "workflow not found in source cluster, proceed to cleanup") e.emitReplicationTaskError(wideevents.ReplOperationSyncState, "Workflow not found in source cluster; cleaning up target", err, map[string]any{ @@ -773,8 +770,7 @@ func (e *ExecutableTaskImpl) SyncState( ), ) } - var failedPreconditionErr *serviceerror.FailedPrecondition - if !errors.As(err, &failedPreconditionErr) { + if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); !ok { return false, err } // Unable to perform sync state. Transition history maybe disabled in source cluster. @@ -860,8 +856,7 @@ func (e *ExecutableTaskImpl) DeleteWorkflow( }, ClosedWorkflowOnly: false, }) - var notFoundErr *serviceerror.NotFound - if errors.As(err, ¬FoundErr) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil } return err diff --git a/service/history/replication/replication_events.go b/service/history/replication/replication_events.go index 7603e718047..c66c04e84c7 100644 --- a/service/history/replication/replication_events.go +++ b/service/history/replication/replication_events.go @@ -542,8 +542,7 @@ func (e *ExecutableVerifyVersionedTransitionTask) emitReplicationVerifyApplied( outcome := "verified" errStr := "" if retErr != nil { - var syncStateErr *serviceerrors.SyncState - if errors.As(retErr, &syncStateErr) { + if _, ok := errors.AsType[*serviceerrors.SyncState](retErr); ok { outcome = "resend_needed" } else { outcome = "error" diff --git a/service/history/replication/stream.go b/service/history/replication/stream.go index 88ef0c90f39..5dd8aa4a385 100644 --- a/service/history/replication/stream.go +++ b/service/history/replication/stream.go @@ -65,8 +65,7 @@ func WrapEventLoop( err := originalEventLoop() if err != nil { - var streamError *StreamError - if errors.As(err, &streamError) { + if streamError, ok := errors.AsType[*StreamError](err); ok { metrics.ReplicationStreamError.With(metricsHandler).Record( int64(1), metrics.ServiceErrorTypeTag(streamError.cause), diff --git a/service/history/workflow/update/registry.go b/service/history/workflow/update/registry.go index 73d9b64b8bf..4d251a451f0 100644 --- a/service/history/workflow/update/registry.go +++ b/service/history/workflow/update/registry.go @@ -463,8 +463,7 @@ func (r *registry) Find(ctx context.Context, id string) *Update { updOutcome, err := r.store.GetUpdateOutcome(ctx, id) // Swallow NotFound error because it means that Update doesn't exist. - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil } diff --git a/service/matching/backlog_manager.go b/service/matching/backlog_manager.go index deb5919a3b4..49c67ab6a8b 100644 --- a/service/matching/backlog_manager.go +++ b/service/matching/backlog_manager.go @@ -115,8 +115,7 @@ func (c *backlogManagerImpl) signalIfFatal(err error) bool { if err == nil { return false } - var condfail *persistence.ConditionFailedError - if errors.As(err, &condfail) { + if _, ok := errors.AsType[*persistence.ConditionFailedError](err); ok { c.metricsHandler.Counter(metrics.ConditionFailedErrorPerTaskQueueCounter.Name()).Record(1) c.skipFinalUpdate.Store(true) c.pqMgr.UnloadFromPartitionManager(unloadCauseConflict) diff --git a/service/matching/fair_backlog_manager.go b/service/matching/fair_backlog_manager.go index 522ce103b75..a905b81c18e 100644 --- a/service/matching/fair_backlog_manager.go +++ b/service/matching/fair_backlog_manager.go @@ -96,8 +96,7 @@ func (c *fairBacklogManagerImpl) signalIfFatal(err error) bool { if err == nil { return false } - var condfail *persistence.ConditionFailedError - if errors.As(err, &condfail) { + if _, ok := errors.AsType[*persistence.ConditionFailedError](err); ok { c.metricsHandler.Counter(metrics.ConditionFailedErrorPerTaskQueueCounter.Name()).Record(1) c.skipFinalUpdate.Store(true) c.pqMgr.UnloadFromPartitionManager(unloadCauseConflict) diff --git a/service/matching/fair_task_reader.go b/service/matching/fair_task_reader.go index 9a27fcfa4ec..33dfcd246da 100644 --- a/service/matching/fair_task_reader.go +++ b/service/matching/fair_task_reader.go @@ -349,8 +349,7 @@ func (tr *fairTaskReader) addErrorBehavior(err error) (drop, retry bool) { // retry here. if tqCtx is closing, addTaskToMatcher will give up. return false, true } - var stickyUnavailable *serviceerrors.StickyWorkerUnavailable - if errors.As(err, &stickyUnavailable) { + if _, ok := errors.AsType[*serviceerrors.StickyWorkerUnavailable](err); ok { return true, false // drop the task } var invalid *serviceerror.InvalidArgument diff --git a/service/matching/pri_backlog_manager.go b/service/matching/pri_backlog_manager.go index 96433844a34..70d03d4fb6e 100644 --- a/service/matching/pri_backlog_manager.go +++ b/service/matching/pri_backlog_manager.go @@ -109,8 +109,7 @@ func (c *priBacklogManagerImpl) signalIfFatal(err error) bool { if err == nil { return false } - var condfail *persistence.ConditionFailedError - if errors.As(err, &condfail) { + if _, ok := errors.AsType[*persistence.ConditionFailedError](err); ok { c.metricsHandler.Counter(metrics.ConditionFailedErrorPerTaskQueueCounter.Name()).Record(1) c.skipFinalUpdate.Store(true) c.pqMgr.UnloadFromPartitionManager(unloadCauseConflict) diff --git a/service/matching/pri_task_reader.go b/service/matching/pri_task_reader.go index 942c44028ba..ceb643422a9 100644 --- a/service/matching/pri_task_reader.go +++ b/service/matching/pri_task_reader.go @@ -337,8 +337,7 @@ func (tr *priTaskReader) addErrorBehavior(err error) (drop, retry bool) { // retry here. if tqCtx is closing, addTaskToMatcher will give up. return false, true } - var stickyUnavailable *serviceerrors.StickyWorkerUnavailable - if errors.As(err, &stickyUnavailable) { + if _, ok := errors.AsType[*serviceerrors.StickyWorkerUnavailable](err); ok { return true, false // drop the task } var invalid *serviceerror.InvalidArgument diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index be19db8a949..fb84e134629 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -701,8 +701,7 @@ func (pm *taskQueuePartitionManagerImpl) processTaskAddHooks(ctx context.Context } func taskAddErrResult(err error) string { - var resourceExhausted *serviceerror.ResourceExhausted - if errors.As(err, &resourceExhausted) { + if _, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { return metrics.TaskAddResultThrottled } return metrics.TaskAddResultFailure diff --git a/service/matching/user_data_manager.go b/service/matching/user_data_manager.go index a74e98ccf1b..35b60d112c4 100644 --- a/service/matching/user_data_manager.go +++ b/service/matching/user_data_manager.go @@ -330,8 +330,7 @@ func (m *userDataManagerImpl) fetchUserData(ctx context.Context) error { if !common.IsContextCanceledErr(err) { m.logger.Error("error fetching user data from parent", tag.Error(err)) } - var unimplErr *serviceerror.Unimplemented - if errors.As(err, &unimplErr) { + if _, ok := errors.AsType[*serviceerror.Unimplemented](err); ok { // This might happen during a deployment. The older version couldn't have had any user data, // so we act as if it just returned an empty response and set ourselves ready. // Return the error so that we backoff with retry, and do not set hasFetchedUserData so that @@ -700,8 +699,7 @@ func (m *userDataManagerImpl) CheckTaskQueueUserDataPropagation( OnlyIfLoaded: true, }) if err != nil { - var failed *serviceerror.FailedPrecondition - if errors.As(err, &failed) { + if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { // this means the partition was not loaded, so skip it (if it loads, it will get the newest data) err = nil } diff --git a/service/worker/batcher/activities.go b/service/worker/batcher/activities.go index d9b1934e891..9c54c716d6a 100644 --- a/service/worker/batcher/activities.go +++ b/service/worker/batcher/activities.go @@ -184,8 +184,7 @@ func fetchPage( Query: config.adjustedQuery, }) if err != nil { - var invalidArgErr *serviceerror.InvalidArgument - if errors.As(err, &invalidArgErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return nil, temporal.NewNonRetryableApplicationError(err.Error(), "InvalidArgument", err) } return nil, err @@ -207,8 +206,7 @@ func fetchPage( Query: config.adjustedQuery, }) if err != nil { - var invalidArgErr *serviceerror.InvalidArgument - if errors.As(err, &invalidArgErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return nil, temporal.NewNonRetryableApplicationError(err.Error(), "InvalidArgument", err) } return nil, err @@ -479,8 +477,7 @@ func (a *activities) BatchActivityWithProtobuf(ctx context.Context, batchParams if err != nil { metrics.BatcherOperationFailures.With(metricsHandler).Record(1) logger.Error("Failed to get estimate execution count", tag.Error(err)) - var invalidArgErr *serviceerror.InvalidArgument - if errors.As(err, &invalidArgErr) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return HeartBeatDetails{}, temporal.NewNonRetryableApplicationError(err.Error(), "InvalidArgument", err) } return HeartBeatDetails{}, err diff --git a/service/worker/deletenamespace/activities.go b/service/worker/deletenamespace/activities.go index 2d5e7b4c84f..52252e20b6f 100644 --- a/service/worker/deletenamespace/activities.go +++ b/service/worker/deletenamespace/activities.go @@ -78,8 +78,7 @@ func (a *localActivities) GetNamespaceInfoActivity(ctx context.Context, nsID nam getNamespaceResponse, err := a.metadataManager.GetNamespace(ctx, getNamespaceRequest) if err != nil { - var nsNotFoundErr *serviceerror.NamespaceNotFound - if stderrors.As(err, &nsNotFoundErr) { + if _, ok := stderrors.AsType[*serviceerror.NamespaceNotFound](err); ok { ns := nsName.String() if ns == "" { ns = nsID.String() diff --git a/service/worker/deletenamespace/errors/errors.go b/service/worker/deletenamespace/errors/errors.go index 4bc2f4d562f..7108fb8b20b 100644 --- a/service/worker/deletenamespace/errors/errors.go +++ b/service/worker/deletenamespace/errors/errors.go @@ -38,8 +38,7 @@ func NewNotDeletedExecutionsStillExist(count int) error { } func ToServiceError(err error, workflowID, runID string) error { - var appErr *temporal.ApplicationError - if errors.As(err, &appErr) { + if appErr, ok := errors.AsType[*temporal.ApplicationError](err); ok { switch appErr.Type() { case InvalidArgumentErrType: return serviceerror.NewInvalidArgument(appErr.Message()) diff --git a/service/worker/deletenamespace/reclaimresources/workflow.go b/service/worker/deletenamespace/reclaimresources/workflow.go index ce678e3f3c0..6a15a5b3993 100644 --- a/service/worker/deletenamespace/reclaimresources/workflow.go +++ b/service/worker/deletenamespace/reclaimresources/workflow.go @@ -247,8 +247,7 @@ func deleteWorkflowExecutions(ctx workflow.Context, logger log.Logger, params Re ctx3 := workflow.WithActivityOptions(ctx, ensureNoExecutionsAdvVisibilityActivityOptions) err = workflow.ExecuteActivity(ctx3, a.EnsureNoExecutionsAdvVisibilityActivity, params.NamespaceID, params.Namespace, der.ErrorCount).Get(ctx, nil) if err != nil { - var appErr *temporal.ApplicationError - if stderrors.As(err, &appErr) { + if appErr, ok := stderrors.AsType[*temporal.ApplicationError](err); ok { switch appErr.Type() { case errors.ExecutionsStillExistErrType, errors.NoProgressErrType, errors.NotDeletedExecutionsStillExistErrType: var notDeletedCount int diff --git a/service/worker/scheduler/activities.go b/service/worker/scheduler/activities.go index fb23804e288..1f9c7a7d01f 100644 --- a/service/worker/scheduler/activities.go +++ b/service/worker/scheduler/activities.go @@ -397,13 +397,11 @@ func (a *activities) MigrateScheduleToChasm(ctx context.Context, req *schedulerp _, err := a.SchedulerClient.CreateFromMigrationState(ctx, req) if err != nil { // Treat "already exists" as success (idempotency). - var alreadyExists *serviceerror.AlreadyExists - if errors.As(err, &alreadyExists) { + if _, ok := errors.AsType[*serviceerror.AlreadyExists](err); ok { return nil } // Sentinel blocking migration is transient; will retry on next workflow wake-up. - var unavailableErr *serviceerror.Unavailable - if errors.As(err, &unavailableErr) { + if _, ok := errors.AsType[*serviceerror.Unavailable](err); ok { return translateError(err, "MigrateScheduleToChasm: blocked by sentinel, will retry") } return translateError(err, "MigrateScheduleToChasm") diff --git a/service/worker/scheduler/query.go b/service/worker/scheduler/query.go index ab8a3418626..25671d0cb3f 100644 --- a/service/worker/scheduler/query.go +++ b/service/worker/scheduler/query.go @@ -118,8 +118,7 @@ func getQueryFields( ).WithChasmMapper(chasmMapper).WithSearchAttributeInterceptor(saInterceptor) _, err = queryConverter.Convert(queryString) if err != nil { - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err @@ -141,8 +140,7 @@ func getQueryFieldsLegacy( queryConverter := elasticsearch.NewQueryConverterLegacy(fnInterceptor, nil, saNameType, chasmMapper) _, err := queryConverter.ConvertWhereOrderBy(queryString) if err != nil { - var converterErr *query.ConverterError - if errors.As(err, &converterErr) { + if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { return nil, converterErr.ToInvalidArgument() } return nil, err diff --git a/service/worker/workerdeployment/activities.go b/service/worker/workerdeployment/activities.go index 9a385ace759..d617b1cba71 100644 --- a/service/worker/workerdeployment/activities.go +++ b/service/worker/workerdeployment/activities.go @@ -304,8 +304,7 @@ func (a *Activities) UpdateWorkerControllerInstanceFromDeployment(ctx context.Co upserts := scalingGroupUpdatesToWCI(input.GetUpsertScalingGroups()) resp, err := a.WorkerControllerInstanceClient.UpdateWorkerControllerInstance(ctx, a.namespace, input.GetVersion(), nil, input.GetIdentity(), upserts, input.GetRemoveScalingGroups()) if err != nil { - var invalidArgs *serviceerror.InvalidArgument - if errors.As(err, &invalidArgs) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return nil, temporal.NewNonRetryableApplicationError(err.Error(), errInvalidComputeConfig, nil) } return nil, err diff --git a/service/worker/workerdeployment/client.go b/service/worker/workerdeployment/client.go index 3b0c3035c31..e455c4754d5 100644 --- a/service/worker/workerdeployment/client.go +++ b/service/worker/workerdeployment/client.go @@ -446,8 +446,7 @@ func (d *ClientImpl) DescribeVersion( versionState, err := d.queryVersionState(ctx, namespaceEntry, deploymentName, buildID) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, nil, serviceerror.NewNotFound("Worker Deployment Version not found") } return nil, nil, err @@ -464,8 +463,7 @@ func (d *ClientImpl) DescribeVersion( wciDesc, _, err := d.workerControllerInstanceClient.DescribeWorkerControllerInstance(ctx, namespaceEntry, apiVersion) if err != nil { // WCI may not exist if no compute config was ever set. - var notFound *serviceerror.NotFound - if !errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); !ok { return nil, nil, err } } else { @@ -555,8 +553,7 @@ func (d *ClientImpl) UpdateVersionComputeConfig( Meta: &updatepb.Meta{UpdateId: "_update_compute_config_" + requestID, Identity: identity}, }) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return serviceerror.NewNotFound(fmt.Sprintf(ErrWorkerDeploymentVersionNotFound, version.GetBuildId(), version.GetDeploymentName())) } return err @@ -601,8 +598,7 @@ func (d *ClientImpl) DescribeWorkerDeployment( res, err := d.queryWorkflowWithRetry(ctx, req) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, nil, serviceerror.NewNotFoundf(ErrWorkerDeploymentNotFound, deploymentName) } var queryFailed *serviceerror.QueryFailed @@ -655,8 +651,7 @@ func (d *ClientImpl) queryCreateRequestID( res, err := d.queryWorkflowWithRetry(ctx, req) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, err } var queryFailed *serviceerror.QueryFailed @@ -700,8 +695,7 @@ func (d *ClientImpl) workerDeploymentExists( }, }) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return false, nil } return false, err @@ -894,8 +888,7 @@ func (d *ClientImpl) SetCurrentVersion( }, ) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, serviceerror.NewNotFoundf(ErrWorkerDeploymentNotFound, deploymentName) } return nil, err @@ -1011,8 +1004,7 @@ func (d *ClientImpl) SetRampingVersion( }, ) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, serviceerror.NewNotFoundf(ErrWorkerDeploymentNotFound, deploymentName) } return nil, err @@ -1152,8 +1144,7 @@ func (d *ClientImpl) DeleteWorkerDeployment( }, ) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil } return err @@ -1277,8 +1268,7 @@ func (d *ClientImpl) ensureWorkerDeploymentDoesNotExist( // right nothing in case of duplicate request ID.) res, err := d.queryCreateRequestID(ctx, namespaceEntry, deploymentName) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return nil, nil } return nil, err @@ -1352,8 +1342,7 @@ func (d *ClientImpl) CreateWorkerDeploymentVersion( updateRequest, ) if err != nil { - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { return serviceerror.NewNotFound(fmt.Sprintf(ErrWorkerDeploymentNotFound, deploymentName)) } return err diff --git a/service/worker/workerdeployment/util.go b/service/worker/workerdeployment/util.go index 76ae805693e..f03fd1e580a 100644 --- a/service/worker/workerdeployment/util.go +++ b/service/worker/workerdeployment/util.go @@ -386,16 +386,14 @@ func isRetryableUpdateError(err error) bool { return true } - var errWfNotReady *serviceerror.WorkflowNotReady - if errors.As(err, &errWfNotReady) { + if _, ok := errors.AsType[*serviceerror.WorkflowNotReady](err); ok { // Update edge cases, can retry. return true } // All updates that are admitted as the workflow is closing due to CaN are considered retryable. // The ErrWorkflowClosing and ResourceExhausted could be nested. - var errMultiOps *serviceerror.MultiOperationExecution - if errors.As(err, &errMultiOps) { + if errMultiOps, ok := errors.AsType[*serviceerror.MultiOperationExecution](err); ok { for _, e := range errMultiOps.OperationErrors() { if e == nil { continue diff --git a/service/worker/workerdeployment/version_activities.go b/service/worker/workerdeployment/version_activities.go index ef4150ad86d..495773dc69d 100644 --- a/service/worker/workerdeployment/version_activities.go +++ b/service/worker/workerdeployment/version_activities.go @@ -43,8 +43,7 @@ func (a *VersionActivities) StartWorkerDeploymentWorkflow( logger.Info("starting worker-deployment workflow", "deploymentName", input.DeploymentName) identity := "deployment-version workflow " + activity.GetInfo(ctx).WorkflowExecution.ID err := a.WorkerDeploymentClient.StartWorkerDeployment(ctx, a.namespace, input.DeploymentName, identity, input.RequestId) - var precond *serviceerror.FailedPrecondition - if errors.As(err, &precond) { + if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { return temporal.NewNonRetryableApplicationError("failed to create deployment", errTooManyDeployments, err) } return err @@ -217,8 +216,7 @@ func (a *VersionActivities) UpdateWorkerControllerInstance(ctx context.Context, upserts := scalingGroupUpdatesToWCI(input.GetUpsertScalingGroups()) resp, err := a.WorkerControllerInstanceClient.UpdateWorkerControllerInstance(ctx, a.namespace, input.GetVersion(), nil, input.GetIdentity(), upserts, input.GetRemoveScalingGroups()) if err != nil { - var invalidArgs *serviceerror.InvalidArgument - if errors.As(err, &invalidArgs) { + if _, ok := errors.AsType[*serviceerror.InvalidArgument](err); ok { return nil, temporal.NewNonRetryableApplicationError(err.Error(), errInvalidComputeConfig, nil) } return nil, err diff --git a/service/worker/workerdeployment/workflow.go b/service/worker/workerdeployment/workflow.go index 05184ec32b1..135914e85e1 100644 --- a/service/worker/workerdeployment/workflow.go +++ b/service/worker/workerdeployment/workflow.go @@ -740,8 +740,7 @@ func (d *WorkflowRunner) handleRegisterWorker(ctx workflow.Context, args *deploy RoutingConfig: routingConfigToSync, }).Get(ctx, nil) if err != nil { - var appError *temporal.ApplicationError - if errors.As(err, &appError) { + if appError, ok := errors.AsType[*temporal.ApplicationError](err); ok { if appError.Type() == errMaxTaskQueuesInVersionType { return temporal.NewApplicationError( fmt.Sprintf("cannot add task queue %v since maximum number of task queues (%d) have been registered in deployment", args.TaskQueueName, args.MaxTaskQueues), diff --git a/tests/activity_workflow_driver.go b/tests/activity_workflow_driver.go index c560e8ff1dd..ba495360c3e 100644 --- a/tests/activity_workflow_driver.go +++ b/tests/activity_workflow_driver.go @@ -95,8 +95,7 @@ func (a *wfaHandle) timeoutInfo(t require.TestingT) activityTimeoutInfo { attempt: pa.GetAttempt(), } } - var timeoutErr *temporal.TimeoutError - if errors.As(a.run.Get(a.testContext(), nil), &timeoutErr) { + if timeoutErr, ok := errors.AsType[*temporal.TimeoutError](a.run.Get(a.testContext(), nil)); ok { return activityTimeoutInfo{timeout: timeoutErr.TimeoutType(), terminal: true} } return activityTimeoutInfo{terminal: true} diff --git a/tests/chasm_test.go b/tests/chasm_test.go index 5f73fa3b082..a127ef083b4 100644 --- a/tests/chasm_test.go +++ b/tests/chasm_test.go @@ -1110,8 +1110,7 @@ func (s *ChasmSuite) TestNamespaceDelete_WithChasmExecutions() { PageSize: 10, Query: visQuery, }) - var notFound *serviceerror.NamespaceNotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); ok { return // namespace fully deleted is also acceptable } require.NoError(t, err) diff --git a/tests/namespace_test.go b/tests/namespace_test.go index 3ce173e87cf..7b81c0c6037 100644 --- a/tests/namespace_test.go +++ b/tests/namespace_test.go @@ -77,8 +77,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_Empty() { _, err := env.FrontendClient().DescribeNamespace(s.Context(), &workflowservice.DescribeNamespaceRequest{ Id: nsID, }) - var notFound *serviceerror.NamespaceNotFound - if !errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); !ok { return false } @@ -121,8 +120,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_OverrideDelay() { _, err := env.FrontendClient().DescribeNamespace(s.Context(), &workflowservice.DescribeNamespaceRequest{ Id: nsID, }) - var notFound *serviceerror.NamespaceNotFound - if !errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); !ok { return false } @@ -164,8 +162,7 @@ func (s *namespaceTestSuite) Test_NamespaceDelete_Empty_WithID() { _, err := env.FrontendClient().DescribeNamespace(s.Context(), &workflowservice.DescribeNamespaceRequest{ Id: nsID, }) - var notFound *serviceerror.NamespaceNotFound - if !errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NamespaceNotFound](err); !ok { return false } diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index db4cadd0f48..fab97d1b34f 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -2509,8 +2509,7 @@ func (s *NexusWorkflowTestSuite) TestNexusAsyncOperationErrorRehydration(chasmEn if !errors.As(err, &opErr) { return nil, fmt.Errorf("expected NexusOperationError, got %w", err) } - var canceledErr *temporal.CanceledError - if !errors.As(opErr, &canceledErr) { + if _, ok := errors.AsType[*temporal.CanceledError](opErr); !ok { return nil, fmt.Errorf("expected CanceledError, got %w", err) } default: diff --git a/tests/testcore/taskpoller.go b/tests/testcore/taskpoller.go index 13821281517..91c2edf3fd1 100644 --- a/tests/testcore/taskpoller.go +++ b/tests/testcore/taskpoller.go @@ -525,8 +525,7 @@ func getQueryResults(queries map[string]*querypb.WorkflowQuery, queryResult *que } func newApplicationFailure(err error, nonRetryable bool, details *commonpb.Payloads) *failurepb.Failure { - var applicationErr *temporal.ApplicationError - if errors.As(err, &applicationErr) { + if applicationErr, ok := errors.AsType[*temporal.ApplicationError](err); ok { nonRetryable = applicationErr.NonRetryable() } diff --git a/tests/versioning_test_env.go b/tests/versioning_test_env.go index ade8d9466de..5649a910bbd 100644 --- a/tests/versioning_test_env.go +++ b/tests/versioning_test_env.go @@ -408,8 +408,7 @@ func (env *VersioningTestEnv) pollUntilRegistered(s parallelsuite.Scope, tv *tes Namespace: env.Namespace().String(), Version: tv.DeploymentVersionString(), }) - var notFound *serviceerror.NotFound - if errors.As(err, ¬Found) { + if _, ok := errors.AsType[*serviceerror.NotFound](err); ok { t.Require().NoError(err) return } diff --git a/tools/elasticsearch/tasks.go b/tools/elasticsearch/tasks.go index 4e013416d3e..b53ef90206e 100644 --- a/tools/elasticsearch/tasks.go +++ b/tools/elasticsearch/tasks.go @@ -76,8 +76,7 @@ func (task *SetupTask) setupIndex(ctx context.Context) error { success, err := task.esClient.CreateIndex(ctx, config.VisibilityIndex, nil) if err != nil { // Check if the error is an Elasticsearch error and if so check if the index already exists. - var esErr *elastic.Error - if errors.As(err, &esErr) { + if esErr, ok := errors.AsType[*elastic.Error](err); ok { if esErr.Status == 400 && esErr.Details != nil && esErr.Details.Type == "resource_already_exists_exception" { task.logger.Info("Index already exists, skipping creation", tag.String("indexName", config.VisibilityIndex)) return nil From d899928abca0c039e80f28be605a0b7da509f652 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 25 Aug 2026 09:10:41 -0700 Subject: [PATCH 058/139] Add log capture to TestLogger (#11748) ## What changed Adds `TestLogger.StartCapture` / `StopCapture` to records log calls and make them queryable. ## Why We want ability to verify certain logs where emitted to verify observability. See #11689 for first use case. --------- Co-authored-by: Claude Opus 5 (1M context) --- common/testing/testlogger/capture.go | 159 +++++++++++++++ common/testing/testlogger/capture_test.go | 194 +++++++++++++++++++ common/testing/testlogger/testlogger.go | 40 ++++ common/testing/testlogger/testlogger_test.go | 12 ++ 4 files changed, 405 insertions(+) create mode 100644 common/testing/testlogger/capture.go create mode 100644 common/testing/testlogger/capture_test.go diff --git a/common/testing/testlogger/capture.go b/common/testing/testlogger/capture.go new file mode 100644 index 00000000000..e84d8646003 --- /dev/null +++ b/common/testing/testlogger/capture.go @@ -0,0 +1,159 @@ +package testlogger + +import ( + "fmt" + "slices" + "strings" + "sync" + + "github.com/google/go-cmp/cmp" + "go.temporal.io/server/common/log/tag" +) + +// CapturedLog is a single log call recorded by a Capture. +type CapturedLog struct { + Level Level + Message string + Tags []tag.Tag +} + +// CapturedLogPattern describes a captured log using formatted tag values. +// Tags match a subset of the captured log's tags; AnyTagValue requires a tag +// without constraining its value. +type CapturedLogPattern struct { + Level Level + Message string + Tags map[string]any +} + +type anyTagValue struct{} + +// AnyTagValue matches any formatted value for a tag that must be present. +var AnyTagValue = &anyTagValue{} + +func (p CapturedLogPattern) matches(record CapturedLog) bool { + if record.Level != p.Level || record.Message != p.Message { + return false + } + + for key, expected := range p.Tags { + if !slices.ContainsFunc(record.Tags, func(actual tag.Tag) bool { + if actual.Key() != key { + return false + } + _, anyValue := expected.(*anyTagValue) + return anyValue || formatValue(actual) == fmt.Sprint(expected) + }) { + return false + } + } + return true +} + +func (p CapturedLogPattern) formattedTags() map[string]string { + formatted := make(map[string]string, len(p.Tags)) + for key, value := range p.Tags { + if _, anyValue := value.(*anyTagValue); anyValue { + formatted[key] = "" + } else { + formatted[key] = fmt.Sprint(value) + } + } + return formatted +} + +type captureFilterTag struct { + key string + value string +} + +// Capture is an opt-in recording of TestLogger calls. +type Capture struct { + filterTags map[captureFilterTag]struct{} + + mu sync.Mutex + records []CapturedLog +} + +func newCapture(anyTags []tag.Tag) *Capture { + capture := &Capture{} + if len(anyTags) == 0 { + return capture + } + capture.filterTags = make(map[captureFilterTag]struct{}, len(anyTags)) + for _, t := range anyTags { + capture.filterTags[captureFilterTag{key: t.Key(), value: formatValue(t)}] = struct{}{} + } + return capture +} + +// Snapshot returns a defensive copy of the captured log calls. +func (c *Capture) Snapshot() []CapturedLog { + c.mu.Lock() + defer c.mu.Unlock() + + records := make([]CapturedLog, len(c.records)) + for i, record := range c.records { + records[i] = record + records[i].Tags = slices.Clone(record.Tags) + } + return records +} + +// RequireContains fails the test with tag diffs when the capture does not include a matching log. +func (c *Capture) RequireContains(t TestingT, pattern CapturedLogPattern) { + t.Helper() + records := c.Snapshot() + if slices.ContainsFunc(records, pattern.matches) { + return + } + + var failure strings.Builder + fmt.Fprintf(&failure, "captured log pattern not found: level=%s message=%q", pattern.Level, pattern.Message) + expectedTags := pattern.formattedTags() + candidateCount := 0 + for _, record := range records { + if record.Level != pattern.Level || record.Message != pattern.Message { + continue + } + candidateCount++ + actualTags := make(map[string]string, len(pattern.Tags)) + for _, actual := range record.Tags { + key := actual.Key() + if _, anyValue := pattern.Tags[key].(*anyTagValue); anyValue { + actualTags[key] = "" + } else if _, expected := pattern.Tags[key]; expected { + actualTags[key] = formatValue(actual) + } + } + fmt.Fprintf(&failure, "\n\ncandidate %d tag mismatch (-want +got):\n%s", candidateCount, cmp.Diff(expectedTags, actualTags)) + } + if candidateCount == 0 { + failure.WriteString("\n\nno captured log had the expected level and message; captured logs:") + for _, record := range records { + fmt.Fprintf(&failure, "\n- %s %q", record.Level, record.Message) + for _, actual := range record.Tags { + fmt.Fprintf(&failure, " %s=%s", actual.Key(), formatValue(actual)) + } + } + } + t.Fatalf("%s", failure.String()) +} + +func (c *Capture) record(record CapturedLog) { + if len(c.filterTags) > 0 { + matched := false + for _, t := range record.Tags { + if _, ok := c.filterTags[captureFilterTag{key: t.Key(), value: formatValue(t)}]; ok { + matched = true + break + } + } + if !matched { + return + } + } + c.mu.Lock() + c.records = append(c.records, record) + c.mu.Unlock() +} diff --git a/common/testing/testlogger/capture_test.go b/common/testing/testlogger/capture_test.go new file mode 100644 index 00000000000..ae4335f3793 --- /dev/null +++ b/common/testing/testlogger/capture_test.go @@ -0,0 +1,194 @@ +package testlogger_test + +import ( + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/testing/testlogger" +) + +func TestCaptureLifecycle(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + testLogger.Info("before") // won't be captured + + capture := testLogger.StartCapture() + testLogger.Debug("debug", tag.String("key", "value")) + testLogger.Info("info") + testLogger.Warn("warn") + testLogger.Error("error") + + testLogger.StopCapture(capture) + testLogger.Info("after") // won't be captured + + require.Equal(t, []testlogger.CapturedLog{ + {Level: testlogger.Debug, Message: "debug", Tags: []tag.Tag{tag.String("key", "value")}}, + {Level: testlogger.Info, Message: "info"}, + {Level: testlogger.Warn, Message: "warn"}, + {Level: testlogger.Error, Message: "error"}, + }, capture.Snapshot()) +} + +func TestCaptureSharesStateWithDerivedLoggers(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + capture := testLogger.StartCapture() + + log.With(testLogger, tag.String("inherited", "value")).Info("with") + testLogger.WithTags(tag.String("direct", "value")).Info("with-tags") + + require.Equal(t, []testlogger.CapturedLog{ + {Level: testlogger.Info, Message: "with", Tags: []tag.Tag{tag.String("inherited", "value")}}, + {Level: testlogger.Info, Message: "with-tags", Tags: []tag.Tag{tag.String("direct", "value")}}, + }, capture.Snapshot()) +} + +func TestCaptureFiltersIndependently(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + all := testLogger.StartCapture() + matching := testLogger.StartCapture(tag.String("keep", "true")) + + testLogger.Info("excluded") + testLogger.Info("included", tag.String("keep", "true")) + + require.Len(t, all.Snapshot(), 2) + require.Equal(t, []testlogger.CapturedLog{{ + Level: testlogger.Info, + Message: "included", + Tags: []tag.Tag{tag.String("keep", "true")}, + }}, matching.Snapshot()) +} + +func TestCaptureSnapshotIsDefensiveCopy(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + capture := testLogger.StartCapture() + tags := []tag.Tag{tag.String("key", "original")} + + testLogger.Info("message", tags...) + tags[0] = tag.String("key", "mutated input") + + // mutate the snapshot + first := capture.Snapshot() + first[0].Message = "mutated snapshot" + first[0].Tags[0] = tag.String("key", "mutated snapshot") + + require.Equal(t, []testlogger.CapturedLog{{ + Level: testlogger.Info, + Message: "message", + Tags: []tag.Tag{tag.String("key", "original")}, + }}, capture.Snapshot()) +} + +func TestCaptureRequireContains(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + capture := testLogger.StartCapture() + testLogger.Error("failed", tag.String("operation", "StartOperation"), tag.Time("attempt-start", time.Now()), tag.Error(errors.New("failure"))) + + t.Run("matching pattern", func(t *testing.T) { + t.Parallel() + + capture.RequireContains(t, testlogger.CapturedLogPattern{ + Level: testlogger.Error, + Message: "failed", + Tags: map[string]any{ + "operation": "StartOperation", + "attempt-start": testlogger.AnyTagValue, + "error": "failure", + }, + }) + }) + + t.Run("mismatched tag", func(t *testing.T) { + t.Parallel() + + recorder := &mockT{T: t} + capture.RequireContains(recorder, testlogger.CapturedLogPattern{ + Level: testlogger.Error, + Message: "failed", + Tags: map[string]any{ + "operation": "CancelOperation", // mismatch! + "attempt-start": testlogger.AnyTagValue, + "error": "failure", + }, + }) + failure := recorder.failure.Load() + require.NotNil(t, failure) + require.Contains(t, *failure, "candidate 1 tag mismatch") + require.Contains(t, *failure, "CancelOperation") + require.Contains(t, *failure, "StartOperation") + }) + + // A pattern with an extra tag matches nothing. + t.Run("extra tag", func(t *testing.T) { + t.Parallel() + + recorder := &mockT{T: t} + capture.RequireContains(recorder, testlogger.CapturedLogPattern{ + Level: testlogger.Error, + Message: "failed", + Tags: map[string]any{ + "operation": "StartOperation", + "attempt-start": testlogger.AnyTagValue, + "error": "failure", + "unexpected": "value", // extra! + }, + }) + failure := recorder.failure.Load() + require.NotNil(t, failure) + require.Contains(t, *failure, "unexpected") + }) + + t.Run("missing level and message", func(t *testing.T) { + t.Parallel() + + recorder := &mockT{T: t} + capture.RequireContains(recorder, testlogger.CapturedLogPattern{ + Level: testlogger.Warn, + Message: "missing", + // missing! + }) + failure := recorder.failure.Load() + require.NotNil(t, failure) + require.Contains(t, *failure, `Error "failed" operation=StartOperation`) + require.NotContains(t, *failure, "zapcore") + }) +} + +func TestCaptureRecordsConcurrentDerivedLoggers(t *testing.T) { + t.Parallel() + + testLogger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly, + testlogger.WrapLogger(log.NewNoopLogger()), + ) + capture := testLogger.StartCapture() + + const ( + loggerCount = 10 + recordsPerLog = 20 + ) + var wg sync.WaitGroup + for loggerID := range loggerCount { + derived := log.With(testLogger, tag.Int("logger-id", loggerID)) + wg.Go(func() { + for range recordsPerLog { + derived.Info("concurrent") + } + }) + } + wg.Wait() + + require.Len(t, capture.Snapshot(), loggerCount*recordsPerLog) +} diff --git a/common/testing/testlogger/testlogger.go b/common/testing/testlogger/testlogger.go index 4161ff315cf..382ebd8ad0f 100644 --- a/common/testing/testlogger/testlogger.go +++ b/common/testing/testlogger/testlogger.go @@ -142,6 +142,7 @@ type sharedTestLoggerState struct { mu struct { sync.RWMutex expectations map[Level]*list.List // Map[Level]List[matcher] + captures map[*Capture]struct{} closed bool } mode Mode @@ -280,6 +281,7 @@ func NewTestLogger(t TestingT, mode Mode, opts ...LoggerOption) *TestLogger { }, } tl.state.mu.expectations = make(map[Level]*list.List) + tl.state.mu.captures = make(map[*Capture]struct{}) tl.state.failOnError.Store(true) tl.state.failOnDPanic.Store(true) tl.state.failOnFatal.Store(true) @@ -360,6 +362,23 @@ func (tl *TestLogger) Forget(e *Expectation) { tl.state.mu.expectations[e.lvl].Remove(e.e) } +// StartCapture starts recording log calls that contain any of the provided exact tags. +// It records all calls when no tags are provided. +func (tl *TestLogger) StartCapture(anyTags ...tag.Tag) *Capture { + capture := newCapture(anyTags) + tl.state.mu.Lock() + tl.state.mu.captures[capture] = struct{}{} + tl.state.mu.Unlock() + return capture +} + +// StopCapture stops recording log calls for capture. +func (tl *TestLogger) StopCapture(capture *Capture) { + tl.state.mu.Lock() + delete(tl.state.mu.captures, capture) + tl.state.mu.Unlock() +} + func (tl *TestLogger) shouldFailTest(level Level, msg string, tags []tag.Tag) bool { // Only check expectations if they've been registered for this level. if expectations, found := tl.state.mu.expectations[level]; found { @@ -400,6 +419,20 @@ func (tl *TestLogger) recordExpectationMatches(level Level, msg string, tags []t } } +func (tl *TestLogger) recordCaptures(level Level, msg string, tags []tag.Tag) { + if len(tl.state.mu.captures) == 0 { + return + } + record := CapturedLog{ + Level: level, + Message: msg, + Tags: slices.Clone(tags), + } + for capture := range tl.state.mu.captures { + capture.record(record) + } +} + // FailOnDPanic overrides the behavior of this logger. It returns the previous value // so that it can be restored later. func (tl *TestLogger) FailOnDPanic(b bool) bool { @@ -474,6 +507,7 @@ func (tl *TestLogger) DPanic(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(DPanic, msg, tags) + tl.recordCaptures(DPanic, msg, tags) // note, actual panic'ing in wrapped is turned off so we can control. tl.wrapped.DPanic(msg, tags...) if tl.state.failOnDPanic.Load() && tl.shouldFailTest(DPanic, msg, tags) { @@ -491,6 +525,7 @@ func (tl *TestLogger) Debug(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Debug, msg, tags) + tl.recordCaptures(Debug, msg, tags) tl.wrapped.Debug(msg, tags...) } @@ -503,6 +538,7 @@ func (tl *TestLogger) Error(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Error, msg, tags) + tl.recordCaptures(Error, msg, tags) if !tl.shouldFailTest(Error, msg, tags) { tl.wrapped.Error(msg, tags...) tl.state.mu.RUnlock() @@ -529,6 +565,7 @@ func (tl *TestLogger) Fatal(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Fatal, msg, tags) + tl.recordCaptures(Fatal, msg, tags) tl.state.t.Helper() if tl.state.failOnFatal.Load() && tl.shouldFailTest(Fatal, msg, tags) { tl.failTest(Fatal, msg, tags...) @@ -552,6 +589,7 @@ func (tl *TestLogger) Info(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Info, msg, tags) + tl.recordCaptures(Info, msg, tags) tl.wrapped.Info(msg, tags...) } @@ -564,6 +602,7 @@ func (tl *TestLogger) Panic(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Panic, msg, tags) + tl.recordCaptures(Panic, msg, tags) tl.state.t.Helper() // Forcibly fail the test when required as otherwise panics can be caught. if tl.shouldFailTest(Panic, msg, tags) { @@ -582,6 +621,7 @@ func (tl *TestLogger) Warn(msg string, tags ...tag.Tag) { } tags = tl.mergeWithLoggerTags(tags) tl.recordExpectationMatches(Warn, msg, tags) + tl.recordCaptures(Warn, msg, tags) tl.wrapped.Warn(msg, tags...) } diff --git a/common/testing/testlogger/testlogger_test.go b/common/testing/testlogger/testlogger_test.go index 3521d133367..60204abe4eb 100644 --- a/common/testing/testlogger/testlogger_test.go +++ b/common/testing/testlogger/testlogger_test.go @@ -78,6 +78,8 @@ func assertMatches(t *testing.T, level testlogger.Level, msg string, tags []tag. } func TestTestLogger_ExpectationsMatch(t *testing.T) { + t.Parallel() + for _, level := range []testlogger.Level{testlogger.Error, testlogger.DPanic, testlogger.Panic, testlogger.Fatal} { t.Run(level.String()+" with tags", func(t *testing.T) { assertMatches(t, level, "message with tags", []tag.Tag{tag.String("key", "value")}) @@ -93,6 +95,8 @@ func TestTestLogger_ExpectationsMatch(t *testing.T) { } func TestTestLogger_InfoExpectationMatchCount(t *testing.T) { + t.Parallel() + tl := testlogger.NewTestLogger(t, testlogger.FailOnAnyUnexpectedError) expected := tl.Expect(testlogger.Info, "expected info", tag.String("key", "value")) require.False(t, expected.Matched()) @@ -133,6 +137,8 @@ func assertFails(t *testing.T, level testlogger.Level, msg string, tags []tag.Ta } func TestTestLogger_Uncaught(t *testing.T) { + t.Parallel() + // Non-panicking levels for _, level := range []testlogger.Level{testlogger.Error, testlogger.DPanic} { t.Run(level.String()+" with tags", func(t *testing.T) { @@ -169,6 +175,8 @@ func TestTestLogger_Uncaught(t *testing.T) { // the first failure-worthy log in FailOnAnyUnexpectedError mode and that // subsequent failures do not overwrite it (first-failure-wins via CAS). func TestTestLogger_Failure_StickyOnAnyUnexpected(t *testing.T) { + t.Parallel() + mt := &mockT{T: t} tl := testlogger.NewTestLogger(mt, testlogger.FailOnAnyUnexpectedError) require.Nil(t, tl.Failure()) @@ -188,6 +196,8 @@ func TestTestLogger_Failure_StickyOnAnyUnexpected(t *testing.T) { // FailOnExpectedErrorOnly mode, an Error matching a registered expectation // (e.g. tag.FailedAssertion) should mark Failure(). func TestTestLogger_Failure_OnExpectedMatch(t *testing.T) { + t.Parallel() + mt := &mockT{T: t} tl := testlogger.NewTestLogger(mt, testlogger.FailOnExpectedErrorOnly) tl.Expect(testlogger.Error, ".*", tag.FailedAssertion) @@ -203,6 +213,8 @@ func TestTestLogger_Failure_OnExpectedMatch(t *testing.T) { // TestTestLogger_Failure_NoMatch verifies that FailOnExpectedErrorOnly remains an // escape hatch: an Error with no matching expectation does not flip Failure(). func TestTestLogger_Failure_NoMatch(t *testing.T) { + t.Parallel() + mt := &mockT{T: t} tl := testlogger.NewTestLogger(mt, testlogger.FailOnExpectedErrorOnly) require.Nil(t, tl.Failure()) From 92a8402adae63c692e1535f76394f3d78a3773cc Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Tue, 25 Aug 2026 09:18:06 -0700 Subject: [PATCH 059/139] XDC coverage for parent-child replication edge cases (#11690) ## What changed? - Added a reusable parent-child XDC test harness for controlling replication task application, delay, ordering, and omission around namespace failover. - Added five functional scenarios covering orphaned children, missing or incomplete children, task discard, missing parent completion, and parent resend recovery. - Added unit tests for the replication gate and legacy/transition-history task decoding. Configured two history shards per cluster and placed parent and child workflows on different shards. - Updated XDC synchronization checks to support multiple history shards. ## Why? Parent-child replication failures depend on rare cross-shard ordering and failover timing, making them difficult to reproduce reliably. These tests deterministically construct the relevant partial states while still exercising real Temporal services, persistence, replication, verification RPCs, task retry/discard behavior, and namespace failover. The harness also makes future scenarios easier to add and review. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- tests/xdc/base.go | 43 +- tests/xdc/parent_child_harness.go | 1262 ++++++++++++++++++++++++ tests/xdc/parent_child_harness_test.go | 398 ++++++++ tests/xdc/parent_child_test.go | 523 ++++++++++ 4 files changed, 2207 insertions(+), 19 deletions(-) create mode 100644 tests/xdc/parent_child_harness.go create mode 100644 tests/xdc/parent_child_harness_test.go create mode 100644 tests/xdc/parent_child_test.go diff --git a/tests/xdc/base.go b/tests/xdc/base.go index c3c6c6bbafa..ee5d891e378 100644 --- a/tests/xdc/base.go +++ b/tests/xdc/base.go @@ -25,6 +25,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/historyrequire" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/tests/testcore" @@ -61,6 +62,7 @@ type ( startTime time.Time onceClusterConnect sync.Once + numHistoryShards int32 enableTransitionHistory bool @@ -82,6 +84,7 @@ func (s *xdcBaseSuite) clusterReplicationConfig() []*replicationpb.ClusterReplic func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { params := testcore.ApplyTestClusterOptions(opts) + s.numHistoryShards = cmp.Or(params.NumHistoryShards, int32(1)) if s.logger == nil { s.logger = log.NewTestLogger() @@ -139,7 +142,7 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { }, }, HistoryConfig: testcore.HistoryConfig{ - NumHistoryShards: cmp.Or(params.NumHistoryShards, 1), + NumHistoryShards: s.numHistoryShards, }, Persistence: persistenceDefaults, DynamicConfigOverrides: s.dynamicConfigOverrides, @@ -184,27 +187,29 @@ func (s *xdcBaseSuite) waitForClusterConnected( sourceCluster *testcore.TestCluster, targetClusterName string, ) { + expectedNumHistoryShards := cmp.Or(s.numHistoryShards, int32(1)) s.logger.Info("wait for clusters to be synced", tag.SourceCluster(sourceCluster.ClusterName()), tag.TargetCluster(targetClusterName)) - s.EventuallyWithT(func(c *assert.CollectT) { + await.Require(context.Background(), s.T(), func(c *await.T) { s.logger.Info("check if clusters are synced", tag.SourceCluster(sourceCluster.ClusterName()), tag.TargetCluster(targetClusterName)) - resp, err := sourceCluster.HistoryClient().GetReplicationStatus(context.Background(), &historyservice.GetReplicationStatusRequest{}) + resp, err := sourceCluster.HistoryClient().GetReplicationStatus(c.Context(), &historyservice.GetReplicationStatusRequest{}) require.NoError(c, err) - require.Lenf(c, resp.Shards, 1, "test cluster has only one history shard") - - shard := resp.Shards[0] - require.NotNil(c, shard) - require.Positive(c, shard.MaxReplicationTaskId) - require.NotNil(c, shard.ShardLocalTime) - require.WithinRange(c, shard.ShardLocalTime.AsTime(), s.startTime, time.Now()) - require.NotNil(c, shard.RemoteClusters) - - standbyAckInfo, ok := shard.RemoteClusters[targetClusterName] - require.True(c, ok) - require.NotNil(c, standbyAckInfo) - require.LessOrEqual(c, shard.MaxReplicationTaskId, standbyAckInfo.AckedTaskId) - require.NotNil(c, standbyAckInfo.AckedTaskVisibilityTime) - require.WithinRange(c, standbyAckInfo.AckedTaskVisibilityTime.AsTime(), s.startTime, time.Now()) - }, 90*time.Second, 1*time.Second) + require.Lenf(c, resp.Shards, int(expectedNumHistoryShards), "unexpected history shard count") + + for _, shard := range resp.Shards { + require.NotNil(c, shard) + require.Positive(c, shard.MaxReplicationTaskId) + require.NotNil(c, shard.ShardLocalTime) + require.WithinRange(c, shard.ShardLocalTime.AsTime(), s.startTime, time.Now()) + require.NotNil(c, shard.RemoteClusters) + + standbyAckInfo, ok := shard.RemoteClusters[targetClusterName] + require.True(c, ok) + require.NotNil(c, standbyAckInfo) + require.LessOrEqual(c, shard.MaxReplicationTaskId, standbyAckInfo.AckedTaskId) + require.NotNil(c, standbyAckInfo.AckedTaskVisibilityTime) + require.WithinRange(c, standbyAckInfo.AckedTaskVisibilityTime.AsTime(), s.startTime, time.Now()) + } + }, 90*time.Second, time.Second) s.logger.Info("clusters synced", tag.SourceCluster(sourceCluster.ClusterName()), tag.TargetCluster(targetClusterName)) } diff --git a/tests/xdc/parent_child_harness.go b/tests/xdc/parent_child_harness.go new file mode 100644 index 00000000000..4588e521e50 --- /dev/null +++ b/tests/xdc/parent_child_harness.go @@ -0,0 +1,1262 @@ +package xdc + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + replicationpb "go.temporal.io/api/replication/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/taskpoller" + "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/common/testing/testvars" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +var ( + errParentChildReplicationGateClosed = errors.New("parent-child replication gate is closed") + errParentChildTaskAlreadyResolved = errors.New("parent-child replication task is already resolved") +) + +type parentChildXDCTestSuite struct{ xdcBaseSuite } + +type ( + parentChildWorkflow int + + parentChildCluster int + + parentChildReplicationTaskAction int + + parentChildScenario struct { + steps []parentChildScenarioStep + expectations []parentChildExpectation + } + + parentChildScenarioStep struct { + name string + run func(context.Context, *parentChildScenarioRuntime) error + } + + parentChildExpectation struct { + name string + check func(context.Context, *parentChildScenarioRuntime) error + } + + parentChildScenarioRuntime struct { + suite *parentChildXDCTestSuite + + namespace string + namespaceID string + parentID string + parentRunID string + childID string + childRunID string + parentTestVars *testvars.TestVars + childTestVars *testvars.TestVars + + activeClusterIndex int + gates [2]*parentChildReplicationGate + removeHooks []func() + cleanups []func() + delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask + metricCaptures [2]parentChildMetricCapture + trace []string + } + + parentChildMetricCapture struct { + handler *metricstest.CaptureHandler + capture *metricstest.Capture + } + + parentChildReplicationLane struct { + targetClusterIndex int + workflow parentChildWorkflow + } + + parentChildReplicationGate struct { + namespaceID string + workflowIDs map[string]struct{} + pending chan *parentChildReplicationTask + buffered map[string][]*parentChildReplicationTask + stop chan struct{} + stopOnce sync.Once + } + + parentChildReplicationTask struct { + task *replicationspb.ReplicationTask + metadata parentChildReplicationTaskMetadata + execute func() error + result chan error + + mu sync.Mutex + resolved bool + } + + parentChildReplicationTaskMetadata struct { + namespaceID string + workflowID string + runID string + } +) + +const ( + parentWorkflow parentChildWorkflow = iota + childWorkflow +) + +const ( + initialActiveCluster parentChildCluster = iota + initialStandbyCluster +) + +const ( + applyReplicationTask parentChildReplicationTaskAction = iota + delayReplicationTaskApply + ackReplicationTaskWithoutApplying +) + +const ( + historyClientVerifyChildCompletion = "HistoryClientVerifyChildExecutionCompletionRecorded" + historyClientVerifyFirstWorkflowTask = "HistoryClientVerifyFirstWorkflowTaskScheduled" +) + +func (s *parentChildXDCTestSuite) runParentChildScenario(scenario parentChildScenario) { + runtime := newParentChildScenarioRuntime(s) + defer runtime.close() + defer func() { + if s.T().Failed() { + s.T().Logf("parent-child scenario trace:\n%s", strings.Join(runtime.trace, "\n")) + } + }() + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + err := runtime.initialize(ctx) + cancel() + s.Require().NoError(err) + + for index, step := range scenario.steps { + runtime.tracef("step %d: %s", index+1, step.name) + ctx, cancel = context.WithTimeout(context.Background(), testTimeout) + err = step.run(ctx, runtime) + cancel() + s.Require().NoError(err, step.name) + } + + for index, expectation := range scenario.expectations { + runtime.tracef("expectation %d: %s", index+1, expectation.name) + await.Require(context.Background(), s.T(), func(t *await.T) { + require.NoError(t, expectation.check(t.Context(), runtime)) + }, testTimeout, replicationCheckInterval) + } +} + +func newParentChildScenarioRuntime(s *parentChildXDCTestSuite) *parentChildScenarioRuntime { + return &parentChildScenarioRuntime{ + suite: s, + delayedTasks: make(map[parentChildReplicationLane]*parentChildReplicationTask), + } +} + +func (r *parentChildScenarioRuntime) initialize(ctx context.Context) error { + if len(r.suite.clusters) != len(r.gates) { + return fmt.Errorf("parent-child scenarios require exactly %d clusters, got %d", len(r.gates), len(r.suite.clusters)) + } + + r.namespace = r.suite.createGlobalNamespace() + nsResp, err := r.suite.clusters[0].FrontendClient().DescribeNamespace(ctx, &workflowservice.DescribeNamespaceRequest{ + Namespace: r.namespace, + }) + if err != nil { + return err + } + r.namespaceID = nsResp.GetNamespaceInfo().GetId() + if r.suite.numHistoryShards < 2 { + return fmt.Errorf("parent-child scenarios require at least 2 history shards, got %d", r.suite.numHistoryShards) + } + + var parentShardID int32 + var childShardID int32 + r.parentID, r.childID, parentShardID, childShardID = workflowIDsOnDifferentShards(r.namespaceID, r.suite.numHistoryShards) + r.tracef("topology: parent shard=%d, child shard=%d", parentShardID, childShardID) + + for clusterIndex, cluster := range r.suite.clusters { + gate := newParentChildReplicationGate(r.namespaceID, r.parentID, r.childID) + r.gates[clusterIndex] = gate + r.removeHooks = append(r.removeHooks, cluster.InjectHook( + r.suite.T(), + testhooks.NewHook(testhooks.HistoryReplicationTaskInterceptor, gate.intercept), + testhooks.GlobalScope, + )) + + metricsHandler, ok := cluster.Host().GetMetricsHandler().(*metricstest.CaptureHandler) + if !ok { + return fmt.Errorf("cluster %d metrics handler does not support capture", clusterIndex) + } + r.metricCaptures[clusterIndex] = parentChildMetricCapture{ + handler: metricsHandler, + capture: metricsHandler.StartCapture(), + } + } + + r.parentTestVars = testvars.New(r.suite.T()).WithTaskQueue("parent-child-xdc-parent-task-queue") + r.childTestVars = testvars.New(r.suite.T()).WithTaskQueue("parent-child-xdc-child-task-queue") + return nil +} + +func (r *parentChildScenarioRuntime) close() { + for _, gate := range r.gates { + if gate != nil { + gate.close() + } + } + for index := len(r.removeHooks) - 1; index >= 0; index-- { + r.removeHooks[index]() + } + for _, capture := range r.metricCaptures { + if capture.handler != nil && capture.capture != nil { + capture.handler.StopCapture(capture.capture) + } + } + for index := len(r.cleanups) - 1; index >= 0; index-- { + r.cleanups[index]() + } +} + +func useLegacyHistoryReplication() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "use legacy history replication for this scenario", + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + for _, cluster := range runtime.suite.clusters { + runtime.cleanups = append(runtime.cleanups, cluster.OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.EnableTransitionHistory, + false, + )) + } + return nil + }, + } +} + +func setLocalParentVerificationGrace( + cluster parentChildCluster, + duration time.Duration, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("set local parent verification grace on %s to %s", cluster, duration), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.MaxLocalParentWorkflowVerificationDuration, + duration, + )) + return nil + }, + } +} + +func setStandbyClusterDelay( + cluster parentChildCluster, + duration time.Duration, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("set standby cluster delay on %s to %s", cluster, duration), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.StandbyClusterDelay, + duration, + )) + return nil + }, + } +} + +func setStandbyTaskDiscardDelay( + cluster parentChildCluster, + taskType enumsspb.TaskType, + duration time.Duration, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("set standby %s discard delay on %s to %s", taskType, cluster, duration), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.StandbyTaskMissingEventsDiscardDelay, + []dynamicconfig.ConstrainedValue{{ + Constraints: dynamicconfig.Constraints{TaskType: taskType}, + Value: duration, + }}, + )) + return nil + }, + } +} + +func startParentWorkflow() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "start parent workflow on the active cluster", + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.startParentWorkflow(ctx) + }, + } +} + +// Event checkpoints identify a replication task; each action applies to the entire task. +func applyReplicationThroughTaskContainingEvent( + targetCluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildScenarioStep { + return replicationTaskStep(applyReplicationTask, targetCluster, workflow, eventType) +} + +func delayReplicationAtTaskContainingEvent( + targetCluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildScenarioStep { + return replicationTaskStep(delayReplicationTaskApply, targetCluster, workflow, eventType) +} + +func applyDelayedReplication( + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("apply delayed %s replication to %s", workflow, targetCluster), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.applyDelayedReplication(targetCluster, workflow) + }, + } +} + +func acknowledgeReplicationTaskContainingEventWithoutApplying( + targetCluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildScenarioStep { + return replicationTaskStep(ackReplicationTaskWithoutApplying, targetCluster, workflow, eventType) +} + +func replicationTaskStep( + action parentChildReplicationTaskAction, + targetCluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("%s %s replication to %s at task containing %s", action, workflow, targetCluster, eventType), + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.processReplicationThroughTaskContainingEvent(ctx, targetCluster, workflow, eventType, action) + }, + } +} + +func completeParentWorkflowTaskWithStartChildCommand() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "complete parent workflow task with a StartChild command", + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.completeParentWorkflowTaskWithStartChildCommand(ctx) + }, + } +} + +func completeChildWorkflowTask() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "complete the child workflow task", + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.completeChildWorkflowTask(ctx) + }, + } +} + +func waitForWorkflowEventOnCluster( + cluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildScenarioStep { + expectation := workflowHasEventOnCluster(cluster, workflow, eventType) + return parentChildScenarioStep{ + name: "wait until " + expectation.name, + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.waitForExpectation(ctx, expectation) + }, + } +} + +func confirmWorkflowIsMissingOnCluster( + cluster parentChildCluster, + workflow parentChildWorkflow, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("confirm %s is missing on %s", workflow, cluster), + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.confirmWorkflowMissing(ctx, cluster, workflow) + }, + } +} + +func forceFailoverNamespaceTo(targetCluster parentChildCluster) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("force fail over the namespace to %s", targetCluster), + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.forceFailover(ctx, targetCluster) + }, + } +} + +func currentWorkflowHasStatusOnCluster( + cluster parentChildCluster, + workflow parentChildWorkflow, + status enumspb.WorkflowExecutionStatus, +) parentChildExpectation { + return parentChildExpectation{ + name: fmt.Sprintf("current %s has status %s on %s", workflow, status, cluster), + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + workflowID, err := runtime.workflowID(workflow) + if err != nil { + return err + } + resp, err := runtime.suite.clusters[clusterIndex].FrontendClient().DescribeWorkflowExecution( + ctx, + &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: runtime.namespace, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID}, + }, + ) + if err != nil { + return err + } + info := resp.GetWorkflowExecutionInfo() + if runID := info.GetExecution().GetRunId(); runID != runtime.workflowRunID(workflow) { + return fmt.Errorf("%s run is %q, want %q", workflow, runID, runtime.workflowRunID(workflow)) + } + if actualStatus := info.GetStatus(); actualStatus != status { + return fmt.Errorf("%s status is %s, want %s", workflow, actualStatus, status) + } + return nil + }, + } +} + +func workflowIsMissingOnCluster( + cluster parentChildCluster, + workflow parentChildWorkflow, +) parentChildExpectation { + return parentChildExpectation{ + name: fmt.Sprintf("%s is missing on %s", workflow, cluster), + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.confirmWorkflowMissing(ctx, cluster, workflow) + }, + } +} + +func workflowHasEventOnCluster( + cluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, +) parentChildExpectation { + return parentChildExpectation{ + name: fmt.Sprintf("%s contains %s on %s", workflow, eventType, cluster), + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + events, err := runtime.workflowHistoryOnCluster(ctx, cluster, workflow) + if err != nil { + return err + } + if !historyContainsEvent(events, eventType) { + return fmt.Errorf("%s has no %s on %s", workflow, eventType, cluster) + } + return nil + }, + } +} + +func historyVerificationFailedOnCluster( + cluster parentChildCluster, + operation string, + expectedError error, +) parentChildExpectation { + errorType := metrics.ServiceErrorTypeTag(expectedError).Value + return parentChildExpectation{ + name: fmt.Sprintf("%s fails with %s on %s", operation, errorType, cluster), + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + namespaceTag := metrics.NamespaceTag(runtime.namespace) + serviceRoleTag := metrics.ServiceRoleTag(metrics.HistoryRoleTagValue) + return runtime.requireCapturedMetric(cluster, metrics.ClientFailures.Name(), map[string]string{ + metrics.OperationTagName: operation, + metrics.ErrorTypeTagName: errorType, + namespaceTag.Key: namespaceTag.Value, + serviceRoleTag.Key: serviceRoleTag.Value, + }) + }, + } +} + +func waitForHistoryVerificationFailureOnCluster( + cluster parentChildCluster, + operation string, + expectedError error, +) parentChildScenarioStep { + expectation := historyVerificationFailedOnCluster(cluster, operation, expectedError) + return parentChildScenarioStep{ + name: "wait until " + expectation.name, + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.waitForExpectation(ctx, expectation) + }, + } +} + +func taskWasDiscardedOnCluster( + cluster parentChildCluster, + taskType string, +) parentChildExpectation { + return parentChildExpectation{ + name: fmt.Sprintf("%s is discarded on %s", taskType, cluster), + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + namespaceTag := metrics.NamespaceTag(runtime.namespace) + return runtime.requireCapturedMetric(cluster, metrics.TaskDiscarded.Name(), map[string]string{ + metrics.OperationTagName: taskType, + metrics.TaskTypeTagName: taskType, + namespaceTag.Key: namespaceTag.Value, + }) + }, + } +} + +func (r *parentChildScenarioRuntime) startParentWorkflow(ctx context.Context) error { + if r.parentRunID != "" { + return fmt.Errorf("parent workflow is already started with run ID %q", r.parentRunID) + } + startResp, err := r.activeCluster().FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: r.namespace, + WorkflowId: r.parentID, + WorkflowType: &commonpb.WorkflowType{Name: "parent-workflow"}, + TaskQueue: r.parentTestVars.TaskQueue(), + RequestId: uuid.NewString(), + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: r.parentTestVars.WorkerIdentity(), + }) + if err != nil { + return err + } + r.parentRunID = startResp.GetRunId() + r.tracef(" started parent %s/%s on cluster %d", r.parentID, r.parentRunID, r.activeClusterIndex) + return nil +} + +func (r *parentChildScenarioRuntime) processReplicationThroughTaskContainingEvent( + ctx context.Context, + targetCluster parentChildCluster, + workflow parentChildWorkflow, + eventType enumspb.EventType, + targetAction parentChildReplicationTaskAction, +) error { + targetClusterIndex := int(targetCluster) + if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { + return fmt.Errorf("unknown parent-child cluster %d", targetCluster) + } + lane := parentChildReplicationLane{targetClusterIndex: targetClusterIndex, workflow: workflow} + workflowID, err := r.workflowID(workflow) + if err != nil { + return err + } + + for { + task, delayed := r.delayedTasks[lane] + if !delayed { + task, err = r.gates[targetClusterIndex].nextForWorkflow(ctx, workflowID) + if err != nil { + return err + } + } + + events, err := decodeParentChildReplicationEvents(task.task) + if err != nil { + return err + } + containsCheckpoint := historyContainsEvent(events, eventType) + action := applyReplicationTask + if containsCheckpoint { + action = targetAction + } + + r.tracef( + " %s task %d to cluster %d for %s [%s]", + action, + task.task.GetSourceTaskId(), + targetClusterIndex, + workflow, + formatParentChildReplicationTask(task.task, events), + ) + switch action { + case applyReplicationTask: + delete(r.delayedTasks, lane) + if err := task.apply(); err != nil { + return err + } + r.recordChildRunIDFromAppliedTask(workflow, task, events) + case delayReplicationTaskApply: + r.delayedTasks[lane] = task + case ackReplicationTaskWithoutApplying: + delete(r.delayedTasks, lane) + if err := task.acknowledgeWithoutApplying(); err != nil { + return err + } + default: + return fmt.Errorf("unknown replication task action %d", action) + } + + if containsCheckpoint { + return nil + } + } +} + +func (r *parentChildScenarioRuntime) applyDelayedReplication( + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) error { + targetClusterIndex := int(targetCluster) + if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { + return fmt.Errorf("unknown parent-child cluster %d", targetCluster) + } + if _, err := r.workflowID(workflow); err != nil { + return err + } + + lane := parentChildReplicationLane{targetClusterIndex: targetClusterIndex, workflow: workflow} + task, delayed := r.delayedTasks[lane] + if !delayed { + return fmt.Errorf("no delayed %s replication task to %s", workflow, targetCluster) + } + events, err := decodeParentChildReplicationEvents(task.task) + if err != nil { + return err + } + + r.tracef( + " apply delayed task %d to cluster %d for %s [%s]", + task.task.GetSourceTaskId(), + targetClusterIndex, + workflow, + formatParentChildReplicationTask(task.task, events), + ) + delete(r.delayedTasks, lane) + if err := task.apply(); err != nil { + return err + } + r.recordChildRunIDFromAppliedTask(workflow, task, events) + return nil +} + +func (r *parentChildScenarioRuntime) completeParentWorkflowTaskWithStartChildCommand(ctx context.Context) error { + if r.parentRunID == "" { + return errors.New("parent workflow is not started") + } + poller := taskpoller.New(r.suite.T(), r.activeCluster().FrontendClient(), r.namespace) + _, err := poller.PollAndHandleWorkflowTask(r.parentTestVars, func( + task *workflowservice.PollWorkflowTaskQueueResponse, + ) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + if task.GetWorkflowExecution().GetWorkflowId() != r.parentID || task.GetWorkflowExecution().GetRunId() != r.parentRunID { + return nil, fmt.Errorf( + "polled workflow %s/%s, want parent %s/%s", + task.GetWorkflowExecution().GetWorkflowId(), + task.GetWorkflowExecution().GetRunId(), + r.parentID, + r.parentRunID, + ) + } + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_StartChildWorkflowExecutionCommandAttributes{ + StartChildWorkflowExecutionCommandAttributes: &commandpb.StartChildWorkflowExecutionCommandAttributes{ + WorkflowId: r.childID, + WorkflowType: &commonpb.WorkflowType{Name: "child-workflow"}, + TaskQueue: r.childTestVars.TaskQueue(), + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + ParentClosePolicy: enumspb.PARENT_CLOSE_POLICY_ABANDON, + }, + }, + }}, + }, nil + }, taskpoller.WithTimeout(testTimeout)) + return err +} + +func (r *parentChildScenarioRuntime) completeChildWorkflowTask(ctx context.Context) error { + poller := taskpoller.New(r.suite.T(), r.activeCluster().FrontendClient(), r.namespace) + _, err := poller.PollAndHandleWorkflowTask(r.childTestVars, func( + task *workflowservice.PollWorkflowTaskQueueResponse, + ) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + execution := task.GetWorkflowExecution() + if execution.GetWorkflowId() != r.childID { + return nil, fmt.Errorf("polled workflow %s/%s, want child %s", execution.GetWorkflowId(), execution.GetRunId(), r.childID) + } + if r.childRunID == "" { + r.childRunID = execution.GetRunId() + } + if execution.GetRunId() != r.childRunID { + return nil, fmt.Errorf("polled child run %s, want %s", execution.GetRunId(), r.childRunID) + } + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + }}, + }, nil + }, taskpoller.WithTimeout(testTimeout)) + return err +} + +func (r *parentChildScenarioRuntime) workflowHistoryOnCluster( + ctx context.Context, + cluster parentChildCluster, + workflow parentChildWorkflow, +) ([]*historypb.HistoryEvent, error) { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(r.suite.clusters) { + return nil, fmt.Errorf("unknown parent-child cluster %d", cluster) + } + workflowID, err := r.workflowID(workflow) + if err != nil { + return nil, err + } + runID := r.workflowRunID(workflow) + resp, err := r.suite.clusters[clusterIndex].FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: r.namespace, + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + }) + if err != nil { + return nil, err + } + return resp.GetHistory().GetEvents(), nil +} + +func (r *parentChildScenarioRuntime) workflowMutableState( + ctx context.Context, + cluster parentChildCluster, + workflow parentChildWorkflow, +) (*persistencespb.WorkflowMutableState, error) { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(r.suite.clusters) { + return nil, fmt.Errorf("unknown parent-child cluster %d", cluster) + } + workflowID, err := r.workflowID(workflow) + if err != nil { + return nil, err + } + resp, err := r.suite.clusters[clusterIndex].HistoryClient().DescribeMutableState(ctx, &historyservice.DescribeMutableStateRequest{ + NamespaceId: r.namespaceID, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: workflowID, + RunId: r.workflowRunID(workflow), + }, + ArchetypeId: chasm.WorkflowArchetypeID, + }) + if err != nil { + return nil, err + } + if resp.GetDatabaseMutableState() == nil { + return nil, fmt.Errorf("%s database mutable state is nil on %s", workflow, cluster) + } + return resp.GetDatabaseMutableState(), nil +} + +func (r *parentChildScenarioRuntime) confirmWorkflowMissing( + ctx context.Context, + cluster parentChildCluster, + workflow parentChildWorkflow, +) error { + _, err := r.workflowMutableState(ctx, cluster, workflow) + var notFound *serviceerror.NotFound + if errors.As(err, ¬Found) { + return nil + } + if err == nil { + return fmt.Errorf("%s unexpectedly exists on %s", workflow, cluster) + } + return err +} + +func (r *parentChildScenarioRuntime) requireCapturedMetric( + cluster parentChildCluster, + metricName string, + tags map[string]string, +) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(r.metricCaptures) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + capture := r.metricCaptures[clusterIndex].capture + if capture == nil { + return fmt.Errorf("metrics capture is not initialized for %s", cluster) + } + recordings := capture.Snapshot()[metricName] + for _, recording := range recordings { + matches := true + for key, value := range tags { + if recording.Tags[key] != value { + matches = false + break + } + } + if matches { + return nil + } + } + recordedTags := make([]map[string]string, 0, len(recordings)) + for _, recording := range recordings { + recordedTags = append(recordedTags, recording.Tags) + } + return fmt.Errorf("metric %q with tags %v was not captured on %s; recorded tags: %v", metricName, tags, cluster, recordedTags) +} + +func (r *parentChildScenarioRuntime) waitForExpectation( + ctx context.Context, + expectation parentChildExpectation, +) error { + ticker := time.NewTicker(replicationCheckInterval) + defer ticker.Stop() + for { + err := expectation.check(ctx, r) + if err == nil { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("waiting for %s: %w; last observation: %v", expectation.name, ctx.Err(), err) + case <-ticker.C: + } + } +} + +func (r *parentChildScenarioRuntime) forceFailover(ctx context.Context, target parentChildCluster) error { + targetClusterIndex := int(target) + if targetClusterIndex < 0 || targetClusterIndex >= len(r.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", target) + } + if targetClusterIndex == r.activeClusterIndex { + return fmt.Errorf("cluster %s is already active", target) + } + targetCluster := r.suite.clusters[targetClusterIndex].ClusterName() + _, err := r.activeCluster().FrontendClient().UpdateNamespace(ctx, &workflowservice.UpdateNamespaceRequest{ + Namespace: r.namespace, + ReplicationConfig: &replicationpb.NamespaceReplicationConfig{ + ActiveClusterName: targetCluster, + }, + }) + if err != nil { + return err + } + + await.Require(ctx, r.suite.T(), func(t *await.T) { + for _, cluster := range r.suite.clusters { + resp, describeErr := cluster.FrontendClient().DescribeNamespace(t.Context(), &workflowservice.DescribeNamespaceRequest{ + Namespace: r.namespace, + }) + require.NoError(t, describeErr) + require.Equal(t, targetCluster, resp.GetReplicationConfig().GetActiveClusterName()) + } + }, replicationWaitTime, replicationCheckInterval) + r.suite.waitForNamespaceCacheRefresh() + r.activeClusterIndex = targetClusterIndex + r.tracef(" active cluster is now %d (%s)", targetClusterIndex, targetCluster) + return nil +} + +func (r *parentChildScenarioRuntime) activeWorkflowHistory( + ctx context.Context, + workflow parentChildWorkflow, +) ([]*historypb.HistoryEvent, error) { + return r.workflowHistoryOnCluster(ctx, parentChildCluster(r.activeClusterIndex), workflow) +} + +func (r *parentChildScenarioRuntime) activeCluster() *testcore.TestCluster { + return r.suite.clusters[r.activeClusterIndex] +} + +func (r *parentChildScenarioRuntime) workflowID(workflow parentChildWorkflow) (string, error) { + switch workflow { + case parentWorkflow: + return r.parentID, nil + case childWorkflow: + return r.childID, nil + default: + return "", fmt.Errorf("unknown parent-child workflow %d", workflow) + } +} + +func (r *parentChildScenarioRuntime) workflowRunID(workflow parentChildWorkflow) string { + switch workflow { + case parentWorkflow: + return r.parentRunID + case childWorkflow: + return r.childRunID + default: + return "" + } +} + +func (r *parentChildScenarioRuntime) recordChildRunIDFromAppliedTask( + workflow parentChildWorkflow, + task *parentChildReplicationTask, + events []*historypb.HistoryEvent, +) { + if workflow == childWorkflow && historyContainsEvent(events, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED) { + r.childRunID = task.metadata.runID + } +} + +func (r *parentChildScenarioRuntime) tracef(format string, args ...any) { + r.trace = append(r.trace, fmt.Sprintf(format, args...)) +} + +func newParentChildReplicationGate(namespaceID string, workflowIDs ...string) *parentChildReplicationGate { + gate := &parentChildReplicationGate{ + namespaceID: namespaceID, + workflowIDs: make(map[string]struct{}, len(workflowIDs)), + pending: make(chan *parentChildReplicationTask, 32), + buffered: make(map[string][]*parentChildReplicationTask), + stop: make(chan struct{}), + } + for _, workflowID := range workflowIDs { + gate.workflowIDs[workflowID] = struct{}{} + } + return gate +} + +func (g *parentChildReplicationGate) intercept( + task *replicationspb.ReplicationTask, + execute func() error, +) error { + metadata, ok := getParentChildReplicationTaskMetadata(task) + if !ok || metadata.namespaceID != g.namespaceID { + return execute() + } + if _, workflowSelected := g.workflowIDs[metadata.workflowID]; !workflowSelected { + return execute() + } + + bufferedTask := &parentChildReplicationTask{ + task: task, + metadata: metadata, + execute: execute, + result: make(chan error, 1), + } + select { + case g.pending <- bufferedTask: + case <-g.stop: + return nil + } + select { + case err := <-bufferedTask.result: + return err + case <-g.stop: + return nil + } +} + +func (g *parentChildReplicationGate) nextForWorkflow( + ctx context.Context, + workflowID string, +) (*parentChildReplicationTask, error) { + select { + case <-g.stop: + return nil, errParentChildReplicationGateClosed + default: + } + + if tasks := g.buffered[workflowID]; len(tasks) > 0 { + task := tasks[0] + g.buffered[workflowID] = tasks[1:] + return task, nil + } + + for { + select { + case task := <-g.pending: + taskWorkflowID := task.metadata.workflowID + if taskWorkflowID == workflowID { + return task, nil + } + g.buffered[taskWorkflowID] = append(g.buffered[taskWorkflowID], task) + case <-g.stop: + return nil, errParentChildReplicationGateClosed + case <-ctx.Done(): + return nil, ctx.Err() + } + } +} + +func (g *parentChildReplicationGate) close() { + g.stopOnce.Do(func() { + close(g.stop) + }) +} + +func (t *parentChildReplicationTask) apply() error { + return t.resolve(true) +} + +func (t *parentChildReplicationTask) acknowledgeWithoutApplying() error { + return t.resolve(false) +} + +func (t *parentChildReplicationTask) resolve(shouldApply bool) error { + t.mu.Lock() + if t.resolved { + t.mu.Unlock() + return errParentChildTaskAlreadyResolved + } + t.resolved = true + t.mu.Unlock() + + var err error + if shouldApply { + err = t.execute() + } + t.result <- err + return err +} + +func getParentChildReplicationTaskMetadata( + task *replicationspb.ReplicationTask, +) (parentChildReplicationTaskMetadata, bool) { + if task == nil { + return parentChildReplicationTaskMetadata{}, false + } + if info := task.GetRawTaskInfo(); info != nil { + return parentChildReplicationTaskMetadata{ + namespaceID: info.GetNamespaceId(), + workflowID: info.GetWorkflowId(), + runID: info.GetRunId(), + }, true + } + + var metadata parentChildReplicationTaskMetadata + switch attrs := task.GetAttributes().(type) { + case *replicationspb.ReplicationTask_SyncWorkflowStateTaskAttributes: + state := attrs.SyncWorkflowStateTaskAttributes.GetWorkflowState() + metadata.namespaceID = state.GetExecutionInfo().GetNamespaceId() + metadata.workflowID = state.GetExecutionInfo().GetWorkflowId() + metadata.runID = state.GetExecutionState().GetRunId() + case *replicationspb.ReplicationTask_SyncActivityTaskAttributes: + metadata.namespaceID = attrs.SyncActivityTaskAttributes.GetNamespaceId() + metadata.workflowID = attrs.SyncActivityTaskAttributes.GetWorkflowId() + metadata.runID = attrs.SyncActivityTaskAttributes.GetRunId() + case *replicationspb.ReplicationTask_HistoryTaskAttributes: + metadata.namespaceID = attrs.HistoryTaskAttributes.GetNamespaceId() + metadata.workflowID = attrs.HistoryTaskAttributes.GetWorkflowId() + metadata.runID = attrs.HistoryTaskAttributes.GetRunId() + case *replicationspb.ReplicationTask_SyncHsmAttributes: + metadata.namespaceID = attrs.SyncHsmAttributes.GetNamespaceId() + metadata.workflowID = attrs.SyncHsmAttributes.GetWorkflowId() + metadata.runID = attrs.SyncHsmAttributes.GetRunId() + case *replicationspb.ReplicationTask_BackfillHistoryTaskAttributes: + metadata.namespaceID = attrs.BackfillHistoryTaskAttributes.GetNamespaceId() + metadata.workflowID = attrs.BackfillHistoryTaskAttributes.GetWorkflowId() + metadata.runID = attrs.BackfillHistoryTaskAttributes.GetRunId() + case *replicationspb.ReplicationTask_VerifyVersionedTransitionTaskAttributes: + metadata.namespaceID = attrs.VerifyVersionedTransitionTaskAttributes.GetNamespaceId() + metadata.workflowID = attrs.VerifyVersionedTransitionTaskAttributes.GetWorkflowId() + metadata.runID = attrs.VerifyVersionedTransitionTaskAttributes.GetRunId() + case *replicationspb.ReplicationTask_SyncVersionedTransitionTaskAttributes: + metadata.namespaceID = attrs.SyncVersionedTransitionTaskAttributes.GetNamespaceId() + metadata.workflowID = attrs.SyncVersionedTransitionTaskAttributes.GetWorkflowId() + metadata.runID = attrs.SyncVersionedTransitionTaskAttributes.GetRunId() + default: + return parentChildReplicationTaskMetadata{}, false + } + return metadata, true +} + +func decodeParentChildReplicationEvents(task *replicationspb.ReplicationTask) ([]*historypb.HistoryEvent, error) { + var blobs []*commonpb.DataBlob + switch attrs := task.GetAttributes().(type) { + case *replicationspb.ReplicationTask_HistoryTaskAttributes: + blobs = attrs.HistoryTaskAttributes.GetEventsBatches() + if attrs.HistoryTaskAttributes.GetEvents() != nil { + blobs = []*commonpb.DataBlob{attrs.HistoryTaskAttributes.GetEvents()} + } + if len(blobs) == 0 { + return nil, errors.New("history replication task has no event batch") + } + case *replicationspb.ReplicationTask_SyncVersionedTransitionTaskAttributes: + artifact := attrs.SyncVersionedTransitionTaskAttributes.GetVersionedTransitionArtifact() + if artifact == nil { + return nil, errors.New("sync versioned transition task has no artifact") + } + blobs = artifact.GetEventBatches() + case *replicationspb.ReplicationTask_BackfillHistoryTaskAttributes: + blobs = attrs.BackfillHistoryTaskAttributes.GetEventBatches() + if len(blobs) == 0 { + return nil, errors.New("backfill history replication task has no event batch") + } + case *replicationspb.ReplicationTask_VerifyVersionedTransitionTaskAttributes, + *replicationspb.ReplicationTask_SyncWorkflowStateTaskAttributes, + *replicationspb.ReplicationTask_SyncActivityTaskAttributes, + *replicationspb.ReplicationTask_SyncHsmAttributes: + return nil, nil + default: + if _, ok := getParentChildReplicationTaskMetadata(task); ok { + return nil, nil + } + return nil, errors.New("replication task has no workflow metadata") + } + + var events []*historypb.HistoryEvent + for _, blob := range blobs { + batch, err := serialization.DefaultDecoder.DeserializeEvents(blob) + if err != nil { + return nil, err + } + events = append(events, batch...) + } + return events, nil +} + +func historyContainsEvent(events []*historypb.HistoryEvent, eventType enumspb.EventType) bool { + for _, event := range events { + if event.GetEventType() == eventType { + return true + } + } + return false +} + +func findHistoryEvent( + events []*historypb.HistoryEvent, + eventType enumspb.EventType, + matches func(*historypb.HistoryEvent) bool, +) *historypb.HistoryEvent { + for _, event := range events { + if event.GetEventType() == eventType && (matches == nil || matches(event)) { + return event + } + } + return nil +} + +func formatParentChildEventTypes(events []*historypb.HistoryEvent) string { + eventTypes := make([]string, 0, len(events)) + for _, event := range events { + eventTypes = append(eventTypes, strings.TrimPrefix(event.GetEventType().String(), "EVENT_TYPE_")) + } + return strings.Join(eventTypes, ", ") +} + +func formatParentChildReplicationTask( + task *replicationspb.ReplicationTask, + events []*historypb.HistoryEvent, +) string { + if len(events) != 0 { + return formatParentChildEventTypes(events) + } + switch task.GetAttributes().(type) { + case *replicationspb.ReplicationTask_SyncWorkflowStateTaskAttributes: + return "SyncWorkflowState" + case *replicationspb.ReplicationTask_SyncActivityTaskAttributes: + return "SyncActivity" + case *replicationspb.ReplicationTask_SyncHsmAttributes: + return "SyncHSM" + case *replicationspb.ReplicationTask_VerifyVersionedTransitionTaskAttributes: + return "VerifyVersionedTransition" + case *replicationspb.ReplicationTask_SyncVersionedTransitionTaskAttributes: + return "SyncVersionedTransition" + default: + return task.GetTaskType().String() + } +} + +func workflowIDsOnDifferentShards( + namespaceID string, + numHistoryShards int32, +) (parentID string, childID string, parentShardID int32, childShardID int32) { + parentID = "parent-" + uuid.NewString() + parentShardID = common.WorkflowIDToHistoryShard(namespaceID, parentID, numHistoryShards) + for { + childID = "child-" + uuid.NewString() + childShardID = common.WorkflowIDToHistoryShard(namespaceID, childID, numHistoryShards) + if childShardID != parentShardID { + return parentID, childID, parentShardID, childShardID + } + } +} + +func (workflow parentChildWorkflow) String() string { + switch workflow { + case parentWorkflow: + return "parent" + case childWorkflow: + return "child" + default: + return fmt.Sprintf("workflow(%d)", workflow) + } +} + +func (cluster parentChildCluster) String() string { + switch cluster { + case initialActiveCluster: + return "initial active cluster" + case initialStandbyCluster: + return "initial standby cluster" + default: + return fmt.Sprintf("cluster(%d)", cluster) + } +} + +func (action parentChildReplicationTaskAction) String() string { + switch action { + case applyReplicationTask: + return "apply" + case delayReplicationTaskApply: + return "delay-apply" + case ackReplicationTaskWithoutApplying: + return "ack-without-apply" + default: + return fmt.Sprintf("action(%d)", action) + } +} diff --git a/tests/xdc/parent_child_harness_test.go b/tests/xdc/parent_child_harness_test.go new file mode 100644 index 00000000000..c787a243d4c --- /dev/null +++ b/tests/xdc/parent_child_harness_test.go @@ -0,0 +1,398 @@ +package xdc + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/protorequire" +) + +func TestParentChildReplicationGate(t *testing.T) { + t.Run("passes unrelated tasks through", func(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + defer gate.close() + + executeErr := errors.New("execute failed") + var executions atomic.Int32 + err := gate.intercept( + newParentChildHistoryReplicationTask("other-namespace", "parent", 1), + func() error { + executions.Add(1) + return executeErr + }, + ) + require.ErrorIs(t, err, executeErr) + require.Equal(t, int32(1), executions.Load()) + + err = gate.intercept( + newParentChildHistoryReplicationTask("namespace", "other-workflow", 2), + func() error { + executions.Add(1) + return executeErr + }, + ) + require.ErrorIs(t, err, executeErr) + require.Equal(t, int32(2), executions.Load()) + }) + + t.Run("applies and acknowledges without applying exactly once", func(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + defer gate.close() + + applyErr := errors.New("apply failed") + var applied atomic.Int32 + applyResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "parent", 1), + func() error { + applied.Add(1) + return applyErr + }, + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + deferredTask, err := gate.nextForWorkflow(ctx, "parent") + cancel() + require.NoError(t, err) + require.Zero(t, applied.Load()) + require.ErrorIs(t, deferredTask.apply(), applyErr) + require.ErrorIs(t, receiveParentChildInterceptResult(t, applyResult), applyErr) + require.ErrorIs(t, deferredTask.apply(), errParentChildTaskAlreadyResolved) + require.Equal(t, int32(1), applied.Load()) + + var appliedWithoutPermission atomic.Int32 + ackResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "child", 2), + func() error { + appliedWithoutPermission.Add(1) + return nil + }, + ) + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + deferredTask, err = gate.nextForWorkflow(ctx, "child") + cancel() + require.NoError(t, err) + require.NoError(t, deferredTask.acknowledgeWithoutApplying()) + require.NoError(t, receiveParentChildInterceptResult(t, ackResult)) + require.ErrorIs(t, deferredTask.acknowledgeWithoutApplying(), errParentChildTaskAlreadyResolved) + require.Zero(t, appliedWithoutPermission.Load()) + }) + + t.Run("keeps tasks for the other workflow buffered", func(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + defer gate.close() + + childResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "child", 1), + func() error { return nil }, + ) + await.RequireTrue(t, func() bool { return len(gate.pending) == 1 }, time.Second, time.Millisecond) + parentResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "parent", 2), + func() error { return nil }, + ) + await.RequireTrue(t, func() bool { return len(gate.pending) == 2 }, time.Second, time.Millisecond) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + parentTask, err := gate.nextForWorkflow(ctx, "parent") + cancel() + require.NoError(t, err) + require.Equal(t, int64(2), parentTask.task.GetSourceTaskId()) + require.Len(t, gate.buffered["child"], 1) + require.NoError(t, parentTask.acknowledgeWithoutApplying()) + require.NoError(t, receiveParentChildInterceptResult(t, parentResult)) + + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + childTask, err := gate.nextForWorkflow(ctx, "child") + cancel() + require.NoError(t, err) + require.Equal(t, int64(1), childTask.task.GetSourceTaskId()) + require.NoError(t, childTask.acknowledgeWithoutApplying()) + require.NoError(t, receiveParentChildInterceptResult(t, childResult)) + }) + + t.Run("close releases delayed and queued tasks", func(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + delayedResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "parent", 1), + func() error { return nil }, + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + _, err := gate.nextForWorkflow(ctx, "parent") + cancel() + require.NoError(t, err) + + queuedResult := interceptParentChildTask( + gate, + newParentChildHistoryReplicationTask("namespace", "child", 2), + func() error { return nil }, + ) + await.RequireTrue(t, func() bool { return len(gate.pending) == 1 }, time.Second, time.Millisecond) + + gate.close() + gate.close() + require.NoError(t, receiveParentChildInterceptResult(t, delayedResult)) + require.NoError(t, receiveParentChildInterceptResult(t, queuedResult)) + + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + _, err = gate.nextForWorkflow(ctx, "child") + cancel() + require.ErrorIs(t, err, errParentChildReplicationGateClosed) + }) +} + +func TestParentChildHarnessAppliesPreviouslyDelayedTaskAfterActiveClusterChanges(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + defer gate.close() + + events := []*historypb.HistoryEvent{{ + EventId: 3, + EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, + }} + blob, err := serialization.NewSerializer().SerializeEvents(events) + require.NoError(t, err) + task := newParentChildHistoryReplicationTask("namespace", "parent", 1) + task.GetHistoryTaskAttributes().Events = blob + + var executions atomic.Int32 + interceptResult := interceptParentChildTask(gate, task, func() error { + executions.Add(1) + return nil + }) + runtime := &parentChildScenarioRuntime{ + parentID: "parent", + gates: [2]*parentChildReplicationGate{nil, gate}, + delayedTasks: make(map[parentChildReplicationLane]*parentChildReplicationTask), + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + err = delayReplicationAtTaskContainingEvent(initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED).run(ctx, runtime) + cancel() + require.NoError(t, err) + require.Zero(t, executions.Load()) + + runtime.activeClusterIndex = int(initialStandbyCluster) + ctx, cancel = context.WithTimeout(context.Background(), time.Second) + err = applyDelayedReplication(initialStandbyCluster, parentWorkflow).run(ctx, runtime) + cancel() + require.NoError(t, err) + require.NoError(t, receiveParentChildInterceptResult(t, interceptResult)) + require.Equal(t, int32(1), executions.Load()) + require.Contains(t, runtime.trace, " apply delayed task 1 to cluster 1 for parent [WorkflowTaskStarted]") +} + +func TestParentChildHarnessApplyDelayedReplicationRequiresDelayedTask(t *testing.T) { + runtime := &parentChildScenarioRuntime{ + parentID: "parent", + delayedTasks: make(map[parentChildReplicationLane]*parentChildReplicationTask), + } + + err := applyDelayedReplication(initialStandbyCluster, parentWorkflow).run(context.Background(), runtime) + require.ErrorContains(t, err, "no delayed parent replication task to initial standby cluster") +} + +func TestParentChildHarnessVersionedTransitionApplyThrough(t *testing.T) { + gate := newParentChildReplicationGate("namespace", "parent", "child") + defer gate.close() + + var verifyExecutions atomic.Int32 + verifyResult := interceptParentChildTask( + gate, + newParentChildVerifyVersionedTransitionTask("namespace", "parent", "run", 1), + func() error { + verifyExecutions.Add(1) + return nil + }, + ) + await.RequireTrue(t, func() bool { return len(gate.pending) == 1 }, time.Second, time.Millisecond) + + events := []*historypb.HistoryEvent{ + {EventId: 3, EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED}, + {EventId: 4, EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED}, + } + blob, err := serialization.NewSerializer().SerializeEvents(events) + require.NoError(t, err) + var syncExecutions atomic.Int32 + syncResult := interceptParentChildTask( + gate, + newParentChildSyncVersionedTransitionTask("namespace", "parent", "run", 2, blob), + func() error { + syncExecutions.Add(1) + return nil + }, + ) + await.RequireTrue(t, func() bool { return len(gate.pending) == 2 }, time.Second, time.Millisecond) + + runtime := &parentChildScenarioRuntime{ + parentID: "parent", + gates: [2]*parentChildReplicationGate{nil, gate}, + delayedTasks: make(map[parentChildReplicationLane]*parentChildReplicationTask), + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + err = acknowledgeReplicationTaskContainingEventWithoutApplying(initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED).run(ctx, runtime) + cancel() + require.NoError(t, err) + require.NoError(t, receiveParentChildInterceptResult(t, verifyResult)) + require.NoError(t, receiveParentChildInterceptResult(t, syncResult)) + require.Equal(t, int32(1), verifyExecutions.Load()) + require.Zero(t, syncExecutions.Load()) + require.Contains(t, runtime.trace, " apply task 1 to cluster 1 for parent [VerifyVersionedTransition]") + require.Contains(t, runtime.trace, " ack-without-apply task 2 to cluster 1 for parent [WorkflowTaskStarted, WorkflowTaskCompleted]") +} + +func TestParentChildHarnessDecodeReplicationEvents(t *testing.T) { + events := []*historypb.HistoryEvent{ + {EventId: 1, EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED}, + {EventId: 2, EventType: enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED}, + } + blob, err := serialization.NewSerializer().SerializeEvents(events) + require.NoError(t, err) + + t.Run("legacy history", func(t *testing.T) { + task := newParentChildHistoryReplicationTask("namespace", "parent", 1) + task.GetHistoryTaskAttributes().Events = blob + actual, err := decodeParentChildReplicationEvents(task) + require.NoError(t, err) + protorequire.ProtoSliceEqual(t, events, actual) + }) + + t.Run("sync versioned transition", func(t *testing.T) { + actual, err := decodeParentChildReplicationEvents( + newParentChildSyncVersionedTransitionTask("namespace", "parent", "run", 1, blob), + ) + require.NoError(t, err) + protorequire.ProtoSliceEqual(t, events, actual) + }) + + t.Run("backfill history", func(t *testing.T) { + actual, err := decodeParentChildReplicationEvents( + newParentChildBackfillHistoryTask("namespace", "parent", "run", 1, blob), + ) + require.NoError(t, err) + protorequire.ProtoSliceEqual(t, events, actual) + }) + + t.Run("verify versioned transition", func(t *testing.T) { + actual, err := decodeParentChildReplicationEvents( + newParentChildVerifyVersionedTransitionTask("namespace", "parent", "run", 1), + ) + require.NoError(t, err) + require.Empty(t, actual) + }) +} + +func newParentChildHistoryReplicationTask( + namespaceID string, + workflowID string, + sourceTaskID int64, +) *replicationspb.ReplicationTask { + return &replicationspb.ReplicationTask{ + SourceTaskId: sourceTaskID, + Attributes: &replicationspb.ReplicationTask_HistoryTaskAttributes{ + HistoryTaskAttributes: &replicationspb.HistoryTaskAttributes{ + NamespaceId: namespaceID, + WorkflowId: workflowID, + }, + }, + } +} + +func newParentChildSyncVersionedTransitionTask( + namespaceID string, + workflowID string, + runID string, + sourceTaskID int64, + eventBatches ...*commonpb.DataBlob, +) *replicationspb.ReplicationTask { + return &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_SYNC_VERSIONED_TRANSITION_TASK, + SourceTaskId: sourceTaskID, + Attributes: &replicationspb.ReplicationTask_SyncVersionedTransitionTaskAttributes{ + SyncVersionedTransitionTaskAttributes: &replicationspb.SyncVersionedTransitionTaskAttributes{ + NamespaceId: namespaceID, + WorkflowId: workflowID, + RunId: runID, + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{ + EventBatches: eventBatches, + }, + }, + }, + } +} + +func newParentChildVerifyVersionedTransitionTask( + namespaceID string, + workflowID string, + runID string, + sourceTaskID int64, +) *replicationspb.ReplicationTask { + return &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_VERIFY_VERSIONED_TRANSITION_TASK, + SourceTaskId: sourceTaskID, + Attributes: &replicationspb.ReplicationTask_VerifyVersionedTransitionTaskAttributes{ + VerifyVersionedTransitionTaskAttributes: &replicationspb.VerifyVersionedTransitionTaskAttributes{ + NamespaceId: namespaceID, + WorkflowId: workflowID, + RunId: runID, + }, + }, + } +} + +func newParentChildBackfillHistoryTask( + namespaceID string, + workflowID string, + runID string, + sourceTaskID int64, + eventBatches ...*commonpb.DataBlob, +) *replicationspb.ReplicationTask { + return &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_BACKFILL_HISTORY_TASK, + SourceTaskId: sourceTaskID, + Attributes: &replicationspb.ReplicationTask_BackfillHistoryTaskAttributes{ + BackfillHistoryTaskAttributes: &replicationspb.BackfillHistoryTaskAttributes{ + NamespaceId: namespaceID, + WorkflowId: workflowID, + RunId: runID, + EventBatches: eventBatches, + }, + }, + } +} + +func interceptParentChildTask( + gate *parentChildReplicationGate, + task *replicationspb.ReplicationTask, + execute func() error, +) <-chan error { + result := make(chan error, 1) + go func() { + result <- gate.intercept(task, execute) + }() + return result +} + +func receiveParentChildInterceptResult(t *testing.T, result <-chan error) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(time.Second): + t.Fatal("timed out waiting for replication interceptor") + return nil + } +} diff --git a/tests/xdc/parent_child_test.go b/tests/xdc/parent_child_test.go new file mode 100644 index 00000000000..d98c5ade1ed --- /dev/null +++ b/tests/xdc/parent_child_test.go @@ -0,0 +1,523 @@ +package xdc + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/suite" + enumspb "go.temporal.io/api/enums/v1" + historypb "go.temporal.io/api/history/v1" + "go.temporal.io/api/serviceerror" + enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/tests/testcore" +) + +func TestParentChildXDCTestSuite(t *testing.T) { + t.Parallel() + suite.Run(t, new(parentChildXDCTestSuite)) +} + +func (s *parentChildXDCTestSuite) SetupSuite() { + s.enableTransitionHistory = true + s.setupSuite(testcore.WithNumHistoryShards(2)) +} + +func (s *parentChildXDCTestSuite) SetupTest() { + s.setupTest() +} + +func (s *parentChildXDCTestSuite) TearDownSuite() { + s.tearDownSuite() +} + +// TestReproOrphanedChildAfterForceFailover covers the cross-shard ordering where the child start +// reaches the passive while parent replication from WorkflowTaskStarted onward, including +// StartChildWorkflowExecutionInitiated, remains delayed, followed by force failover. +// +// | parent on target | child on target | active | outcome +// -----------------------+------------------------------------------+---------------------------------+-----------------+------------------------------ +// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix +// child start arrives | parent update remains delayed | RUNNING, points to old version | initial active | cross-shard partial state +// force failover | incomplete branch becomes current | unchanged | initial standby | target begins active recovery +// retry StartChild | same initiated event ID at a new version | unchanged | initial standby | WORKFLOW_ALREADY_EXISTS +// assert | no ChildWorkflowExecutionStarted | still points to losing version | initial standby | child is orphaned +// +// Event checkpoints select an entire replication task, not an individual event. The delayed parent +// task intentionally remains unapplied through the assertions. +func (s *parentChildXDCTestSuite) TestReproOrphanedChildAfterForceFailover() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + // Create the parent and its first workflow task on the initial active cluster. + startParentWorkflow(), + // Give the passive a common parent prefix before introducing the cross-shard gap. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + // Complete the parent task with StartChild, creating the child on the source. + completeParentWorkflowTaskWithStartChildCommand(), + // Stop parent replication before its StartChild branch is complete on the passive. + delayReplicationAtTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, + ), + // Let the child arrive independently, still pointing to the old parent branch. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Promote the incomplete passive state without draining the delayed parent task. + forceFailoverNamespaceTo(initialStandbyCluster), + // Retry StartChild on the new active branch, where the child ID already exists. + completeParentWorkflowTaskWithStartChildCommand(), + }, + expectations: []parentChildExpectation{ + { + name: "parent StartChild fails because the child already exists", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + events, err := runtime.activeWorkflowHistory(ctx, parentWorkflow) + if err != nil { + return err + } + failedEvent := findHistoryEvent( + events, + enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED, + func(event *historypb.HistoryEvent) bool { + attrs := event.GetStartChildWorkflowExecutionFailedEventAttributes() + return attrs.GetWorkflowId() == runtime.childID && + attrs.GetCause() == enumspb.START_CHILD_WORKFLOW_EXECUTION_FAILED_CAUSE_WORKFLOW_ALREADY_EXISTS + }, + ) + if failedEvent == nil { + return fmt.Errorf("parent has no WORKFLOW_ALREADY_EXISTS failure for child %q", runtime.childID) + } + return nil + }, + }, + currentWorkflowHasStatusOnCluster( + initialStandbyCluster, + childWorkflow, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + ), + { + name: "child remains attached to the losing parent branch", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + if runtime.childRunID == "" { + return errors.New("child WorkflowExecutionStarted was not applied") + } + childEvents, err := runtime.activeWorkflowHistory(ctx, childWorkflow) + if err != nil { + return err + } + childStartedEvent := findHistoryEvent( + childEvents, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + nil, + ) + if childStartedEvent == nil { + return fmt.Errorf("child run %q has no persisted WorkflowExecutionStarted event", runtime.childRunID) + } + childStartedAttrs := childStartedEvent.GetWorkflowExecutionStartedEventAttributes() + if childStartedAttrs == nil { + return errors.New("persisted child start event has no attributes") + } + parentExecution := childStartedAttrs.GetParentWorkflowExecution() + if parentExecution.GetWorkflowId() != runtime.parentID || parentExecution.GetRunId() != runtime.parentRunID { + return fmt.Errorf( + "child parent is %s/%s, want %s/%s", + parentExecution.GetWorkflowId(), + parentExecution.GetRunId(), + runtime.parentID, + runtime.parentRunID, + ) + } + + parentEvents, err := runtime.activeWorkflowHistory(ctx, parentWorkflow) + if err != nil { + return err + } + currentInitiatedEvent := findHistoryEvent( + parentEvents, + enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED, + func(event *historypb.HistoryEvent) bool { + attrs := event.GetStartChildWorkflowExecutionInitiatedEventAttributes() + return attrs.GetWorkflowId() == runtime.childID && + event.GetEventId() == childStartedAttrs.GetParentInitiatedEventId() + }, + ) + currentChildStartedEvent := findHistoryEvent( + parentEvents, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + func(event *historypb.HistoryEvent) bool { + attrs := event.GetChildWorkflowExecutionStartedEventAttributes() + return attrs.GetWorkflowExecution().GetWorkflowId() == runtime.childID && + attrs.GetWorkflowExecution().GetRunId() == runtime.childRunID + }, + ) + if currentChildStartedEvent != nil { + return errors.New("current parent branch owns the applied child") + } + if currentInitiatedEvent == nil { + return fmt.Errorf( + "current parent branch has no StartChild event %d for child %q", + childStartedAttrs.GetParentInitiatedEventId(), + runtime.childID, + ) + } + if currentInitiatedEvent.GetVersion() == childStartedAttrs.GetParentInitiatedEventVersion() { + return fmt.Errorf( + "child and current parent branch have the same initiation version %d", + currentInitiatedEvent.GetVersion(), + ) + } + runtime.tracef( + "orphan confirmed: child parent pointer=(%d, v%d), current parent initiation=(%d, v%d)", + childStartedAttrs.GetParentInitiatedEventId(), + childStartedAttrs.GetParentInitiatedEventVersion(), + currentInitiatedEvent.GetEventId(), + currentInitiatedEvent.GetVersion(), + ) + return nil + }, + }, + }, + }) +} + +// TestStandbyVerifiesMissingChild covers the cross-shard ordering where the parent update +// identifying a started child reaches the passive while the child's WorkflowExecutionStarted +// task remains delayed. +// +// | parent on target | child on target | active | outcome +// ---------------------------+---------------------------------+-----------------+----------------+--------------------------------------------- +// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix +// child start is delayed | unchanged | does not exist | initial active | child start remains unapplied +// parent child-start arrives | ChildWorkflowExecutionStarted | does not exist | initial active | standby verifies the child's first WFT +// verification fails | child-start relationship exists | does not exist | initial active | VerifyFirstWorkflowTaskScheduled: NotFound +// discard window expires | unchanged | does not exist | initial active | standby StartChild task is discarded +// +// Event checkpoints select an entire replication task, not an individual event. The delayed child +// start task intentionally remains unapplied while the standby StartChild task is allowed to expire. +func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + // Track source time without the production standby lag so task expiration is observable quickly. + setStandbyClusterDelay(initialStandbyCluster, 0), + // Create the parent and its first workflow task on the initial active cluster. + startParentWorkflow(), + // Establish the parent on the passive before replicating its child relationship. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + // Complete the parent task with StartChild, creating the child on the source. + completeParentWorkflowTaskWithStartChildCommand(), + // Keep the child start off the passive so the child is locally missing there. + delayReplicationAtTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Apply the parent's child-start record, triggering verification of the missing child. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + // Observe the exact missing-child error before shortening the task's discard window. + waitForHistoryVerificationFailureOnCluster( + initialStandbyCluster, + historyClientVerifyFirstWorkflowTask, + &serviceerror.NotFound{}, + ), + // Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded. + setStandbyTaskDiscardDelay( + initialStandbyCluster, + enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, + 0, + ), + }, + expectations: []parentChildExpectation{ + workflowIsMissingOnCluster(initialStandbyCluster, childWorkflow), + taskWasDiscardedOnCluster( + initialStandbyCluster, + metrics.TaskTypeTransferStandbyTaskStartChildExecution, + ), + }, + }) +} + +// TestStandbyVerifiesChildWithoutFirstWorkflowTask covers the cross-shard ordering where the +// child's WorkflowExecutionStarted reaches the passive, its first WorkflowTaskScheduled remains +// delayed, and the parent update identifying the started child then arrives. +// +// | parent on target | child on target | active | outcome +// ---------------------------+---------------------------------+----------------------------------+----------------+---------------------------------------------- +// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix +// child start arrives | unchanged | CREATED/RUNNING, no first WFT | initial active | child execution exists on the target +// child first WFT is delayed | unchanged | unchanged | initial active | WorkflowTaskScheduled remains unapplied +// parent child-start arrives | ChildWorkflowExecutionStarted | unchanged | initial active | standby verifies the child's first WFT +// verification fails | child-start relationship exists | next event ID 2, no scheduled ID | initial active | VerifyFirstWorkflowTaskScheduled: WorkflowNotReady +// discard window expires | unchanged | unchanged | initial active | standby StartChild task is discarded +// +// This scenario uses legacy history replication because it fixes each transaction's event range. +// The gate can therefore apply WorkflowExecutionStarted while keeping the separate +// WorkflowTaskScheduled task delayed through the assertions. +func (s *parentChildXDCTestSuite) TestStandbyVerifiesChildWithoutFirstWorkflowTask() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + // Keep child Started and its first WFT in separate event-range replication tasks. + useLegacyHistoryReplication(), + // Track source time without the production standby lag so task expiration is observable quickly. + setStandbyClusterDelay(initialStandbyCluster, 0), + // Create the parent and its first workflow task on the initial active cluster. + startParentWorkflow(), + // Establish the parent on the passive before replicating its child relationship. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + // Complete the parent task with StartChild, creating the child on the source. + completeParentWorkflowTaskWithStartChildCommand(), + // Create the child on the passive without advancing through its first WFT. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Hold the child's first WFT so its passive mutable state remains not ready. + delayReplicationAtTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + // Apply the parent's child-start record, triggering verification of that partial child. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + // Observe the exact not-ready error before shortening the task's discard window. + waitForHistoryVerificationFailureOnCluster( + initialStandbyCluster, + historyClientVerifyFirstWorkflowTask, + &serviceerror.WorkflowNotReady{}, + ), + // Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded. + setStandbyTaskDiscardDelay( + initialStandbyCluster, + enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, + 0, + ), + }, + expectations: []parentChildExpectation{ + { + name: "child exists without its first workflow task on the initial standby cluster", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + mutableState, err := runtime.workflowMutableState(ctx, initialStandbyCluster, childWorkflow) + if err != nil { + return err + } + if state := mutableState.GetExecutionState().GetState(); state != enumsspb.WORKFLOW_EXECUTION_STATE_CREATED { + return fmt.Errorf("child state is %s, want %s", state, enumsspb.WORKFLOW_EXECUTION_STATE_CREATED) + } + if status := mutableState.GetExecutionState().GetStatus(); status != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING { + return fmt.Errorf("child status is %s, want %s", status, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING) + } + if scheduledEventID := mutableState.GetExecutionInfo().GetWorkflowTaskScheduledEventId(); scheduledEventID != common.EmptyEventID { + return fmt.Errorf("child workflow task scheduled event ID is %d, want %d", scheduledEventID, common.EmptyEventID) + } + if nextEventID := mutableState.GetNextEventId(); nextEventID != common.FirstEventID+1 { + return fmt.Errorf("child next event ID is %d, want %d", nextEventID, common.FirstEventID+1) + } + return nil + }, + }, + taskWasDiscardedOnCluster( + initialStandbyCluster, + metrics.TaskTypeTransferStandbyTaskStartChildExecution, + ), + }, + }) +} + +// TestStandbyDiscardsChildCloseTaskWhenParentCompletionIsMissing covers the cross-shard ordering +// where the child's completion reaches the passive while the parent's corresponding +// ChildWorkflowExecutionCompleted update remains delayed. +// +// | parent on target | child on target | active | outcome +// ---------------------------+-----------------------------------+-----------------+----------------+---------------------------------------------- +// parent-child state arrives | ChildWorkflowExecutionStarted | RUNNING | initial active | both sides initially agree that child is running +// child closes on source | unchanged on target | unchanged | initial active | source parent records child completion +// parent completion delayed | still tracks the running child | unchanged | initial active | parent completion remains unapplied +// child completion arrives | no child-completion event | COMPLETED | initial active | VerifyChildExecutionCompletionRecorded: WorkflowNotReady +// discard window expires | unchanged | COMPLETED | initial active | standby CloseExecution task is discarded +// +// The local parent-verification grace is kept longer than the compressed discard window so this +// scenario exercises the retry-and-discard fallback rather than parent state resend. The delayed +// parent completion task remains unapplied through the assertions. +func (s *parentChildXDCTestSuite) TestStandbyDiscardsChildCloseTaskWhenParentCompletionIsMissing() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + // Track source time without the production standby lag so task expiration is observable quickly. + setStandbyClusterDelay(initialStandbyCluster, 0), + // Create the parent and its first workflow task on the initial active cluster. + startParentWorkflow(), + // Establish the parent on the passive before replicating its child relationship. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + // Start the child on the source and produce both sides of the relationship. + completeParentWorkflowTaskWithStartChildCommand(), + // Replicate the child so its later completion can run the passive CloseExecution task. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Let the passive parent record that the child started and is still running. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + // Close the child on the source, which also records completion in the source parent. + completeChildWorkflowTask(), + // Wait for and hold the parent completion so the passive still considers the child running. + delayReplicationAtTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED, + ), + // Prevent this scenario from switching to parent state resend before the task is discarded. + setLocalParentVerificationGrace(initialStandbyCluster, time.Hour), + // Apply child completion, triggering verification against the incomplete passive parent. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED, + ), + // Observe the exact not-ready error before shortening the task's discard window. + waitForHistoryVerificationFailureOnCluster( + initialStandbyCluster, + historyClientVerifyChildCompletion, + &serviceerror.WorkflowNotReady{}, + ), + // Expire only the pending standby CloseExecution task; its next attempt should be discarded. + setStandbyTaskDiscardDelay( + initialStandbyCluster, + enumsspb.TASK_TYPE_TRANSFER_CLOSE_EXECUTION, + 0, + ), + }, + expectations: []parentChildExpectation{ + currentWorkflowHasStatusOnCluster( + initialStandbyCluster, + childWorkflow, + enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + ), + { + name: "parent completion remains delayed on the initial standby cluster", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + events, err := runtime.workflowHistoryOnCluster(ctx, initialStandbyCluster, parentWorkflow) + if err != nil { + return err + } + if historyContainsEvent(events, enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED) { + return errors.New("passive parent already contains ChildWorkflowExecutionCompleted") + } + return nil + }, + }, + taskWasDiscardedOnCluster( + initialStandbyCluster, + metrics.TaskTypeTransferStandbyTaskCloseExecution, + ), + }, + }) +} + +// TestStandbyResendsMissingParentWhenChildCloses covers the cross-shard ordering where child +// replication reaches the passive while ordinary parent replication remains delayed. When the +// child's completion arrives, standby verification finds the parent missing and resends its state. +// +// | parent on target | child on target | active | outcome +// -------------------------+-------------------------------------+--------------------------+----------------+----------------------------------------------- +// parent start is delayed | does not exist | does not exist | initial active | ordinary parent replication remains unapplied +// child start arrives | does not exist | RUNNING, points to parent | initial active | cross-shard partial state +// child closes on source | does not exist | unchanged on target | initial active | source parent records child completion +// confirm missing parent | does not exist | unchanged | initial active | establishes the resend precondition +// child completion arrives | restored by parent state sync | COMPLETED | initial active | VerifyChild detects the missing parent +// assert | has ChildWorkflowExecutionCompleted | COMPLETED | initial active | resend observed; parent state restored +// +// The original parent replication task remains delayed. The local verification grace is set to zero +// so VerifyChildExecutionCompletionRecorded requests a parent resend within the test timeout. +func (s *parentChildXDCTestSuite) TestStandbyResendsMissingParentWhenChildCloses() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + // Create the parent and its first workflow task on the initial active cluster. + startParentWorkflow(), + // Keep all ordinary parent replication off the passive. + delayReplicationAtTaskContainingEvent( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Start the child on the source while its parent remains absent on the passive. + completeParentWorkflowTaskWithStartChildCommand(), + // Replicate only the child, creating a child whose parent is locally missing. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, + ), + // Close the child on the source to produce child-completion verification work. + completeChildWorkflowTask(), + // Ensure a state resend will include the parent's child-completion event. + waitForWorkflowEventOnCluster( + initialActiveCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED, + ), + // Confirm the passive still lacks the parent before delivering child completion. + confirmWorkflowIsMissingOnCluster(initialStandbyCluster, parentWorkflow), + // Skip the normal local retry window so the next verification requests a resend. + setLocalParentVerificationGrace(initialStandbyCluster, 0), + // Deliver child completion, causing standby verification to resend the missing parent. + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED, + ), + }, + expectations: []parentChildExpectation{ + { + name: "parent workflow resend is attempted on the initial standby cluster", + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.requireCapturedMetric( + initialStandbyCluster, + metrics.ParentWorkflowResendAttempts.Name(), + nil, + ) + }, + }, + workflowHasEventOnCluster( + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED, + ), + }, + }) +} From ee5b0d8c1a2be4e681f95424ad6dcac97ba9d40a Mon Sep 17 00:00:00 2001 From: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Date: Tue, 25 Aug 2026 10:15:40 -0700 Subject: [PATCH 060/139] Fix batch activity unpause visibility query scope (#11725) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Preserved the caller-provided visibility scope when starting type-based batch activity unpause operations. The activity-type predicate is now safely combined with the original query. ## Why? The server previously replaced the caller’s visibility query with the activity-type predicate. This could broaden the batch scope and unpause activities in workflows the caller did not select. ## How did you test it? - [X] built - [ ] run locally and tested manually - [ ] covered by existing tests - [X] added new unit test(s) - [ ] added new functional test(s) ## Potential risks Type-based unpause batches now strictly honor the caller’s visibility scope, so they may process fewer workflows than before, but this is the correct design --- service/frontend/workflow_handler.go | 52 +++++++-- service/frontend/workflow_handler_test.go | 135 ++++++++++++++++++++++ 2 files changed, 179 insertions(+), 8 deletions(-) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 4db0c5a4053..b2441d39caa 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -69,6 +69,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/persistence/visibility/manager" + "go.temporal.io/server/common/persistence/visibility/store/query" "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/priorities" @@ -79,6 +80,7 @@ import ( "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" + "go.temporal.io/server/common/sqlquery" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/util" @@ -5796,7 +5798,7 @@ func (wh *WorkflowHandler) StartBatchOperation( // Malformed queries (e.g. "()") would otherwise cause the batch activity // to retry indefinitely since the error is not marked non-retryable. if q := request.GetVisibilityQuery(); len(q) > 0 { - if _, err := sqlparser.Parse("select * from dummy where " + q); err != nil { + if _, err := sqlparser.Parse(fmt.Sprintf(sqlquery.QueryTemplate, q)); err != nil { return nil, serviceerror.NewInvalidArgumentf("invalid visibility query: %v", err) } } @@ -5881,13 +5883,13 @@ func (wh *WorkflowHandler) StartBatchOperation( input.BatchType = enumspb.BATCH_OPERATION_TYPE_UNPAUSE_ACTIVITY identity = op.UnpauseActivitiesOperation.GetIdentity() - switch a := op.UnpauseActivitiesOperation.GetActivity().(type) { - case *batchpb.BatchOperationUnpauseActivities_Type: - searchValue := fmt.Sprintf("property:activityType=%s", a.Type) - escapedSearchValue := sqlparser.String(sqlparser.NewStrVal([]byte(searchValue))) - input.Request.VisibilityQuery = fmt.Sprintf("%s = %s", sadefs.TemporalPauseInfo, escapedSearchValue) - case *batchpb.BatchOperationUnpauseActivities_MatchAll: - input.Request.VisibilityQuery = visibilityQuery + if a, ok := op.UnpauseActivitiesOperation.GetActivity().(*batchpb.BatchOperationUnpauseActivities_Type); ok { + unpauseQuery, err := buildUnpauseActivityVisibilityQuery(visibilityQuery, a.Type) + if err != nil { + return nil, err + } + input.Request = proto.CloneOf(request) + input.Request.VisibilityQuery = unpauseQuery } case *workflowservice.StartBatchOperationRequest_ResetActivitiesOperation: input.BatchType = enumspb.BATCH_OPERATION_TYPE_RESET_ACTIVITY @@ -5960,6 +5962,40 @@ func (wh *WorkflowHandler) StartBatchOperation( return &workflowservice.StartBatchOperationResponse{}, nil } +// buildUnpauseActivityVisibilityQuery narrows the caller's scope to workflows containing a +// paused activity of the requested type. Only the WHERE expression is returned because the batch +// worker appends its own execution-status filter. +func buildUnpauseActivityVisibilityQuery(q string, activityType string) (string, error) { + if q == "" { + // An empty query means the caller targeted executions explicitly, so there is no scope to + // narrow. + return "", nil + } + + // Visibility queries are predicate fragments, so wrap the query in a synthetic SELECT to obtain + // a WHERE expression. Reject set operations (for example, UNION) because they do not expose a + // single WHERE clause to extend. + stmt, err := sqlparser.Parse(fmt.Sprintf(sqlquery.QueryTemplate, q)) + if err != nil { + return "", serviceerror.NewInvalidArgumentf("invalid visibility query: %v", err) + } + selectStmt, ok := stmt.(*sqlparser.Select) + if !ok { + return "", serviceerror.NewInvalidArgument("invalid visibility query: set operations are not supported") + } + + activityTypeExpr := &sqlparser.ComparisonExpr{ + Operator: sqlparser.EqualStr, + Left: query.NewColName(sadefs.TemporalPauseInfo), + Right: sqlparser.NewStrVal(fmt.Appendf(nil, "property:activityType=%s", activityType)), + } + selectStmt.Where.Expr = &sqlparser.AndExpr{ + Left: activityTypeExpr, + Right: &sqlparser.ParenExpr{Expr: selectStmt.Where.Expr}, + } + return sqlparser.String(selectStmt.Where.Expr), nil +} + // snakeCaseBatchType maps a batch operation type enum to the canonical string // stored in the batcher workflow memo (see batcher.BatchType* constants). // Workflow operation strings retain their legacy values so old frontends can diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index b58abcc4a81..a896ebb6e5e 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -76,6 +76,7 @@ import ( "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/grpc/health" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/fieldmaskpb" "google.golang.org/protobuf/types/known/timestamppb" @@ -2868,6 +2869,140 @@ func (s *WorkflowHandlerSuite) TestStartBatchOperation_Signal() { s.NoError(err) } +func TestBuildUnpauseActivityVisibilityQuery(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + query string + activityType string + want string + wantErr bool + }{ + { + name: "explicit targets", + activityType: "ActivityFunc", + }, + { + name: "scoped query", + query: "WorkflowType='ScopedWorkflow'", + activityType: "ActivityFunc", + want: "TemporalPauseInfo = 'property:activityType=ActivityFunc' and (WorkflowType = 'ScopedWorkflow')", + }, + { + name: "boolean query preserves precedence and escapes activity type", + query: "WorkflowType='ScopedWorkflow' OR WorkflowType='OtherWorkflow'", + activityType: "Activity'Func", + want: "TemporalPauseInfo = 'property:activityType=Activity\\'Func' and (WorkflowType = 'ScopedWorkflow' or WorkflowType = 'OtherWorkflow')", + }, + { + name: "order by is dropped", + query: "WorkflowType='ScopedWorkflow' ORDER BY StartTime DESC", + activityType: "ActivityFunc", + want: "TemporalPauseInfo = 'property:activityType=ActivityFunc' and (WorkflowType = 'ScopedWorkflow')", + }, + { + name: "set operation", + query: "WorkflowId='test' UNION SELECT * FROM other", + activityType: "ActivityFunc", + wantErr: true, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := buildUnpauseActivityVisibilityQuery(tt.query, tt.activityType) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func (s *WorkflowHandlerSuite) TestStartBatchOperation_UnpauseActivitiesPreservesTargetScope() { + testNamespace := namespace.Name("test-namespace") + targetExecution := &commonpb.Execution{ + Type: enumspb.EXECUTION_TYPE_WORKFLOW, + BusinessId: uuid.NewString(), + RunId: uuid.NewString(), + } + testCases := []struct { + name string + request *workflowservice.StartBatchOperationRequest + wantQuery string + wantTarget *commonpb.Execution + }{ + { + name: "visibility query", + request: &workflowservice.StartBatchOperationRequest{ + Namespace: testNamespace.String(), + VisibilityQuery: "WorkflowType='ScopedWorkflow'", + JobId: uuid.NewString(), + Reason: "test", + Operation: &workflowservice.StartBatchOperationRequest_UnpauseActivitiesOperation{ + UnpauseActivitiesOperation: &batchpb.BatchOperationUnpauseActivities{ + Activity: &batchpb.BatchOperationUnpauseActivities_Type{Type: "ActivityFunc"}, + }, + }, + }, + wantQuery: "TemporalPauseInfo = 'property:activityType=ActivityFunc' and (WorkflowType = 'ScopedWorkflow')", + }, + { + name: "explicit target", + request: &workflowservice.StartBatchOperationRequest{ + Namespace: testNamespace.String(), + TargetExecutions: []*commonpb.Execution{targetExecution}, + JobId: uuid.NewString(), + Reason: "test", + Operation: &workflowservice.StartBatchOperationRequest_UnpauseActivitiesOperation{ + UnpauseActivitiesOperation: &batchpb.BatchOperationUnpauseActivities{ + Activity: &batchpb.BatchOperationUnpauseActivities_Type{Type: "ActivityFunc"}, + }, + }, + }, + wantTarget: targetExecution, + }, + } + + for _, tt := range testCases { + s.Run(tt.name, func() { + namespaceID := namespace.ID(uuid.NewString()) + wh := s.getWorkflowHandler(s.newConfig()) + originalRequest := proto.Clone(tt.request).(*workflowservice.StartBatchOperationRequest) + var input batchspb.BatchOperationInput + + s.mockNamespaceCache.EXPECT().GetNamespaceID(gomock.Any()).Return(namespaceID, nil).AnyTimes() + s.mockVisibilityMgr.EXPECT().CountWorkflowExecutions(gomock.Any(), gomock.Any()).Return(&manager.CountWorkflowExecutionsResponse{Count: 0}, nil) + s.mockHistoryClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn( + func( + _ context.Context, + request *historyservice.StartWorkflowExecutionRequest, + _ ...grpc.CallOption, + ) (*historyservice.StartWorkflowExecutionResponse, error) { + s.Require().NoError(payloads.Decode(request.StartRequest.Input, &input)) + return &historyservice.StartWorkflowExecutionResponse{}, nil + }, + ) + + _, err := wh.StartBatchOperation(context.Background(), tt.request) + s.Require().NoError(err) + s.ProtoEqual(originalRequest, tt.request) + s.Equal(tt.wantQuery, input.Request.GetVisibilityQuery()) + if tt.wantTarget == nil { + s.Empty(input.Request.GetTargetExecutions()) + } else { + s.Require().Len(input.Request.GetTargetExecutions(), 1) + s.ProtoEqual(tt.wantTarget, input.Request.GetTargetExecutions()[0]) + } + }) + } +} + func (s *WorkflowHandlerSuite) TestStartBatchOperation_WorkflowExecutions_Signal() { testNamespace := namespace.Name("test-namespace") namespaceID := namespace.ID(uuid.NewString()) From cc4199dd0c8d3d63bb86e1930378e76de951ecc0 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 25 Aug 2026 14:38:16 -0700 Subject: [PATCH 061/139] Clean up LRU cache test assertions (#11785) Address hundreds of linter issues. --- common/cache/lru_test.go | 413 ++++++++++++++++++------------------ common/cache/simple_test.go | 55 +++-- 2 files changed, 228 insertions(+), 240 deletions(-) diff --git a/common/cache/lru_test.go b/common/cache/lru_test.go index bb73bc38ab4..f84adf4ec59 100644 --- a/common/cache/lru_test.go +++ b/common/cache/lru_test.go @@ -7,12 +7,12 @@ import ( "time" "github.com/google/uuid" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/testing/await" ) type ( @@ -37,52 +37,52 @@ func TestLRU(t *testing.T) { cache := NewWithMetrics(4, nil, metricsHandler) cache.Put("A", "Foo") - assert.Equal(t, "Foo", cache.Get("A")) - assert.Nil(t, cache.Get("B")) - assert.Equal(t, 1, cache.Size()) + require.Equal(t, "Foo", cache.Get("A")) + require.Nil(t, cache.Get("B")) + require.Equal(t, 1, cache.Size()) snapshot := capture.Snapshot() - assert.Equal(t, float64(4), snapshot[metrics.CacheSize.Name()][0].Value) - assert.Equal(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value) + require.InDelta(t, float64(4), snapshot[metrics.CacheSize.Name()][0].Value, 0) + require.InDelta(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value, 0) capture = metricsHandler.StartCapture() cache.Put("B", "Bar") cache.Put("C", "Cid") cache.Put("D", "Delt") - assert.Equal(t, 4, cache.Size()) + require.Equal(t, 4, cache.Size()) snapshot = capture.Snapshot() - assert.Equal(t, float64(4), snapshot[metrics.CacheUsage.Name()][2].Value) + require.InDelta(t, float64(4), snapshot[metrics.CacheUsage.Name()][2].Value, 0) - assert.Equal(t, "Bar", cache.Get("B")) - assert.Equal(t, "Cid", cache.Get("C")) - assert.Equal(t, "Delt", cache.Get("D")) + require.Equal(t, "Bar", cache.Get("B")) + require.Equal(t, "Cid", cache.Get("C")) + require.Equal(t, "Delt", cache.Get("D")) cache.Put("A", "Foo2") - assert.Equal(t, "Foo2", cache.Get("A")) - assert.Equal(t, 4, cache.Size()) + require.Equal(t, "Foo2", cache.Get("A")) + require.Equal(t, 4, cache.Size()) capture = metricsHandler.StartCapture() cache.Put("E", "Epsi") - assert.Equal(t, "Epsi", cache.Get("E")) - assert.Equal(t, "Foo2", cache.Get("A")) - assert.Nil(t, cache.Get("B")) // Oldest, should be evicted - assert.Equal(t, 4, cache.Size()) + require.Equal(t, "Epsi", cache.Get("E")) + require.Equal(t, "Foo2", cache.Get("A")) + require.Nil(t, cache.Get("B")) // Oldest, should be evicted + require.Equal(t, 4, cache.Size()) snapshot = capture.Snapshot() - assert.Equal(t, 2, len(snapshot[metrics.CacheUsage.Name()])) - assert.Equal(t, float64(4), snapshot[metrics.CacheUsage.Name()][1].Value) + require.Len(t, snapshot[metrics.CacheUsage.Name()], 2) + require.InDelta(t, float64(4), snapshot[metrics.CacheUsage.Name()][1].Value, 0) // Access C, D is now LRU cache.Get("C") cache.Put("F", "Felp") - assert.Nil(t, cache.Get("D")) - assert.Equal(t, 4, cache.Size()) + require.Nil(t, cache.Get("D")) + require.Equal(t, 4, cache.Size()) capture = metricsHandler.StartCapture() cache.Delete("A") - assert.Nil(t, cache.Get("A")) - assert.Equal(t, 3, cache.Size()) + require.Nil(t, cache.Get("A")) + require.Equal(t, 3, cache.Size()) snapshot = capture.Snapshot() - assert.Equal(t, 1, len(snapshot[metrics.CacheUsage.Name()])) - assert.Equal(t, float64(3), snapshot[metrics.CacheUsage.Name()][0].Value) + require.Len(t, snapshot[metrics.CacheUsage.Name()], 1) + require.InDelta(t, float64(3), snapshot[metrics.CacheUsage.Name()][0].Value, 0) } func TestGenerics(t *testing.T) { @@ -97,20 +97,20 @@ func TestGenerics(t *testing.T) { cache := NewLRU(5, metrics.NoopMetricsHandler) cache.Put(key, value) - assert.Equal(t, value, cache.Get(key)) - assert.Equal(t, value, cache.Get(keyType{ + require.Equal(t, value, cache.Get(key)) + require.Equal(t, value, cache.Get(keyType{ dummyString: "some random key", dummyInt: 59, })) - assert.Nil(t, cache.Get(keyType{ + require.Nil(t, cache.Get(keyType{ dummyString: "some other random key", dummyInt: 56, })) - assert.Equal(t, 1, cache.Size()) + require.Equal(t, 1, cache.Size()) cache.Put(key, "some other random value") - assert.Equal(t, "some other random value", cache.Get(key)) - assert.Equal(t, 1, cache.Size()) + require.Equal(t, "some other random value", cache.Get(key)) + require.Equal(t, 1, cache.Size()) } func TestLRUWithTTL(t *testing.T) { @@ -127,20 +127,20 @@ func TestLRUWithTTL(t *testing.T) { metricsHandler, ) cache.Put("A", "foo") - assert.Equal(t, "foo", cache.Get("A")) + require.Equal(t, "foo", cache.Get("A")) snapshot := capture.Snapshot() - assert.Equal(t, float64(5), snapshot[metrics.CacheSize.Name()][0].Value) - assert.Equal(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value) - assert.Equal(t, time.Millisecond*100, snapshot[metrics.CacheTtl.Name()][0].Value) - assert.Equal(t, time.Duration(0), snapshot[metrics.CacheEntryAgeOnGet.Name()][0].Value) + require.InDelta(t, float64(5), snapshot[metrics.CacheSize.Name()][0].Value, 0) + require.InDelta(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value, 0) + require.Equal(t, time.Millisecond*100, snapshot[metrics.CacheTtl.Name()][0].Value) + require.Equal(t, time.Duration(0), snapshot[metrics.CacheEntryAgeOnGet.Name()][0].Value) timeSource.Advance(time.Millisecond * 300) - assert.Nil(t, cache.Get("A")) + require.Nil(t, cache.Get("A")) snapshot = capture.Snapshot() - assert.Equal(t, 2, len(snapshot[metrics.CacheUsage.Name()])) - assert.Equal(t, float64(0), snapshot[metrics.CacheUsage.Name()][1].Value) - assert.Equal(t, 0, cache.Size()) - assert.Equal(t, 1, len(snapshot[metrics.CacheEntryAgeOnGet.Name()])) - assert.Equal(t, time.Millisecond*300, snapshot[metrics.CacheEntryAgeOnEviction.Name()][0].Value) + require.Len(t, snapshot[metrics.CacheUsage.Name()], 2) + require.InDelta(t, float64(0), snapshot[metrics.CacheUsage.Name()][1].Value, 0) + require.Equal(t, 0, cache.Size()) + require.Len(t, snapshot[metrics.CacheEntryAgeOnGet.Name()], 1) + require.Equal(t, time.Millisecond*300, snapshot[metrics.CacheEntryAgeOnEviction.Name()][0].Value) } func TestLRUCacheConcurrentAccess(t *testing.T) { @@ -162,24 +162,18 @@ func TestLRUCacheConcurrentAccess(t *testing.T) { start := make(chan struct{}) var wg sync.WaitGroup for range 20 { - wg.Add(2) - // concurrent get and put - go func() { - defer wg.Done() - + wg.Go(func() { <-start for range 1000 { cache.Get("A") cache.Put("A", "fooo") } - }() + }) // concurrent iteration - go func() { - defer wg.Done() - + wg.Go(func() { <-start for range 50 { @@ -189,7 +183,7 @@ func TestLRUCacheConcurrentAccess(t *testing.T) { } it.Close() } - }() + }) } close(start) @@ -208,9 +202,9 @@ func TestTTL(t *testing.T) { ) cache.Put("A", t) - assert.Equal(t, t, cache.Get("A")) + require.Equal(t, t, cache.Get("A")) timeSource.Advance(time.Millisecond * 100) - assert.Nil(t, cache.Get("A")) + require.Nil(t, cache.Get("A")) } func TestTTLWithPin(t *testing.T) { @@ -229,27 +223,27 @@ func TestTTLWithPin(t *testing.T) { capture := metricsHandler.StartCapture() _, err := cache.PutIfNotExist("A", t) - assert.NoError(t, err) - assert.Equal(t, t, cache.Get("A")) - assert.Equal(t, 1, cache.Size()) + require.NoError(t, err) + require.Equal(t, t, cache.Get("A")) + require.Equal(t, 1, cache.Size()) snapshot := capture.Snapshot() - assert.Equal(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value) - assert.Equal(t, float64(1), snapshot[metrics.CachePinnedUsage.Name()][0].Value) + require.InDelta(t, float64(1), snapshot[metrics.CacheUsage.Name()][0].Value, 0) + require.InDelta(t, float64(1), snapshot[metrics.CachePinnedUsage.Name()][0].Value, 0) capture = metricsHandler.StartCapture() timeSource.Advance(time.Millisecond * 100) - assert.Equal(t, t, cache.Get("A")) - assert.Equal(t, 1, cache.Size()) + require.Equal(t, t, cache.Get("A")) + require.Equal(t, 1, cache.Size()) // release 3 time since put if not exist also increase the counter cache.Release("A") cache.Release("A") cache.Release("A") snapshot = capture.Snapshot() - assert.Equal(t, float64(0), snapshot[metrics.CachePinnedUsage.Name()][0].Value) - assert.Nil(t, cache.Get("A")) - assert.Equal(t, 0, cache.Size()) + require.InDelta(t, float64(0), snapshot[metrics.CachePinnedUsage.Name()][0].Value, 0) + require.Nil(t, cache.Get("A")) + require.Equal(t, 0, cache.Size()) snapshot = capture.Snapshot() // cache.Release() will emit cacheUsage 3 times. cache.Get() will emit cacheUsage once. - assert.Equal(t, float64(0), snapshot[metrics.CacheUsage.Name()][3].Value) + require.InDelta(t, float64(0), snapshot[metrics.CacheUsage.Name()][3].Value, 0) } func TestMaxSizeWithPin_MidItem(t *testing.T) { @@ -265,39 +259,39 @@ func TestMaxSizeWithPin_MidItem(t *testing.T) { ) _, err := cache.PutIfNotExist("A", t) - assert.NoError(t, err) - assert.Equal(t, 1, cache.Size()) + require.NoError(t, err) + require.Equal(t, 1, cache.Size()) _, err = cache.PutIfNotExist("B", t) - assert.NoError(t, err) - assert.Equal(t, 2, cache.Size()) + require.NoError(t, err) + require.Equal(t, 2, cache.Size()) _, err = cache.PutIfNotExist("C", t) - assert.Error(t, err) - assert.Equal(t, 2, cache.Size()) + require.Error(t, err) + require.Equal(t, 2, cache.Size()) - assert.Equal(t, t, cache.Get("A")) + require.Equal(t, t, cache.Get("A")) cache.Release("A") // get will also increase the ref count - assert.Equal(t, t, cache.Get("B")) + require.Equal(t, t, cache.Get("B")) cache.Release("B") // get will also increase the ref count - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) cache.Release("B") // B's ref count is 0 _, err = cache.PutIfNotExist("C", t) - assert.NoError(t, err) - assert.Equal(t, t, cache.Get("C")) + require.NoError(t, err) + require.Equal(t, t, cache.Get("C")) cache.Release("C") // get will also increase the ref count - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) cache.Release("A") // A's ref count is 0 cache.Release("C") // C's ref count is 0 - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) timeSource.Advance(time.Millisecond * 100) - assert.Nil(t, cache.Get("A")) - assert.Nil(t, cache.Get("B")) - assert.Nil(t, cache.Get("C")) - assert.Equal(t, 0, cache.Size()) + require.Nil(t, cache.Get("A")) + require.Nil(t, cache.Get("B")) + require.Nil(t, cache.Get("C")) + require.Equal(t, 0, cache.Size()) } func TestMaxSizeWithPin_LastItem(t *testing.T) { @@ -313,39 +307,39 @@ func TestMaxSizeWithPin_LastItem(t *testing.T) { ) _, err := cache.PutIfNotExist("A", t) - assert.NoError(t, err) - assert.Equal(t, 1, cache.Size()) + require.NoError(t, err) + require.Equal(t, 1, cache.Size()) _, err = cache.PutIfNotExist("B", t) - assert.NoError(t, err) - assert.Equal(t, 2, cache.Size()) + require.NoError(t, err) + require.Equal(t, 2, cache.Size()) _, err = cache.PutIfNotExist("C", t) - assert.Error(t, err) - assert.Equal(t, 2, cache.Size()) + require.Error(t, err) + require.Equal(t, 2, cache.Size()) - assert.Equal(t, t, cache.Get("A")) + require.Equal(t, t, cache.Get("A")) cache.Release("A") // get will also increase the ref count - assert.Equal(t, t, cache.Get("B")) + require.Equal(t, t, cache.Get("B")) cache.Release("B") // get will also increase the ref count - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) cache.Release("A") // A's ref count is 0 _, err = cache.PutIfNotExist("C", t) - assert.NoError(t, err) - assert.Equal(t, t, cache.Get("C")) + require.NoError(t, err) + require.Equal(t, t, cache.Get("C")) cache.Release("C") // get will also increase the ref count - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) cache.Release("B") // B's ref count is 0 cache.Release("C") // C's ref count is 0 - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) timeSource.Advance(time.Millisecond * 100) - assert.Nil(t, cache.Get("A")) - assert.Nil(t, cache.Get("B")) - assert.Nil(t, cache.Get("C")) - assert.Equal(t, 0, cache.Size()) + require.Nil(t, cache.Get("A")) + require.Nil(t, cache.Get("B")) + require.Nil(t, cache.Get("C")) + require.Equal(t, 0, cache.Size()) } func TestIterator(t *testing.T) { @@ -372,7 +366,7 @@ func TestIterator(t *testing.T) { actual[entry.Key().(string)] = entry.Value().(string) } it.Close() - assert.Equal(t, expected, actual) + require.Equal(t, expected, actual) it = cache.Iterator() for i := 0; i < len(expected); i++ { @@ -380,7 +374,7 @@ func TestIterator(t *testing.T) { actual[entry.Key().(string)] = entry.Value().(string) } it.Close() - assert.Equal(t, expected, actual) + require.Equal(t, expected, actual) } func TestZeroSizeCache(t *testing.T) { @@ -388,18 +382,18 @@ func TestZeroSizeCache(t *testing.T) { cache := NewLRU(0, metrics.NoopMetricsHandler) _, err := cache.PutIfNotExist("A", t) - assert.NoError(t, err) - assert.Equal(t, nil, cache.Get("A")) - assert.Equal(t, 0, cache.Size()) + require.NoError(t, err) + require.Nil(t, cache.Get("A")) + require.Equal(t, 0, cache.Size()) it := cache.Iterator() - assert.False(t, it.HasNext()) + require.False(t, it.HasNext()) it.Close() cache.Release("A") cache.Delete("A") v, err := cache.PutIfNotExist("A", t) - assert.Equal(t, v, t) - assert.Nil(t, err) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, t, v) + require.NoError(t, err) + require.Equal(t, 0, cache.Size()) } func TestCache_ItemSizeTooLarge(t *testing.T) { @@ -409,13 +403,13 @@ func TestCache_ItemSizeTooLarge(t *testing.T) { cache := NewLRU(maxTotalBytes, metrics.NoopMetricsHandler) res := cache.Put(uuid.New(), &testEntryWithCacheSize{maxTotalBytes}) - assert.Equal(t, res, nil) - assert.Equal(t, 10, cache.Size()) + require.Nil(t, res) + require.Equal(t, 10, cache.Size()) res, err := cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{maxTotalBytes + 1}) - assert.Equal(t, err, ErrCacheItemTooLarge) - assert.Equal(t, res, nil) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, ErrCacheItemTooLarge, err) + require.Nil(t, res) + require.Equal(t, 10, cache.Size()) } @@ -433,9 +427,10 @@ func TestCache_ItemHasCacheSizeDefined(t *testing.T) { startWG.Add(numPuts) endWG.Add(numPuts) + sizeDuringPuts := make(chan int) go func() { startWG.Wait() - assert.True(t, cache.Size() < maxTotalBytes) + await.Snd(t, sizeDuringPuts, cache.Size()) }() for range numPuts { go func() { @@ -448,7 +443,9 @@ func TestCache_ItemHasCacheSizeDefined(t *testing.T) { startWG.Done() } + size := await.Rcv(t, sizeDuringPuts) endWG.Wait() + require.Less(t, size, maxTotalBytes) } func TestCache_ItemHasCacheSizeDefined_PutWithNewKeys(t *testing.T) { @@ -459,21 +456,21 @@ func TestCache_ItemHasCacheSizeDefined_PutWithNewKeys(t *testing.T) { // Put with new key and value size greater than cache size, should not be added to cache cache.Put(uuid.New(), &testEntryWithCacheSize{15}) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, 0, cache.Size()) // Put with new key and value size less than cache size, should be added to cache cache.Put(uuid.New(), &testEntryWithCacheSize{5}) - assert.Equal(t, 5, cache.Size()) + require.Equal(t, 5, cache.Size()) // Put with new key and value size less than cache size, should evict 0 ref items and added to cache cache.Put(uuid.New(), &testEntryWithCacheSize{10}) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, 10, cache.Size()) // Put with new key and value size less than cache size, should evict 0 ref items until enough spaces and added to cache cache.Put(uuid.New(), &testEntryWithCacheSize{3}) - assert.Equal(t, 3, cache.Size()) + require.Equal(t, 3, cache.Size()) cache.Put(uuid.New(), &testEntryWithCacheSize{7}) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, 10, cache.Size()) } func TestCache_ItemHasCacheSizeDefined_PutWithSameKeyAndDifferentSizes(t *testing.T) { @@ -484,32 +481,32 @@ func TestCache_ItemHasCacheSizeDefined_PutWithSameKeyAndDifferentSizes(t *testin key1 := "A" cache.Put(key1, &testEntryWithCacheSize{4}) - assert.Equal(t, 4, cache.Size()) + require.Equal(t, 4, cache.Size()) key2 := "B" cache.Put(key2, &testEntryWithCacheSize{4}) // 4 + 4 = 8 < 10 should not evict any items - assert.Equal(t, 8, cache.Size()) + require.Equal(t, 8, cache.Size()) // put same key with smaller size, should not evict any items cache.Put(key2, &testEntryWithCacheSize{3}) - assert.Equal(t, cache.Get(key1), &testEntryWithCacheSize{4}) + require.Equal(t, &testEntryWithCacheSize{4}, cache.Get(key1)) // 8 - 4 + 3 = 7 < 10, should not evict any items - assert.Equal(t, 7, cache.Size()) + require.Equal(t, 7, cache.Size()) // put same key with larger size, but below cache size, should not evict any items cache.Put(key2, &testEntryWithCacheSize{6}) // 7 - 3 + 6 = 10 =< 10, should not evict any items - assert.Equal(t, 10, cache.Size()) + require.Equal(t, 10, cache.Size()) // get key1 after to make it the most recently used - assert.Equal(t, cache.Get(key2), &testEntryWithCacheSize{6}) - assert.Equal(t, cache.Get(key1), &testEntryWithCacheSize{4}) + require.Equal(t, &testEntryWithCacheSize{6}, cache.Get(key2)) + require.Equal(t, &testEntryWithCacheSize{4}, cache.Get(key1)) // put same key with larger size, but take all cache size, should evict all items cache.Put(key2, &testEntryWithCacheSize{10}) // 10 - 4 - 6 + 10 = 10 =< 10, should evict all items - assert.Equal(t, 10, cache.Size()) - assert.Equal(t, cache.Get(key1), nil) - assert.Equal(t, cache.Get(key2), &testEntryWithCacheSize{10}) + require.Equal(t, 10, cache.Size()) + require.Nil(t, cache.Get(key1)) + require.Equal(t, &testEntryWithCacheSize{10}, cache.Get(key2)) } func TestCache_ItemHasCacheSizeDefined_PutWithSameKey(t *testing.T) { @@ -522,21 +519,21 @@ func TestCache_ItemHasCacheSizeDefined_PutWithSameKey(t *testing.T) { // Put with same key and value size greater than cache size, should not be added to cache cache.Put(key, &testEntryWithCacheSize{15}) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, 0, cache.Size()) // Put with same key and value size less than cache size, should be added to cache cache.Put(key, &testEntryWithCacheSize{5}) - assert.Equal(t, 5, cache.Size()) + require.Equal(t, 5, cache.Size()) // Put with same key and value size less than cache size, should be evicted until enough space and added to cache cache.Put(key, &testEntryWithCacheSize{10}) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, 10, cache.Size()) // Put with same key and value size less than cache size, should be evicted until enough space and added to cache cache.Put(key, &testEntryWithCacheSize{3}) - assert.Equal(t, 3, cache.Size()) + require.Equal(t, 3, cache.Size()) cache.Put(key, &testEntryWithCacheSize{7}) - assert.Equal(t, 7, cache.Size()) + require.Equal(t, 7, cache.Size()) } func TestCache_ItemHasCacheSizeDefined_PutIfNotExistWithNewKeys(t *testing.T) { @@ -547,27 +544,27 @@ func TestCache_ItemHasCacheSizeDefined_PutIfNotExistWithNewKeys(t *testing.T) { // PutIfNotExist with new keys with size greater than cache size, should return error and not add to cache val, err := cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{15}) - assert.Equal(t, ErrCacheItemTooLarge, err) - assert.Nil(t, val) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, ErrCacheItemTooLarge, err) + require.Nil(t, val) + require.Equal(t, 0, cache.Size()) // PutIfNotExist with new keys with size less than cache size, should add to cache val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{5}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{5}, val) - assert.Equal(t, 5, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{5}, val) + require.Equal(t, 5, cache.Size()) // PutIfNotExist with new keys with size less than cache size, should evict item and add to cache val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{10}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{10}, val) - assert.Equal(t, 10, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{10}, val) + require.Equal(t, 10, cache.Size()) // PutIfNotExist with new keys with size less than cache size, should evict item and add to cache val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{5}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{5}, val) - assert.Equal(t, 5, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{5}, val) + require.Equal(t, 5, cache.Size()) } func TestCache_ItemHasCacheSizeDefined_PutIfNotExistWithSameKey(t *testing.T) { @@ -579,21 +576,21 @@ func TestCache_ItemHasCacheSizeDefined_PutIfNotExistWithSameKey(t *testing.T) { // PutIfNotExist with new keys with size greater than cache size, should return error and not add to cache val, err := cache.PutIfNotExist(key, &testEntryWithCacheSize{15}) - assert.Equal(t, ErrCacheItemTooLarge, err) - assert.Nil(t, val) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, ErrCacheItemTooLarge, err) + require.Nil(t, val) + require.Equal(t, 0, cache.Size()) // PutIfNotExist with new keys with size less than cache size, should add to cache val, err = cache.PutIfNotExist(key, &testEntryWithCacheSize{5}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{5}, val) - assert.Equal(t, 5, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{5}, val) + require.Equal(t, 5, cache.Size()) // PutIfNotExist with same keys with size less than cache size, should not be added to cache val, err = cache.PutIfNotExist(key, &testEntryWithCacheSize{10}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{5}, val) - assert.Equal(t, 5, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{5}, val) + require.Equal(t, 5, cache.Size()) } func TestCache_PutIfNotExistWithNewKeys_Pin(t *testing.T) { @@ -603,24 +600,24 @@ func TestCache_PutIfNotExistWithNewKeys_Pin(t *testing.T) { cache := New(maxTotalBytes, &Options{Pin: true}) val, err := cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{15}) - assert.Equal(t, ErrCacheItemTooLarge, err) - assert.Nil(t, val) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, ErrCacheItemTooLarge, err) + require.Nil(t, val) + require.Equal(t, 0, cache.Size()) val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{3}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{3}, val) - assert.Equal(t, 3, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{3}, val) + require.Equal(t, 3, cache.Size()) val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{7}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{7}, val) - assert.Equal(t, 10, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{7}, val) + require.Equal(t, 10, cache.Size()) val, err = cache.PutIfNotExist(uuid.New(), &testEntryWithCacheSize{8}) - assert.Equal(t, ErrCacheFull, err) - assert.Nil(t, val) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, ErrCacheFull, err) + require.Nil(t, val) + require.Equal(t, 10, cache.Size()) } func TestCache_PutIfNotExistWithSameKeys_Pin(t *testing.T) { @@ -631,19 +628,19 @@ func TestCache_PutIfNotExistWithSameKeys_Pin(t *testing.T) { key := uuid.New() val, err := cache.PutIfNotExist(key, &testEntryWithCacheSize{15}) - assert.Equal(t, ErrCacheItemTooLarge, err) - assert.Nil(t, val) - assert.Equal(t, 0, cache.Size()) + require.Equal(t, ErrCacheItemTooLarge, err) + require.Nil(t, val) + require.Equal(t, 0, cache.Size()) val, err = cache.PutIfNotExist(key, &testEntryWithCacheSize{3}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{3}, val) - assert.Equal(t, 3, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{3}, val) + require.Equal(t, 3, cache.Size()) val, err = cache.PutIfNotExist(key, &testEntryWithCacheSize{7}) - assert.NoError(t, err) - assert.Equal(t, &testEntryWithCacheSize{3}, val) - assert.Equal(t, 3, cache.Size()) + require.NoError(t, err) + require.Equal(t, &testEntryWithCacheSize{3}, val) + require.Equal(t, 3, cache.Size()) } func TestCache_ItemSizeChangeBeforeRelease(t *testing.T) { @@ -663,19 +660,19 @@ func TestCache_ItemSizeChangeBeforeRelease(t *testing.T) { } key1 := uuid.New() _, err := cache.PutIfNotExist(key1, entry1) - assert.NoError(t, err) - assert.Equal(t, 1, cache.Size()) + require.NoError(t, err) + require.Equal(t, 1, cache.Size()) entry1.cacheSize = 5 cache.Release(key1) - assert.Equal(t, 5, cache.Size()) + require.Equal(t, 5, cache.Size()) _, err = cache.PutIfNotExist(key1, entry1) - assert.NoError(t, err) - assert.Equal(t, 5, cache.Size()) + require.NoError(t, err) + require.Equal(t, 5, cache.Size()) entry1.cacheSize = 10 cache.Release(key1) - assert.Equal(t, 10, cache.Size()) + require.Equal(t, 10, cache.Size()) // Inserting another entry when cache is full. entry1 should be evicted from cache. entry2 := &testEntryWithCacheSize{ @@ -683,19 +680,19 @@ func TestCache_ItemSizeChangeBeforeRelease(t *testing.T) { } key2 := uuid.New() _, err = cache.PutIfNotExist(key2, entry2) - assert.NoError(t, err) - assert.Equal(t, 2, cache.Size()) + require.NoError(t, err) + require.Equal(t, 2, cache.Size()) // Inserting entry1 again to make cache full again. entry1.cacheSize = 8 _, err = cache.PutIfNotExist(key1, entry1) - assert.NoError(t, err) - assert.Equal(t, 10, cache.Size()) + require.NoError(t, err) + require.Equal(t, 10, cache.Size()) // Increasing the size of entry1 before releasing. This will make the cache size > max limit. entry1.cacheSize = 10 cache.Release(key1) // Cache should have evicted entry1 to bring cache size under max limit. - assert.Equal(t, 2, cache.Size()) + require.Equal(t, 2, cache.Size()) } func TestCache_InvokeLifecycleCallbacks(t *testing.T) { @@ -709,11 +706,11 @@ func TestCache_InvokeLifecycleCallbacks(t *testing.T) { TTL: ttl, TimeSource: timeSource, OnPut: func(val any) { - require.Equal(t, val, "value") + require.Equal(t, "value", val) onPut++ }, OnEvict: func(val any) { - require.Equal(t, val, "value") + require.Equal(t, "value", val) onEvict++ }, }, @@ -732,7 +729,7 @@ func TestCache_InvokeLifecycleCallbacks(t *testing.T) { cache.Put("key", "value") timeSource.Advance(2 * ttl) - assert.Nil(t, cache.Get("key")) + require.Nil(t, cache.Get("key")) require.Equal(t, 2, onEvict, "expected OnEvict callback to be invoked") } @@ -761,9 +758,9 @@ func TestCache_UnusedExpiry(t *testing.T) { cache.Put(1, 1) r.Equal(1, cache.Size()) - r.Eventually(func() bool { + await.Require(t.Context(), t, func(t *await.T) { timeSource.Advance(loopInterval) - return cache.Size() == 0 + require.Equal(t, 0, cache.Size()) }, 2*time.Second, 100*time.Millisecond) cache.Put(2, 2) @@ -771,14 +768,18 @@ func TestCache_UnusedExpiry(t *testing.T) { cache.Put(3, 3) r.Equal(2, cache.Size()) - r.Eventually(func() bool { + await.Require(t.Context(), t, func(t *await.T) { timeSource.Advance(loopInterval) - return cache.Size() == 1 && cache.Get(2) == nil && cache.Get(3) == 3 + require.Equal(t, 1, cache.Size()) + require.Nil(t, cache.Get(2)) + require.Equal(t, 3, cache.Get(3)) }, 2*time.Second, 100*time.Millisecond) - r.Eventually(func() bool { + await.Require(t.Context(), t, func(t *await.T) { timeSource.Advance(loopInterval) - return cache.Size() == 0 && cache.Get(2) == nil && cache.Get(3) == nil + require.Equal(t, 0, cache.Size()) + require.Nil(t, cache.Get(2)) + require.Nil(t, cache.Get(3)) }, 2*time.Second, 100*time.Millisecond) // Stop the background goroutine, confirm no active expiration. @@ -791,19 +792,12 @@ func TestCache_UnusedExpiry(t *testing.T) { l.loops.Wait() close(c) }() - r.Eventually(func() bool { - select { - case <-c: - return true - default: - return false - } - }, 2*time.Second, 100*time.Millisecond) + await.Rcv(t, c) timeSource.Advance(ttl + 1*time.Second) // The cache should still have entry 4, r.Equal(1, cache.Size()) // but this Get call will check the (hard) ttl & expire it. - r.Equal(nil, cache.Get(4)) + r.Nil(cache.Get(4)) } func TestCache_UnusedExpiryPin(t *testing.T) { @@ -837,15 +831,16 @@ func TestCache_UnusedExpiryPin(t *testing.T) { r.NoError(err) r.Equal(2, cache.Size()) - r.Eventually(func() bool { + await.Require(t.Context(), t, func(t *await.T) { timeSource.Advance(loopInterval) - return cache.Size() == 1 && cache.Get(1) == nil + require.Equal(t, 1, cache.Size()) + require.Nil(t, cache.Get(1)) }, 1*time.Second, 100*time.Millisecond) cache.Release(2) - r.Eventually(func() bool { + await.Require(t.Context(), t, func(t *await.T) { timeSource.Advance(loopInterval) - return cache.Size() == 0 + require.Equal(t, 0, cache.Size()) }, 1*time.Second, 100*time.Millisecond) } diff --git a/common/cache/simple_test.go b/common/cache/simple_test.go index 7c89e3bf8da..4bbdc2c2ec2 100644 --- a/common/cache/simple_test.go +++ b/common/cache/simple_test.go @@ -5,35 +5,35 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestSimple(t *testing.T) { cache := NewSimple(nil) cache.Put("A", "Foo") - assert.Equal(t, "Foo", cache.Get("A")) - assert.Nil(t, cache.Get("B")) - assert.Equal(t, 1, cache.Size()) + require.Equal(t, "Foo", cache.Get("A")) + require.Nil(t, cache.Get("B")) + require.Equal(t, 1, cache.Size()) cache.Put("B", "Bar") cache.Put("C", "Cid") cache.Put("D", "Delt") - assert.Equal(t, 4, cache.Size()) + require.Equal(t, 4, cache.Size()) - assert.Equal(t, "Bar", cache.Get("B")) - assert.Equal(t, "Cid", cache.Get("C")) - assert.Equal(t, "Delt", cache.Get("D")) + require.Equal(t, "Bar", cache.Get("B")) + require.Equal(t, "Cid", cache.Get("C")) + require.Equal(t, "Delt", cache.Get("D")) cache.Put("A", "Foo2") - assert.Equal(t, "Foo2", cache.Get("A")) + require.Equal(t, "Foo2", cache.Get("A")) cache.Put("E", "Epsi") - assert.Equal(t, "Epsi", cache.Get("E")) - assert.Equal(t, "Foo2", cache.Get("A")) + require.Equal(t, "Epsi", cache.Get("E")) + require.Equal(t, "Foo2", cache.Get("A")) cache.Delete("A") - assert.Nil(t, cache.Get("A")) + require.Nil(t, cache.Get("A")) } func TestSimpleGenerics(t *testing.T) { @@ -46,12 +46,12 @@ func TestSimpleGenerics(t *testing.T) { cache := NewSimple(nil) cache.Put(key, value) - assert.Equal(t, value, cache.Get(key)) - assert.Equal(t, value, cache.Get(keyType{ + require.Equal(t, value, cache.Get(key)) + require.Equal(t, value, cache.Get(keyType{ dummyString: "some random key", dummyInt: 59, })) - assert.Nil(t, cache.Get(keyType{ + require.Nil(t, cache.Get(keyType{ dummyString: "some other random key", dummyInt: 56, })) @@ -74,24 +74,18 @@ func TestSimpleCacheConcurrentAccess(t *testing.T) { start := make(chan struct{}) var wg sync.WaitGroup for range 20 { - wg.Add(2) - // concurrent get and put - go func() { - defer wg.Done() - + wg.Go(func() { <-start for range 1000 { cache.Get("A") cache.Put("A", "fooo") } - }() + }) // concurrent iteration - go func() { - defer wg.Done() - + wg.Go(func() { <-start for range 50 { @@ -101,7 +95,7 @@ func TestSimpleCacheConcurrentAccess(t *testing.T) { } it.Close() } - }() + }) } close(start) @@ -113,19 +107,18 @@ func TestSimpleRemoveFunc(t *testing.T) { cache := NewSimple(&SimpleOptions{ RemovedFunc: func(i any) { _, ok := i.(*testing.T) - assert.True(t, ok) - ch <- true + ch <- ok }, }) cache.Put("testing", t) cache.Delete("testing") - assert.Nil(t, cache.Get("testing")) + require.Nil(t, cache.Get("testing")) timeout := time.NewTimer(time.Millisecond * 300) select { case b := <-ch: - assert.True(t, b) + require.True(t, b) case <-timeout.C: t.Error("RemovedFunc did not send true on channel ch") } @@ -154,7 +147,7 @@ func TestSimpleIterator(t *testing.T) { actual[entry.Key().(string)] = entry.Value().(string) } it.Close() - assert.Equal(t, expected, actual) + require.Equal(t, expected, actual) it = cache.Iterator() for i := 0; i < len(expected); i++ { @@ -163,5 +156,5 @@ func TestSimpleIterator(t *testing.T) { actual[entry.Key().(string)] = entry.Value().(string) } it.Close() - assert.Equal(t, expected, actual) + require.Equal(t, expected, actual) } From 0e6194c52cfcfc7ae99b0917f4d5e2435635006b Mon Sep 17 00:00:00 2001 From: Alex Stanfield <13949480+chaptersix@users.noreply.github.com> Date: Tue, 25 Aug 2026 17:17:45 -0500 Subject: [PATCH 062/139] Align CHASM ALLOW_ALL lifecycle with V1 (#11631) ## What changed? - Keep attaching completion callbacks to every CHASM scheduler workflow start, including actions whose resolved overlap policy is `ALLOW_ALL`, so start requests remain safe across rolling upgrades. - Exclude scheduler-wide last-completion result/failure input from new `ALLOW_ALL` actions. - After `StartWorkflowExecution` succeeds, remove the new `ALLOW_ALL` `BufferedStart` in the same transaction. Separately, copy its start data to `ScheduleInfo.RecentActions` as a start-only `RUNNING` history record. - Resolve an unspecified pending-start policy while building the V1-to-V2 migration request, so every V2 dispatch and retry uses the policy persisted on the buffered start. - For a pre-existing callback, use the policy stamped on that start rather than resolving against the schedule's current policy; an unstamped V1-migrated running workflow remains tracked, as it was in V1. - Preserve compatibility for previously persisted `ALLOW_ALL` callbacks without updating last-completion state or `PauseOnFailure`. - Carry start-only history through CHASM-to-V1 migration. - Merge start-only and completion-tracked history by actual start time, retaining the newest ten actions across both sources. - Re-run generation after recording a start-only action, so a finite or manual-only schedule rearms its idle timer from that start. ## Behavior This makes CHASM match modern V1: - A new `ALLOW_ALL` workflow starts normally, but is absent from `DescribeSchedule.Info.RunningWorkflows` and does not consume active-buffer capacity. - It appears in `DescribeSchedule.Info.RecentActions` (and List's recent actions) with its start time, workflow execution, and `RUNNING` status. - Its completion callback remains attached for rolling-upgrade compatibility. Because the start has already moved out of active buffered state, the callback is ignored and success or failure cannot change shared completion input or `PauseOnFailure`; its recent status therefore remains `RUNNING`. - For a final `ALLOW_ALL` action, an idle task armed before the start is invalidated by the newer start time; generation immediately arms its replacement, so the schedule still closes after `IdleTime`. - If an older handler retained an `ALLOW_ALL` start, its callback remains a compatibility path: the terminal action record is retained, but its completion cannot update shared completion state or pause the schedule once handled by this version. - `DescribeSchedule.Info.RecentActions` is ordered by actual start time and bounded to the newest ten actions across start-only and completion-tracked history. Newer retained actions evict the oldest; completion does not remove a start-only action. - Non-`ALLOW_ALL` actions remain active and appear in `RunningWorkflows` until their completion is handled. - A V1 pending start with an unspecified override snapshots the schedule's effective policy when its V2 migration request is built. - A V1-migrated `RunningWorkflows` entry has an unspecified policy but is nevertheless tracked: its completion updates sequential state and may pause the schedule, regardless of the schedule's current `ALLOW_ALL` default. ## Why? `ALLOW_ALL` actions are independent executions. Tracking their completions made scheduler-wide last result/failure and `PauseOnFailure` depend on callback arrival order, and retaining them as active could affect overlap and capacity behavior. Callbacks remain attached because `StartWorkflowExecution` deduplicates by request ID without reconciling callback differences. Keeping the request callback-compatible prevents a mixed-version retry from retaining a start that waits for a callback the workflow never received. ## How did you test it? - [x] added unit coverage - [x] added shared V1/CHASM functional coverage Commands: - `go test -tags test_dep ./chasm/lib/scheduler/... -count=1` - `go test -tags test_dep ./tests -run '^TestSchedule(CHASM|V1)/TestAllowAllDescribeContract$' -count=1` - `make fmt-imports` - `git diff --check` - `env GOCACHE=/tmp/sch-038-gocache go vet -tags disable_grpc_modules,,test_dep -vettool=.bin/errortype -style-check=false ./chasm/lib/scheduler/...` The functional test uses workflow signals to control completion. It asserts counters, buffer size, recent-action status/timestamps, active workflows, and failure/pause isolation across both backends. A CHASM functional idle-close case verifies that a final `ALLOW_ALL` action still closes after `IdleTime`. The migration regression and callback-reason metric test drive real component transactions through the CHASM test engine. `make lint-code` currently exits before analysis with `no go files to analyze` from its `--new-from-rev` filter, despite the Go diff; package `go vet` is clean. ## Additional observability Ignored callbacks are tagged as either `unrecognized_request_id` or `already_completed`. A newly started `ALLOW_ALL` callback is expected to be unrecognized after its buffered start moves to start-only history. An already-completed callback is a valid redelivery (for example, after a workflow reset). ~~Both preserve scheduler state while emitting a warning and counter increment.~~ Missing request IDs are now metric-only because they include expected `ALLOW_ALL` callbacks; known `already_completed` redeliveries still emit the warning, event, and counter. ## Potential risks Keeping callbacks attached avoids permanently orphaning buffered starts when old and new binaries race on the same request ID. During a rolling upgrade, however, old and new requests still differ in whether scheduler-wide last-completion result/failure input is included for `ALLOW_ALL`; the request that wins deduplication determines whether that workflow receives the legacy input. An older callback handler can also temporarily apply the legacy completion and pause semantics. These mixed-version differences end once the rollout completes, and the callback ensures an older handler cannot wait indefinitely. New `ALLOW_ALL` terminal status is intentionally not reflected in schedule Describe/List results, as in modern V1. ~~Its expected late callback is recorded as `unrecognized_request_id`, which adds callback delivery plus warning/metric volume compared with omitting callbacks.~~ Its expected late callback remains recorded as `unrecognized_request_id`, but only as a metric; warning and event logging are suppressed until these callbacks can be safely omitted. Migration resolves an unspecified pending-start policy at the V1-to-V2 boundary. If the schedule policy changes while that start remains pending and the schedule then rolls back to V1, the explicit migrated policy is preserved instead of inheriting the newer schedule policy. This is a narrow semantic difference that keeps V2 dispatch and retry behavior stable. --------- Co-authored-by: David Porter Co-authored-by: Claude Sonnet 5 --- chasm/chasmtest/test_engine.go | 7 + chasm/lib/scheduler/export_test.go | 9 +- chasm/lib/scheduler/helper_test.go | 6 + chasm/lib/scheduler/internal/overlap.go | 24 ++ chasm/lib/scheduler/internal/overlap_test.go | 44 ++++ chasm/lib/scheduler/invoker.go | 80 +++++-- .../scheduler/invoker_execute_task_test.go | 211 ++++++++++++++++-- chasm/lib/scheduler/invoker_tasks.go | 24 +- .../lib/scheduler/migration/identity_test.go | 49 ++++ chasm/lib/scheduler/migration/migration.go | 29 ++- .../lib/scheduler/migration/migration_test.go | 12 +- chasm/lib/scheduler/scheduler.go | 148 ++++++++---- .../scheduler/scheduler_idle_tasks_test.go | 56 +++++ .../scheduler_nexus_completion_test.go | 166 ++++++++++++++ chasm/lib/scheduler/scheduler_test.go | 64 ++++++ tests/schedule_test.go | 141 +++++++++++- 16 files changed, 978 insertions(+), 92 deletions(-) create mode 100644 chasm/lib/scheduler/internal/overlap.go create mode 100644 chasm/lib/scheduler/internal/overlap_test.go diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index 7e36c5bc3e7..b0a47953a35 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -78,6 +78,13 @@ func WithTimeSource(ts clock.TimeSource) EngineOption { } } +// WithMetricsHandler overrides the engine's default no-op metrics handler. +func WithMetricsHandler(handler metrics.Handler) EngineOption { + return func(e *Engine) { + e.metrics = handler + } +} + var defaultTransitionOptions = chasm.TransitionOptions{ ReusePolicy: chasm.BusinessIDReusePolicyAllowDuplicate, ConflictPolicy: chasm.BusinessIDConflictPolicyFail, diff --git a/chasm/lib/scheduler/export_test.go b/chasm/lib/scheduler/export_test.go index 8880a4efd63..a3f2cba5f51 100644 --- a/chasm/lib/scheduler/export_test.go +++ b/chasm/lib/scheduler/export_test.go @@ -65,13 +65,20 @@ func (i *Invoker) RecordExecuteResult( ctx chasm.MutableContext, completed []*schedulespb.BufferedStart, retryable []*schedulespb.BufferedStart, -) (newlyStarted, droppedDuplicates int) { +) (newlyStarted, droppedDuplicates int, startOnlyActions []*schedulespb.BufferedStart) { return i.recordExecuteResult(ctx, &executeResult{ CompletedStarts: completed, RetryableStarts: retryable, }) } +func (s *Scheduler) RecordStartOnlyActions( + ctx chasm.MutableContext, + starts []*schedulespb.BufferedStart, +) { + s.recordStartOnlyActions(ctx, starts) +} + func (b *BackfillerTaskHandler) ProcessBackfill( scheduler *Scheduler, backfiller *Backfiller, diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 2345aa02ec2..4a3467399f1 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -216,6 +216,12 @@ func withEngineTimeSource(ts *clock.EventTimeSource) engineTestOption { } } +func withEngineMetricsHandler(handler metrics.Handler) engineTestOption { + return func(c *engineTestConfig) { + c.engineOpts = append(c.engineOpts, chasmtest.WithMetricsHandler(handler)) + } +} + func newEngineTestConfig(opts ...engineTestOption) *engineTestConfig { config := &engineTestConfig{} for _, opt := range opts { diff --git a/chasm/lib/scheduler/internal/overlap.go b/chasm/lib/scheduler/internal/overlap.go new file mode 100644 index 00000000000..567f1ba4f7d --- /dev/null +++ b/chasm/lib/scheduler/internal/overlap.go @@ -0,0 +1,24 @@ +package internal + +import enumspb "go.temporal.io/api/enums/v1" + +// ResolveOverlapPolicy applies a per-action override, then the schedule policy, +// then the API default. +func ResolveOverlapPolicy( + overlapPolicy enumspb.ScheduleOverlapPolicy, + schedulePolicy enumspb.ScheduleOverlapPolicy, +) enumspb.ScheduleOverlapPolicy { + if overlapPolicy != enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED { + return overlapPolicy + } + if schedulePolicy != enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED { + return schedulePolicy + } + return enumspb.SCHEDULE_OVERLAP_POLICY_SKIP +} + +// TracksCompletionResult reports whether an action participates in scheduler-wide +// completion state and overlap resolution after it starts. +func TracksCompletionResult(overlapPolicy enumspb.ScheduleOverlapPolicy) bool { + return overlapPolicy != enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL +} diff --git a/chasm/lib/scheduler/internal/overlap_test.go b/chasm/lib/scheduler/internal/overlap_test.go new file mode 100644 index 00000000000..b8f6277e0f1 --- /dev/null +++ b/chasm/lib/scheduler/internal/overlap_test.go @@ -0,0 +1,44 @@ +package internal + +import ( + "testing" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" +) + +func TestTracksCompletionResult(t *testing.T) { + require.False(t, TracksCompletionResult(enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL)) + require.True(t, TracksCompletionResult(enumspb.SCHEDULE_OVERLAP_POLICY_SKIP)) +} + +func TestResolveOverlapPolicy(t *testing.T) { + tests := []struct { + name string + overlapPolicy enumspb.ScheduleOverlapPolicy + schedulePolicy enumspb.ScheduleOverlapPolicy + want enumspb.ScheduleOverlapPolicy + }{ + { + name: "action override", + overlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + schedulePolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + { + name: "schedule policy", + schedulePolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + }, + { + name: "default", + want: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, ResolveOverlapPolicy(tt.overlapPolicy, tt.schedulePolicy)) + }) + } +} diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index f9980dde6ea..d26669fb172 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -11,6 +11,8 @@ import ( schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" + "go.temporal.io/server/common" "go.temporal.io/server/common/util" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -154,10 +156,13 @@ func (e *executeResult) Append(o executeResult) executeResult { // recordExecuteResult updates the Invoker's internal state with the results of a // completed InvokerExecuteTask. It returns the number of *new* actions recorded -// (starts that transitioned from "no RunId" to "has RunId" in this call) and -// the number of completed results that were dropped because they were previously -// recorded. -func (i *Invoker) recordExecuteResult(ctx chasm.MutableContext, result *executeResult) (newlyStarted, droppedDuplicates int) { +// (starts that transitioned from "no RunId" to "has RunId" in this call), the +// number of completed results that were dropped because they were previously +// recorded, and starts that do not remain active while awaiting completion. +func (i *Invoker) recordExecuteResult( + ctx chasm.MutableContext, + result *executeResult, +) (newlyStarted, droppedDuplicates int, startOnlyActions []*schedulespb.BufferedStart) { completed := make(map[string]*schedulespb.BufferedStart) // request ID -> BufferedStart with RunId/StartTime failed := make(map[string]bool) // request ID -> is present retryable := make(map[string]*schedulespb.BufferedStart) // request ID -> *BufferedStart @@ -183,6 +188,7 @@ func (i *Invoker) recordExecuteResult(ctx chasm.MutableContext, result *executeR // Remove failed (non-retryable) starts from the buffer. removedStarts := 0 retriedStarts := 0 + startedUntracked := make(map[string]struct{}) i.BufferedStarts = slices.DeleteFunc(i.GetBufferedStarts(), func(start *schedulespb.BufferedStart) bool { failed := failed[start.RequestId] if failed { @@ -214,10 +220,16 @@ func (i *Invoker) recordExecuteResult(ctx chasm.MutableContext, result *executeR continue } if completedStart, ok := completed[start.RequestId]; ok { + newlyStarted++ + if !internal.TracksCompletionResult(start.GetOverlapPolicy()) { + startOnlyActions = append(startOnlyActions, completedStart) + startedUntracked[start.RequestId] = struct{}{} + removedStarts++ + continue + } start.RunId = completedStart.GetRunId() start.StartTime = completedStart.GetStartTime() start.HasCallback = true - newlyStarted++ } if retry, ok := retryable[start.RequestId]; ok { start.Attempt++ @@ -225,7 +237,10 @@ func (i *Invoker) recordExecuteResult(ctx chasm.MutableContext, result *executeR retriedStarts++ } } - + i.BufferedStarts = slices.DeleteFunc(i.GetBufferedStarts(), func(start *schedulespb.BufferedStart) bool { + _, remove := startedUntracked[start.GetRequestId()] + return remove + }) i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("recordExecuteResult kicked off %d starts, removed %d starts, retried %d starts", newlyStarted, @@ -233,7 +248,7 @@ func (i *Invoker) recordExecuteResult(ctx chasm.MutableContext, result *executeR retriedStarts)) i.addTasks(ctx) - return newlyStarted, droppedDuplicates + return newlyStarted, droppedDuplicates, startOnlyActions } // runningWorkflowID returns the workflow ID associated with the given @@ -260,13 +275,24 @@ func (i *Invoker) recordCompletedAction( i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("recording completed action: %s", requestID)) // Find the BufferedStart and mark it as completed. + completedUntracked := "" for _, start := range i.BufferedStarts { if start.GetRequestId() == requestID { scheduleTime = start.DesiredTime.AsTime() - start.Completed = completed + if !internal.TracksCompletionResult(start.GetOverlapPolicy()) { + i.Scheduler.Get(ctx).recordRecentAction(start, completed.GetStatus()) + completedUntracked = requestID + } else { + start.Completed = completed + } break } } + if completedUntracked != "" { + i.BufferedStarts = slices.DeleteFunc(i.BufferedStarts, func(start *schedulespb.BufferedStart) bool { + return start.GetRequestId() == completedUntracked + }) + } // Re-enable deferred starts (Attempt == -1) so they can be re-processed by // ProcessBuffer now that a workflow has completed. This allows the overlap @@ -380,7 +406,8 @@ func (i *Invoker) getEligibleBufferedStarts() []*schedulespb.BufferedStart { func (i *Invoker) runningWorkflowExecutions() []*commonpb.WorkflowExecution { var running []*commonpb.WorkflowExecution for _, start := range i.GetBufferedStarts() { - if start.GetRunId() != "" && start.GetCompleted() == nil { + if start.GetRunId() != "" && start.GetCompleted() == nil && + internal.TracksCompletionResult(start.GetOverlapPolicy()) { running = append(running, &commonpb.WorkflowExecution{ WorkflowId: start.GetWorkflowId(), RunId: start.GetRunId(), @@ -390,11 +417,13 @@ func (i *Invoker) runningWorkflowExecutions() []*commonpb.WorkflowExecution { return running } -// recentActions returns started/completed actions as ScheduleActionResults. -// This includes both running workflows (with status RUNNING) and completed -// workflows (with their final status). -func (i *Invoker) recentActions() []*schedulepb.ScheduleActionResult { - var results []*schedulepb.ScheduleActionResult +// recentActions combines stored start-only actions with completion-tracked actions +// represented by BufferedStarts. +func (i *Invoker) recentActions(storedActions []*schedulepb.ScheduleActionResult) []*schedulepb.ScheduleActionResult { + results := make([]*schedulepb.ScheduleActionResult, 0, len(storedActions)+len(i.GetBufferedStarts())) + for _, action := range storedActions { + results = append(results, common.CloneProto(action)) + } for _, start := range i.GetBufferedStarts() { // Only include workflows that have been started (have a RunId). if start.GetRunId() == "" { @@ -414,7 +443,28 @@ func (i *Invoker) recentActions() []*schedulepb.ScheduleActionResult { StartWorkflowStatus: status, }) } - return results + slices.SortFunc(results, func(a, b *schedulepb.ScheduleActionResult) int { + return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) + }) + return util.SliceTail(results, recentActionCount) +} + +// bufferedStartsCount returns the actions whose successful StartWorkflowExecution +// result has not yet been recorded. BufferedStarts also retains running and completed +// actions for lifecycle tracking and history, so its length is not the API buffer size. +// This preserves V1's distinction: V1 removes selected starts from BufferedStarts +// before recording them as running or recent, while CHASM uses a recorded RunId as +// the durable boundary between those states. +// Count starts without a RunId directly because recent actions include start-only +// ALLOW_ALL records stored outside BufferedStarts and are capped independently. +func (i *Invoker) bufferedStartsCount() int { + count := 0 + for _, start := range i.GetBufferedStarts() { + if start.GetRunId() == "" { + count++ + } + } + return count } // applyCompletedRetention removes the oldest completed BufferedStarts beyond diff --git a/chasm/lib/scheduler/invoker_execute_task_test.go b/chasm/lib/scheduler/invoker_execute_task_test.go index 1122e394ebc..f6e9f8663d3 100644 --- a/chasm/lib/scheduler/invoker_execute_task_test.go +++ b/chasm/lib/scheduler/invoker_execute_task_test.go @@ -10,16 +10,19 @@ import ( commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" "go.temporal.io/api/serviceerror" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/historyservicemock/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/chasmtest" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" + "go.temporal.io/server/common/testing/testlogger" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/proto" @@ -150,6 +153,10 @@ func executeTaskOnce(t *testing.T, env *invokerExecuteTestEnv, ctx chasm.Mutable // Execute success case. func TestExecuteTask_Basic(t *testing.T) { env := newInvokerExecuteTestEnv(t) + env.Scheduler.LastCompletionResult = chasm.NewDataField(env.MutableContext(), &schedulerpb.LastCompletionResult{ + Success: &commonpb.Payload{Data: []byte("previous-result")}, + Failure: &failurepb.Failure{Message: "previous-failure"}, + }) startTime := timestamppb.New(env.TimeSource.Now()) bufferedStarts := []*schedulespb.BufferedStart{ { @@ -175,21 +182,153 @@ func TestExecuteTask_Basic(t *testing.T) { // Expect both buffered starts to result in workflow executions. env.mockFrontendClient.EXPECT(). StartWorkflowExecution(gomock.Any(), gomock.Any()). - Times(2). - Return(&workflowservice.StartWorkflowExecutionResponse{ - RunId: "run-id", - }, nil) + DoAndReturn(func(_ context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + require.Len(t, req.GetCompletionCallbacks(), 1) + require.Empty(t, req.GetLastCompletionResult().GetPayloads()) + require.Nil(t, req.GetContinuedFailure()) + return &workflowservice.StartWorkflowExecutionResponse{RunId: "run-id"}, nil + }). + Times(2) - // After execution, both BufferedStarts are kept (with RunId set). - // They become "running" workflows. runExecuteTestCase(t, env, &executeTestCase{ InitialBufferedStarts: bufferedStarts, - ExpectedBufferedStarts: 2, // kept after starting - ExpectedRunningWorkflows: 2, + ExpectedBufferedStarts: 0, + ExpectedRunningWorkflows: 0, ExpectedActionCount: 2, + ValidateInvoker: func(t *testing.T, _ *scheduler.Invoker, env *invokerExecuteTestEnv) { + listInfo := env.Scheduler.ListInfo(env.ReadContext()) + require.Len(t, listInfo.GetRecentActions(), 2) + for _, action := range listInfo.GetRecentActions() { + require.Equal(t, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, action.GetStartWorkflowStatus()) + } + }, }) } +func TestExecuteTask_AllowAllRemovedAcrossEngineTransaction(t *testing.T) { + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + engine, engineCtx := newTestEngineContext(t, logger) + result, err := chasm.StartExecution( + engineCtx, + chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: scheduleID}, + scheduler.CreateScheduler, + &schedulerpb.CreateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: namespace, + ScheduleId: scheduleID, + Schedule: defaultSchedule(), + }, + }, + ) + require.NoError(t, err) + rootRef := chasm.NewComponentRef[*scheduler.Scheduler](result.ExecutionKey) + + startTime := timestamppb.Now() + var invoker *scheduler.Invoker + _, _, err = chasm.UpdateComponent( + engineCtx, + rootRef, + func(sched *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + invoker = sched.Invoker.Get(ctx) + invoker.LastProcessedTime = startTime + invoker.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: startTime, ActualTime: startTime, DesiredTime: startTime, + RequestId: "allow-all-request", WorkflowId: "allow-all-workflow", + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, Attempt: 1, + }} + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) + + ctrl := gomock.NewController(t) + frontendClient := workflowservicemock.NewMockWorkflowServiceClient(ctrl) + frontendClient.EXPECT().StartWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + require.Len(t, req.GetCompletionCallbacks(), 1) + return &workflowservice.StartWorkflowExecutionResponse{RunId: "allow-all-run"}, nil + }) + handler := scheduler.NewInvokerExecuteTaskHandler(scheduler.InvokerTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: logger, + FrontendClient: frontendClient, + }) + + dropped, err := chasmtest.ExecuteSideEffectTask( + context.Background(), + engine, + invoker, + handler, + chasm.TaskAttributes{}, + &schedulerpb.InvokerExecuteTask{}, + ) + require.NoError(t, err) + require.False(t, dropped) + + _, err = chasm.ReadComponent( + engineCtx, + rootRef, + func(sched *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + persistedInvoker := sched.Invoker.Get(ctx) + require.Empty(t, persistedInvoker.GetBufferedStarts()) + require.Len(t, sched.Info.GetRecentActions(), 1) + require.Equal(t, "allow-all-run", sched.Info.GetRecentActions()[0].GetStartWorkflowResult().GetRunId()) + describe, describeErr := sched.Describe(ctx, &schedulerpb.DescribeScheduleRequest{}, newLegacySpecBuilder(0, 0)) + require.NoError(t, describeErr) + require.Empty(t, describe.GetFrontendResponse().GetInfo().GetRunningWorkflows()) + require.Len(t, describe.GetFrontendResponse().GetInfo().GetRecentActions(), 1) + require.Zero(t, describe.GetFrontendResponse().GetInfo().GetBufferSize()) + return struct{}{}, nil + }, + struct{}{}, + ) + require.NoError(t, err) +} + +func TestRecordExecuteResult_AllowAllRecentActionsBounded(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + invoker := env.Scheduler.Invoker.Get(ctx) + base := env.TimeSource.Now().Add(-time.Hour) + var completed []*schedulespb.BufferedStart + for idx := range scheduler.RecentActionCount + 2 { + startTime := timestamppb.New(base.Add(time.Duration(idx) * time.Minute)) + start := &schedulespb.BufferedStart{ + NominalTime: startTime, ActualTime: startTime, DesiredTime: startTime, + RequestId: fmt.Sprintf("req-%d", idx), WorkflowId: fmt.Sprintf("wf-%d", idx), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, Attempt: 1, + } + invoker.BufferedStarts = append(invoker.BufferedStarts, start) + result := proto.Clone(start).(*schedulespb.BufferedStart) + result.RunId = fmt.Sprintf("run-%d", idx) + result.StartTime = startTime + completed = append(completed, result) + } + + newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, completed, nil) + env.Scheduler.RecordStartOnlyActions(ctx, startOnlyActions) + require.Equal(t, scheduler.RecentActionCount+2, newlyStarted) + require.Zero(t, droppedDuplicates) + require.Empty(t, invoker.GetBufferedStarts()) + require.Len(t, env.Scheduler.Info.GetRecentActions(), scheduler.RecentActionCount) + require.Equal(t, "run-2", env.Scheduler.Info.GetRecentActions()[0].GetStartWorkflowResult().GetRunId()) + require.Equal(t, "run-11", env.Scheduler.Info.GetRecentActions()[scheduler.RecentActionCount-1].GetStartWorkflowResult().GetRunId()) + recentRunIDs := make(map[string]struct{}, scheduler.RecentActionCount) + for _, action := range env.Scheduler.Info.GetRecentActions() { + recentRunIDs[action.GetStartWorkflowResult().GetRunId()] = struct{}{} + } + require.NotContains(t, recentRunIDs, "run-0") + bufferedRequestIDs := make(map[string]struct{}, len(invoker.GetBufferedStarts())) + for _, start := range invoker.GetBufferedStarts() { + bufferedRequestIDs[start.GetRequestId()] = struct{}{} + } + require.NotContains(t, bufferedRequestIDs, "req-0") + require.Len(t, env.Scheduler.ListInfo(ctx).GetRecentActions(), 5) +} + func TestExecuteTask_ForwardsVersioningOverride(t *testing.T) { tests := map[string]struct { override *workflowpb.VersioningOverride @@ -259,8 +398,8 @@ func TestExecuteTask_ForwardsVersioningOverride(t *testing.T) { OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, Attempt: 1, }}, - ExpectedBufferedStarts: 1, - ExpectedRunningWorkflows: 1, + ExpectedBufferedStarts: 0, + ExpectedRunningWorkflows: 0, ExpectedActionCount: 1, }) }) @@ -311,6 +450,33 @@ func TestExecuteTask_DistinctRequestsCanReuseCompletedWorkflowID(t *testing.T) { }) } +func TestExecuteTask_UsesBufferedOverlapPolicy(t *testing.T) { + env := newInvokerExecuteTestEnv(t) + env.Scheduler.Schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + startTime := timestamppb.New(env.TimeSource.Now()) + + env.mockFrontendClient.EXPECT(). + StartWorkflowExecution(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *workflowservice.StartWorkflowExecutionRequest, _ ...grpc.CallOption) (*workflowservice.StartWorkflowExecutionResponse, error) { + require.Len(t, req.GetCompletionCallbacks(), 1) + return &workflowservice.StartWorkflowExecutionResponse{RunId: "run-id"}, nil + }) + + runExecuteTestCase(t, env, &executeTestCase{ + InitialBufferedStarts: []*schedulespb.BufferedStart{{ + NominalTime: startTime, ActualTime: startTime, DesiredTime: startTime, + RequestId: "request-id", WorkflowId: "workflow-id", Attempt: 1, + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + }}, + ExpectedBufferedStarts: 1, + ExpectedRunningWorkflows: 1, + ExpectedActionCount: 1, + ValidateInvoker: func(t *testing.T, invoker *scheduler.Invoker, _ *invokerExecuteTestEnv) { + require.Equal(t, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, invoker.BufferedStarts[0].GetOverlapPolicy()) + }, + }) +} + // Execute is scheduled with an empty buffer. func TestExecuteTask_Empty(t *testing.T) { env := newInvokerExecuteTestEnv(t) @@ -361,11 +527,11 @@ func TestExecuteTask_RetryableFailure(t *testing.T) { // After execution: // - Failed start stays in buffer with backoff (pending) - // - Successful start stays in buffer with RunId set (running) + // - Successful ALLOW_ALL start moves to recent-action history runExecuteTestCase(t, env, &executeTestCase{ InitialBufferedStarts: bufferedStarts, - ExpectedBufferedStarts: 2, // both kept: 1 failed (backoff) + 1 running - ExpectedRunningWorkflows: 1, + ExpectedBufferedStarts: 1, + ExpectedRunningWorkflows: 0, ExpectedActionCount: 1, ValidateInvoker: func(t *testing.T, invoker *scheduler.Invoker, env *invokerExecuteTestEnv) { // Find the failed start (no RunId, has backoff). @@ -461,8 +627,8 @@ func TestExecuteTask_SucceedsOnFinalAttempt(t *testing.T) { OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, Attempt: scheduler.InvokerMaxStartAttempts, }}, - ExpectedBufferedStarts: 1, - ExpectedRunningWorkflows: 1, + ExpectedBufferedStarts: 0, + ExpectedRunningWorkflows: 0, ExpectedActionCount: 1, }) } @@ -659,11 +825,11 @@ func TestExecuteTask_ExceedsMaxActionsPerExecution(t *testing.T) { RunId: "run-id", }, nil) - // All BufferedStarts are kept: maxStarts get RunId set (running), the rest stay pending. + // Started ALLOW_ALL actions move to recent history; only the unexecuted half remains buffered. runExecuteTestCase(t, env, &executeTestCase{ InitialBufferedStarts: bufferedStarts, - ExpectedBufferedStarts: maxStarts * 2, // all kept: started + pending - ExpectedRunningWorkflows: maxStarts, // only started ones + ExpectedBufferedStarts: maxStarts, + ExpectedRunningWorkflows: 0, ExpectedActionCount: int64(maxStarts), }) } @@ -700,9 +866,10 @@ func TestExecuteTask_RecordResultIdempotentOnRace(t *testing.T) { StartTime: loserStartTime, }} - newlyStarted, droppedDuplicates := invoker.RecordExecuteResult(ctx, loser, nil) + newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, loser, nil) require.Equal(t, 0, newlyStarted, "duplicate RunId-set start must not be counted") require.Equal(t, 1, droppedDuplicates, "the dropped completion must be reported for observability") + require.Empty(t, startOnlyActions) live := invoker.BufferedStarts[0] require.Equal(t, winning, live.RunId, "live RunId must not be stomped") require.Equal(t, startTime.AsTime(), live.StartTime.AsTime(), "live StartTime must not be stomped") @@ -723,9 +890,10 @@ func TestExecuteTask_RecordResultIdempotentOnRace(t *testing.T) { RunId: "first-run", StartTime: startTime, }} - newlyStarted, droppedDuplicates = invoker.RecordExecuteResult(ctx, first, nil) + newlyStarted, droppedDuplicates, startOnlyActions = invoker.RecordExecuteResult(ctx, first, nil) require.Equal(t, 1, newlyStarted, "first-time RunId assignment must be counted") require.Equal(t, 0, droppedDuplicates, "no duplicate was dropped") + require.Empty(t, startOnlyActions) freshlyStarted := invoker.BufferedStarts[1] require.Equal(t, "first-run", freshlyStarted.RunId) require.Equal(t, startTime.AsTime(), freshlyStarted.StartTime.AsTime()) @@ -763,9 +931,10 @@ func TestExecuteTask_RecordResultIdempotentOnRetryableRace(t *testing.T) { BackoffTime: loserBackoff, }} - newlyStarted, droppedDuplicates := invoker.RecordExecuteResult(ctx, nil, retryable) + newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, nil, retryable) require.Equal(t, 0, newlyStarted) require.Equal(t, 0, droppedDuplicates, "retryable drops aren't counted as duplicates - only completed-side drops are") + require.Empty(t, startOnlyActions) live := invoker.BufferedStarts[0] require.Equal(t, int64(1), live.Attempt, "Attempt must not be incremented on a started entry") require.Nil(t, live.BackoffTime, "BackoffTime must not be set on a started entry") diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index e3f654c4361..2f06a07d668 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -16,6 +16,7 @@ import ( schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" @@ -243,7 +244,8 @@ func (h *InvokerExecuteTaskHandler) Execute( s := i.Scheduler.Get(ctx) // Use newlyStarted (not len(result.CompletedStarts)) so a concurrent // ExecuteTask's duplicate StartWorkflow can't inflate ActionCount. - newlyStarted, droppedDuplicates := i.recordExecuteResult(ctx, &result) + newlyStarted, droppedDuplicates, startOnlyActions := i.recordExecuteResult(ctx, &result) + s.recordStartOnlyActions(ctx, startOnlyActions) s.recordActionResult(&schedulerActionResult{actionCount: int64(newlyStarted)}) if droppedDuplicates > 0 { h.recordDuplicateExecuteDrops(s, droppedDuplicates) @@ -378,8 +380,8 @@ func (h *InvokerExecuteTaskHandler) startWorkflows( break } - // Clone start before concurrent access. The clone will have RunId/StartTime - // set by startWorkflow, then copied back to the original in recordExecuteResult. + // Clone start before concurrent access. Buffered starts carry the policy + // resolved when they entered CHASM, including through migration. start = common.CloneProto(start) // Run all starts concurrently. @@ -660,14 +662,18 @@ func (h *InvokerExecuteTaskHandler) startWorkflow( reusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE } + tracksCompletionResult := internal.TracksCompletionResult(start.GetOverlapPolicy()) var lcr []*commonpb.Payload - if lastCompletionState.Success != nil { + continuedFailure := lastCompletionState.Failure + if tracksCompletionResult && lastCompletionState.Success != nil { lcr = append(lcr, lastCompletionState.Success) } - // Build the completion callback with this start's request ID packed into its token, so the - // completion is matched by a request ID that rides in the callback header and survives - // continue-as-new, rather than the started workflow's callback state which is re-stamped on each - // new run. + if !tracksCompletionResult { + continuedFailure = nil + } + // Always attach callbacks so StartWorkflowExecution requests remain compatible + // across mixed server versions. ALLOW_ALL callbacks become harmless late deliveries + // after their starts move to start-only history. callback, err := chasm.GenerateNexusCallback(schedulerRef, start.RequestId, h.config.EncodeInternalTokenWithEnvelope(scheduler.Namespace)) if err != nil { return err @@ -691,7 +697,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflow( WorkflowTaskTimeout: requestSpec.WorkflowTaskTimeout, WorkflowType: requestSpec.WorkflowType, Priority: requestSpec.Priority, - ContinuedFailure: lastCompletionState.Failure, + ContinuedFailure: continuedFailure, LastCompletionResult: &commonpb.Payloads{ Payloads: lcr, }, diff --git a/chasm/lib/scheduler/migration/identity_test.go b/chasm/lib/scheduler/migration/identity_test.go index 793e9a875cf..a828b774ae0 100644 --- a/chasm/lib/scheduler/migration/identity_test.go +++ b/chasm/lib/scheduler/migration/identity_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" schedulespb "go.temporal.io/server/api/schedule/v1" schedulerinternal "go.temporal.io/server/chasm/lib/scheduler/internal" "google.golang.org/protobuf/types/known/timestamppb" @@ -24,6 +25,7 @@ func TestSameTimePendingStartsReceiveUniqueIdentities(t *testing.T) { "schedule-id", 1, "workflow-id", + enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, ) require.Len(t, converted, 2) require.NotEqual(t, converted[0].GetRequestId(), converted[1].GetRequestId(), @@ -51,6 +53,7 @@ func TestMigratedStartsPreserveExistingIdentities(t *testing.T) { "schedule-id", 1, "workflow-id", + enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, ) require.Len(t, converted, 2) for i, start := range converted { @@ -60,3 +63,49 @@ func TestMigratedStartsPreserveExistingIdentities(t *testing.T) { "identities carried over from V1 must not be suffixed") } } + +func TestMigratedStartsResolveOverlapPolicy(t *testing.T) { + when := timestamppb.New(time.Date(2026, 7, 19, 12, 0, 0, 0, time.UTC)) + tests := []struct { + name string + startPolicy enumspb.ScheduleOverlapPolicy + schedulePolicy enumspb.ScheduleOverlapPolicy + want enumspb.ScheduleOverlapPolicy + }{ + { + name: "explicit start policy", + startPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + schedulePolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + want: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + }, + { + name: "inherited schedule policy", + schedulePolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + want: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + { + name: "default policy", + want: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + converted := convertBufferedStartsLegacyToCHASM( + []*schedulespb.BufferedStart{{ + NominalTime: when, + ActualTime: when, + OverlapPolicy: tt.startPolicy, + }}, + "namespace-id", + "schedule-id", + 1, + "workflow-id", + tt.schedulePolicy, + ) + + require.Len(t, converted, 1) + require.Equal(t, tt.want, converted[0].GetOverlapPolicy()) + }) + } +} diff --git a/chasm/lib/scheduler/migration/migration.go b/chasm/lib/scheduler/migration/migration.go index 316edd1ed8f..2affc16234f 100644 --- a/chasm/lib/scheduler/migration/migration.go +++ b/chasm/lib/scheduler/migration/migration.go @@ -3,6 +3,7 @@ package migration import ( "fmt" "maps" + "slices" "strconv" "time" @@ -15,9 +16,12 @@ import ( schedulerinternal "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common" "go.temporal.io/server/common/searchattribute/sadefs" + "go.temporal.io/server/common/util" "google.golang.org/protobuf/types/known/timestamppb" ) +const legacyRecentActionCount = 10 + // LegacyToCreateFromMigrationStateRequest converts legacy (workflow-backed) scheduler // state to a CreateFromMigrationStateRequest proto. This is the primary V1-to-V2 // migration function. @@ -37,8 +41,8 @@ import ( // - High water mark (becomes Generator.LastProcessedTime) // - Search attributes and memo // -// Note: In V2, RunningWorkflows and RecentActions are computed on-demand from -// BufferedStarts by the Invoker, rather than being stored separately in ScheduleInfo. +// Note: In V2, completion-tracked RunningWorkflows and RecentActions are computed +// on-demand from BufferedStarts. Start-only recent actions remain in ScheduleInfo. func LegacyToCreateFromMigrationStateRequest( schedule *schedulepb.Schedule, info *schedulepb.ScheduleInfo, @@ -47,7 +51,7 @@ func LegacyToCreateFromMigrationStateRequest( memo *commonpb.Memo, migrationTime time.Time, ) *schedulerpb.CreateFromMigrationStateRequest { - // V2 computes RunningWorkflows/RecentActions on-demand from BufferedStarts + // Imported recent actions are represented by BufferedStarts in V2. infoClone := common.CloneProto(info) infoClone.RunningWorkflows = nil infoClone.RecentActions = nil @@ -73,6 +77,7 @@ func LegacyToCreateFromMigrationStateRequest( state.ScheduleId, state.ConflictToken, getWorkflowID(schedule), + schedule.GetPolicies().GetOverlapPolicy(), ) runningBufferedStarts := convertRunningWorkflowsToBufferedStarts( @@ -164,6 +169,17 @@ func CHASMToLegacyStartScheduleArgs( invokerBuffered = invoker.GetBufferedStarts() } bufferedStarts, running, recent := splitBufferedStartsForLegacy(invokerBuffered) + if len(info.GetRecentActions()) > 0 { + storedRecent := make([]*schedulepb.ScheduleActionResult, 0, len(info.GetRecentActions())) + for _, action := range info.GetRecentActions() { + storedRecent = append(storedRecent, common.CloneProto(action)) + } + recent = append(storedRecent, recent...) + slices.SortFunc(recent, func(a, b *schedulepb.ScheduleActionResult) int { + return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) + }) + recent = util.SliceTail(recent, legacyRecentActionCount) + } ongoingBackfills, triggerStarts := convertBackfillersCHASMToLegacy(backfillers, migrationTime) bufferedStarts = append(bufferedStarts, triggerStarts...) @@ -208,6 +224,7 @@ func convertBufferedStartsLegacyToCHASM( namespaceID, scheduleID string, conflictToken int64, baseWorkflowID string, + scheduleOverlapPolicy enumspb.ScheduleOverlapPolicy, ) []*schedulespb.BufferedStart { if len(v1Starts) == 0 { return nil @@ -250,6 +267,10 @@ func convertBufferedStartsLegacyToCHASM( v2Start.Attempt = 0 v2Start.BackoffTime = nil + v2Start.OverlapPolicy = schedulerinternal.ResolveOverlapPolicy( + v2Start.GetOverlapPolicy(), + scheduleOverlapPolicy, + ) v2Starts[i] = v2Start } @@ -466,7 +487,7 @@ func splitBufferedStartsForLegacy( // to later non-ALLOW_ALL starts. They still appear in RecentActions above, // matching V1. if start.GetCompleted() == nil && - start.GetOverlapPolicy() != enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL { + schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { running = append(running, &commonpb.WorkflowExecution{ WorkflowId: start.GetWorkflowId(), RunId: start.GetRunId(), diff --git a/chasm/lib/scheduler/migration/migration_test.go b/chasm/lib/scheduler/migration/migration_test.go index 1242b367a4d..fc3ee84c784 100644 --- a/chasm/lib/scheduler/migration/migration_test.go +++ b/chasm/lib/scheduler/migration/migration_test.go @@ -232,6 +232,15 @@ func TestCHASMToLegacyStartScheduleArgs(t *testing.T) { Info: &schedulepb.ScheduleInfo{ActionCount: 12}, } generator := &schedulerpb.GeneratorState{LastProcessedTime: timestamppb.New(now.Add(-time.Minute))} + scheduler.Info.RecentActions = []*schedulepb.ScheduleActionResult{{ + ScheduleTime: timestamppb.New(now.Add(-4 * time.Minute)), + ActualTime: timestamppb.New(now.Add(-4 * time.Minute)), + StartWorkflowResult: &commonpb.WorkflowExecution{ + WorkflowId: "wf-start-only", + RunId: "run-start-only", + }, + StartWorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + }} invoker := &schedulerpb.InvokerState{ BufferedStarts: []*schedulespb.BufferedStart{ { @@ -299,7 +308,8 @@ func TestCHASMToLegacyStartScheduleArgs(t *testing.T) { require.Equal(t, "wf-running", args.Info.RunningWorkflows[0].WorkflowId) require.Equal(t, "run-running", args.Info.RunningWorkflows[0].RunId) - require.Len(t, args.Info.RecentActions, 2) + require.Len(t, args.Info.RecentActions, 3) + require.Equal(t, "run-start-only", args.Info.RecentActions[2].GetStartWorkflowResult().GetRunId()) require.Len(t, args.State.BufferedStarts, 2) // pending + trigger require.Len(t, args.State.OngoingBackfills, 1) require.Equal(t, backfillProgress.AsTime(), args.State.OngoingBackfills[0].StartTime.AsTime()) diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index 017b7cef668..d11e8f10af0 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -18,6 +18,7 @@ import ( schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common" "go.temporal.io/server/common/contextutil" "go.temporal.io/server/common/log/tag" @@ -67,6 +68,11 @@ var ( _ (chasm.VisibilityMemoProvider) = (*Scheduler)(nil) ) +const ( + callbackIgnoredUnrecognizedRequest metrics.ReasonString = "unrecognized_request_id" + callbackIgnoredAlreadyCompleted metrics.ReasonString = "already_completed" +) + var ( executionStatusRunning = "Running" executionStatusCompleted = "Completed" @@ -475,18 +481,14 @@ func (s *Scheduler) identity() string { } func (s *Scheduler) overlapPolicy() enumspb.ScheduleOverlapPolicy { - policy := s.Schedule.GetPolicies().GetOverlapPolicy() - if policy == enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED { - policy = enumspb.SCHEDULE_OVERLAP_POLICY_SKIP - } - return policy + return internal.ResolveOverlapPolicy( + enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, + s.Schedule.GetPolicies().GetOverlapPolicy(), + ) } func (s *Scheduler) resolveOverlapPolicy(overlapPolicy enumspb.ScheduleOverlapPolicy) enumspb.ScheduleOverlapPolicy { - if overlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED { - overlapPolicy = s.overlapPolicy() - } - return overlapPolicy + return internal.ResolveOverlapPolicy(overlapPolicy, s.overlapPolicy()) } // validateCachedState clears cached fields whenever the Scheduler's @@ -520,7 +522,7 @@ func (s *Scheduler) getLastEventTime(ctx chasm.Context) time.Time { ) // The recentActions list is unsorted. - for _, a := range s.Invoker.Get(ctx).recentActions() { + for _, a := range s.recentActions(ctx) { latest = util.MaxTime(latest, a.GetActualTime().AsTime()) } @@ -581,13 +583,51 @@ type schedulerActionResult struct { } // recordActionResult updates the Scheduler's customer-facing metrics. -// RunningWorkflows and RecentActions are computed from BufferedStarts. +// RunningWorkflows are computed from BufferedStarts. RecentActions also includes +// start-only records for actions that are removed from active state after starting. func (s *Scheduler) recordActionResult(result *schedulerActionResult) { s.Info.ActionCount += result.actionCount s.Info.OverlapSkipped += result.overlapSkipped s.Info.MissedCatchupWindow += result.missedCatchupWindow } +func (s *Scheduler) recordRecentAction( + start *schedulespb.BufferedStart, + status enumspb.WorkflowExecutionStatus, +) { + s.Info.RecentActions = append(s.Info.RecentActions, &schedulepb.ScheduleActionResult{ + ScheduleTime: start.GetActualTime(), + ActualTime: start.GetStartTime(), + StartWorkflowResult: &commonpb.WorkflowExecution{ + WorkflowId: start.GetWorkflowId(), + RunId: start.GetRunId(), + }, + StartWorkflowStatus: status, + }) + slices.SortFunc(s.Info.RecentActions, func(a, b *schedulepb.ScheduleActionResult) int { + return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) + }) + s.Info.RecentActions = util.SliceTail(s.Info.RecentActions, recentActionCount) +} + +func (s *Scheduler) recordStartOnlyActions( + ctx chasm.MutableContext, + starts []*schedulespb.BufferedStart, +) { + for _, start := range starts { + s.recordRecentAction(start, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING) + } + if len(starts) > 0 { + // These starts have no completion callback to rearm an idle task after + // their start-only history advances the idle deadline. + s.Generator.Get(ctx).Generate(ctx) + } +} + +func (s *Scheduler) recentActions(ctx chasm.Context) []*schedulepb.ScheduleActionResult { + return s.Invoker.Get(ctx).recentActions(s.Info.GetRecentActions()) +} + var _ chasm.NexusCompletionHandler = &Scheduler{} func executionStatusFromFailure(failure *failurepb.Failure) enumspb.WorkflowExecutionStatus { @@ -613,6 +653,18 @@ func countsAsFailureForPause(status enumspb.WorkflowExecutionStatus) bool { } } +func (s *Scheduler) recordIgnoredCallback( + ctx chasm.MutableContext, + metricsHandler metrics.Handler, + requestID string, + reason metrics.ReasonString, + message string, +) { + s.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("%s: %s", message, requestID)) + ctx.Logger().Warn(message, tag.RequestID(requestID), tag.ScheduleID(s.ScheduleId)) + metricsHandler.Counter(metrics.ScheduleCallbackIgnored.Name()).Record(1, metrics.ReasonTag(reason)) +} + // HandleNexusCompletion allows Scheduler to record workflow completions from // worfklows started by the same scheduler tree's Invoker. func (s *Scheduler) HandleNexusCompletion( @@ -622,19 +674,37 @@ func (s *Scheduler) HandleNexusCompletion( invoker := s.Invoker.Get(ctx) metricsHandler := newTaggedMetricsHandler(ctx.MetricsHandler(), s) - workflowID := invoker.runningWorkflowID(info.RequestId) - if workflowID == "" { - // If the request ID was removed, the request must have already been processed; - // fast-succeed. - msg := "handled Nexus completion with an unrecognized request ID" - s.getOrCreateEventLog(ctx).LogEvent(ctx, - fmt.Sprintf("%s: %s", msg, info.RequestId)) - ctx.Logger().Warn(msg, - tag.RequestID(info.RequestId), - tag.ScheduleID(s.ScheduleId)) - metricsHandler.Counter(metrics.ScheduleCallbackIgnored.Name()).Record(1) + var start *schedulespb.BufferedStart + for _, bufferedStart := range invoker.GetBufferedStarts() { + if bufferedStart.GetRequestId() == info.RequestId { + start = bufferedStart + break + } + } + if start == nil { + // Missing request IDs are expected for start-only ALLOW_ALL actions because + // their callbacks remain attached for rolling-upgrade compatibility. + // TODO: Restore warning and event logging once those callbacks can be safely omitted. + metricsHandler.Counter(metrics.ScheduleCallbackIgnored.Name()).Record( + 1, + metrics.ReasonTag(callbackIgnoredUnrecognizedRequest), + ) + return nil + } + if start.GetCompleted() != nil { + // Completion callbacks may be validly redelivered, for example after a workflow reset. + // Preserve state but keep the duplicate observable through the log and metric. + s.recordIgnoredCallback( + ctx, + metricsHandler, + info.RequestId, + callbackIgnoredAlreadyCompleted, + "handled Nexus completion for an already-completed buffered start", + ) return nil } + workflowID := start.GetWorkflowId() + tracksCompletionResult := internal.TracksCompletionResult(start.GetOverlapPolicy()) // Record how long it took for the callback to arrive after the action completed. // Use ctx.Now instead of time.Since to use a consistent time source across nodes, @@ -650,23 +720,27 @@ func (s *Scheduler) HandleNexusCompletion( var wfStatus enumspb.WorkflowExecutionStatus switch outcome := info.Outcome.(type) { case *persistencespb.ChasmNexusCompletion_Failure: - previousResult := s.LastCompletionResult.Get(ctx) // Most-recent success is kept after failure. wfStatus = executionStatusFromFailure(outcome.Failure) - s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ - Failure: outcome.Failure, - Success: previousResult.Success, - }) + if tracksCompletionResult { + previousResult := s.LastCompletionResult.Get(ctx) // Most-recent success is kept after failure. + s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ + Failure: outcome.Failure, + Success: previousResult.Success, + }) + } case *persistencespb.ChasmNexusCompletion_Success: wfStatus = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED - s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ - Success: outcome.Success, - }) + if tracksCompletionResult { + s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ + Success: outcome.Success, + }) + } default: wfStatus = enumspb.WORKFLOW_EXECUTION_STATUS_FAILED } // Handle pause-on-failure. - if countsAsFailureForPause(wfStatus) && + if tracksCompletionResult && countsAsFailureForPause(wfStatus) && s.Schedule.Policies.PauseOnFailure && !s.Schedule.State.Paused { s.Schedule.State.Paused = true s.Schedule.State.Notes = fmt.Sprintf( @@ -735,11 +809,10 @@ func (s *Scheduler) Describe( invoker := s.Invoker.Get(ctx) info := common.CloneProto(s.Info) info.RunningWorkflows = invoker.runningWorkflowExecutions() - info.RecentActions = invoker.recentActions() + info.RecentActions = s.recentActions(ctx) info.FutureActionTimes = futureActionTimes - // BufferedStarts holds waiting, running, and recently-completed entries; only the - // waiting portion (those not yet surfaced via RecentActions) counts as buffered. - info.BufferSize = int64(len(invoker.GetBufferedStarts()) - len(info.RecentActions)) + // Only starts that have not reached StartWorkflowExecution count as buffered. + info.BufferSize = int64(invoker.bufferedStartsCount()) executionInfo := ctx.ExecutionInfo() info.StateSizeBytes = int64(executionInfo.ApproximateStateSize) @@ -1017,7 +1090,7 @@ func (s *Scheduler) SearchAttributes(ctx chasm.Context) []chasm.SearchAttributeK invoker := s.Invoker.Get(ctx) runningWorkflowCount := int64(len(invoker.runningWorkflowExecutions())) - bufferedStartsCount := int64(len(invoker.GetBufferedStarts()) - len(invoker.recentActions())) + bufferedStartsCount := int64(invoker.bufferedStartsCount()) // Emitted even when zero so that exact and range queries both work. out = append(out, @@ -1054,11 +1127,10 @@ func (s *Scheduler) ListInfo( spec.StructuredCalendar = util.SliceHead(spec.StructuredCalendar, listInfoSpecFieldLimit) generator := s.Generator.Get(ctx) - invoker := s.Invoker.Get(ctx) // Hard-cap the memo's recent-action list by length after sorting by actual time // (ascending towards most recent). - recentActions := invoker.recentActions() + recentActions := s.recentActions(ctx) slices.SortFunc(recentActions, func(a, b *schedulepb.ScheduleActionResult) int { return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) }) diff --git a/chasm/lib/scheduler/scheduler_idle_tasks_test.go b/chasm/lib/scheduler/scheduler_idle_tasks_test.go index 17f6dd03767..7af50a8b4bb 100644 --- a/chasm/lib/scheduler/scheduler_idle_tasks_test.go +++ b/chasm/lib/scheduler/scheduler_idle_tasks_test.go @@ -5,10 +5,13 @@ import ( "time" "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" @@ -99,6 +102,59 @@ func TestIdleTask_ExecuteInitializesEventLogMissingFromOlderTree(t *testing.T) { require.Equal(t, "schedule closed from idle timer", eventLog.Events[0].Message) } +func TestIdleTask_AllowAllStartRearmsIdleTimer(t *testing.T) { + base := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) + timeSource := clock.NewEventTimeSource() + timeSource.Update(base) + + schedule := defaultSchedule() + schedule.Spec = &schedulepb.ScheduleSpec{} + testEngine := newSchedulerTestEngine(t, schedule, withEngineTimeSource(timeSource)) + + _, err := testEngine.engine.FirePureTasks(testEngine.rootRef, base) + require.NoError(t, err) + oldIdleDeadline := base.Add(scheduler.DefaultTweakables.IdleTime) + require.NoError(t, testEngine.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.Equal(t, oldIdleDeadline, s.IdleCloseTime.AsTime()) + return nil + })) + + startTime := base.Add(time.Minute) + timeSource.Update(startTime) + require.NoError(t, testEngine.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { + invoker := s.Invoker.Get(ctx) + invoker.BufferedStarts = append(invoker.BufferedStarts, &schedulespb.BufferedStart{ + RequestId: "allow-all-request", + WorkflowId: "allow-all-workflow", + NominalTime: timestamppb.New(startTime), + ActualTime: timestamppb.New(startTime), + DesiredTime: timestamppb.New(startTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + Attempt: 1, + }) + _, _, startOnlyActions := invoker.RecordExecuteResult(ctx, []*schedulespb.BufferedStart{{ + RequestId: "allow-all-request", + RunId: "allow-all-run", + StartTime: timestamppb.New(startTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }}, nil) + s.RecordStartOnlyActions(ctx, startOnlyActions) + return nil + })) + + _, err = testEngine.engine.FirePureTasks(testEngine.rootRef, oldIdleDeadline) + require.NoError(t, err) + + newIdleDeadline := startTime.Add(scheduler.DefaultTweakables.IdleTime) + timeSource.Update(newIdleDeadline) + _, err = testEngine.engine.FirePureTasks(testEngine.rootRef, newIdleDeadline) + require.NoError(t, err) + require.NoError(t, testEngine.readScheduler(func(s *scheduler.Scheduler, _ chasm.Context) error { + require.True(t, s.Closed) + return nil + })) +} + func TestIdleTask_Validate_SchedulerNotIdle(t *testing.T) { env := newTestEnv(t) now := env.TimeSource.Now() diff --git a/chasm/lib/scheduler/scheduler_nexus_completion_test.go b/chasm/lib/scheduler/scheduler_nexus_completion_test.go index 40d3edc655c..f5c993a25f0 100644 --- a/chasm/lib/scheduler/scheduler_nexus_completion_test.go +++ b/chasm/lib/scheduler/scheduler_nexus_completion_test.go @@ -9,12 +9,17 @@ import ( commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" + schedulepb "go.temporal.io/api/schedule/v1" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/migration" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/testing/testlogger" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -128,6 +133,167 @@ func TestHandleNexusCompletion_Success(t *testing.T) { executeNexusCompletion(t, tc) } +func TestHandleNexusCompletion_ExistingAllowAllDoesNotUpdateCompletionState(t *testing.T) { + for _, tc := range []struct { + name string + manual bool + }{ + {name: "generator", manual: false}, + {name: "backfiller", manual: true}, + } { + t.Run(tc.name, func(t *testing.T) { + sched, ctx, node := setupSchedulerForTest(t) + initial := &schedulerpb.LastCompletionResult{Success: &commonpb.Payload{Data: []byte("previous-result")}} + sched.LastCompletionResult = chasm.NewDataField(ctx, initial) + sched.Schedule.Policies.PauseOnFailure = true + sched.Invoker.Get(ctx).BufferedStarts = []*schedulespb.BufferedStart{{ + RequestId: "req-1", WorkflowId: "wf-1", RunId: "run-1", Attempt: 1, + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + ActualTime: timestamppb.New(time.Now().Add(-time.Minute)), + StartTime: timestamppb.New(time.Now().Add(-30 * time.Second)), + Manual: tc.manual, + }} + + err := sched.HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + RequestId: "req-1", + Outcome: &persistencespb.ChasmNexusCompletion_Failure{ + Failure: &failurepb.Failure{Message: "allow-all failure"}, + }, + CloseTime: timestamppb.Now(), + }) + require.NoError(t, err) + _, err = node.CloseTransaction() + require.NoError(t, err) + + readCtx := chasm.NewContext(context.Background(), node) + require.Equal(t, initial, sched.LastCompletionResult.Get(readCtx)) + require.False(t, sched.Schedule.State.Paused) + invoker := sched.Invoker.Get(readCtx) + require.Empty(t, invoker.GetBufferedStarts()) + require.Len(t, sched.Info.GetRecentActions(), 1) + require.Equal(t, enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, sched.Info.GetRecentActions()[0].GetStartWorkflowStatus()) + }) + } +} + +func TestHandleNexusCompletion_IgnoredReasonMetric(t *testing.T) { + for _, tc := range []struct { + name string + setup func(*scheduler.Invoker) + reason string + }{ + { + name: "unrecognized request", + reason: "unrecognized_request_id", + }, + { + name: "already completed", + setup: func(invoker *scheduler.Invoker) { + invoker.BufferedStarts = []*schedulespb.BufferedStart{{ + RequestId: "request-id", + Completed: &schedulespb.CompletedResult{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + }, + }} + }, + reason: "already_completed", + }, + } { + t.Run(tc.name, func(t *testing.T) { + recorder := metricstest.NewCaptureHandler() + capture := recorder.StartCapture() + defer recorder.StopCapture(capture) + + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + _, engineCtx := newTestEngineContext(t, logger, withEngineMetricsHandler(recorder)) + _, err := chasm.StartExecution( + engineCtx, + chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: scheduleID}, + scheduler.CreateScheduler, + &schedulerpb.CreateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: namespace, ScheduleId: scheduleID, Schedule: defaultSchedule(), RequestId: "create-request", + }, + }, + ) + require.NoError(t, err) + rootRef := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: scheduleID}) + + _, _, err = chasm.UpdateComponent(engineCtx, rootRef, + func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + if tc.setup != nil { + tc.setup(s.Invoker.Get(ctx)) + } + return struct{}{}, s.HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{RequestId: "request-id"}) + }, struct{}{}) + require.NoError(t, err) + + recordings := capture.Snapshot()[metrics.ScheduleCallbackIgnored.Name()] + require.Len(t, recordings, 1) + require.Equal(t, tc.reason, recordings[0].Tags["reason"]) + }) + } +} + +func TestHandleNexusCompletion_MigratedRunningWorkflowKeepsCompletionState(t *testing.T) { + now := time.Now().UTC() + v1Schedule := defaultSchedule() + v1Schedule.Policies.OverlapPolicy = enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL + v1Schedule.Policies.PauseOnFailure = true + v1Info := &schedulepb.ScheduleInfo{ + RunningWorkflows: []*commonpb.WorkflowExecution{{WorkflowId: "wf-tracked", RunId: "run-tracked"}}, + } + v1State := &schedulespb.InternalState{ + Namespace: namespace, + NamespaceId: namespaceID, + ScheduleId: scheduleID, + ConflictToken: 42, + LastProcessedTime: timestamppb.New(now), + } + req := migration.LegacyToCreateFromMigrationStateRequest(v1Schedule, v1Info, v1State, nil, nil, now) + + var migrated *schedulespb.BufferedStart + for _, start := range req.GetState().GetInvokerState().GetBufferedStarts() { + if start.GetWorkflowId() == "wf-tracked" { + migrated = start + break + } + } + require.NotNil(t, migrated) + require.Equal(t, enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED, migrated.GetOverlapPolicy()) + + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + _, engineCtx := newTestEngineContext(t, logger) + handler := scheduler.NewTestHandler(logger) + _, err := handler.TestCreateFromMigrationState(engineCtx, req) + require.NoError(t, err) + + rootRef := chasm.NewComponentRef[*scheduler.Scheduler](chasm.ExecutionKey{ + NamespaceID: namespaceID, + BusinessID: scheduleID, + }) + _, _, err = chasm.UpdateComponent(engineCtx, rootRef, + func(s *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + return struct{}{}, s.HandleNexusCompletion(ctx, &persistencespb.ChasmNexusCompletion{ + RequestId: migrated.GetRequestId(), + Outcome: &persistencespb.ChasmNexusCompletion_Failure{ + Failure: &failurepb.Failure{Message: "tracked workflow failed"}, + }, + CloseTime: timestamppb.New(now), + }) + }, struct{}{}) + require.NoError(t, err) + + _, err = chasm.ReadComponent(engineCtx, rootRef, + func(s *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + require.Equal(t, "tracked workflow failed", s.LastCompletionResult.Get(ctx).GetFailure().GetMessage()) + require.True(t, s.Schedule.State.Paused) + return struct{}{}, nil + }, struct{}{}) + require.NoError(t, err) +} + // TestHandleNexusCompletion_Failure verifies that a failed workflow completion // is properly recorded with the failure payload and workflow status is updated. func TestHandleNexusCompletion_Failure(t *testing.T) { diff --git a/chasm/lib/scheduler/scheduler_test.go b/chasm/lib/scheduler/scheduler_test.go index 571cd4efb3f..1ecb268ad0f 100644 --- a/chasm/lib/scheduler/scheduler_test.go +++ b/chasm/lib/scheduler/scheduler_test.go @@ -156,6 +156,70 @@ func TestListInfo_RecentActionsCapped(t *testing.T) { } } +func TestDescribe_RecentActionsCappedAcrossSources(t *testing.T) { + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + _, engineCtx := newTestEngineContext(t, logger) + result, err := chasm.StartExecution( + engineCtx, + chasm.ExecutionKey{NamespaceID: namespaceID, BusinessID: scheduleID}, + scheduler.CreateScheduler, + &schedulerpb.CreateScheduleRequest{ + NamespaceId: namespaceID, + FrontendRequest: &workflowservice.CreateScheduleRequest{ + Namespace: namespace, + ScheduleId: scheduleID, + Schedule: defaultSchedule(), + }, + }, + ) + require.NoError(t, err) + + rootRef := chasm.NewComponentRef[*scheduler.Scheduler](result.ExecutionKey) + base := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC) + _, _, err = chasm.UpdateComponent(engineCtx, rootRef, + func(sched *scheduler.Scheduler, ctx chasm.MutableContext, _ struct{}) (struct{}, error) { + invoker := sched.Invoker.Get(ctx) + for i := range scheduler.RecentActionCount { + storedTime := timestamppb.New(base.Add(time.Duration(2*i) * time.Minute)) + sched.Info.RecentActions = append(sched.Info.RecentActions, &schedulepb.ScheduleActionResult{ + ScheduleTime: storedTime, + ActualTime: storedTime, + StartWorkflowResult: &commonpb.WorkflowExecution{ + WorkflowId: fmt.Sprintf("stored-workflow-%d", i), + RunId: fmt.Sprintf("stored-run-%d", i), + }, + }) + + trackedTime := timestamppb.New(base.Add(time.Duration(2*i+1) * time.Minute)) + invoker.BufferedStarts = append(invoker.BufferedStarts, &schedulespb.BufferedStart{ + WorkflowId: fmt.Sprintf("tracked-workflow-%d", i), + RunId: fmt.Sprintf("tracked-run-%d", i), + StartTime: trackedTime, + }) + } + return struct{}{}, nil + }, struct{}{}) + require.NoError(t, err) + + _, err = chasm.ReadComponent(engineCtx, rootRef, + func(sched *scheduler.Scheduler, ctx chasm.Context, _ struct{}) (struct{}, error) { + response, err := sched.Describe(ctx, &schedulerpb.DescribeScheduleRequest{}, newLegacySpecBuilder(0, 0)) + require.NoError(t, err) + actions := response.GetFrontendResponse().GetInfo().GetRecentActions() + require.Len(t, actions, scheduler.RecentActionCount) + for i, action := range actions { + wantIndex := i/2 + scheduler.RecentActionCount/2 + if i%2 == 0 { + require.Equal(t, fmt.Sprintf("stored-run-%d", wantIndex), action.GetStartWorkflowResult().GetRunId()) + } else { + require.Equal(t, fmt.Sprintf("tracked-run-%d", wantIndex), action.GetStartWorkflowResult().GetRunId()) + } + } + return struct{}{}, nil + }, struct{}{}) + require.NoError(t, err) +} + func TestCreateSchedulerFromMigration(t *testing.T) { now := time.Now().UTC() _, _, node := setupSchedulerForTest(t) diff --git a/tests/schedule_test.go b/tests/schedule_test.go index 0018ca1cce9..2e4db2068d4 100644 --- a/tests/schedule_test.go +++ b/tests/schedule_test.go @@ -521,6 +521,7 @@ func runSharedScheduleTests(t *testing.T, newContext contextFactory) { t.Run("TestBasics", func(t *testing.T) { t.Parallel(); testBasics(t, newContext) }) t.Run("TestInput", func(t *testing.T) { t.Parallel(); testInput(t, newContext) }) t.Run("TestLastCompletionAndError", func(t *testing.T) { t.Parallel(); testLastCompletionAndError(t, newContext) }) + t.Run("TestAllowAllDescribeContract", func(t *testing.T) { t.Parallel(); testAllowAllDescribeContract(t, newContext) }) t.Run("TestScheduleContinuesAfterWorkflowRetryFailure", func(t *testing.T) { t.Parallel(); testScheduleContinuesAfterWorkflowRetryFailure(t, newContext) }) t.Run("TestListSchedulesReturnsWorkflowStatus", func(t *testing.T) { t.Parallel(); testListSchedulesReturnsWorkflowStatus(t, newContext) }) t.Run("TestListSchedulesRecentActionsCapped", func(t *testing.T) { t.Parallel(); testListSchedulesRecentActionsCapped(t, newContext) }) @@ -567,6 +568,127 @@ func runSharedScheduleTests(t *testing.T, newContext contextFactory) { t.Run("TestBufferOneDeferredFiresAfterCompletion", func(t *testing.T) { t.Parallel(); testBufferOneDeferredFiresAfterCompletion(t, newContext) }) } +// testAllowAllDescribeContract verifies the customer-facing Describe state shared by V1 and CHASM. +// ALLOW_ALL executions appear in RecentActions but not RunningWorkflows; sequential executions remain active. +func testAllowAllDescribeContract(t *testing.T, newContext contextFactory) { + s := newScheduleEnv(t, scheduleCommonOpts(t)...) + sid := testcore.RandomizeStr("sched-allow-all-active") + wid := testcore.RandomizeStr("sched-allow-all-active-wf") + wt := testcore.RandomizeStr("sched-allow-all-active-wt") + + var runs atomic.Int32 + s.SdkWorker().RegisterWorkflowWithOptions(func(ctx workflow.Context) error { + _ = workflow.SideEffect(ctx, func(workflow.Context) any { runs.Add(1); return 0 }) + failed := false + selector := workflow.NewSelector(ctx) + selector.AddReceive(workflow.GetSignalChannel(ctx, "complete"), func(c workflow.ReceiveChannel, _ bool) { + c.Receive(ctx, nil) + }) + selector.AddReceive(workflow.GetSignalChannel(ctx, "fail"), func(c workflow.ReceiveChannel, _ bool) { + c.Receive(ctx, nil) + failed = true + }) + selector.Select(ctx) + if failed { + return errors.New("allow-all failure") + } + return nil + }, workflow.RegisterOptions{Name: wt}) + + ctx := newContext(testcore.NewContext()) + createSchedule(ctx, t, s, sid, &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{}, + Action: startWorkflowAction(s, wid, wt), + Policies: &schedulepb.SchedulePolicies{PauseOnFailure: true}, + }) + + patchSchedule(ctx, t, s, sid, triggerPatch(enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL)) + var allowAllRun *commonpb.WorkflowExecution + var allowAllDescribe *workflowservice.DescribeScheduleResponse + require.Eventually(t, func() bool { + desc, err := s.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: s.Namespace().String(), ScheduleId: sid, + }) + if err != nil || runs.Load() != 1 || desc.GetInfo().GetActionCount() != 1 || + desc.GetInfo().GetBufferSize() != 0 || len(desc.GetInfo().GetRecentActions()) != 1 || + len(desc.GetInfo().GetRunningWorkflows()) != 0 { + return false + } + recent := desc.GetInfo().GetRecentActions()[0] + if recent.GetStartWorkflowStatus() != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING || + recent.GetScheduleTime() == nil || recent.GetActualTime() == nil { + return false + } + allowAllRun = recent.GetStartWorkflowResult() + if allowAllRun.GetRunId() == "" { + return false + } + allowAllDescribe = desc + return true + }, awaitTimeout, pollInterval, "ALLOW_ALL Describe state should be recent, running, and not active") + require.Empty(t, allowAllDescribe.GetInfo().GetRunningWorkflows()) + require.Equal(t, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, allowAllDescribe.GetInfo().GetRecentActions()[0].GetStartWorkflowStatus()) + + allowAllNominal, err := time.Parse(time.RFC3339, strings.TrimPrefix(allowAllRun.GetWorkflowId(), wid+"-")) + require.NoError(t, err) + require.Eventually(t, func() bool { + return time.Now().UTC().Truncate(time.Second).After(allowAllNominal) + }, awaitTimeout, pollInterval, "next trigger should receive a distinct timestamp-based workflow ID") + + patchSchedule(ctx, t, s, sid, triggerPatch(enumspb.SCHEDULE_OVERLAP_POLICY_SKIP)) + var sequentialRun *commonpb.WorkflowExecution + var sequentialDescribe *workflowservice.DescribeScheduleResponse + require.Eventually(t, func() bool { + desc, err := s.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: s.Namespace().String(), ScheduleId: sid, + }) + if err != nil || runs.Load() != 2 || desc.GetInfo().GetActionCount() != 2 || + desc.GetInfo().GetBufferSize() != 0 || len(desc.GetInfo().GetRecentActions()) != 2 || + len(desc.GetInfo().GetRunningWorkflows()) != 1 { + return false + } + sequentialRun = desc.GetInfo().GetRunningWorkflows()[0] + if sequentialRun.GetRunId() == "" || sequentialRun.GetRunId() == allowAllRun.GetRunId() { + return false + } + for _, recent := range desc.GetInfo().GetRecentActions() { + if recent.GetStartWorkflowStatus() != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING || + recent.GetScheduleTime() == nil || recent.GetActualTime() == nil { + return false + } + } + sequentialDescribe = desc + return true + }, awaitTimeout, pollInterval, "ALLOW_ALL execution should not block a sequential trigger") + require.Equal(t, []*commonpb.WorkflowExecution{sequentialRun}, sequentialDescribe.GetInfo().GetRunningWorkflows()) + require.ElementsMatch(t, []*commonpb.WorkflowExecution{allowAllRun, sequentialRun}, []*commonpb.WorkflowExecution{ + sequentialDescribe.GetInfo().GetRecentActions()[0].GetStartWorkflowResult(), + sequentialDescribe.GetInfo().GetRecentActions()[1].GetStartWorkflowResult(), + }) + + require.NoError(t, s.SdkClient().SignalWorkflow(ctx, allowAllRun.GetWorkflowId(), allowAllRun.GetRunId(), "fail", nil)) + require.Eventually(t, func() bool { + resp, err := s.FrontendClient().DescribeWorkflowExecution(ctx, &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: s.Namespace().String(), Execution: allowAllRun, + }) + return err == nil && resp.GetWorkflowExecutionInfo().GetStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_FAILED + }, awaitTimeout, pollInterval, "ALLOW_ALL workflow should fail") + + desc, err := s.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: s.Namespace().String(), ScheduleId: sid, + }) + require.NoError(t, err) + require.False(t, desc.GetSchedule().GetState().GetPaused()) + require.Equal(t, int64(2), desc.GetInfo().GetActionCount()) + require.Zero(t, desc.GetInfo().GetBufferSize()) + require.Equal(t, []*commonpb.WorkflowExecution{sequentialRun}, desc.GetInfo().GetRunningWorkflows()) + require.Len(t, desc.GetInfo().GetRecentActions(), 2) + for _, recent := range desc.GetInfo().GetRecentActions() { + require.Equal(t, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, recent.GetStartWorkflowStatus()) + } + require.NoError(t, s.SdkClient().SignalWorkflow(ctx, sequentialRun.GetWorkflowId(), sequentialRun.GetRunId(), "complete", nil)) +} + // testBufferSizeReportedWhenBuffered verifies that ScheduleInfo.BufferSize is // populated by both the V1 and V2 (CHASM) schedulers when at least one fire is // queued behind a still-running workflow. A schedule with a 1s interval and @@ -4842,6 +4964,7 @@ type scheduleClosesCase struct { name string prefix string state *schedulepb.ScheduleState + policies *schedulepb.SchedulePolicies expectedRuns int32 // buildSpec receives the current time at the moment the schedule is created @@ -4892,6 +5015,17 @@ func testScheduleClosesFromIdle(t *testing.T, newContext contextFactory) { state: &schedulepb.ScheduleState{LimitedActions: true, RemainingActions: 2}, strictRunCount: true, }, + { + name: "FinalAllowAllAction", + prefix: "sched-final-allow-all-closes", + expectedRuns: 1, + buildSpec: func(_ time.Time) *schedulepb.ScheduleSpec { + return intervalSpec(fastInterval) + }, + state: &schedulepb.ScheduleState{LimitedActions: true, RemainingActions: 1}, + policies: &schedulepb.SchedulePolicies{OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL}, + strictRunCount: true, + }, { name: "IntervalEndTime", prefix: "sched-interval-end-closes", @@ -4923,9 +5057,10 @@ func runScheduleClosesFromIdleCase(t *testing.T, newContext contextFactory, c sc ctx := newContext(s.Context()) createSchedule(ctx, t, s, sid, &schedulepb.Schedule{ - Spec: c.buildSpec(time.Now().UTC()), - State: c.state, - Action: startWorkflowAction(s, wid, wt), + Spec: c.buildSpec(time.Now().UTC()), + Policies: c.policies, + State: c.state, + Action: startWorkflowAction(s, wid, wt), }) // A hard action budget must land on exactly expectedRuns; time-bounded specs From a02e33e5717ad6b35be95a84dea866e21df522fe Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Tue, 25 Aug 2026 15:22:54 -0700 Subject: [PATCH 063/139] Add version to deletion workflow replication task (#11411) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? **1. Delete execution replication tasks carry a failover version.** Stamped with the source cluster's failover version at generation (`shard/context_impl.go:1013`), carried in the existing `ReplicationTaskInfo.version` (no proto change), and skipped on apply when older than the target's namespace failover version (`executable_delete_execution_task.go:130`). Still applied: unversioned tasks (queued pre-upgrade), tasks newer than the target's namespace entry, and deletions synthesized by versioned-transition tasks. **2. `DeleteWorkflowExecution` is rejected on a cluster passive for the workflow** (`workflow_handler.go:2513`), with the usual `NamespaceNotActive`. In the frontend, because the history path is shared with replication apply, which must delete on passive clusters — that path, the delete-namespace worker, and admin force-delete are untouched. ## Why? An unversioned delete task generated before a failover kept being applied afterwards, deleting a run the new active cluster owns. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ``` temporal --address :7233 workflow -n global-ns delete -w pay-invoice-0 -r 019fcf5b-996e-7416-9a2d-4ffb0ae7fc13 --grpc-meta xdc-redirection=false WARNING: Deleting Workflow Executions in a global Namespace removes them from all replicas. Requests sent to a passive cluster are forwarded to the active cluster by default; to target the passive cluster directly, specify `--grpc-meta xdc-redirection=false`. Delete Workflow "pay-invoice-0" with Run ID "019fcf5b-996e-7416-9a2d-4ffb0ae7fc13"? y/N y Error: failed to delete workflow: Namespace: global-ns is active in cluster: cluster-b, while current cluster cluster-a is a standby cluster. ``` ## Potential risks - A deletion issued just before a failover is dropped on targets; cleanup waits for the new active cluster's retention timer. Delayed, not leaked — intended trade-off. - API behavior change: deleting against a passive cluster now fails, including batch delete (`batcher/activities.go:704`). No killswitch; admin force-delete is the escape hatch. --------- Co-authored-by: Claude Opus 5 (1M context) --- .../serialization/task_serializers.go | 2 + .../serialization/task_serializers_test.go | 1 + service/frontend/workflow_handler.go | 37 ++++- service/frontend/workflow_handler_test.go | 69 ++++++++- .../history/deletemanager/delete_manager.go | 9 ++ .../deletemanager/delete_manager_test.go | 9 ++ service/history/interfaces/shard_context.go | 1 + .../history/interfaces/shard_context_mock.go | 16 +- .../executable_delete_execution_task.go | 74 ++++++++- .../executable_delete_execution_task_test.go | 145 +++++++++++++++++- service/history/shard/context_impl.go | 2 + service/history/shard/context_test.go | 18 ++- .../delete_execution_replication_task.go | 9 +- tests/xdc/stream_based_replication_test.go | 13 +- 14 files changed, 380 insertions(+), 25 deletions(-) diff --git a/common/persistence/serialization/task_serializers.go b/common/persistence/serialization/task_serializers.go index beec7718f7d..c1ce03ebda9 100644 --- a/common/persistence/serialization/task_serializers.go +++ b/common/persistence/serialization/task_serializers.go @@ -1582,6 +1582,7 @@ func replicationDeleteExecutionTaskToProto( TaskId: task.TaskID, VisibilityTime: timestamppb.New(task.VisibilityTimestamp), ArchetypeId: task.ArchetypeID, + Version: task.Version, } } @@ -1601,5 +1602,6 @@ func replicationDeleteExecutionTaskFromProto( VisibilityTimestamp: visibilityTimestamp, TaskID: info.TaskId, ArchetypeID: info.ArchetypeId, + Version: info.Version, } } diff --git a/common/persistence/serialization/task_serializers_test.go b/common/persistence/serialization/task_serializers_test.go index 34719e38f9c..87197b217e1 100644 --- a/common/persistence/serialization/task_serializers_test.go +++ b/common/persistence/serialization/task_serializers_test.go @@ -457,6 +457,7 @@ func (s *taskSerializerSuite) TestDeleteExecutionReplicationTask() { VisibilityTimestamp: time.Unix(0, 0).UTC(), // go == compare for location as well which is striped during marshaling/unmarshaling TaskID: rand.Int63(), ArchetypeID: rand.Uint32(), + Version: rand.Int63(), } s.assertEqualTasks(deleteExecutionReplicationTask) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index b2441d39caa..ea04af7495c 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -2518,13 +2518,17 @@ func (wh *WorkflowHandler) DeleteWorkflowExecution(ctx context.Context, request return nil, err } - namespaceID, err := wh.namespaceRegistry.GetNamespaceID(namespace.Name(request.GetNamespace())) + namespaceEntry, err := wh.namespaceRegistry.GetNamespace(namespace.Name(request.GetNamespace())) if err != nil { return nil, err } + if err := wh.validateWorkflowDeletionCluster(namespaceEntry, request.GetWorkflowExecution().GetWorkflowId()); err != nil { + return nil, err + } + _, err = wh.historyClient.DeleteWorkflowExecution(ctx, &historyservice.DeleteWorkflowExecutionRequest{ - NamespaceId: namespaceID.String(), + NamespaceId: namespaceEntry.ID().String(), WorkflowExecution: request.GetWorkflowExecution(), ClosedWorkflowOnly: false, }) @@ -2535,6 +2539,35 @@ func (wh *WorkflowHandler) DeleteWorkflowExecution(ctx context.Context, request return &workflowservice.DeleteWorkflowExecutionResponse{}, nil } +// validateWorkflowDeletionCluster rejects a deletion that targets a cluster which is passive for the +// workflow. A deletion performed on a passive cluster is not replicated: it only drops the local copy +// while the active cluster still holds the execution and keeps replicating it back, so the two +// clusters diverge (and the local copy can even be resurrected by a later replication task). The +// caller must delete on the active cluster, which replicates the deletion to every other cluster. +// +// When XDC redirection is enabled the request is forwarded to the active cluster before it gets here, +// so this only rejects requests that would otherwise be served locally on a passive cluster. Deleting +// local state on a passive cluster is still possible through the admin ForceDeleteWorkflowExecution +// API, which does not go through this handler. +func (wh *WorkflowHandler) validateWorkflowDeletionCluster( + namespaceEntry *namespace.Namespace, + workflowID string, +) error { + if !namespaceEntry.IsGlobalNamespace() { + return nil + } + currentCluster := wh.clusterMetadata.GetCurrentClusterName() + activeCluster := namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: workflowID}) + if activeCluster == currentCluster { + return nil + } + return serviceerror.NewNamespaceNotActive( + namespaceEntry.Name().String(), + currentCluster, + activeCluster, + ) +} + // ListOpenWorkflowExecutions is a visibility API to list the open executions in a specific namespace. func (wh *WorkflowHandler) ListOpenWorkflowExecutions(ctx context.Context, request *workflowservice.ListOpenWorkflowExecutionsRequest) (_ *workflowservice.ListOpenWorkflowExecutionsResponse, retError error) { defer log.CapturePanic(wh.logger, &retError) diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index a896ebb6e5e..386e18fadea 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -4696,7 +4696,8 @@ func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution() { // History call failed. s.mockResource.HistoryClient.EXPECT().DeleteWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, errors.New("random error")) - s.mockResource.NamespaceCache.EXPECT().GetNamespaceID(namespace.Name("test-namespace")).Return(namespace.ID("test-namespace-id"), nil) + s.mockResource.NamespaceCache.EXPECT().GetNamespace(namespace.Name("test-namespace")). + Return(s.localNamespaceEntry("test-namespace", "test-namespace-id"), nil) resp, err := wh.DeleteWorkflowExecution(ctx, &workflowservice.DeleteWorkflowExecutionRequest{ Namespace: "test-namespace", WorkflowExecution: &commonpb.WorkflowExecution{ @@ -4710,7 +4711,8 @@ func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution() { // Success case. s.mockResource.HistoryClient.EXPECT().DeleteWorkflowExecution(gomock.Any(), gomock.Any()).Return(&historyservice.DeleteWorkflowExecutionResponse{}, nil) - s.mockResource.NamespaceCache.EXPECT().GetNamespaceID(namespace.Name("test-namespace")).Return(namespace.ID("test-namespace-id"), nil) + s.mockResource.NamespaceCache.EXPECT().GetNamespace(namespace.Name("test-namespace")). + Return(s.localNamespaceEntry("test-namespace", "test-namespace-id"), nil) resp, err = wh.DeleteWorkflowExecution(ctx, &workflowservice.DeleteWorkflowExecutionRequest{ Namespace: "test-namespace", WorkflowExecution: &commonpb.WorkflowExecution{ @@ -4722,6 +4724,69 @@ func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution() { s.NotNil(resp) } +// A deletion is only replicated when it happens on the active cluster, so a request that lands on a +// passive cluster (XDC redirection disabled) must be rejected instead of deleting the local copy only. +func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution_PassiveCluster() { + wh := s.getWorkflowHandler(s.newConfig()) + + s.mockResource.NamespaceCache.EXPECT().GetNamespace(namespace.Name("test-namespace")). + Return(s.globalNamespaceEntry("test-namespace", "test-namespace-id", cluster.TestAlternativeClusterName), nil) + s.mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() + // No DeleteWorkflowExecution call is expected on the history client. + + resp, err := wh.DeleteWorkflowExecution(context.Background(), &workflowservice.DeleteWorkflowExecutionRequest{ + Namespace: "test-namespace", + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: "test-workflow-id", + }, + }) + s.Nil(resp) + var notActiveErr *serviceerror.NamespaceNotActive + s.ErrorAs(err, ¬ActiveErr) +} + +func (s *WorkflowHandlerSuite) Test_DeleteWorkflowExecution_ActiveCluster() { + wh := s.getWorkflowHandler(s.newConfig()) + + s.mockResource.NamespaceCache.EXPECT().GetNamespace(namespace.Name("test-namespace")). + Return(s.globalNamespaceEntry("test-namespace", "test-namespace-id", cluster.TestCurrentClusterName), nil) + s.mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockResource.HistoryClient.EXPECT().DeleteWorkflowExecution(gomock.Any(), gomock.Any()). + Return(&historyservice.DeleteWorkflowExecutionResponse{}, nil) + + resp, err := wh.DeleteWorkflowExecution(context.Background(), &workflowservice.DeleteWorkflowExecutionRequest{ + Namespace: "test-namespace", + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: "test-workflow-id", + }, + }) + s.NoError(err) + s.NotNil(resp) +} + +func (s *WorkflowHandlerSuite) localNamespaceEntry(nsName string, nsID string) *namespace.Namespace { + return namespace.NewLocalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: nsID, Name: nsName}, + nil, + cluster.TestCurrentClusterName, + ) +} + +func (s *WorkflowHandlerSuite) globalNamespaceEntry(nsName string, nsID string, activeCluster string) *namespace.Namespace { + return namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: nsID, Name: nsName}, + nil, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: activeCluster, + Clusters: []string{ + cluster.TestCurrentClusterName, + cluster.TestAlternativeClusterName, + }, + }, + cluster.TestCurrentClusterInitialFailoverVersion, + ) +} + func (s *WorkflowHandlerSuite) TestExecuteMultiOperation() { ctx := context.Background() config := s.newConfig() diff --git a/service/history/deletemanager/delete_manager.go b/service/history/deletemanager/delete_manager.go index 7fa66c44e7c..af4e26d649b 100644 --- a/service/history/deletemanager/delete_manager.go +++ b/service/history/deletemanager/delete_manager.go @@ -6,6 +6,7 @@ import ( "context" commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/metrics" @@ -181,6 +182,13 @@ func (m *DeleteManagerImpl) deleteWorkflowExecutionInternal( } executionInfo := ms.GetExecutionInfo() + lastWriteVersion := common.EmptyVersion + if !retentionDelete { + lastWriteVersion, err = ms.GetLastWriteVersion() + if err != nil { + return err + } + } if err := m.shardContext.DeleteWorkflowExecution( ctx, definition.WorkflowKey{ @@ -189,6 +197,7 @@ func (m *DeleteManagerImpl) deleteWorkflowExecutionInternal( RunID: we.GetRunId(), }, ms.ChasmTree().ArchetypeID(), + lastWriteVersion, currentBranchToken, executionInfo.GetCloseVisibilityTaskId(), closeTime, diff --git a/service/history/deletemanager/delete_manager_test.go b/service/history/deletemanager/delete_manager_test.go index f5cb3be51cf..5e806a08105 100644 --- a/service/history/deletemanager/delete_manager_test.go +++ b/service/history/deletemanager/delete_manager_test.go @@ -11,6 +11,7 @@ import ( "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/definition" @@ -96,6 +97,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteDeletedWorkflowExecution() { }) mockMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{}) mockMutableState.EXPECT().GetWorkflowCloseTime(gomock.Any()).Return(time.Unix(0, 0).UTC(), nil) + mockMutableState.EXPECT().GetLastWriteVersion().Return(tests.Version, nil) mockMutableState.EXPECT().ChasmTree().Return(workflow.NoopChasmTree).AnyTimes() stage := tasks.DeleteWorkflowExecutionStageNone @@ -107,6 +109,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteDeletedWorkflowExecution() { RunID: tests.RunID, }, chasm.WorkflowArchetypeID, + tests.Version, []byte{22, 8, 78}, closeExecutionVisibilityTaskID, time.Unix(0, 0).UTC(), @@ -142,6 +145,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteDeletedWorkflowExecution_Error() }) mockMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{}) mockMutableState.EXPECT().GetWorkflowCloseTime(gomock.Any()).Return(time.Unix(0, 0).UTC(), nil) + mockMutableState.EXPECT().GetLastWriteVersion().Return(tests.Version, nil) mockMutableState.EXPECT().ChasmTree().Return(workflow.NoopChasmTree).AnyTimes() stage := tasks.DeleteWorkflowExecutionStageNone @@ -153,6 +157,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteDeletedWorkflowExecution_Error() RunID: tests.RunID, }, chasm.WorkflowArchetypeID, + tests.Version, []byte{22, 8, 78}, closeExecutionVisibilityTaskID, time.Unix(0, 0).UTC(), @@ -198,6 +203,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteWorkflowExecutionByRetention_Skip RunID: tests.RunID, }, workflow.NoopChasmTree.ArchetypeID(), + common.EmptyVersion, []byte{22, 8, 78}, closeExecutionVisibilityTaskID, time.Unix(0, 0).UTC(), @@ -208,6 +214,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteWorkflowExecutionByRetention_Skip _ context.Context, _ definition.WorkflowKey, _ chasm.ArchetypeID, + _ int64, _ []byte, _ int64, _ time.Time, @@ -248,6 +255,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteWorkflowExecution_OpenWorkflow() }) mockMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{}) mockMutableState.EXPECT().GetWorkflowCloseTime(gomock.Any()).Return(time.Unix(0, 0).UTC(), nil) + mockMutableState.EXPECT().GetLastWriteVersion().Return(tests.Version, nil) mockMutableState.EXPECT().ChasmTree().Return(workflow.NoopChasmTree).AnyTimes() stage := tasks.DeleteWorkflowExecutionStageNone @@ -259,6 +267,7 @@ func (s *deleteManagerWorkflowSuite) TestDeleteWorkflowExecution_OpenWorkflow() RunID: tests.RunID, }, chasm.WorkflowArchetypeID, + tests.Version, []byte{22, 8, 78}, closeExecutionVisibilityTaskID, time.Unix(0, 0).UTC(), diff --git a/service/history/interfaces/shard_context.go b/service/history/interfaces/shard_context.go index d0a773435c6..b853db94032 100644 --- a/service/history/interfaces/shard_context.go +++ b/service/history/interfaces/shard_context.go @@ -105,6 +105,7 @@ type ( ctx context.Context, workflowKey definition.WorkflowKey, archetypeID chasm.ArchetypeID, + lastWriteVersion int64, branchToken []byte, closeExecutionVisibilityTaskID int64, workflowCloseTime time.Time, diff --git a/service/history/interfaces/shard_context_mock.go b/service/history/interfaces/shard_context_mock.go index 0fefd04a306..d6ad1af2c94 100644 --- a/service/history/interfaces/shard_context_mock.go +++ b/service/history/interfaces/shard_context_mock.go @@ -211,17 +211,17 @@ func (mr *MockShardContextMockRecorder) CurrentVectorClock() *gomock.Call { } // DeleteWorkflowExecution mocks base method. -func (m *MockShardContext) DeleteWorkflowExecution(ctx context.Context, workflowKey definition.WorkflowKey, archetypeID chasm.ArchetypeID, branchToken []byte, closeExecutionVisibilityTaskID int64, workflowCloseTime, workflowStartTime time.Time, stage *tasks.DeleteWorkflowExecutionStage, retentionDelete bool) error { +func (m *MockShardContext) DeleteWorkflowExecution(ctx context.Context, workflowKey definition.WorkflowKey, archetypeID chasm.ArchetypeID, lastWriteVersion int64, branchToken []byte, closeExecutionVisibilityTaskID int64, workflowCloseTime, workflowStartTime time.Time, stage *tasks.DeleteWorkflowExecutionStage, retentionDelete bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteWorkflowExecution", ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) + ret := m.ctrl.Call(m, "DeleteWorkflowExecution", ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) ret0, _ := ret[0].(error) return ret0 } // DeleteWorkflowExecution indicates an expected call of DeleteWorkflowExecution. -func (mr *MockShardContextMockRecorder) DeleteWorkflowExecution(ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete any) *gomock.Call { +func (mr *MockShardContextMockRecorder) DeleteWorkflowExecution(ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteWorkflowExecution", reflect.TypeOf((*MockShardContext)(nil).DeleteWorkflowExecution), ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteWorkflowExecution", reflect.TypeOf((*MockShardContext)(nil).DeleteWorkflowExecution), ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) } // EndpointRegistry mocks base method. @@ -1058,17 +1058,17 @@ func (mr *MockControllableContextMockRecorder) CurrentVectorClock() *gomock.Call } // DeleteWorkflowExecution mocks base method. -func (m *MockControllableContext) DeleteWorkflowExecution(ctx context.Context, workflowKey definition.WorkflowKey, archetypeID chasm.ArchetypeID, branchToken []byte, closeExecutionVisibilityTaskID int64, workflowCloseTime, workflowStartTime time.Time, stage *tasks.DeleteWorkflowExecutionStage, retentionDelete bool) error { +func (m *MockControllableContext) DeleteWorkflowExecution(ctx context.Context, workflowKey definition.WorkflowKey, archetypeID chasm.ArchetypeID, lastWriteVersion int64, branchToken []byte, closeExecutionVisibilityTaskID int64, workflowCloseTime, workflowStartTime time.Time, stage *tasks.DeleteWorkflowExecutionStage, retentionDelete bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteWorkflowExecution", ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) + ret := m.ctrl.Call(m, "DeleteWorkflowExecution", ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) ret0, _ := ret[0].(error) return ret0 } // DeleteWorkflowExecution indicates an expected call of DeleteWorkflowExecution. -func (mr *MockControllableContextMockRecorder) DeleteWorkflowExecution(ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete any) *gomock.Call { +func (mr *MockControllableContextMockRecorder) DeleteWorkflowExecution(ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteWorkflowExecution", reflect.TypeOf((*MockControllableContext)(nil).DeleteWorkflowExecution), ctx, workflowKey, archetypeID, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteWorkflowExecution", reflect.TypeOf((*MockControllableContext)(nil).DeleteWorkflowExecution), ctx, workflowKey, archetypeID, lastWriteVersion, branchToken, closeExecutionVisibilityTaskID, workflowCloseTime, workflowStartTime, stage, retentionDelete) } // EndpointRegistry mocks base method. diff --git a/service/history/replication/executable_delete_execution_task.go b/service/history/replication/executable_delete_execution_task.go index 91a48af1e0b..039424a0069 100644 --- a/service/history/replication/executable_delete_execution_task.go +++ b/service/history/replication/executable_delete_execution_task.go @@ -12,8 +12,10 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -27,6 +29,11 @@ type ExecutableDeleteExecutionTask struct { chasm.ComponentRef ExecutableTask + + // lastWriteVersion is the source execution's last write version when it was deleted. + // It is common.EmptyVersion when the source did not stamp one, i.e. for tasks generated before + // the version was introduced and for deletions synthesized from another replication task. + lastWriteVersion int64 } var _ ctasks.Task = (*ExecutableDeleteExecutionTask)(nil) @@ -50,6 +57,14 @@ func NewExecutableDeleteExecutionTask( softassert.That(processToolBox.Logger, false, "delete execution replication task has unspecified archetype ID") } + // Only take the version from a genuine delete execution replication task. Other replication + // tasks (sync/verify versioned transition) synthesize a deletion out of their own task, whose + // version describes a different operation and must not be interpreted as a deletion version. + lastWriteVersion := common.EmptyVersion + if rawInfo.GetTaskType() == enumsspb.TASK_TYPE_REPLICATION_DELETE_EXECUTION { + lastWriteVersion = rawInfo.GetVersion() + } + return &ExecutableDeleteExecutionTask{ ProcessToolBox: processToolBox, ComponentRef: chasm.NewComponentRefByArchetypeID( @@ -70,6 +85,7 @@ func NewExecutableDeleteExecutionTask( sourceShardKey, replicationTask, ), + lastWriteVersion: lastWriteVersion, } } @@ -115,7 +131,9 @@ func (e *ExecutableDeleteExecutionTask) Execute() error { return err } currentCluster := e.ClusterMetadata.GetCurrentClusterName() - if namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: e.BusinessID}) == currentCluster { + // Legacy tasks have no execution-state fence and remain unsafe to apply on an active cluster. + if e.lastWriteVersion == common.EmptyVersion && + namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: e.BusinessID}) == currentCluster { e.Logger.Warn("Skipping delete execution replication task on active cluster", tag.WorkflowNamespaceID(e.NamespaceID), tag.WorkflowID(e.BusinessID), @@ -138,6 +156,28 @@ func (e *ExecutableDeleteExecutionTask) Execute() error { if err != nil { return err } + if e.lastWriteVersion != common.EmptyVersion { + targetLastWriteVersion, err := e.getLastWriteVersion(ctx, archetypeID) + if err != nil { + return err + } + if e.lastWriteVersion != targetLastWriteVersion { + e.Logger.Warn("Skipping delete execution replication task due to last write version mismatch", + tag.WorkflowNamespaceID(e.NamespaceID), + tag.WorkflowID(e.BusinessID), + tag.WorkflowRunID(e.RunID), + tag.TaskID(e.TaskID()), + tag.IncomingVersion(e.lastWriteVersion), + tag.CurrentVersion(targetLastWriteVersion), + ) + metrics.ReplicationTasksSkipped.With(e.MetricsHandler).Record( + 1, + metrics.OperationTag(metrics.DeleteExecutionReplicationTaskScope), + metrics.NamespaceTag(namespaceName), + ) + return nil + } + } switch archetypeID { case chasm.WorkflowArchetypeID: return e.deleteWorkflowExecution(ctx) @@ -173,6 +213,38 @@ func (e *ExecutableDeleteExecutionTask) deleteChasmExecution(ctx context.Context return e.ChasmEngine.DeleteExecution(ctx, e.ComponentRef, chasm.DeleteExecutionRequest{}) } +func (e *ExecutableDeleteExecutionTask) getLastWriteVersion( + ctx context.Context, + archetypeID chasm.ArchetypeID, +) (_ int64, retError error) { + namespaceID := namespace.ID(e.NamespaceID) + shardContext, err := e.ShardController.GetShardByNamespaceWorkflow(namespaceID, e.BusinessID) + if err != nil { + return common.EmptyVersion, err + } + workflowContext, release, err := e.WorkflowCache.GetOrCreateChasmExecution( + ctx, + shardContext, + namespaceID, + &commonpb.WorkflowExecution{ + WorkflowId: e.BusinessID, + RunId: e.RunID, + }, + archetypeID, + locks.PriorityLow, + ) + if err != nil { + return common.EmptyVersion, err + } + defer func() { release(retError) }() + + mutableState, err := workflowContext.LoadMutableState(ctx, shardContext) + if err != nil { + return common.EmptyVersion, err + } + return mutableState.GetLastWriteVersion() +} + func (e *ExecutableDeleteExecutionTask) HandleErr(err error) error { metrics.ReplicationTasksErrorByType.With(e.MetricsHandler).Record( 1, diff --git a/service/history/replication/executable_delete_execution_task_test.go b/service/history/replication/executable_delete_execution_task_test.go index 94dfc7fa84d..c298ca401c8 100644 --- a/service/history/replication/executable_delete_execution_task_test.go +++ b/service/history/replication/executable_delete_execution_task_test.go @@ -14,7 +14,9 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -22,6 +24,7 @@ import ( historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" + wcache "go.temporal.io/server/service/history/workflow/cache" "go.uber.org/mock/gomock" ) @@ -34,6 +37,8 @@ type ( clusterMetadata *cluster.MockMetadata shardController *shard.MockController namespaceCache *namespace.MockRegistry + workflowCache *wcache.MockCache + chasmEngine *chasm.MockEngine metricsHandler metrics.Handler logger log.Logger config *configs.Config @@ -63,6 +68,8 @@ func (s *executableDeleteExecutionTaskSuite) SetupTest() { s.clusterMetadata = cluster.NewMockMetadata(s.controller) s.shardController = shard.NewMockController(s.controller) s.namespaceCache = namespace.NewMockRegistry(s.controller) + s.workflowCache = wcache.NewMockCache(s.controller) + s.chasmEngine = chasm.NewMockEngine(s.controller) s.metricsHandler = metrics.NoopMetricsHandler s.logger = log.NewNoopLogger() s.config = tests.NewDynamicConfig() @@ -81,10 +88,12 @@ func (s *executableDeleteExecutionTaskSuite) SetupTest() { Config: s.config, ClusterMetadata: s.clusterMetadata, ShardController: s.shardController, + ChasmEngine: s.chasmEngine, NamespaceCache: s.namespaceCache, MetricsHandler: s.metricsHandler, Logger: s.logger, ThrottledLogger: s.logger, + WorkflowCache: s.workflowCache, } s.clusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() @@ -105,6 +114,116 @@ func (s *executableDeleteExecutionTaskSuite) TestExecute_CurrentClusterPassive_D namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + s.expectDeleteWorkflowExecution() + + s.NoError(s.newTask().Execute()) +} + +func (s *executableDeleteExecutionTaskSuite) TestExecute_MatchingLastWriteVersion_DeletesWorkflowExecution() { + lastWriteVersion := cluster.TestAlternativeClusterInitialFailoverVersion + namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectLastWriteVersion(chasm.WorkflowArchetypeID, lastWriteVersion) + s.expectDeleteWorkflowExecution() + + s.NoError(s.newTaskWithVersion(lastWriteVersion).Execute()) +} + +func (s *executableDeleteExecutionTaskSuite) TestExecute_MismatchedLastWriteVersion_SkipsTask() { + lastWriteVersion := cluster.TestAlternativeClusterInitialFailoverVersion + namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectLastWriteVersion(chasm.WorkflowArchetypeID, lastWriteVersion+cluster.TestFailoverVersionIncrement) + + s.NoError(s.newTaskWithVersion(lastWriteVersion).Execute()) +} + +func (s *executableDeleteExecutionTaskSuite) TestExecute_VersionedTaskOnActiveCluster_DeletesWorkflowExecution() { + lastWriteVersion := cluster.TestAlternativeClusterInitialFailoverVersion + namespaceEntry := s.namespaceEntry(cluster.TestCurrentClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectLastWriteVersion(chasm.WorkflowArchetypeID, lastWriteVersion) + s.expectDeleteWorkflowExecution() + + s.NoError(s.newTaskWithVersion(lastWriteVersion).Execute()) +} + +func (s *executableDeleteExecutionTaskSuite) TestExecute_MatchingLastWriteVersion_DeletesChasmExecution() { + lastWriteVersion := cluster.TestAlternativeClusterInitialFailoverVersion + archetypeID := chasm.ArchetypeID(42) + namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectLastWriteVersion(archetypeID, lastWriteVersion) + s.chasmEngine.EXPECT().DeleteExecution(gomock.Any(), gomock.Any(), chasm.DeleteExecutionRequest{}).Return(nil) + + s.NoError(s.newTaskWithVersionAndArchetype(lastWriteVersion, archetypeID).Execute()) +} + +// Tasks generated before the version was introduced carry no version and must keep being applied. +func (s *executableDeleteExecutionTaskSuite) TestExecute_UnversionedTask_DeletesWorkflowExecution() { + namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectDeleteWorkflowExecution() + + s.NoError(s.newTaskWithVersion(common.EmptyVersion).Execute()) +} + +// Sync/verify versioned transition tasks synthesize a deletion out of their own replication task. +// Their version describes a different operation and must not be treated as a deletion version. +func (s *executableDeleteExecutionTaskSuite) TestExecute_SynthesizedDeletion_IgnoresSourceTaskVersion() { + namespaceEntry := s.namespaceEntry(cluster.TestAlternativeClusterName) + s.namespaceCache.EXPECT().GetNamespaceByID(s.namespaceID).Return(namespaceEntry, nil).Times(3) + + s.expectDeleteWorkflowExecution() + + task := NewExecutableDeleteExecutionTask( + s.processToolBox, + s.taskID, + s.taskCreationTime, + s.sourceClusterName, + s.sourceShardKey, + &replicationspb.ReplicationTask{ + TaskType: enumsspb.REPLICATION_TASK_TYPE_SYNC_VERSIONED_TRANSITION_TASK, + RawTaskInfo: &persistencespb.ReplicationTaskInfo{ + NamespaceId: s.namespaceID.String(), + WorkflowId: s.workflowID, + RunId: s.runID, + TaskId: s.taskID, + TaskType: enumsspb.TASK_TYPE_REPLICATION_SYNC_VERSIONED_TRANSITION, + ArchetypeId: chasm.WorkflowArchetypeID, + Version: cluster.TestAlternativeClusterInitialFailoverVersion, + }, + }, + ) + s.NoError(task.Execute()) +} + +func (s *executableDeleteExecutionTaskSuite) expectLastWriteVersion( + archetypeID chasm.ArchetypeID, + lastWriteVersion int64, +) { + shardContext := historyi.NewMockShardContext(s.controller) + workflowContext := historyi.NewMockWorkflowContext(s.controller) + mutableState := historyi.NewMockMutableState(s.controller) + s.shardController.EXPECT().GetShardByNamespaceWorkflow(s.namespaceID, s.workflowID).Return(shardContext, nil) + s.workflowCache.EXPECT().GetOrCreateChasmExecution( + gomock.Any(), + shardContext, + s.namespaceID, + &commonpb.WorkflowExecution{WorkflowId: s.workflowID, RunId: s.runID}, + archetypeID, + locks.PriorityLow, + ).Return(workflowContext, func(error) {}, nil) + workflowContext.EXPECT().LoadMutableState(gomock.Any(), shardContext).Return(mutableState, nil) + mutableState.EXPECT().GetLastWriteVersion().Return(lastWriteVersion, nil) +} + +func (s *executableDeleteExecutionTaskSuite) expectDeleteWorkflowExecution() { shardContext := historyi.NewMockShardContext(s.controller) engine := historyi.NewMockEngine(s.controller) s.shardController.EXPECT().GetShardByNamespaceWorkflow(s.namespaceID, s.workflowID).Return(shardContext, nil) @@ -116,11 +235,20 @@ func (s *executableDeleteExecutionTaskSuite) TestExecute_CurrentClusterPassive_D RunId: s.runID, }, }).Return(&historyservice.DeleteWorkflowExecutionResponse{}, nil) - - s.NoError(s.newTask().Execute()) } func (s *executableDeleteExecutionTaskSuite) newTask() *ExecutableDeleteExecutionTask { + return s.newTaskWithVersion(common.EmptyVersion) +} + +func (s *executableDeleteExecutionTaskSuite) newTaskWithVersion(version int64) *ExecutableDeleteExecutionTask { + return s.newTaskWithVersionAndArchetype(version, chasm.WorkflowArchetypeID) +} + +func (s *executableDeleteExecutionTaskSuite) newTaskWithVersionAndArchetype( + version int64, + archetypeID chasm.ArchetypeID, +) *ExecutableDeleteExecutionTask { return NewExecutableDeleteExecutionTask( s.processToolBox, s.taskID, @@ -134,13 +262,22 @@ func (s *executableDeleteExecutionTaskSuite) newTask() *ExecutableDeleteExecutio WorkflowId: s.workflowID, RunId: s.runID, TaskId: s.taskID, - ArchetypeId: chasm.WorkflowArchetypeID, + TaskType: enumsspb.TASK_TYPE_REPLICATION_DELETE_EXECUTION, + ArchetypeId: archetypeID, + Version: version, }, }, ) } func (s *executableDeleteExecutionTaskSuite) namespaceEntry(activeCluster string) *namespace.Namespace { + return s.namespaceEntryWithFailoverVersion(activeCluster, cluster.TestCurrentClusterInitialFailoverVersion) +} + +func (s *executableDeleteExecutionTaskSuite) namespaceEntryWithFailoverVersion( + activeCluster string, + failoverVersion int64, +) *namespace.Namespace { detail := &persistencespb.NamespaceDetail{ Info: &persistencespb.NamespaceInfo{ Id: s.namespaceID.String(), @@ -154,7 +291,7 @@ func (s *executableDeleteExecutionTaskSuite) namespaceEntry(activeCluster string cluster.TestAlternativeClusterName, }, }, - FailoverVersion: cluster.TestCurrentClusterInitialFailoverVersion, + FailoverVersion: failoverVersion, } namespaceEntry, err := namespace.FromPersistentState( detail, diff --git a/service/history/shard/context_impl.go b/service/history/shard/context_impl.go index b55b29c23e0..74e48d5c387 100644 --- a/service/history/shard/context_impl.go +++ b/service/history/shard/context_impl.go @@ -911,6 +911,7 @@ func (s *ContextImpl) DeleteWorkflowExecution( ctx context.Context, key definition.WorkflowKey, archetypeID chasm.ArchetypeID, + lastWriteVersion int64, branchToken []byte, closeVisibilityTaskId int64, workflowCloseTime time.Time, @@ -1010,6 +1011,7 @@ func (s *ContextImpl) DeleteWorkflowExecution( &tasks.DeleteExecutionReplicationTask{ WorkflowKey: key, ArchetypeID: archetypeID, + Version: lastWriteVersion, }, } } diff --git a/service/history/shard/context_test.go b/service/history/shard/context_test.go index abaef132d95..0ed4489ab57 100644 --- a/service/history/shard/context_test.go +++ b/service/history/shard/context_test.go @@ -208,6 +208,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -236,6 +237,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_Continue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -253,6 +255,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_Continue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -269,6 +272,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_Continue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -296,6 +300,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_ErrorAndContinue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -312,6 +317,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_ErrorAndContinue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -328,6 +334,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_ErrorAndContinue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -343,6 +350,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_ErrorAndContinue_Success() { context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -355,7 +363,8 @@ func (s *contextSuite) TestDeleteWorkflowExecution_ErrorAndContinue_Success() { } func (s *contextSuite) TestDeleteWorkflowExecution_EmitsReplicationTaskWhenWorkflowActiveInCurrentCluster() { - captured := s.runDeleteWorkflowExecutionForReplicationCheck(cluster.TestCurrentClusterName) + lastWriteVersion := tests.Version + 100 + captured := s.runDeleteWorkflowExecutionForReplicationCheck(cluster.TestCurrentClusterName, lastWriteVersion) replicationTasks := captured.Tasks[tasks.CategoryReplication] s.Require().Len(replicationTasks, 1, "expected a DeleteExecutionReplicationTask when workflow is active in current cluster") @@ -363,10 +372,11 @@ func (s *contextSuite) TestDeleteWorkflowExecution_EmitsReplicationTaskWhenWorkf s.True(ok, "task should be *DeleteExecutionReplicationTask") s.Equal(captured.WorkflowID, deleteTask.WorkflowID) s.Equal(captured.NamespaceID, deleteTask.NamespaceID) + s.Equal(lastWriteVersion, deleteTask.Version) } func (s *contextSuite) TestDeleteWorkflowExecution_NoReplicationTaskWhenWorkflowActiveInOtherCluster() { - captured := s.runDeleteWorkflowExecutionForReplicationCheck(cluster.TestAlternativeClusterName) + captured := s.runDeleteWorkflowExecutionForReplicationCheck(cluster.TestAlternativeClusterName, tests.Version) s.Empty(captured.Tasks[tasks.CategoryReplication], "expected no DeleteExecutionReplicationTask when workflow is active in another cluster") @@ -374,6 +384,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_NoReplicationTaskWhenWorkflow func (s *contextSuite) runDeleteWorkflowExecutionForReplicationCheck( workflowActiveCluster string, + lastWriteVersion int64, ) *persistence.AddHistoryTasksRequest { nsID := namespace.NewID() nsEntry := namespace.NewGlobalNamespaceForTest( @@ -412,6 +423,7 @@ func (s *contextSuite) runDeleteWorkflowExecutionForReplicationCheck( context.Background(), workflowKey, chasm.WorkflowArchetypeID, + lastWriteVersion, []byte("branchToken"), 0, time.Time{}, @@ -439,6 +451,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_DeleteVisibilityTaskNotificti context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, @@ -457,6 +470,7 @@ func (s *contextSuite) TestDeleteWorkflowExecution_DeleteVisibilityTaskNotificti context.Background(), workflowKey, chasm.WorkflowArchetypeID, + tests.Version, branchToken, 0, time.Time{}, diff --git a/service/history/tasks/delete_execution_replication_task.go b/service/history/tasks/delete_execution_replication_task.go index 160529a2f74..31e735a0727 100644 --- a/service/history/tasks/delete_execution_replication_task.go +++ b/service/history/tasks/delete_execution_replication_task.go @@ -15,6 +15,10 @@ type DeleteExecutionReplicationTask struct { VisibilityTimestamp time.Time TaskID int64 ArchetypeID uint32 + // Version is the namespace failover version of the cluster that deleted the execution. + // The target cluster uses it to drop deletions that were generated before a failover. + // It is common.EmptyVersion for tasks generated before this field was introduced. + Version int64 } func (a *DeleteExecutionReplicationTask) GetKey() Key { @@ -22,10 +26,11 @@ func (a *DeleteExecutionReplicationTask) GetKey() Key { } func (a *DeleteExecutionReplicationTask) GetVersion() int64 { - return 0 + return a.Version } -func (a *DeleteExecutionReplicationTask) SetVersion(_ int64) { +func (a *DeleteExecutionReplicationTask) SetVersion(version int64) { + a.Version = version } func (a *DeleteExecutionReplicationTask) GetTaskID() int64 { diff --git a/tests/xdc/stream_based_replication_test.go b/tests/xdc/stream_based_replication_test.go index cb9f2646640..af5e124a5e7 100644 --- a/tests/xdc/stream_based_replication_test.go +++ b/tests/xdc/stream_based_replication_test.go @@ -473,17 +473,22 @@ func (s *streamBasedReplicationTestSuite) TestForceReplicateResetWorkflow_BaseWo }) s.NoError(err) + // Wipe the local copies on the passive cluster. The frontend DeleteWorkflowExecution API rejects + // deletions on a cluster that is passive for the workflow (they would not be replicated), so go + // through the history service directly, which is the same path replication apply uses. The admin + // force-delete API is not usable here: it deletes the DB rows without clearing the workflow cache, + // so the target keeps serving the deleted runs from cache. client1 := s.clusters[1].FrontendClient() - _, err = client1.DeleteWorkflowExecution(testcore.NewContext(), &workflowservice.DeleteWorkflowExecutionRequest{ - Namespace: ns, + _, err = s.clusters[1].HistoryClient().DeleteWorkflowExecution(testcore.NewContext(), &historyservice.DeleteWorkflowExecutionRequest{ + NamespaceId: resp.NamespaceInfo.GetId(), WorkflowExecution: &commonpb.WorkflowExecution{ WorkflowId: id, RunId: we.GetRunId(), }, }) s.NoError(err) - _, err = client1.DeleteWorkflowExecution(testcore.NewContext(), &workflowservice.DeleteWorkflowExecutionRequest{ - Namespace: ns, + _, err = s.clusters[1].HistoryClient().DeleteWorkflowExecution(testcore.NewContext(), &historyservice.DeleteWorkflowExecutionRequest{ + NamespaceId: resp.NamespaceInfo.GetId(), WorkflowExecution: &commonpb.WorkflowExecution{ WorkflowId: id, RunId: resetResp.GetRunId(), From 534045279e15856b4274058ff4758dd5591417b1 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Tue, 25 Aug 2026 16:00:13 -0700 Subject: [PATCH 064/139] Add wide events for parent-child workflow lifecycle debugging (#11707) ## What changed? - Added `parent_child_lifecycle` wide events at critical parent-child failure and recovery points. - Recorded parent/child execution identities, initiated event ID/version, local task metadata, errors, and locally observed workflow state such as `Zombie`. - Added visibility into standby verification and parent resend scheduling, execution, deduplication, and completion. - Reused the existing `history.emitReplicationLifecycleEvents` flag. - Added and extended unit tests for event payloads and emission paths. ## Why? Parent-child failures across shards and clusters are difficult to diagnose from existing metrics alone. These events help distinguish missing workflows, missing first workflow tasks, completion verification failures, workflow ID conflicts, orphaned children, and parent resend progress. They can also be correlated with replication lifecycle and task-discard events. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ### output from local e2e test
Wide Events for Parent resend scenario ``` [ { "attempt": 1, "details": { "child_workflow_state": "Completed", "event_type": "parent_child_lifecycle", "local_task_id": 1048587, "message": "Standby child completion verification started with parent resend requested", "operation": "standby_verification", "outcome": "started", "parent_initiated_version": 1, "parent_namespace_id": "691a916b-7f7a-4284-93af-782495421958", "phase": "verify_child_completion", "version": 1 }, "event_name": "replication_lifecycle", "namespace": "test-namespace-974d10a5-e0ef-451a-bac1-0834ec45f658", "namespace_id": "691a916b-7f7a-4284-93af-782495421958", "parent_initiated_id": 5, "parent_run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "parent_workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee", "phase": "executing", "run_id": "01a034ee-475b-7e96-9e3f-4239bd23f79a", "shard": 2, "task_type": "TransferCloseExecution", "workflow_id": "child-211e426f-e194-4492-8534-32d30fefb827" }, { "attempt": 0, "details": { "child_run_id": "01a034ee-475b-7e96-9e3f-4239bd23f79a", "child_workflow_id": "child-211e426f-e194-4492-8534-32d30fefb827", "event_type": "parent_child_lifecycle", "initial_error_type": "serviceerror.NotFound", "local_cluster": "standby_annmr", "message": "Parent workflow resend checkpoint", "operation": "standby_verification_sync_state", "outcome": "started", "parent_initiated_id": 5, "parent_initiated_version": 1, "parent_namespace_id": "691a916b-7f7a-4284-93af-782495421958", "parent_run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "parent_workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee", "parent_workflow_state": "", "phase": "parent_resend", "stage": "sync_workflow_state" }, "event_name": "replication_lifecycle", "namespace": "test-namespace-974d10a5-e0ef-451a-bac1-0834ec45f658", "namespace_id": "691a916b-7f7a-4284-93af-782495421958", "phase": "executing", "run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "shard": 1, "source_cluster": "active_annmr", "task_type": "sync_workflow_state", "workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee" }, { "details": { "child_run_id": "01a034ee-475b-7e96-9e3f-4239bd23f79a", "child_workflow_id": "child-211e426f-e194-4492-8534-32d30fefb827", "event_type": "parent_child_lifecycle", "initial_error_type": "serviceerror.NotFound", "local_cluster": "standby_annmr", "message": "Parent workflow resend checkpoint", "operation": "standby_verification_sync_state", "outcome": "succeeded", "parent_initiated_id": 5, "parent_initiated_version": 1, "parent_namespace_id": "691a916b-7f7a-4284-93af-782495421958", "parent_run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "parent_workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee", "parent_workflow_state": "Running", "phase": "parent_resend" }, "event_name": "replication_lifecycle", "namespace": "test-namespace-974d10a5-e0ef-451a-bac1-0834ec45f658", "namespace_id": "691a916b-7f7a-4284-93af-782495421958", "outcome": "verified", "phase": "applied", "run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "shard": 1, "source_cluster": "active_annmr", "task_type": "sync_workflow_state", "workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee" }, { "details": { "attempt": 1, "child_workflow_state": "Completed", "event_type": "parent_child_lifecycle", "local_task_id": 1048587, "message": "Standby child completion verification completed", "operation": "standby_verification", "outcome": "verified", "parent_initiated_version": 1, "parent_namespace_id": "691a916b-7f7a-4284-93af-782495421958", "phase": "verify_child_completion", "version": 1 }, "event_name": "replication_lifecycle", "namespace": "test-namespace-974d10a5-e0ef-451a-bac1-0834ec45f658", "namespace_id": "691a916b-7f7a-4284-93af-782495421958", "outcome": "verified", "parent_initiated_id": 5, "parent_run_id": "01a034ee-4749-795e-875c-07c66ddaf4a7", "parent_workflow_id": "parent-d30f61cf-70b9-4cfd-94af-85bc0af99eee", "phase": "applied", "run_id": "01a034ee-475b-7e96-9e3f-4239bd23f79a", "shard": 2, "task_type": "TransferCloseExecution", "workflow_id": "child-211e426f-e194-4492-8534-32d30fefb827" } ] ```
## Potential risks Enabling replication lifecycle events will produce additional wide-event volume. --- common/dynamicconfig/constants.go | 2 +- common/wideevents/replication_events.go | 21 ++ .../api.go | 175 +++++++++++++-- service/history/history_engine2_test.go | 158 +++++++++++++- service/history/parent_child_events.go | 205 ++++++++++++++++++ service/history/parent_child_events_test.go | 80 +++++++ service/history/shard/context_testutil.go | 16 +- .../transfer_queue_standby_task_executor.go | 31 ++- ...ansfer_queue_standby_task_executor_test.go | 94 +++++++- 9 files changed, 748 insertions(+), 34 deletions(-) create mode 100644 service/history/parent_child_events.go create mode 100644 service/history/parent_child_events_test.go diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index c253135b776..26ba8ec802b 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1739,7 +1739,7 @@ branch token is not the execution's current one, but still serves the read.`, EmitReplicationLifecycleEvents = NewGlobalBoolSetting( "history.emitReplicationLifecycleEvents", false, - `EmitReplicationLifecycleEvents controls whether the history service emits ReplicationLifecycle wide events (sent/executing/applied/skipped/error phases). Cluster-level; default off.`, + `EmitReplicationLifecycleEvents controls whether the history service emits ReplicationLifecycle wide events, including parent-child diagnostics. Cluster-level; default off.`, ) EnableCloseInboundReplicationStreamOnShutdown = NewGlobalBoolSetting( "history.enableCloseInboundReplicationStreamOnShutdown", diff --git a/common/wideevents/replication_events.go b/common/wideevents/replication_events.go index aff0342edfb..3621db16419 100644 --- a/common/wideevents/replication_events.go +++ b/common/wideevents/replication_events.go @@ -104,6 +104,27 @@ const ( ReplOperationDLQWrite = "dlq_write" ) +// Parent-child diagnostics reuse the ReplicationLifecycle event and store this vocabulary in +// details. Keeping the values here gives producers a shared contract without adding an event table. +const ( + ParentChildLifecycleEventType = "parent_child_lifecycle" + + ParentChildPhaseVerifyChildCompletion = "verify_child_completion" + ParentChildPhaseParentResend = "parent_resend" + + ParentChildOutcomeNotFound = "not_found" + ParentChildOutcomeCompletionMissing = "completion_missing" + ParentChildOutcomeScheduled = "scheduled" + ParentChildOutcomeStarted = "started" + ParentChildOutcomeVerified = "verified" + ParentChildOutcomeIgnored = "ignored" + ParentChildOutcomeSucceeded = "succeeded" + ParentChildOutcomeSourceNotFound = "source_not_found" + ParentChildOutcomeDeduplicated = "deduplicated" + ParentChildOutcomeLimited = "limited" + ParentChildOutcomeFailed = "failed" +) + // EmitReplicationError normalizes the common error envelope and emits it into the existing // replication_lifecycle event, not a separate event table. func EmitReplicationError( diff --git a/service/history/api/verifychildworkflowcompletionrecorded/api.go b/service/history/api/verifychildworkflowcompletionrecorded/api.go index 65f01388e55..5a1b59da0c0 100644 --- a/service/history/api/verifychildworkflowcompletionrecorded/api.go +++ b/service/history/api/verifychildworkflowcompletionrecorded/api.go @@ -3,6 +3,7 @@ package verifychildworkflowcompletionrecorded import ( "context" "errors" + "maps" "time" commonpb "go.temporal.io/api/common/v1" @@ -23,6 +24,8 @@ import ( "go.temporal.io/server/common/persistence/transitionhistory" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/rpc" + "go.temporal.io/server/common/util" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -33,7 +36,10 @@ func verifyChildExecution( workflowConsistencyChecker api.WorkflowConsistencyChecker, request *historyservice.VerifyChildExecutionCompletionRecordedRequest, ) (versionedTransition *persistencespb.VersionedTransition, - versionHistories *historyspb.VersionHistories, retError error) { + versionHistories *historyspb.VersionHistories, + parentWorkflowState string, + retError error, +) { workflowLease, err := workflowConsistencyChecker.GetWorkflowLease( ctx, request.Clock, @@ -49,28 +55,29 @@ func verifyChildExecution( locks.PriorityLow, ) if err != nil { - return nil, nil, err + return nil, nil, "", err } defer func() { workflowLease.GetReleaseFn()(retError) }() mutableState := workflowLease.GetMutableState() + parentWorkflowState = mutableState.GetExecutionState().GetState().String() if !mutableState.IsWorkflowExecutionRunning() && mutableState.GetExecutionState().State != enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE { // parent has already completed and can't be blocked after failover. - return nil, nil, nil + return nil, nil, parentWorkflowState, nil } onCurrentBranch, err := api.IsHistoryEventOnCurrentBranch(mutableState, request.ParentInitiatedId, request.ParentInitiatedVersion) if err != nil { // initiated event not found on any branch - return nil, nil, consts.ErrWorkflowNotReady + return nil, nil, parentWorkflowState, consts.ErrWorkflowNotReady } if !onCurrentBranch { // due to conflict resolution, the initiated event may on a different branch of the workflow. // we don't have to do anything and can simply return not found error. Standby logic // after seeing this error will give up verification. - return nil, nil, consts.ErrChildExecutionNotFound + return nil, nil, parentWorkflowState, consts.ErrChildExecutionNotFound } ci, isRunning := mutableState.GetChildExecutionInfo(request.ParentInitiatedId) @@ -78,15 +85,15 @@ func verifyChildExecution( if ci.StartedEventId != common.EmptyEventID && ci.GetStartedWorkflowId() != request.ChildExecution.GetWorkflowId() { // this can happen since we may not have the initiated version - return nil, nil, consts.ErrChildExecutionNotFound + return nil, nil, parentWorkflowState, consts.ErrChildExecutionNotFound } - return nil, nil, consts.ErrWorkflowNotReady + return nil, nil, parentWorkflowState, consts.ErrWorkflowNotReady } versionedTransition = transitionhistory.CopyVersionedTransition(transitionhistory.LastVersionedTransition(mutableState.GetExecutionInfo().TransitionHistory)) versionHistories = versionhistory.CopyVersionHistories(mutableState.GetExecutionInfo().VersionHistories) - return versionedTransition, versionHistories, nil + return versionedTransition, versionHistories, parentWorkflowState, nil } func Invoke( @@ -102,7 +109,7 @@ func Invoke( } resendParent := false - versionedTransition, versionHistories, errVerify := verifyChildExecution(ctx, workflowConsistencyChecker, request) + versionedTransition, versionHistories, parentWorkflowState, errVerify := verifyChildExecution(ctx, workflowConsistencyChecker, request) switch errVerify.(type) { case nil: return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil @@ -114,12 +121,14 @@ func Invoke( } metricsHandler := shardContext.GetMetricsHandler() + emitLifecycle := shardContext.GetConfig().EmitReplicationLifecycleEvents() + asyncResend := shardContext.GetConfig().EnableAsyncParentWorkflowResend() // The measured resend, run either inline or in the background. resend := func(ctx context.Context) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { metrics.ParentWorkflowResendAttempts.With(metricsHandler).Record(1) startTime := time.Now().UTC() - resp, err := resendParentAndVerify(ctx, request, workflowConsistencyChecker, shardContext, namespaceID, versionedTransition, versionHistories, errVerify) + resp, err := resendParentAndVerify(ctx, request, workflowConsistencyChecker, shardContext, namespaceID, versionedTransition, versionHistories, errVerify, parentWorkflowState, emitLifecycle) metrics.ParentWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime)) if err != nil { recordResendFailure(shardContext, metricsHandler, request, err) @@ -127,7 +136,7 @@ func Invoke( return resp, err } - if !shardContext.GetConfig().EnableAsyncParentWorkflowResend() { + if !asyncResend { return resend(ctx) } @@ -138,9 +147,15 @@ func Invoke( // retry while an earlier resend runs, so without these a stale parent, or a namespace with many // of them, would spawn goroutines without bound. parentKey := definition.NewWorkflowKey(request.NamespaceId, request.ParentExecution.WorkflowId, request.ParentExecution.RunId) - claimed, atCapacity := inFlightResends.tryClaim(parentKey, shardContext.GetConfig().ParentWorkflowResendMaxInFlight()) + maxInFlight := shardContext.GetConfig().ParentWorkflowResendMaxInFlight() + claimed, atCapacity := inFlightResends.tryClaim(parentKey, maxInFlight) if atCapacity { metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1) + if emitLifecycle { + details := parentResendEventDetails(errVerify) + details["max_in_flight"] = maxInFlight + emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, wideevents.ParentChildOutcomeLimited, nil, details) + } shardContext.GetLogger().Warn("Dropped parent workflow resend, shard is at its in-flight limit", tag.WorkflowNamespaceID(request.GetNamespaceId()), tag.NewStringTag("parent-workflow-id", request.ParentExecution.GetWorkflowId()), @@ -153,8 +168,28 @@ func Invoke( } if !claimed { metrics.ParentWorkflowResendSkipped.With(metricsHandler).Record(1) + if emitLifecycle { + emitParentResendLifecycleEvent( + shardContext, + request, + parentWorkflowState, + wideevents.ParentChildOutcomeDeduplicated, + nil, + parentResendEventDetails(errVerify), + ) + } return nil, errVerify } + if emitLifecycle { + emitParentResendLifecycleEvent( + shardContext, + request, + parentWorkflowState, + wideevents.ParentChildOutcomeScheduled, + nil, + parentResendEventDetails(errVerify), + ) + } // The context is detached from the request, which gRPC cancels when this handler returns, and // rooted at the shard lifecycle so the work stops with the shard. @@ -205,26 +240,47 @@ func resendParentAndVerify( versionedTransition *persistencespb.VersionedTransition, versionHistories *historyspb.VersionHistories, errVerify error, + parentWorkflowState string, + emitLifecycle bool, ) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { // Resend parent workflow from source cluster clusterMetadata := shardContext.GetClusterMetadata() targetClusterInfo := clusterMetadata.GetAllClusterInfo()[clusterMetadata.GetCurrentClusterName()] + activeClusterName := "" + emitResult := func(outcome string, eventErr error, stage string) { + if !emitLifecycle { + return + } + details := parentResendEventDetails(errVerify) + if activeClusterName != "" { + details["source_cluster"] = activeClusterName + } + if stage != "" { + details["stage"] = stage + } + emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, outcome, eventErr, details) + } namespaceEntry, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespaceID) if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_namespace") return nil, err } - activeClusterName := namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: request.ParentExecution.WorkflowId}) + activeClusterName = namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: request.ParentExecution.WorkflowId}) if activeClusterName == clusterMetadata.GetCurrentClusterName() { - return nil, errors.New("namespace becomes active when processing task as standby") + err = errors.New("namespace becomes active when processing task as standby") + emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_source_cluster") + return nil, err } remoteAdminClient, err := shardContext.GetRemoteAdminClient(activeClusterName) if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_remote_client") return nil, err } + emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state") resp, err := remoteAdminClient.SyncWorkflowState(ctx, &adminservice.SyncWorkflowStateRequest{ NamespaceId: request.NamespaceId, @@ -243,30 +299,119 @@ func resendParentAndVerify( // parent workflow is not found on source cluster, // we can return empty response to indicate that verification is done // TODO: add parent workflow to workflowNotFoundCache + emitResult(wideevents.ParentChildOutcomeSourceNotFound, err, "sync_workflow_state") return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil } if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { // Unable to perform sync state. Transition history maybe disabled in source cluster. + emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state") return nil, errVerify } + emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state") return nil, err } engine, err := shardContext.GetEngine(ctx) if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "get_engine") return nil, err } err = engine.ReplicateVersionedTransition(ctx, chasm.WorkflowArchetypeID, resp.VersionedTransitionArtifact, activeClusterName) if err != nil { if !errors.Is(err, consts.ErrDuplicate) { + emitResult(wideevents.ParentChildOutcomeFailed, err, "replicate_versioned_transition") return nil, err } } // Verify child execution again after resending parent workflow - _, _, err = verifyChildExecution(ctx, workflowConsistencyChecker, request) + _, _, observedParentWorkflowState, err := verifyChildExecution(ctx, workflowConsistencyChecker, request) + if observedParentWorkflowState != "" { + parentWorkflowState = observedParentWorkflowState + } if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "verify_after_resend") return nil, err } + emitResult(wideevents.ParentChildOutcomeSucceeded, nil, "") return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil } + +func parentResendEventDetails(initialError error) map[string]any { + return map[string]any{ + "initial_error_type": util.ErrorType(initialError), + } +} + +func emitParentResendLifecycleEvent( + shardContext historyi.ShardContext, + request *historyservice.VerifyChildExecutionCompletionRecordedRequest, + parentWorkflowState string, + outcome string, + err error, + details map[string]any, +) { + eventDetails := make(map[string]any, len(details)+11) + maps.Copy(eventDetails, details) + eventDetails["event_type"] = wideevents.ParentChildLifecycleEventType + eventDetails["phase"] = wideevents.ParentChildPhaseParentResend + eventDetails["outcome"] = outcome + eventDetails["local_cluster"] = shardContext.GetClusterMetadata().GetCurrentClusterName() + eventDetails["parent_namespace_id"] = request.GetNamespaceId() + eventDetails["parent_workflow_id"] = request.GetParentExecution().GetWorkflowId() + eventDetails["parent_run_id"] = request.GetParentExecution().GetRunId() + eventDetails["parent_workflow_state"] = parentWorkflowState + eventDetails["child_workflow_id"] = request.GetChildExecution().GetWorkflowId() + eventDetails["child_run_id"] = request.GetChildExecution().GetRunId() + eventDetails["parent_initiated_id"] = request.GetParentInitiatedId() + eventDetails["parent_initiated_version"] = request.GetParentInitiatedVersion() + + namespaceName := "" + if entry, namespaceErr := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(request.GetNamespaceId())); namespaceErr == nil { + namespaceName = entry.Name().String() + } + sourceCluster, ok := eventDetails["source_cluster"].(string) + if !ok { + sourceCluster = "" + } + delete(eventDetails, "source_cluster") + if err != nil { + eventDetails["error"] = err.Error() + eventDetails["error_type"] = util.ErrorType(err) + } + payload := wideevents.ReplicationLifecyclePayload{ + TaskType: wideevents.ReplTaskSyncWorkflowState, + Shard: shardContext.GetShardID(), + Namespace: namespaceName, + NamespaceID: request.GetNamespaceId(), + WorkflowID: request.GetParentExecution().GetWorkflowId(), + RunID: request.GetParentExecution().GetRunId(), + SourceCluster: sourceCluster, + } + switch outcome { + case wideevents.ParentChildOutcomeScheduled, + wideevents.ParentChildOutcomeStarted, + wideevents.ParentChildOutcomeDeduplicated: + eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState + eventDetails["message"] = "Parent workflow resend checkpoint" + payload.Phase = wideevents.ReplicationExecuting + payload.Details = eventDetails + wideevents.Emit(shardContext.GetEventLogger(), payload) + case wideevents.ParentChildOutcomeSucceeded, wideevents.ParentChildOutcomeSourceNotFound: + eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState + eventDetails["message"] = "Parent workflow resend checkpoint" + payload.Phase = wideevents.ReplicationApplied + payload.Outcome = wideevents.ParentChildOutcomeVerified + payload.Details = eventDetails + wideevents.Emit(shardContext.GetEventLogger(), payload) + default: + wideevents.EmitReplicationError( + shardContext.GetEventLogger(), + payload, + wideevents.ReplOperationStandbyVerificationSyncState, + "Parent workflow resend checkpoint", + err, + eventDetails, + ) + } +} diff --git a/service/history/history_engine2_test.go b/service/history/history_engine2_test.go index 6304ecc9645..de14cf86c76 100644 --- a/service/history/history_engine2_test.go +++ b/service/history/history_engine2_test.go @@ -48,8 +48,11 @@ import ( "go.temporal.io/server/common/searchattribute/sadefs" serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testvars" + "go.temporal.io/server/common/util" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -89,6 +92,7 @@ type ( mockClusterMetadata *cluster.MockMetadata mockVisibilityManager *manager.MockVisibilityManager mockWorkflowStateReplicator *ndc.MockWorkflowStateReplicator + parentChildEventCapture *parentChildEventCapture workflowCache wcache.Cache historyEngine *historyEngineImpl @@ -142,6 +146,7 @@ func (s *engine2Suite) SetupTest() { s.mockMemoryScheduledQueue.EXPECT().NotifyNewTasks(gomock.Any()).AnyTimes() s.config = tests.NewDynamicConfig() + s.parentChildEventCapture = &parentChildEventCapture{} mockShard := shard.NewTestContext( s.controller, &persistencespb.ShardInfo{ @@ -149,6 +154,7 @@ func (s *engine2Suite) SetupTest() { RangeId: 1, }, s.config, + s.parentChildEventCapture, ) reg := hsm.NewRegistry() err := workflow.RegisterStateMachine(reg) @@ -2684,6 +2690,8 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowNotExi func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() { // Inline resend: the RPC pulls and re-verifies before returning. s.config.EnableAsyncParentWorkflowResend = func() bool { return false } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: tests.ParentNamespaceID.String(), @@ -2705,7 +2713,7 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() mockClusterMetadata := cluster.NewMockMetadata(s.controller) mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() - mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() s.mockShard.SetClusterMetadata(mockClusterMetadata) @@ -2740,17 +2748,13 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() ).Return(resp, nil) s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition(gomock.Any(), chasm.WorkflowArchetypeID, resp.VersionedTransitionArtifact, cluster.TestCurrentClusterName).Return(nil) - // prepare closed workflow + // The parent can land as a zombie after conflict resolution; preserve that state in the event. ms := workflow.TestGlobalMutableState(s.historyEngine.shardContext, s.mockEventsCache, log.NewTestLogger(), tests.Version, tests.WorkflowID, tests.RunID) addWorkflowExecutionStartedEvent(ms, &commonpb.WorkflowExecution{ WorkflowId: tests.WorkflowID, RunId: tests.RunID, }, "wType", "testTaskQueue", payloads.EncodeString("input"), 25*time.Second, 20*time.Second, 200*time.Second, "identity") - _, err := ms.AddTimeoutWorkflowEvent( - enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, - uuid.NewString(), - ) - s.NoError(err) + ms.GetExecutionState().State = enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE ms.GetExecutionInfo().VersionHistories = &historyspb.VersionHistories{ CurrentVersionHistoryIndex: 0, Histories: []*historyspb.VersionHistory{ @@ -2767,14 +2771,79 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs} s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil) - _, err = s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(context.Background()), request) - s.NoError(err) + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(context.Background()), request) + s.Require().NoError(err) + + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeSucceeded), + }, parentChildOutcomes(capture)) + records := parentChildRecords(capture) + s.Require().Len(records, 2) + attributes := wideEventAttributes(records[0]) + details := wideEventDetails(records[0]) + s.Equal(string(wideevents.ReplicationExecuting), attributes["phase"].AsString()) + s.Equal(wideevents.ParentChildPhaseParentResend, details["phase"]) + s.Equal(cluster.TestAlternativeClusterName, details["local_cluster"]) + s.Equal(int64(1), attributes["shard"].AsInt64()) + s.Equal(request.GetParentExecution().GetWorkflowId(), attributes["workflow_id"].AsString()) + s.Equal(request.GetParentExecution().GetWorkflowId(), details["parent_workflow_id"]) + s.Equal(request.GetChildExecution().GetWorkflowId(), details["child_workflow_id"]) + s.InDelta(float64(request.GetParentInitiatedId()), details["parent_initiated_id"], 0) + s.InDelta(float64(request.GetParentInitiatedVersion()), details["parent_initiated_version"], 0) + s.Equal(cluster.TestCurrentClusterName, attributes["source_cluster"].AsString()) + s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) + s.Equal("sync_workflow_state", details["stage"]) + s.Equal(string(wideevents.ReplicationApplied), wideEventAttributes(records[1])["phase"].AsString()) + s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(records[1])["outcome"].AsString()) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE.String(), wideEventDetails(records[1])["parent_workflow_state"]) + + // Source NotFound is a terminal, successful outcome: there is no parent state left to pull. + sourceMissingRequest := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: request.GetNamespaceId(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: "parent missing on source", + RunId: uuid.NewString(), + }, + ChildExecution: request.GetChildExecution(), + ParentInitiatedId: request.GetParentInitiatedId(), + ParentInitiatedVersion: request.GetParentInitiatedVersion(), + ResendParent: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState( + gomock.Any(), + &adminservice.SyncWorkflowStateRequest{ + NamespaceId: sourceMissingRequest.NamespaceId, + Execution: sourceMissingRequest.ParentExecution, + ArchetypeId: chasm.WorkflowArchetypeID, + TargetClusterId: int32( + cluster.TestAlternativeClusterInitialFailoverVersion, + ), + }, + ).Return(nil, serviceerror.NewNotFound("parent missing on source")) + + _, err = s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(context.Background()), sourceMissingRequest) + s.Require().NoError(err) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeSucceeded), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeSourceNotFound), + }, parentChildOutcomes(capture)) + sourceNotFoundRecord := parentChildRecords(capture)[3] + s.Equal(string(wideevents.ReplicationApplied), wideEventAttributes(sourceNotFoundRecord)["phase"].AsString()) + s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(sourceNotFoundRecord)["outcome"].AsString()) + sourceNotFoundDetails := wideEventDetails(sourceNotFoundRecord) + s.Equal(util.ErrorType(serviceerror.NewNotFound("")), sourceNotFoundDetails["error_type"]) } // Async resend: the RPC returns the verification error immediately and the pull runs in the // background. func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAsync() { s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: tests.ParentNamespaceID.String(), @@ -2819,12 +2888,60 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAs case <-time.After(10 * time.Second): s.Fail("background resend did not call SyncWorkflowState") } + await.RequireTrue(s.T(), func() bool { + return len(parentChildOutcomes(capture)) >= 3 + }, 10*time.Second, 10*time.Millisecond) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeFailed), + }, parentChildOutcomes(capture)) + records := parentChildRecords(capture) + attributes := wideEventAttributes(records[2]) + details := wideEventDetails(records[2]) + s.Equal(util.ErrorType(serviceerror.NewUnavailable("source cluster unavailable")), details["error_type"]) + s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) + s.Equal("sync_workflow_state", details["stage"]) + s.Equal(cluster.TestCurrentClusterName, attributes["source_cluster"].AsString()) +} + +func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLimited() { + s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + s.config.ParentWorkflowResendMaxInFlight = func() int { return 0 } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture + + request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: tests.ParentNamespaceID.String(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ChildExecution: &commonpb.WorkflowExecution{ + WorkflowId: "child workflowId", + RunId: "child runId", + }, + ParentInitiatedId: 123, + ParentInitiatedVersion: 100, + ResendParent: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(context.Background()), request) + var notFound *serviceerror.NotFound + s.Require().ErrorAs(err, ¬Found) + s.Require().Equal([]string{string(wideevents.ParentChildOutcomeLimited)}, parentChildOutcomes(capture)) + details := wideEventDetails(parentChildRecords(capture)[0]) + s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) + s.InDelta(0, details["max_in_flight"], 0) } // TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped asserts that a second attempt for // the same parent does not start a concurrent resend while the first is still running. func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped() { s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: tests.ParentNamespaceID.String(), @@ -2853,6 +2970,12 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe // Times(1): only one resend may reach the source. It blocks so the second attempt overlaps. entered := make(chan struct{}) release := make(chan struct{}) + released := false + defer func() { + if !released { + close(release) + } + }() s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). DoAndReturn(func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { close(entered) @@ -2873,8 +2996,25 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe // Second attempt while the first is in flight: must not start another resend. _, err = s.historyEngine.VerifyChildExecutionCompletionRecorded(ctx, request) s.Error(err) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeDeduplicated), + }, parentChildOutcomes(capture)) + deduplicatedRecord := parentChildRecords(capture)[2] + s.Equal(string(wideevents.ReplicationExecuting), wideEventAttributes(deduplicatedRecord)["phase"].AsString()) close(release) + released = true + await.RequireTrue(s.T(), func() bool { + return len(parentChildOutcomes(capture)) >= 4 + }, 10*time.Second, 10*time.Millisecond) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeDeduplicated), + string(wideevents.ParentChildOutcomeFailed), + }, parentChildOutcomes(capture)) } func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowClosed() { diff --git a/service/history/parent_child_events.go b/service/history/parent_child_events.go new file mode 100644 index 00000000000..5f7dd2fa711 --- /dev/null +++ b/service/history/parent_child_events.go @@ -0,0 +1,205 @@ +package history + +import ( + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/wideevents" + historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/tasks" +) + +func emitChildCompletionVerificationStarted( + shardContext historyi.ShardContext, + task *tasks.CloseExecutionTask, + parentNamespaceID string, + parentWorkflowID string, + parentRunID string, + parentInitiatedID int64, + parentInitiatedVersion int64, + childWorkflowState string, + resendParent bool, + attempt int, +) { + if !resendParent || !parentChildLifecycleEnabled(shardContext) { + return + } + payload, details := parentChildEventForCloseTask( + shardContext, + task, + parentNamespaceID, + parentWorkflowID, + parentRunID, + parentInitiatedID, + parentInitiatedVersion, + childWorkflowState, + attempt, + ) + details["phase"] = wideevents.ParentChildPhaseVerifyChildCompletion + details["outcome"] = wideevents.ParentChildOutcomeStarted + emitParentChildReplicationEvent( + shardContext, + payload, + wideevents.ReplicationExecuting, + wideevents.ReplOperationStandbyVerification, + "Standby child completion verification started with parent resend requested", + nil, + details, + ) +} + +func emitChildCompletionVerificationResult( + shardContext historyi.ShardContext, + task *tasks.CloseExecutionTask, + parentNamespaceID string, + parentWorkflowID string, + parentRunID string, + parentInitiatedID int64, + parentInitiatedVersion int64, + childWorkflowState string, + resendParent bool, + attempt int, + err error, +) { + if !resendParent || !parentChildLifecycleEnabled(shardContext) { + return + } + outcome, emit := parentChildVerificationOutcome( + err, + wideevents.ParentChildOutcomeNotFound, + wideevents.ParentChildOutcomeCompletionMissing, + ) + switch err.(type) { + case nil: + outcome, emit = wideevents.ParentChildOutcomeVerified, true + case *serviceerror.NamespaceNotFound, *serviceerror.Unimplemented: + outcome, emit = wideevents.ParentChildOutcomeIgnored, true + default: + } + if !emit { + return + } + payload, details := parentChildEventForCloseTask( + shardContext, + task, + parentNamespaceID, + parentWorkflowID, + parentRunID, + parentInitiatedID, + parentInitiatedVersion, + childWorkflowState, + attempt, + ) + details["phase"] = wideevents.ParentChildPhaseVerifyChildCompletion + details["outcome"] = outcome + if attempt >= 0 { + details["attempt"] = attempt + } + replicationPhase := wideevents.ReplicationError + if err == nil || outcome == wideevents.ParentChildOutcomeIgnored { + replicationPhase = wideevents.ReplicationApplied + } + emitParentChildReplicationEvent( + shardContext, + payload, + replicationPhase, + wideevents.ReplOperationStandbyVerification, + "Standby child completion verification completed", + err, + details, + ) +} + +func parentChildEventForCloseTask( + shardContext historyi.ShardContext, + task *tasks.CloseExecutionTask, + parentNamespaceID string, + parentWorkflowID string, + parentRunID string, + parentInitiatedID int64, + parentInitiatedVersion int64, + childWorkflowState string, + attempt int, +) (wideevents.ReplicationLifecyclePayload, map[string]any) { + payload, details := parentChildEventForTask(shardContext, task, attempt) + payload.ParentWorkflowID = parentWorkflowID + payload.ParentRunID = parentRunID + payload.ParentInitiatedID = parentInitiatedID + details["parent_namespace_id"] = parentNamespaceID + details["child_workflow_state"] = childWorkflowState + details["parent_initiated_version"] = parentInitiatedVersion + return payload, details +} + +func parentChildEventForTask( + shardContext historyi.ShardContext, + task tasks.Task, + attempt int, +) (wideevents.ReplicationLifecyclePayload, map[string]any) { + namespaceName := "" + if name, err := shardContext.GetNamespaceRegistry().GetNamespaceName(namespace.ID(task.GetNamespaceID())); err == nil { + namespaceName = name.String() + } + details := map[string]any{ + "event_type": wideevents.ParentChildLifecycleEventType, + "local_task_id": task.GetTaskID(), + } + if versionedTask, ok := task.(tasks.HasVersion); ok { + details["version"] = versionedTask.GetVersion() + } + return wideevents.ReplicationLifecyclePayload{ + TaskType: task.GetType().String(), + Shard: shardContext.GetShardID(), + Namespace: namespaceName, + NamespaceID: task.GetNamespaceID(), + WorkflowID: task.GetWorkflowID(), + RunID: task.GetRunID(), + Attempt: int32(attempt), + }, details +} + +func emitParentChildReplicationEvent( + shardContext historyi.ShardContext, + payload wideevents.ReplicationLifecyclePayload, + phase wideevents.ReplicationPhase, + operation string, + message string, + err error, + details map[string]any, +) { + if phase == wideevents.ReplicationError { + wideevents.EmitReplicationError(shardContext.GetEventLogger(), payload, operation, message, err, details) + return + } + details["operation"] = operation + details["message"] = message + payload.Phase = phase + payload.Details = details + if phase == wideevents.ReplicationApplied { + payload.Outcome = wideevents.ParentChildOutcomeVerified + } + wideevents.Emit(shardContext.GetEventLogger(), payload) +} + +func parentChildLifecycleEnabled(shardContext historyi.ShardContext) bool { + return shardContext.GetConfig().EmitReplicationLifecycleEvents() +} + +func parentChildVerificationOutcome( + err error, + notFoundOutcome string, + notReadyOutcome string, +) (string, bool) { + if err == nil { + return "", false + } + switch err.(type) { + case *serviceerror.NotFound: + return notFoundOutcome, true + case *serviceerror.WorkflowNotReady: + return notReadyOutcome, true + case *serviceerror.NamespaceNotFound, *serviceerror.Unimplemented: + return "", false + default: + return wideevents.ParentChildOutcomeFailed, true + } +} diff --git a/service/history/parent_child_events_test.go b/service/history/parent_child_events_test.go new file mode 100644 index 00000000000..d31adb82856 --- /dev/null +++ b/service/history/parent_child_events_test.go @@ -0,0 +1,80 @@ +package history + +import ( + "context" + "encoding/json" + "sync" + + otellog "go.opentelemetry.io/otel/log" + "go.opentelemetry.io/otel/log/embedded" + "go.temporal.io/server/common/wideevents" +) + +type parentChildEventCapture struct { + embedded.Logger + + mu sync.RWMutex + records []otellog.Record +} + +func (c *parentChildEventCapture) Emit(_ context.Context, record otellog.Record) { + c.mu.Lock() + defer c.mu.Unlock() + c.records = append(c.records, record.Clone()) +} + +func (*parentChildEventCapture) Enabled(context.Context, otellog.EnabledParameters) bool { + return true +} + +func (c *parentChildEventCapture) snapshot() []otellog.Record { + c.mu.RLock() + defer c.mu.RUnlock() + records := make([]otellog.Record, len(c.records)) + for i, record := range c.records { + records[i] = record.Clone() + } + return records +} + +func wideEventAttributes(record otellog.Record) map[string]otellog.Value { + attributes := make(map[string]otellog.Value, record.AttributesLen()) + record.WalkAttributes(func(attribute otellog.KeyValue) bool { + attributes[attribute.Key] = attribute.Value + return true + }) + return attributes +} + +func parentChildOutcomes(capture *parentChildEventCapture) []string { + var outcomes []string + for _, record := range parentChildRecords(capture) { + if outcome, ok := wideEventDetails(record)["outcome"].(string); ok { + outcomes = append(outcomes, outcome) + } + } + return outcomes +} + +func parentChildRecords(capture *parentChildEventCapture) []otellog.Record { + var records []otellog.Record + for _, record := range capture.snapshot() { + if record.EventName() == wideevents.ReplicationLifecycleEventName && + wideEventDetails(record)["event_type"] == wideevents.ParentChildLifecycleEventType { + records = append(records, record) + } + } + return records +} + +func wideEventDetails(record otellog.Record) map[string]any { + attribute, ok := wideEventAttributes(record)["details"] + if !ok { + return nil + } + var details map[string]any + if err := json.Unmarshal([]byte(attribute.AsString()), &details); err != nil { + return nil + } + return details +} diff --git a/service/history/shard/context_testutil.go b/service/history/shard/context_testutil.go index 7ef7ecbefa4..375b4ade900 100644 --- a/service/history/shard/context_testutil.go +++ b/service/history/shard/context_testutil.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" + otellog "go.opentelemetry.io/otel/log" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" @@ -44,8 +45,9 @@ func NewTestContextWithTimeSource( shardInfo *persistencespb.ShardInfo, config *configs.Config, timeSource clock.TimeSource, + eventLogger ...otellog.Logger, ) *ContextTest { - result := NewTestContext(ctrl, shardInfo, config) + result := NewTestContext(ctrl, shardInfo, config, eventLogger...) result.timeSource = timeSource result.taskKeyManager.generator.timeSource = timeSource result.Resource.TimeSource = timeSource @@ -56,15 +58,21 @@ func NewTestContext( ctrl *gomock.Controller, shardInfo *persistencespb.ShardInfo, config *configs.Config, + eventLogger ...otellog.Logger, ) *ContextTest { + var logger otellog.Logger + if len(eventLogger) > 0 { + logger = eventLogger[0] + } resourceTest := resourcetest.NewTest(ctrl, primitives.HistoryService) eventsCache := events.NewMockCache(ctrl) shard := newTestContext( resourceTest, eventsCache, ContextConfigOverrides{ - ShardInfo: shardInfo, - Config: config, + ShardInfo: shardInfo, + Config: config, + EventLogger: logger, }, ) return &ContextTest{ @@ -80,6 +88,7 @@ type ContextConfigOverrides struct { Registry namespace.Registry ClusterMetadata cluster.Metadata ExecutionManager persistence.ExecutionManager + EventLogger otellog.Logger } type StubContext struct { @@ -141,6 +150,7 @@ func newTestContext(t *resourcetest.Test, eventsCache events.Cache, config Conte lifecycleCtx: lifecycleCtx, lifecycleCancel: lifecycleCancel, queueMetricEmitter: sync.Once{}, + eventLogger: config.EventLogger, state: contextStateAcquired, engineFuture: future.NewFuture[historyi.Engine](), diff --git a/service/history/transfer_queue_standby_task_executor.go b/service/history/transfer_queue_standby_task_executor.go index 9e0f1df07ff..e8c309d382b 100644 --- a/service/history/transfer_queue_standby_task_executor.go +++ b/service/history/transfer_queue_standby_task_executor.go @@ -103,7 +103,7 @@ func (t *transferQueueStandbyTaskExecutor) Execute( // TODO: add error logs err = nil case *tasks.CloseExecutionTask: - err = t.processCloseExecution(ctx, task) + err = t.processCloseExecution(ctx, task, executable.Attempt()) case *tasks.DeleteExecutionTask: err = t.processDeleteExecutionTask(ctx, task, false) case *tasks.ChasmTask: @@ -294,6 +294,7 @@ func (t *transferQueueStandbyTaskExecutor) processWorkflowTask( func (t *transferQueueStandbyTaskExecutor) processCloseExecution( ctx context.Context, transferTask *tasks.CloseExecutionTask, + attempt int, ) error { processTaskIfClosed := true actionFn := func(ctx context.Context, wfContext historyi.WorkflowContext, mutableState historyi.MutableState, release historyi.ReleaseWorkflowContextFunc) (any, error) { @@ -339,10 +340,24 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution( parentInitiatedID := executionInfo.ParentInitiatedId parentInitiatedVersion := executionInfo.ParentInitiatedVersion parentClock := executionInfo.ParentClock + childWorkflowState := mutableState.GetExecutionState().GetState().String() // no need for mutable state anymore, release workflow lock release(nil) + emitChildCompletionVerificationStarted( + t.shardContext, + transferTask, + parentNamespaceID, + parentWorkflowID, + parentRunID, + parentInitiatedID, + parentInitiatedVersion, + childWorkflowState, + resendParent, + attempt, + ) + _, err := t.historyRawClient.VerifyChildExecutionCompletionRecorded(ctx, &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: parentNamespaceID, ParentExecution: &commonpb.WorkflowExecution{ @@ -358,6 +373,19 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution( Clock: parentClock, ResendParent: resendParent, }) + emitChildCompletionVerificationResult( + t.shardContext, + transferTask, + parentNamespaceID, + parentWorkflowID, + parentRunID, + parentInitiatedID, + parentInitiatedVersion, + childWorkflowState, + resendParent, + attempt, + err, + ) switch err.(type) { case nil, *serviceerror.NamespaceNotFound, *serviceerror.Unimplemented: // Case 1: Target workflow is in the desired state. @@ -491,7 +519,6 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( childStartedWorkflowID := childWorkflowInfo.StartedWorkflowId childStartedRunID := childWorkflowInfo.StartedRunId childClock := childWorkflowInfo.Clock - // no need for mutable state anymore, release workflow lock release(nil) diff --git a/service/history/transfer_queue_standby_task_executor_test.go b/service/history/transfer_queue_standby_task_executor_test.go index ab7d6221504..81729905e2b 100644 --- a/service/history/transfer_queue_standby_task_executor_test.go +++ b/service/history/transfer_queue_standby_task_executor_test.go @@ -30,6 +30,7 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -41,6 +42,8 @@ import ( "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" "go.temporal.io/server/common/testing/protomock" + "go.temporal.io/server/common/util" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" @@ -76,9 +79,10 @@ type ( mockRemoteAdminClient *adminservicemock.MockAdminServiceClient mockChasmEngine chasm.Engine - mockExecutionMgr *persistence.MockExecutionManager - mockArchivalMetadata archiver.MetadataMock - mockArchiverProvider *provider.MockArchiverProvider + mockExecutionMgr *persistence.MockExecutionManager + mockArchivalMetadata archiver.MetadataMock + mockArchiverProvider *provider.MockArchiverProvider + parentChildEventCapture *parentChildEventCapture workflowCache wcache.Cache logger log.Logger @@ -124,6 +128,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) SetupTest() { s.chasmDiscardDuration = config.ChasmStandbyTaskDiscardDelay("") s.controller = gomock.NewController(s.T()) + s.parentChildEventCapture = &parentChildEventCapture{} s.mockShard = shard.NewTestContextWithTimeSource( s.controller, &persistencespb.ShardInfo{ @@ -131,6 +136,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) SetupTest() { }, config, s.timeSource, + s.parentChildEventCapture, ) reg := hsm.NewRegistry() @@ -725,6 +731,9 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessWorkflowTask_StampMis } func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() { + capture := s.parentChildEventCapture + s.mockShard.GetConfig().EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + execution := &commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", RunId: uuid.NewString(), @@ -841,8 +850,17 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() { s.ErrorAs(resp.ExecutionErr, &verificationErr) var resourceExhaustedErr *serviceerror.ResourceExhausted s.ErrorAs(resp.ExecutionErr, &resourceExhaustedErr) + s.Require().Empty(parentChildRecords(capture)) s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.localVerificationDuration)) + s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, nil) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) + + s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, serviceerror.NewUnimplemented("not implemented")) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) + s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, consts.ErrWorkflowNotReady) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) @@ -862,6 +880,68 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() { resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.ErrorAs(resp.ExecutionErr, &verificationErr) s.Equal(randomErr, verificationErr.Unwrap()) + + s.Equal([]string{ + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeVerified), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeIgnored), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeCompletionMissing), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeCompletionMissing), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeCompletionMissing), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeFailed), + }, parentChildOutcomes(capture)) + records := parentChildRecords(capture) + s.Require().Len(records, 12) + attributes := wideEventAttributes(records[0]) + details := wideEventDetails(records[0]) + s.Equal(string(wideevents.ReplicationExecuting), attributes["phase"].AsString()) + s.Equal(wideevents.ParentChildPhaseVerifyChildCompletion, details["phase"]) + s.Equal(wideevents.ParentChildOutcomeStarted, details["outcome"]) + s.Equal(s.namespaceID.String(), attributes["namespace_id"].AsString()) + s.Equal(parentExecution.GetWorkflowId(), attributes["parent_workflow_id"].AsString()) + s.Equal(parentExecution.GetRunId(), attributes["parent_run_id"].AsString()) + s.Equal(execution.GetWorkflowId(), attributes["workflow_id"].AsString()) + s.Equal(execution.GetRunId(), attributes["run_id"].AsString()) + s.Equal(transferTask.GetType().String(), attributes["task_type"].AsString()) + s.Equal(int64(1), attributes["attempt"].AsInt64()) + s.Equal("Completed", details["child_workflow_state"]) + s.Equal(parentInitiatedID, attributes["parent_initiated_id"].AsInt64()) + s.Equal(parentNamespaceID, details["parent_namespace_id"]) + s.InDelta(float64(parentInitiatedVersion), details["parent_initiated_version"], 0) + s.InDelta(float64(taskID), details["local_task_id"], 0) + s.InDelta(float64(transferTask.GetVersion()), details["version"], 0) + s.Require().NotContains(details, "attempt") + for _, key := range []string{ + "child_namespace_id", + "child_workflow_id", + "child_run_id", + "local_cluster", + "local_task_type", + "parent_initiated_id", + "parent_run_id", + "parent_workflow_id", + "resend_parent_requested", + "verification_scope", + } { + for _, record := range records { + s.Require().NotContains(wideEventDetails(record), key) + } + } + missingDetails := wideEventDetails(records[5]) + s.Equal(wideevents.ParentChildOutcomeCompletionMissing, missingDetails["outcome"]) + s.Equal(util.ErrorType(consts.ErrWorkflowNotReady), missingDetails["error_type"]) + s.InDelta(1, missingDetails["attempt"], 0) + verifiedAttributes := wideEventAttributes(records[1]) + s.Require().NotContains(verifiedAttributes, "attempt") + s.InDelta(1, wideEventDetails(records[1])["attempt"], 0) + ignoredAttributes := wideEventAttributes(records[3]) + s.Equal(string(wideevents.ReplicationApplied), ignoredAttributes["phase"].AsString()) + s.Equal(wideevents.ParentChildOutcomeVerified, ignoredAttributes["outcome"].AsString()) } func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCancelExecution_Pending() { @@ -1125,6 +1205,9 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessSignalExecution_Succe } func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_Pending() { + capture := s.parentChildEventCapture + s.mockShard.GetConfig().EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + execution := &commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", RunId: uuid.NewString(), @@ -1185,7 +1268,8 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) - event = addChildWorkflowExecutionStartedEvent(mutableState, event.GetEventId(), childWorkflowID, uuid.NewString(), childWorkflowType, nil) + childRunID := uuid.NewString() + event = addChildWorkflowExecutionStartedEvent(mutableState, event.GetEventId(), childWorkflowID, childRunID, childWorkflowType, nil) mutableState.FlushBufferedEvents() // clear the cache @@ -1230,6 +1314,8 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.NoError(resp.ExecutionErr) + + s.Require().Empty(parentChildRecords(capture)) } func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_Success() { From 30e0884ac084ecb923b717c3f3f446f72d615e4d Mon Sep 17 00:00:00 2001 From: Kannan Date: Tue, 25 Aug 2026 17:18:51 -0700 Subject: [PATCH 065/139] Keep Version workflow open if delete propagation Activity fails for unknown reasons (#11698) **What** Handle terminal errors from asynchronous task-queue delete propagation without marking propagation complete, so the Version workflow stays open until cleanup succeeds. **Why** Retryable activity failures retry indefinitely, but a terminal error can escape. The current code ignores that error and allows the Version workflow to complete without confirming task-queue cleanup. Note: We don't know how this non retryable case can happen. We observed 1 setup in this state; so this PR exports a metric for us to observe when it happens. **How did you test it?** Workflow test covering a non-retryable cleanup failure and verifying the Version workflow remains open. --------- Co-authored-by: Claude Opus 4.6 --- common/metrics/metric_defs.go | 1 + .../workerdeployment/version_workflow.go | 12 +++- .../workerdeployment/version_workflow_test.go | 62 +++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 92808f0bbd4..f1b47c5eed2 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1641,6 +1641,7 @@ var ( WorkerDeploymentVersionVisibilityQueryCount = NewCounterDef("worker_deployment_version_visibility_query_count") WorkerDeploymentVersioningOverrideCounter = NewCounterDef("worker_deployment_versioning_override_count") WorkerDeploymentVersioningOneTimeOverrideCounter = NewCounterDef("worker_deployment_versioning_one_time_override_count") + WorkerDeploymentVersionDeletePropagationFailure = NewCounterDef("worker_deployment_version_delete_propagation_failure") StartDeploymentTransitionCounter = NewCounterDef("start_deployment_transition_count") VersioningDataPropagationLatency = NewTimerDef("versioning_data_propagation_latency") SlowVersioningDataPropagationCounter = NewCounterDef("slow_versioning_data_propagation") diff --git a/service/worker/workerdeployment/version_workflow.go b/service/worker/workerdeployment/version_workflow.go index 8161c7e510a..a1949aa4a4e 100644 --- a/service/worker/workerdeployment/version_workflow.go +++ b/service/worker/workerdeployment/version_workflow.go @@ -609,7 +609,6 @@ func (d *VersionWorkflowRunner) handleDeleteVersion(ctx workflow.Context, args * return nil } -//nolint:revive,errcheck // In async mode the activities retry indefinitely so this function should not return error func (d *VersionWorkflowRunner) deleteVersionFromTaskQueuesAsync(ctx workflow.Context) { // If there are propagations in progress, we ask them to cancel and wait for them to do so. // The reason is that the ongoing upsert propagation may overwrite the delete that we want to send here, unintentionally undoing it. @@ -617,7 +616,16 @@ func (d *VersionWorkflowRunner) deleteVersionFromTaskQueuesAsync(ctx workflow.Co workflow.Await(ctx, func() bool { return d.asyncPropagationsInProgress == 1 }) // delete itself is counted as one d.cancelPropagations = false // need to unset this in case the version is revived - d.deleteVersionFromTaskQueues(ctx, workflow.WithActivityOptions(ctx, propagationActivityOptions)) + // Retryable failures retry indefinitely. + err := d.deleteVersionFromTaskQueues(ctx, workflow.WithActivityOptions(ctx, propagationActivityOptions)) + if err != nil { + // Don't decrement asyncPropagationsInProgress — the workflow must stay open + // because task queues may still have stale version data. The metric below + // enables alerting for manual recovery. + d.logger.Error("failed to delete worker deployment version from task queues", "error", err) + d.metrics.Counter(metrics.WorkerDeploymentVersionDeletePropagationFailure.Name()).Inc(1) + return + } d.asyncPropagationsInProgress-- } diff --git a/service/worker/workerdeployment/version_workflow_test.go b/service/worker/workerdeployment/version_workflow_test.go index b745d5cf3d5..585453d9852 100644 --- a/service/worker/workerdeployment/version_workflow_test.go +++ b/service/worker/workerdeployment/version_workflow_test.go @@ -853,6 +853,68 @@ func (s *VersionWorkflowSuite) Test_DeleteVersion_AsyncPropagation() { s.Require().NoError(s.env.GetWorkflowError()) } +// Test_DeleteVersion_AsyncPropagationFailureKeepsWorkflowOpen verifies that when +// task-queue delete propagation fails with a non-retryable error, the version +// workflow stays open (propagation counter is not decremented) so that the +// system does not lose track of the incomplete cleanup. +func (s *VersionWorkflowSuite) Test_DeleteVersion_AsyncPropagationFailureKeepsWorkflowOpen() { + tv := testvars.New(s.T()) + + var a *VersionActivities + s.env.OnActivity(a.DeleteWorkerControllerInstance, mock.Anything, mock.Anything).Return(nil).Maybe() + s.env.OnActivity(a.CheckIfTaskQueuesHavePollers, mock.Anything, mock.Anything).Return(false, nil).Maybe() + + // Simulate failure: task-queue delete propagation returns a non-retryable error, + // as if the matching service rejected the request. + s.env.OnActivity(a.SyncDeploymentVersionUserData, mock.Anything, mock.Anything). + Return(nil, temporal.NewNonRetryableApplicationError("delete propagation failed", "test-error", nil)). + Once() + + s.env.RegisterDelayedCallback(func() { + s.env.UpdateWorkflow(DeleteVersion, "", &testsuite.TestUpdateCallback{ + OnReject: func(err error) { + s.Fail("delete version should not have been rejected", err) + }, + OnAccept: func() {}, + OnComplete: func(_ any, err error) { + s.Require().NoError(err) + }, + }, &deploymentspb.DeleteVersionArgs{AsyncPropagation: true}) + }, time.Millisecond) + + // Verify consequence: after propagation fails, check the workflow is still open. + observedOpen := false + s.env.RegisterDelayedCallback(func() { + observedOpen = !s.env.IsWorkflowCompleted() + s.env.CancelWorkflow() + }, 100*time.Millisecond) + + s.env.ExecuteWorkflow(WorkerDeploymentVersionWorkflowType, &deploymentspb.WorkerDeploymentVersionWorkflowArgs{ + NamespaceName: tv.NamespaceName().String(), + NamespaceId: tv.NamespaceID().String(), + VersionState: &deploymentspb.VersionLocalState{ + Version: &deploymentspb.WorkerDeploymentVersion{ + DeploymentName: tv.DeploymentSeries(), + BuildId: tv.BuildID(), + }, + TaskQueueFamilies: map[string]*deploymentspb.VersionLocalState_TaskQueueFamilyData{ + tv.TaskQueue().Name: { + TaskQueues: map[int32]*deploymentspb.TaskQueueVersionData{ + int32(enumspb.TASK_QUEUE_TYPE_WORKFLOW): {}, + }, + }, + }, + Status: enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_INACTIVE, + SyncBatchSize: int32(s.workerDeploymentClient.getSyncBatchSize()), + //nolint:staticcheck // SA1019: worker versioning v0.31 + StartedDeploymentWorkflow: true, + }, + }) + + s.True(observedOpen, "workflow should remain open when delete propagation fails") + s.True(temporal.IsCanceledError(s.env.GetWorkflowError())) +} + // Test_DeleteVersion_AsyncPropagation_BlocksWorkerRegistration tests that: // 1. When deletion with async propagation is in progress, worker registration is blocked // 2. Worker registration completes after propagation finishes From 109a38e8ca4827ae8c624fc1a9382290dcae0f69 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 26 Aug 2026 10:00:40 -0700 Subject: [PATCH 066/139] Add `make lint-code-fast` (#11786) When using `git worktree` for agentic workflows during development, the golangci-lint cache is always cold. That means the first invocation scans the entire codebase (consuming all CPU cores) unnecessarily, since almost all lints can be found by just scanning the changed packages. The exceptions are type-based checks, but they are rare. This PR speeds up the agentic development cycle greatly for local development; and anything that slips through is still caught in CI where the full codebase is checked. Example benchmark for cold cache with 1 changed file with 1 finding: - `lint-code-fast`: 8s - `lint-code`: 14min --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Dan Davison --- AGENTS.md | 5 +++-- Makefile | 31 +++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a4807125da7..5d5f73fe9db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -47,7 +47,8 @@ Before starting the implementation of any request, you MUST REVIEW the following - `/service/worker`: worker service implementation ## Important Commands: -- Linting: `make lint-code` +- Fast Go linting (changed packages): `make lint-code-fast` +- Full Go linting (all packages): `make lint-code` - Formatting imports: `make fmt-imports` - Code generation: `make proto` - Update API proto: `make update-go-api` @@ -87,7 +88,7 @@ When requested to perform tasks like fixing bugs, adding features, refactoring, 3. **Implement:** Use the available tools to act on the plan, strictly adhering to the project's established conventions (detailed under 'Core Mandates'). 4. **Regenerate:** If necessary, regenerate code based on your changes. If you alter anything annotated with `//go:generate` or in a `.proto` file you will need to do this. 5. **Verify (Tests):** If applicable and feasible, verify the changes using the project's testing procedures. Identify the correct test commands and frameworks by examining 'README' files, build/package configuration (e.g., 'Makefile'), or existing test execution patterns. NEVER assume standard test commands. -6. **Verify (Standards):** VERY IMPORTANT: After making code changes, execute the project-specific build, linting and type-checking commands (`make lint-code`) +6. **Verify (Standards):** VERY IMPORTANT: After making code changes, execute the project-specific build, linting and type-checking commands (`make lint-code-fast` for development) ## Planning When planning (under 'Software Engineering Tasks'): diff --git a/Makefile b/Makefile index 23f4697b5ab..ccb299f7f22 100644 --- a/Makefile +++ b/Makefile @@ -173,6 +173,7 @@ $(LOCALBIN): @mkdir -p $(LOCALBIN) .PHONY: golangci-lint +LINT_CODE_TARGETS ?= ./... GOLANGCI_LINT_BASE_REV ?= $(MAIN_BRANCH) GOLANGCI_LINT_FIX ?= true GOLANGCI_LINT_VERSION := v2.13.0 @@ -403,10 +404,36 @@ lint-actions: $(ACTIONLINT) @printf $(COLOR) "Linting GitHub actions..." @$(ACTIONLINT) +.PHONY: lint-code lint-code-fast +# --new-from-rev filters reported issues _after_ analysis; this target also reduces package inputs _before_ analysis. +lint-code-fast: + @if ! git rev-parse --verify --quiet "$(GOLANGCI_LINT_BASE_REV)^{commit}" >/dev/null; then \ + printf $(RED) "GOLANGCI_LINT_BASE_REV=$(GOLANGCI_LINT_BASE_REV) is not a known commit; fetch it or override GOLANGCI_LINT_BASE_REV"; \ + exit 1; \ + fi + @base=$$(git merge-base HEAD "$(GOLANGCI_LINT_BASE_REV)"); \ + targets=$$({ \ + git diff --no-renames --name-only "$$base" -- '*.go'; \ + git ls-files --others --exclude-standard -- '*.go'; \ + } | sed 's|^|./|; s|/[^/]*$$||' | sort -u \ + | while read -r dir; do [ -d "$$dir" ] && printf '%s ' "$$dir"; done); \ + if [ -z "$$targets" ]; then \ + printf $(COLOR) "No changed Go packages to lint."; \ + else \ + $(MAKE) GOLANGCI_LINT_BASE_REV="$$base" LINT_CODE_TARGETS="$$targets" lint-code; \ + fi + lint-code: $(GOLANGCI_LINT) $(ERRORTYPE) @printf $(COLOR) "Linting code..." - @$(GOLANGCI_LINT) run --verbose --build-tags $(ALL_TEST_TAGS) --timeout 10m --fix=$(GOLANGCI_LINT_FIX) --new-from-rev=$(GOLANGCI_LINT_BASE_REV) --config=.github/.golangci.yml - @go vet -tags $(ALL_TEST_TAGS) -vettool="$(ERRORTYPE)" -style-check=false ./... + @$(GOLANGCI_LINT) run \ + --verbose \ + --build-tags $(ALL_TEST_TAGS) \ + --timeout 10m \ + --fix=$(GOLANGCI_LINT_FIX) \ + --new-from-rev=$(GOLANGCI_LINT_BASE_REV) \ + --config=.github/.golangci.yml \ + $(LINT_CODE_TARGETS) + @go vet -tags $(ALL_TEST_TAGS) -vettool="$(ERRORTYPE)" -style-check=false $(LINT_CODE_TARGETS) lint-yaml: $(YAMLFMT) @printf $(COLOR) "Checking YAML formatting..." From 00d09082cc807038b666effff2d5ffceb9e233ec Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Wed, 26 Aug 2026 13:02:29 -0400 Subject: [PATCH 067/139] Add data race summary to CI report (#11211) ## What changed? Add data race summary to CI report ## Why? Notify when CI detects data race issues in main. ## How did you test it? - [X] built - [X] run locally and tested manually - [X] covered by existing tests - [X] added new unit test(s) - [ ] added new functional test(s) --- .github/workflows/run-tests.yml | 33 ++++ tools/ci-notify/app.go | 96 ++++++++++++ tools/ci-notify/artifacts.go | 53 +++++-- tools/ci-notify/datarace.go | 254 +++++++++++++++++++++++++++++++ tools/ci-notify/datarace_test.go | 210 +++++++++++++++++++++++++ tools/ci-notify/slack.go | 48 ++++++ 6 files changed, 679 insertions(+), 15 deletions(-) create mode 100644 tools/ci-notify/datarace.go create mode 100644 tools/ci-notify/datarace_test.go diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 45c1ea060d5..d9502a41684 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -551,3 +551,36 @@ jobs: go run ./cmd/tools/ci-notify alert \ --run-id "${{ github.run_id }}" \ --slack-webhook "$SLACK_WEBHOOK" + + notify-data-race: + name: Notify Slack on Data Race + # Runs on both passing and failing main runs: the race detector can trip on + # one attempt and pass on a retry, leaving the run green while a data race + # was still recorded in the test-summary artifacts. + if: | + always() && + github.ref == 'refs/heads/main' && + (needs.test-status.result == 'success' || needs.test-status.result == 'failure') + needs: test-status + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version-file: "go.mod" + cache: true + + - name: Send Slack notification + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + GH_TOKEN: ${{ github.token }} + run: | + go run ./cmd/tools/ci-notify data-race \ + --run-id "${{ github.run_id }}" \ + --slack-webhook "$SLACK_WEBHOOK" diff --git a/tools/ci-notify/app.go b/tools/ci-notify/app.go index 905d4a080a0..ab61242cdea 100644 --- a/tools/ci-notify/app.go +++ b/tools/ci-notify/app.go @@ -51,6 +51,30 @@ func NewCliApp() *cli.App { }, Action: runAlertCommand, }, + { + Name: "data-race", + Usage: "Send a notification when data races are detected in a CI run", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: FlagRunID, + Usage: "GitHub Actions run ID", + Required: true, + EnvVars: []string{"GITHUB_RUN_ID"}, + }, + &cli.StringFlag{ + Name: FlagSlackWebhook, + Usage: "Slack webhook URL", + Required: false, // Not required for dry-run + EnvVars: []string{"SLACK_WEBHOOK"}, + }, + &cli.BoolFlag{ + Name: FlagDryRun, + Usage: "Print message without sending to Slack", + Value: false, + }, + }, + Action: runDataRaceCommand, + }, { Name: "digest", Usage: "Generate digest report for CI runs", @@ -157,6 +181,78 @@ func runAlertCommand(c *cli.Context) error { return nil } +func runDataRaceCommand(c *cli.Context) error { + // Set up logger + logger, err := zap.NewProduction() + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to create logger: %v\n", err) + return nil // Don't fail CI if notification fails + } + defer func() { _ = logger.Sync() }() + + runID := c.String(FlagRunID) + slackWebhook := c.String(FlagSlackWebhook) + dryRun := c.Bool(FlagDryRun) + + logger.Info("Starting `ci-notify data-race`", + zap.String("run_id", runID), + zap.Bool("dry_run", dryRun), + ) + + // Build data race report + report, err := BuildDataRaceReport(runID) + if err != nil { + logger.Error("Failed to build data race report", zap.Error(err)) + // Don't fail CI if notification fails + return nil + } + + logger.Info("Built data race report", + zap.Int("data_races", len(report.DataRaces)), + ) + + // Nothing to report: stay silent so green runs don't spam the channel. + if len(report.DataRaces) == 0 { + logger.Info("No data races detected; skipping notification") + return nil + } + + // Handle dry-run mode + if dryRun { + logger.Info("Dry-run mode: printing message to stdout") + message := BuildDataRaceMessage(report) + fmt.Println(message.RenderMarkdown()) + fmt.Println("\n--- Slack JSON Payload ---") + payload, err := marshalIndent(message) + if err != nil { + logger.Error("Failed to marshal message for display", zap.Error(err)) + return nil + } + fmt.Println(payload) + return nil + } + + // Validate webhook URL + if slackWebhook == "" { + logger.Error("Slack webhook URL is required when not in dry-run mode") + // Don't fail CI if notification fails + return nil + } + + // Build and send Slack message + message := BuildDataRaceMessage(report) + + logger.Info("Sending Slack notification") + if err := message.Send(slackWebhook); err != nil { + logger.Error("Failed to send Slack message", zap.Error(err)) + // Don't fail CI if notification fails + return nil + } + + logger.Info("Slack notification sent successfully") + return nil +} + func runDigestCommand(c *cli.Context) error { // Set up logger logger, err := zap.NewProduction() diff --git a/tools/ci-notify/artifacts.go b/tools/ci-notify/artifacts.go index 26afd98be75..a1f62e7c586 100644 --- a/tools/ci-notify/artifacts.go +++ b/tools/ci-notify/artifacts.go @@ -19,6 +19,7 @@ const ( summaryArtifactPrefix = "test-summary-json--" summaryFileSuffix = "test-summary.json" summaryKindOOM = "OOM" + summaryKindDataRace = "DATA RACE" ) var trailingFailureSuffixRegex = regexp.MustCompile(`\s*\([^)]+\)$`) @@ -28,27 +29,31 @@ type testSummary struct { } type summaryRow struct { - Kind string `json:"kind"` - Name string `json:"name"` - Final bool `json:"final"` + Kind string `json:"kind"` + Name string `json:"name"` + Details string `json:"details"` + Final bool `json:"final"` } -func getFailures(ctx context.Context, runID int64) ([]string, error) { +// forEachSummaryZip downloads every test-summary artifact for the run and +// invokes fn with each artifact's name and the local path of its downloaded zip. +// Artifacts that fail to download are skipped. All downloads share a single temp +// dir that is removed when the function returns. +func forEachSummaryZip(ctx context.Context, runID int64, fn func(artifactName, zipPath string)) error { ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() artifacts, err := github.ListRunArtifacts(ctx, temporalRepository, runID) if err != nil { - return nil, err + return err } tempDir, err := os.MkdirTemp("", "ci-notify-artifacts-*") if err != nil { - return nil, err + return err } defer func() { _ = os.RemoveAll(tempDir) }() - var failures []string for _, artifact := range artifacts { if artifact.Expired || !strings.HasPrefix(artifact.Name, summaryArtifactPrefix) { continue @@ -58,40 +63,58 @@ func getFailures(ctx context.Context, runID int64) ([]string, error) { if err != nil { continue } + fn(artifact.Name, zipPath) + } + return nil +} + +func getFailures(ctx context.Context, runID int64) ([]string, error) { + var failures []string + err := forEachSummaryZip(ctx, runID, func(_, zipPath string) { artifactFailures, err := failuresFromZip(zipPath) if err != nil { - continue + return } failures = append(failures, artifactFailures...) + }) + if err != nil { + return nil, err } - return uniqueSorted(failures), nil } func failuresFromZip(zipPath string) ([]string, error) { + rows, err := summaryRowsFromZip(zipPath) + if err != nil { + return nil, err + } + return reportableFailures(rows), nil +} + +func summaryRowsFromZip(zipPath string) ([]summaryRow, error) { reader, err := zip.OpenReader(zipPath) if err != nil { return nil, fmt.Errorf("failed to open artifact zip %s: %w", zipPath, err) } defer func() { _ = reader.Close() }() - var failures []string + var rows []summaryRow for _, file := range reader.File { if file.FileInfo().IsDir() || !strings.HasSuffix(file.Name, summaryFileSuffix) { continue } - fileFailures, err := failuresFromZipFile(file) + fileRows, err := summaryRowsFromZipFile(file) if err != nil { continue } - failures = append(failures, fileFailures...) + rows = append(rows, fileRows...) } - return failures, nil + return rows, nil } -func failuresFromZipFile(file *zip.File) ([]string, error) { +func summaryRowsFromZipFile(file *zip.File) ([]summaryRow, error) { rc, err := file.Open() if err != nil { return nil, fmt.Errorf("failed to open %s in artifact zip: %w", file.Name, err) @@ -102,7 +125,7 @@ func failuresFromZipFile(file *zip.File) ([]string, error) { if err := json.NewDecoder(rc).Decode(&summary); err != nil { return nil, fmt.Errorf("failed to parse %s in artifact zip: %w", file.Name, err) } - return reportableFailures(summary.Rows), nil + return summary.Rows, nil } func reportableFailures(rows []summaryRow) []string { diff --git a/tools/ci-notify/datarace.go b/tools/ci-notify/datarace.go new file mode 100644 index 00000000000..5fab9c37798 --- /dev/null +++ b/tools/ci-notify/datarace.go @@ -0,0 +1,254 @@ +package cinotify + +import ( + "context" + "fmt" + "regexp" + "slices" + "strings" + "time" + + "go.temporal.io/server/tools/common/github" +) + +// maxRaceSites caps how many conflicting-access lines we surface per race in +// case the test-runner merged several race reports into one alert. +const maxRaceSites = 6 + +var ( + // raceAccessRegex matches a race detector access header. The current access + // is "Read at"/"Write at"; the conflicting one is "Previous read at"/ + // "Previous write at". At least one is always a write, but a race can be + // write/write, so both kinds must be matched. + raceAccessRegex = regexp.MustCompile(`^(Read|Write|Previous read|Previous write) at 0x[0-9a-fA-F]+ by (goroutine \d+|main goroutine)`) + // raceFileLineRegex matches a ".go:" stack frame location. + raceFileLineRegex = regexp.MustCompile(`(\S+\.go):(\d+)`) + // repoPathPrefixRegex strips the CI checkout prefix (e.g. + // /home/runner/work/temporal/temporal/) so paths read as service/history/…. + repoPathPrefixRegex = regexp.MustCompile(`^.*/temporal/temporal/`) +) + +// temporalModulePrefix is trimmed from race locations so we show, e.g., +// "service/history.TestFoo" instead of the fully-qualified package path. +const temporalModulePrefix = "go.temporal.io/server/" + +// DataRace is a single data race detected by the Go race detector during a CI +// run. The test-runner surfaces these into the JUnit ALERTS suite and the +// test-summary.json artifact (see tools/testrunner/junit.go). +type DataRace struct { + // Location is the package-qualified test where the race was detected, + // e.g. "service/history.TestFoo". + Location string + // Details is the race detector's report (stacktraces), possibly truncated. + Details string + // JobID is the GitHub Actions job that reported the race, used to link + // directly to the offending job. Empty when it can't be determined. + JobID string +} + +// DataRaceReport aggregates the data races found in a single CI run along with +// the commit context needed to attribute and link them. +type DataRaceReport struct { + Run github.Run + Author string + Title string + DataRaces []DataRace +} + +// BuildDataRaceReport fetches the run's test summaries, extracts any data races, +// and enriches them with commit metadata. It returns a report with an empty +// DataRaces slice when no races were detected. +func BuildDataRaceReport(runID string) (*DataRaceReport, error) { + run, err := getWorkflowRun(runID) + if err != nil { + return nil, err + } + + races, err := getDataRaces(context.Background(), run.DatabaseID) + if err != nil { + return nil, err + } + + report := &DataRaceReport{ + Run: *run, + Title: run.DisplayTitle, + DataRaces: races, + } + + // Commit metadata is best-effort: a missing author must not suppress the alert. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if commit, err := github.GetCommit(ctx, temporalRepository, run.HeadSHA); err == nil { + report.Author = commit.Commit.Author.Name + if title := commit.Title(); title != "" { + report.Title = title + } + } + + return report, nil +} + +// getDataRaces downloads the run's test-summary artifacts and returns the unique +// set of data races reported across all of them. +func getDataRaces(ctx context.Context, runID int64) ([]DataRace, error) { + var races []DataRace + err := forEachSummaryZip(ctx, runID, func(artifactName, zipPath string) { + rows, err := summaryRowsFromZip(zipPath) + if err != nil { + return + } + races = append(races, dataRacesFromRows(rows, jobIDFromArtifactName(artifactName))...) + }) + if err != nil { + return nil, err + } + return uniqueDataRaces(races), nil +} + +// dataRacesFromRows extracts data race rows from a parsed test summary, tagging +// each with the job that produced the artifact. +func dataRacesFromRows(rows []summaryRow, jobID string) []DataRace { + var races []DataRace + for _, row := range rows { + if row.Kind != summaryKindDataRace { + continue + } + races = append(races, DataRace{ + Location: dataRaceLocation(row.Name), + Details: row.Details, + JobID: jobID, + }) + } + return races +} + +// dataRaceLocation reduces a test-runner alert name like +// "DATA RACE: Data race detected — in go.temporal.io/server/service/history.TestFoo" +// to just the package-qualified test, e.g. "service/history.TestFoo". +func dataRaceLocation(name string) string { + loc := strings.TrimSpace(strings.TrimPrefix(name, summaryKindDataRace+":")) + if _, after, ok := strings.Cut(loc, "— in "); ok { + loc = strings.TrimSpace(after) + } + return strings.TrimPrefix(loc, temporalModulePrefix) +} + +// raceSite is one conflicting memory access from a race detector report. +type raceSite struct { + access string // "Read", "Write", "Previous read", or "Previous write" + goroutine string // "goroutine 8" or "main goroutine" + location string // "service/history/mutable_state.go:127" +} + +// parseRaceSites reduces a race detector report to the conflicting memory +// accesses and the source line each occurred at. It handles read/write and +// write/write races. Returns nil when the report can't be parsed, so callers can +// degrade to just the location and job link. +func parseRaceSites(details string) []raceSite { + lines := strings.Split(details, "\n") + var sites []raceSite + for i, line := range lines { + m := raceAccessRegex.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + continue + } + if loc := firstGoFrame(lines[i+1:]); loc != "" { + sites = append(sites, raceSite{access: m[1], goroutine: m[2], location: loc}) + if len(sites) == maxRaceSites { + break + } + } + } + return sites +} + +// raceSites formats the conflicting accesses for display, e.g. +// +// Read at (goroutine 8): service/history/mutable_state.go:127 +// Previous write at (goroutine 7): service/history/mutable_state.go:130 +func raceSites(details string) []string { + sites := parseRaceSites(details) + out := make([]string, 0, len(sites)) + for _, s := range sites { + out = append(out, fmt.Sprintf("%s at (%s): %s", s.access, s.goroutine, s.location)) + } + return out +} + +// raceAffectedLines returns the sorted, de-duplicated source lines involved in a +// race. Unlike the raw report, this excludes memory addresses and goroutine ids +// (which differ every run), so it identifies a race stably across shards. +func raceAffectedLines(details string) []string { + sites := parseRaceSites(details) + locations := make([]string, 0, len(sites)) + for _, s := range sites { + locations = append(locations, s.location) + } + slices.Sort(locations) + return slices.Compact(locations) +} + +// firstGoFrame returns ".go:" for the first application stack frame +// following an access header, skipping the race detector's runtime shim frames +// (e.g. runtime.raceread in race_amd64.s). It stops at the blank line that ends +// the access's stack. +func firstGoFrame(lines []string) string { + for _, line := range lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" { + return "" // end of this access's stack; no application frame found + } + m := raceFileLineRegex.FindStringSubmatch(trimmed) + if m == nil || strings.Contains(m[1], "/src/runtime/") { + continue // function-name line, assembly shim, or Go runtime internals + } + return trimRepoPath(m[1]) + ":" + m[2] + } + return "" +} + +// trimRepoPath strips the CI checkout prefix from an absolute source path. +func trimRepoPath(path string) string { + return repoPathPrefixRegex.ReplaceAllString(path, "") +} + +// jobIDFromArtifactName extracts the job ID from a test-summary artifact name of +// the form "test-summary-json--------". +// Returns "" when the name doesn't carry a job ID. +func jobIDFromArtifactName(name string) string { + parts := strings.Split(name, "--") + if len(parts) < 3 { + return "" + } + return parts[2] +} + +// uniqueDataRaces removes duplicate races (the same race is reported by every +// shard/attempt that hit it) while preserving first-seen order. Races are keyed +// by their test and affected source lines rather than the raw report, so the +// same race still de-duplicates despite the differing memory addresses and +// goroutine ids the detector prints each run. +func uniqueDataRaces(races []DataRace) []DataRace { + seen := make(map[string]struct{}, len(races)) + var unique []DataRace + for _, race := range races { + key := raceFingerprint(race) + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + unique = append(unique, race) + } + return unique +} + +// raceFingerprint is a stable identity for a race: its test plus the set of +// affected source lines. Falls back to the raw report when no source lines can +// be parsed, so unparseable reports aren't over-merged. +func raceFingerprint(race DataRace) string { + lines := raceAffectedLines(race.Details) + if len(lines) == 0 { + return race.Location + "\n" + race.Details + } + return race.Location + "\n" + strings.Join(lines, "\n") +} diff --git a/tools/ci-notify/datarace_test.go b/tools/ci-notify/datarace_test.go new file mode 100644 index 00000000000..55631387f22 --- /dev/null +++ b/tools/ci-notify/datarace_test.go @@ -0,0 +1,210 @@ +package cinotify + +import ( + "archive/zip" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/tools/common/github" +) + +func TestDataRacesFromRows(t *testing.T) { + rows := []summaryRow{ + {Kind: "Failed", Name: "TestFoo (final)", Final: true}, + { + Kind: "DATA RACE", + Name: "DATA RACE: Data race detected — in go.temporal.io/server/service/history.TestBar", + Details: "WARNING: DATA RACE\nRead at 0x00c0...", + }, + {Kind: "OOM", Name: "OOM prevention"}, + {Kind: "PANIC", Name: "PANIC: boom"}, + } + + races := dataRacesFromRows(rows, "job-1") + + require.Len(t, races, 1) + require.Equal(t, "service/history.TestBar", races[0].Location) + require.Equal(t, "job-1", races[0].JobID) + require.Contains(t, races[0].Details, "WARNING: DATA RACE") +} + +func TestDataRacesFromRowsNone(t *testing.T) { + rows := []summaryRow{ + {Kind: "Failed", Name: "TestFoo (final)", Final: true}, + {Kind: "OOM", Name: "OOM prevention"}, + } + require.Empty(t, dataRacesFromRows(rows, "job-1")) +} + +func TestDataRaceLocation(t *testing.T) { + require.Equal(t, + "service/history.TestBar", + dataRaceLocation("DATA RACE: Data race detected — in go.temporal.io/server/service/history.TestBar"), + ) + // A race not attributed to a temporal package keeps its qualified name. + require.Equal(t, + "example.com/foo.TestBaz", + dataRaceLocation("DATA RACE: Data race detected — in example.com/foo.TestBaz"), + ) + // A name without the "— in" locator falls back to the summary text. + require.Equal(t, "Data race detected", dataRaceLocation("DATA RACE: Data race detected")) +} + +func TestJobIDFromArtifactName(t *testing.T) { + require.Equal(t, "12345", jobIDFromArtifactName("test-summary-json--999--12345--1--unit-test")) + require.Empty(t, jobIDFromArtifactName("test-summary-json--999")) +} + +func TestUniqueDataRaces(t *testing.T) { + // The same race reported by two shards: identical source lines, but the race + // detector prints different memory addresses and goroutine ids each run. + shard1 := readWriteRaceDetails + shard2 := strings.NewReplacer( + "0x00c0001121e8", "0xdeadbeef", + "goroutine 8", "goroutine 42", + "goroutine 7", "goroutine 99", + ).Replace(shard1) + + // A race at a different source line is a distinct race. + otherLine := strings.ReplaceAll(shard1, "mutable_state.go:130", "mutable_state.go:200") + + races := []DataRace{ + {Location: "service/history.TestMS", Details: shard1, JobID: "1"}, + {Location: "service/history.TestMS", Details: shard2, JobID: "2"}, // dup of shard1 + {Location: "service/history.TestMS", Details: otherLine}, // different line + {Location: "service/matching.TestCache", Details: shard1}, // different test + } + + unique := uniqueDataRaces(races) + + require.Len(t, unique, 3) + require.Equal(t, "1", unique[0].JobID) // first occurrence wins + require.Equal(t, "service/history.TestMS", unique[1].Location) + require.Equal(t, "service/matching.TestCache", unique[2].Location) +} + +func TestSummaryRowsFromZipParsesDataRaceDetails(t *testing.T) { + dir := t.TempDir() + zipPath := filepath.Join(dir, "artifact.zip") + file, err := os.Create(zipPath) + require.NoError(t, err) + + writer := zip.NewWriter(file) + summaryFile, err := writer.Create("test-summary.json") + require.NoError(t, err) + _, err = summaryFile.Write([]byte(`{ + "rows": [ + { + "kind": "DATA RACE", + "name": "DATA RACE: Data race detected — in go.temporal.io/server/pkg.TestRacy", + "details": "WARNING: DATA RACE\nWrite at 0x00c000" + } + ] +}`)) + require.NoError(t, err) + require.NoError(t, writer.Close()) + require.NoError(t, file.Close()) + + rows, err := summaryRowsFromZip(zipPath) + require.NoError(t, err) + + races := dataRacesFromRows(rows, "job-9") + require.Len(t, races, 1) + require.Equal(t, "pkg.TestRacy", races[0].Location) + require.Contains(t, races[0].Details, "WARNING: DATA RACE") +} + +// readWriteRaceDetails is a read/write race whose top frame is the race +// detector's runtime shim (which must be skipped) and whose source paths use the +// CI checkout prefix (which must be trimmed). +const readWriteRaceDetails = `================== +WARNING: DATA RACE +Read at 0x00c0001121e8 by goroutine 8: + runtime.raceread() + /usr/local/go/src/runtime/race_amd64.s:260 +0x21 + go.temporal.io/server/service/history.(*ms).Get() + /home/runner/work/temporal/temporal/service/history/mutable_state.go:127 +0x2c + +Previous write at 0x00c0001121e8 by goroutine 7: + runtime.racewrite() + /usr/local/go/src/runtime/race_amd64.s:269 +0x21 + go.temporal.io/server/service/history.(*ms).Set() + /home/runner/work/temporal/temporal/service/history/mutable_state.go:130 +0x3c + +Goroutine 8 (running) created at: + go.temporal.io/server/service/history.TestMutableState() + /home/runner/work/temporal/temporal/service/history/mutable_state_test.go:41 +0x88 +==================` + +// writeWriteRaceDetails is a race between two concurrent writes. +const writeWriteRaceDetails = `================== +WARNING: DATA RACE +Write at 0x00c000000180 by goroutine 7: + go.temporal.io/server/service/matching.(*cache).put() + /home/runner/work/temporal/temporal/service/matching/cache.go:88 +0x66 + +Previous write at 0x00c000000180 by goroutine 12: + go.temporal.io/server/service/matching.(*cache).put() + /home/runner/work/temporal/temporal/service/matching/cache.go:88 +0x66 +==================` + +func TestRaceSites(t *testing.T) { + require.Equal(t, []string{ + "Read at (goroutine 8): service/history/mutable_state.go:127", + "Previous write at (goroutine 7): service/history/mutable_state.go:130", + }, raceSites(readWriteRaceDetails)) + + // A write/write race is still a race; both writes must be surfaced. + require.Equal(t, []string{ + "Write at (goroutine 7): service/matching/cache.go:88", + "Previous write at (goroutine 12): service/matching/cache.go:88", + }, raceSites(writeWriteRaceDetails)) + + // Unparseable details yield no sites; the caller falls back to location + link. + require.Empty(t, raceSites("some unrelated text")) +} + +func TestBuildDataRaceMessageLinksToJob(t *testing.T) { + report := &DataRaceReport{ + Run: github.Run{ + DatabaseID: 123456, + HeadSHA: "abc1234567890defghijk", + URL: "https://github.com/temporalio/temporal/actions/runs/123456", + }, + Author: "Test Author", + Title: "Some commit title", + DataRaces: []DataRace{ + {Location: "service/history.TestMutableState", Details: readWriteRaceDetails, JobID: "789"}, + }, + } + + rendered := BuildDataRaceMessage(report).RenderMarkdown() + + require.Contains(t, rendered, "Data Race Detected on Main Branch") + require.Contains(t, rendered, "Test Author") + require.Contains(t, rendered, "service/history.TestMutableState") + require.Contains(t, rendered, "Read at (goroutine 8): service/history/mutable_state.go:127") + require.Contains(t, rendered, "Previous write at (goroutine 7): service/history/mutable_state.go:130") + require.Contains(t, rendered, "abc1234") // short SHA link + // Links to the specific job, not just the top-level run. + require.Contains(t, rendered, "actions/runs/123456/job/789") + // Runtime shim frames and raw stacktrace noise are not dumped into Slack. + require.NotContains(t, rendered, "race_amd64.s") +} + +func TestBuildDataRaceMessageFallsBackToRunLink(t *testing.T) { + report := &DataRaceReport{ + Run: github.Run{DatabaseID: 123456, HeadSHA: "abc1234567890"}, + DataRaces: []DataRace{{Location: "pkg.TestRacy", Details: "unparseable"}}, // no JobID + } + + rendered := BuildDataRaceMessage(report).RenderMarkdown() + + require.Contains(t, rendered, "Unknown") // author + require.Contains(t, rendered, "pkg.TestRacy") + require.Contains(t, rendered, "actions/runs/123456") + require.NotContains(t, rendered, "/job/") +} diff --git a/tools/ci-notify/slack.go b/tools/ci-notify/slack.go index ec17b2ee878..9deb03b58e2 100644 --- a/tools/ci-notify/slack.go +++ b/tools/ci-notify/slack.go @@ -2,8 +2,10 @@ package cinotify import ( "fmt" + "strconv" "strings" + "go.temporal.io/server/tools/common/github" "go.temporal.io/server/tools/common/slack" ) @@ -70,6 +72,52 @@ func FormatMessageForDebug(report *FailureReport) string { return sb.String() } +// BuildDataRaceMessage creates a Slack message announcing data races on main. +func BuildDataRaceMessage(report *DataRaceReport) *slack.Message { + runID := strconv.FormatInt(report.Run.DatabaseID, 10) + commitURL := github.CommitURL(temporalRepository, report.Run.HeadSHA) + + message := slack.NewMessage(fmt.Sprintf("Data Race Detected on Main (%d)", len(report.DataRaces))) + message.AddSection(":rotating_light: *Data Race Detected on Main Branch* :rotating_light:") + message.AddFields( + fmt.Sprintf("*Commit:*\n<%s|%s>", commitURL, report.Run.ShortSHA()), + fmt.Sprintf("*Author:*\n%s", orUnknown(report.Author)), + ) + if report.Title != "" { + message.AddSection(fmt.Sprintf("*Commit message:*\n%s", report.Title)) + } + + for _, race := range report.DataRaces { + var sb strings.Builder + if race.Location != "" { + fmt.Fprintf(&sb, "*%s*", race.Location) + } + for _, site := range raceSites(race.Details) { + fmt.Fprintf(&sb, "\n• %s", site) + } + fmt.Fprintf(&sb, "\n<%s|View job logs>", raceLink(runID, race)) + message.AddSection(sb.String()) + } + + return message +} + +func orUnknown(s string) string { + if s == "" { + return "Unknown" + } + return s +} + +// raceLink points at the specific job that reported the race so the alert is +// directly actionable, falling back to the run when the job is unknown. +func raceLink(runID string, race DataRace) string { + if race.JobID != "" { + return github.JobURL(temporalRepository, runID, race.JobID) + } + return github.RunURL(temporalRepository, runID) +} + // BuildSuccessReportMessage creates a Slack message for success report func BuildSuccessReportMessage(report *DigestReport) *slack.Message { message := slack.NewMessage(fmt.Sprintf("Weekly CI Report - %s Branch", report.Branch)) From 9309d4a9ca2dd4cfdb0f2caa19e47f0cd681184c Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 26 Aug 2026 10:42:14 -0700 Subject: [PATCH 068/139] Improve object leak diagnostics (#11643) ## What changed? 1. Object leak reports now includes sorted heap addresses for unexpected retained objects. 2. The leak test can optionally capture a raw heap dump with `LEAK_HEAP_DUMP=1`. ## Why? Local debugging; it helps identify the runtime root retaining a reported object. ## Potential risks Raw heap diagnostics can be large and may contain sensitive process data. They are only captured after an object leak failure when `LEAK_HEAP_DUMP=1` is explicitly set. --- common/testing/objectleak/diagnostics.go | 31 +++++++++++ common/testing/objectleak/diagnostics_test.go | 35 ++++++++++++ common/testing/objectleak/leakcheck_test.go | 55 ++++++++++++++++++- common/testing/objectleak/report.go | 20 +++++-- tests/leakcheck/leak_test.go | 8 +++ 5 files changed, 144 insertions(+), 5 deletions(-) create mode 100644 common/testing/objectleak/diagnostics.go create mode 100644 common/testing/objectleak/diagnostics_test.go diff --git a/common/testing/objectleak/diagnostics.go b/common/testing/objectleak/diagnostics.go new file mode 100644 index 00000000000..0babcb1b5e3 --- /dev/null +++ b/common/testing/objectleak/diagnostics.go @@ -0,0 +1,31 @@ +package objectleak + +import ( + "errors" + "os" + "path/filepath" + "runtime/debug" +) + +// WriteHeapDiagnostics writes a raw heap dump to outputDir. It may contain +// sensitive data. +func WriteHeapDiagnostics(outputDir string) error { + return writeHeapDumpFile(outputDir, func(heapDump *os.File) { + debug.WriteHeapDump(heapDump.Fd()) + }) +} + +func writeHeapDumpFile( + outputDir string, + writeHeapDump func(*os.File), +) error { + heapDump, err := os.OpenFile(filepath.Join(outputDir, "heap.dump"), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o600) + if err != nil { + return err + } + if err := heapDump.Chmod(0o600); err != nil { + return errors.Join(err, heapDump.Close()) + } + writeHeapDump(heapDump) + return heapDump.Close() +} diff --git a/common/testing/objectleak/diagnostics_test.go b/common/testing/objectleak/diagnostics_test.go new file mode 100644 index 00000000000..f6f30d2f57d --- /dev/null +++ b/common/testing/objectleak/diagnostics_test.go @@ -0,0 +1,35 @@ +package objectleak + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWriteHeapDiagnostics(t *testing.T) { + outputDir := t.TempDir() + heapDumpPath := filepath.Join(outputDir, "heap.dump") + require.NoError(t, os.WriteFile(heapDumpPath, []byte("old heap"), 0o644)) + require.NoError(t, os.Chmod(heapDumpPath, 0o644)) + + err := writeHeapDumpFile(outputDir, func(heapDump *os.File) { + _, err := heapDump.WriteString("heap") + require.NoError(t, err) + }) + require.NoError(t, err) + + heapDump, err := os.ReadFile(heapDumpPath) + require.NoError(t, err) + require.Equal(t, []byte("heap"), heapDump) + + heapDumpInfo, err := os.Stat(heapDumpPath) + require.NoError(t, err) + require.Equal(t, os.FileMode(0o600), heapDumpInfo.Mode().Perm()) + + entries, err := os.ReadDir(outputDir) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, "heap.dump", entries[0].Name()) +} diff --git a/common/testing/objectleak/leakcheck_test.go b/common/testing/objectleak/leakcheck_test.go index d319ca2d8cc..10e4caf591f 100644 --- a/common/testing/objectleak/leakcheck_test.go +++ b/common/testing/objectleak/leakcheck_test.go @@ -1,8 +1,11 @@ package objectleak import ( + "fmt" "reflect" + "regexp" "runtime" + "slices" "testing" "time" @@ -36,6 +39,8 @@ type tinyZeroLengthPointerArrayValue struct { Value byte } +var diagnosticAddressesPattern = regexp.MustCompile(`(?m)^( addresses: )\[[^\n]*\]$`) + type aliasOuter struct { aliasInner _ [8]byte @@ -68,8 +73,11 @@ retained objects: 4 total, 0 baseline, 0 expected, 4 unexpected unexpected retained objects: 2 objects: *objectleak.graphRoot + addresses: [...] 2 paths, 1 object: Node (*objectleak.graphNode) + addresses: [...] 2 paths, 1 object: Node.Leaf (*objectleak.graphLeaf) + addresses: [...] expected retained objects: none @@ -209,6 +217,7 @@ retained objects: 4 total, 3 baseline, 0 expected, 1 unexpected unexpected retained objects: 1 object: *objectleak.graphRoot + addresses: [...] expected retained objects: none @@ -237,8 +246,11 @@ retained objects: 4 total, 0 baseline, 0 expected, 4 unexpected unexpected retained objects: 2 objects: *objectleak.graphRoot + addresses: [...] 2 paths, 1 object: Node (*objectleak.graphNode) + addresses: [...] 2 paths, 1 object: Node.Leaf (*objectleak.graphLeaf) + addresses: [...] expected retained objects: none @@ -269,6 +281,7 @@ retained objects: 4 total, 3 baseline, 0 expected, 1 unexpected unexpected retained objects: 1 object: *objectleak.graphRoot + addresses: [...] expected retained objects: none @@ -349,6 +362,7 @@ retained objects: 3 total, 2 baseline, 0 expected, 1 unexpected unexpected retained objects: 1 object: *objectleak.graphRoot + addresses: [...] expected retained objects: none @@ -402,11 +416,50 @@ baseline retained objects: for _, expected := range tc.wantErrContains { require.Contains(t, err.Error(), expected) } - require.Equal(t, tc.wantReport, report) + require.Equal(t, tc.wantReport, diagnosticAddressesPattern.ReplaceAllString(report, "${1}[...]")) }) } } +func TestReportStringIncludesSortedUnexpectedAddresses(t *testing.T) { + roots := []*graphRoot{{}, {}} + addresses := []uintptr{ + reflect.ValueOf(roots[0]).Pointer(), + reflect.ValueOf(roots[1]).Pointer(), + } + slices.Sort(addresses) + check, err := NewObjectLeakCheck(WithGCSettleTimeout(testGCSettleTimeout)) + require.NoError(t, err) + check.gcSettleMinWait = testGCSettleMinWait + check.gcSettleQuiet = testGCSettleQuiet + for _, root := range roots { + check.Track(root) + } + + report, err := check.Check(Baseline{}) + runtime.KeepAlive(roots) + require.Error(t, err) + require.Contains(t, report, fmt.Sprintf(" addresses: %#x", addresses)) +} + +func TestReportStringLimitsUnexpectedAddresses(t *testing.T) { + addresses := make(map[uintptr]struct{}, 34) + for address := uintptr(1); address <= 34; address++ { + addresses[address] = struct{}{} + } + r := report{} + r.retained[retentionUnexpected].groups = []objectGroup{{ + typeName: "*objectleak.graphRoot", + addresses: addresses, + }} + + require.Contains( + t, + r.string(), + " addresses: [0x1 0x2 0x3 0x4 0x5 0x6 0x7 0x8 0x9 0xa 0xb 0xc 0xd 0xe 0xf 0x10 0x11 0x12 0x13 0x14 0x15 0x16 0x17 0x18 0x19 0x1a 0x1b 0x1c 0x1d 0x1e 0x1f 0x20] ... and 2 more", + ) +} + func TestObjectLeak_CheckSkipsTinyPointerFreeObjects(t *testing.T) { values := []any{ new(tinyValue), diff --git a/common/testing/objectleak/report.go b/common/testing/objectleak/report.go index f911b925f98..db9c6a920e3 100644 --- a/common/testing/objectleak/report.go +++ b/common/testing/objectleak/report.go @@ -4,12 +4,15 @@ import ( "cmp" "errors" "fmt" + "maps" "slices" "strings" ) type retentionClass int +const maxInlineAddresses = 32 + const ( retentionUnexpected retentionClass = iota retentionExpected @@ -117,7 +120,7 @@ func (r report) string() string { var out strings.Builder r.writeSummary(&out) - writeGroups := func(title string, groups []objectGroup) { + writeGroups := func(title string, groups []objectGroup, includeAddresses bool) { fmt.Fprintf(&out, "%s:\n", title) if len(groups) == 0 { out.WriteString(" none\n") @@ -125,14 +128,23 @@ func (r report) string() string { } for _, group := range groups { fmt.Fprintf(&out, " %s: %s\n", group.counts(), group.name()) + if includeAddresses { + addresses := slices.Sorted(maps.Keys(group.addresses)) + inlineAddresses := addresses[:min(len(addresses), maxInlineAddresses)] + fmt.Fprintf(&out, " addresses: %#x", inlineAddresses) + if remaining := len(addresses) - len(inlineAddresses); remaining > 0 { + fmt.Fprintf(&out, " ... and %d more", remaining) + } + out.WriteByte('\n') + } } } out.WriteByte('\n') - writeGroups("unexpected retained objects", r.retained[retentionUnexpected].groups) + writeGroups("unexpected retained objects", r.retained[retentionUnexpected].groups, true) out.WriteByte('\n') - writeGroups("expected retained objects", r.retained[retentionExpected].groups) + writeGroups("expected retained objects", r.retained[retentionExpected].groups, false) out.WriteByte('\n') - writeGroups("baseline retained objects", r.retained[retentionBaseline].groups) + writeGroups("baseline retained objects", r.retained[retentionBaseline].groups, false) if len(r.unmatchedExpected) > 0 { out.WriteString("\nstale expected patterns:\n") diff --git a/tests/leakcheck/leak_test.go b/tests/leakcheck/leak_test.go index 8c9406e854e..9b8bf02d305 100644 --- a/tests/leakcheck/leak_test.go +++ b/tests/leakcheck/leak_test.go @@ -44,6 +44,7 @@ var objectLeakOpts = []objectleak.Option{ // LEAK_ITERS_WARMUP warmup clusters before snapshotting the baseline // LEAK_OUTPUT_DIR directory for diagnostics on failure // LEAK_GC_SETTLE_TIMEOUT maximum time to settle object leak checks +// LEAK_HEAP_DUMP set to 1 to write a raw heap dump on object leak failure; this file may contain sensitive data func TestClusterShutdownLeak(t *testing.T) { iters, err := strconv.Atoi(os.Getenv("LEAK_ITERS")) if err != nil { @@ -115,6 +116,13 @@ func TestClusterShutdownLeak(t *testing.T) { writeProfile(t, outputDir, "heap", "heap.txt", 1) writeProfile(t, outputDir, "allocs", "allocs.pb.gz", 0) writeProfile(t, outputDir, "allocs", "allocs.txt", 1) + if leakErr != nil && os.Getenv("LEAK_HEAP_DUMP") == "1" { + if err := objectleak.WriteHeapDiagnostics(outputDir); err != nil { + t.Logf("failed to write raw heap diagnostics: %v", err) + } else { + t.Logf("raw heap diagnostics written to %s", outputDir) + } + } if goleakErr != nil { t.Error(goleakErr) From b40bba4360eb634243ce9d777c55c14e5fba7cd9 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 26 Aug 2026 10:42:41 -0700 Subject: [PATCH 069/139] Auto-approve test shard salt updates (#11779) WISOTT --- .github/CODEOWNERS | 3 +++ .github/workflows/optimize-test-sharding.yml | 24 ++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e890a020ffb..3e3fd350385 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -107,3 +107,6 @@ *_mock.go @temporalio/server @temporalio/cgs @temporalio/nexus *_gen.go @temporalio/server @temporalio/cgs @temporalio/nexus *.gen.go @temporalio/server @temporalio/cgs @temporalio/nexus + +# This generated file has no code owner to allow automated updates by CI. +/tests/testcore/shard_salt.txt diff --git a/.github/workflows/optimize-test-sharding.yml b/.github/workflows/optimize-test-sharding.yml index 5e3572096bf..ccfa9ae4547 100644 --- a/.github/workflows/optimize-test-sharding.yml +++ b/.github/workflows/optimize-test-sharding.yml @@ -8,6 +8,7 @@ on: permissions: contents: read + pull-requests: write env: SALT_FILE: tests/testcore/shard_salt.txt @@ -51,8 +52,9 @@ jobs: GH_TOKEN: ${{ steps.generate-token.outputs.token }} - name: Create pull request + id: create-pr run: | - if git diff --quiet; then + if git diff --quiet -- "${{ env.SALT_FILE }}"; then echo "No changes to shard salt" exit 0 fi @@ -69,10 +71,24 @@ jobs: git commit -m "Update test shard salt" git push origin ${{ env.BRANCH }} - gh pr create \ + pr_url=$(gh pr create \ --title "Update test shard salt" \ - --body "Automatically generated by the optimize-test-sharding workflow." + --body "Automatically generated by the optimize-test-sharding workflow.") - gh pr merge ${{ env.BRANCH }} --auto --squash + echo "pr_url=${pr_url}" >> "${GITHUB_OUTPUT}" env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} + + - name: Approve pull request + if: steps.create-pr.outputs.pr_url != '' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_URL: ${{ steps.create-pr.outputs.pr_url }} + run: gh pr review --approve "${PR_URL}" + + - name: Enable auto-merge + if: steps.create-pr.outputs.pr_url != '' + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + PR_URL: ${{ steps.create-pr.outputs.pr_url }} + run: gh pr merge "${PR_URL}" --auto --squash From 871d03858c0ea6ea2eb4b2ba7b41b062274412ee Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 26 Aug 2026 11:26:30 -0700 Subject: [PATCH 070/139] Make the scheduler idle-close deadline monotonic (#11702) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? This adds a lastEventTime tracking value which keeps track of the last value even if they fall out of the buffer. It's a bugfix for the current state which can presently cause multiple idle-tasks to get assigned as the calculation of when to set the idle-task can jump around as actions are removed from the buffer. By tracking the last seen value, it can be forced to be a monotonic value. ### LLM summary: `getLastEventTime` maxed over `Invoker.recentActions()`, which reads `BufferedStarts` — truncated by `applyCompletedRetention` to the last `recentActionCount` (10) completed entries, **ordered by `CloseTime`**. Whenever completion order differs from start order, evicting the start that held the largest `StartTime` drags the value, and therefore the idle deadline, backwards. **Unbounded growth of the mutable state blob and the timer queue.** Idle tasks are reaped only via `Validate`, whose predicate (`idleExpiration.After(scheduledTime)`) drops a task only when the deadline moved *later*. A flat or regressed deadline invalidates nothing, so every Generator tick leaves another task behind — one per action between idle ticks. ### Observed in production ## How did you test it? - [x] built - [x] covered by existing tests - [x] added new unit test(s) New file `chasm/lib/scheduler/scheduler_idle_deadline_test.go`, 10 tests. Verified they fail without the fix by reverting both production changes — 9 of 10 fail, reproducing the production numbers: ``` TestGeneratorTask_IdleTask_NotReArmedWhenDeadlineUnchanged expected: 1 actual: 21 # 20 no-op ticks -> 21 tasks, one per tick TestGeneratorTask_IdleTask_DoesNotAccumulateAcrossActionBurst idle deadline regressed at action 10: 2026-08-28 04:43:25.199231 -> 2026-08-28 04:43:25.197731 ``` A 1.5ms regression at action 10 — landing exactly where `recentActionCount = 10` first evicts. The tenth test (`NotArmedWhileSpecHasWork`) passes before and after by design; it guards against the skip logic swallowing the non-idle branch. Coverage: the root cause independent of the idle machinery (recomputed value regresses on eviction, floored read does not, mark never moves backwards, nil mark degrades to old behaviour); accumulation (repeated no-op ticks, and 43 actions completing one at a time through the retention window); stuck-open safety for the skip guard (deadline advance re-arms and reaps, pause/unpause re-arms, live spec arms nothing); `Validate` no longer entering the regressed branch after an eviction; and a sanity test that the snapshot helper sees armed tasks so the count assertions can't pass vacuously. Whole package green, stable under `-count=3 -test.shuffle=on`, `make lint-code` 0 issues, `go build ./...` and `go test ./chasm/...` clean. --------- Co-authored-by: Claude Opus 5 (1M context) --- chasm/lib/scheduler/export_test.go | 28 +- .../gen/schedulerpb/v1/message.pb.go | 84 ++-- chasm/lib/scheduler/generator_tasks.go | 44 +- chasm/lib/scheduler/helper_test.go | 27 +- chasm/lib/scheduler/invoker.go | 8 +- .../scheduler/invoker_execute_task_test.go | 80 ++++ chasm/lib/scheduler/invoker_tasks.go | 3 +- chasm/lib/scheduler/proto/v1/message.proto | 16 + chasm/lib/scheduler/scheduler.go | 43 +- .../scheduler/scheduler_idle_deadline_test.go | 439 ++++++++++++++++++ .../scheduler/scheduler_idle_tasks_test.go | 1 + chasm/lib/scheduler/scheduler_tasks.go | 9 +- chasm/lib/scheduler/util.go | 5 +- 13 files changed, 731 insertions(+), 56 deletions(-) create mode 100644 chasm/lib/scheduler/scheduler_idle_deadline_test.go diff --git a/chasm/lib/scheduler/export_test.go b/chasm/lib/scheduler/export_test.go index a3f2cba5f51..426bcd44c8e 100644 --- a/chasm/lib/scheduler/export_test.go +++ b/chasm/lib/scheduler/export_test.go @@ -54,6 +54,30 @@ func ContextWithTweakables(ctx chasm.Context, tweakables Tweakables) chasm.Conte // RecentActionCount exposes the completed-retention limit for tests. const RecentActionCount = recentActionCount +// ComputeLastEventTime exposes the non-monotonic, recompute-from-state value so +// tests can pin the regression it is subject to. +func (s *Scheduler) ComputeLastEventTime(ctx chasm.Context) time.Time { + return s.computeLastEventTime(ctx) +} + +// GetLastEventTimeFloored exposes the monotonic read path (recomputed value +// floored at the persisted high water mark). +func (s *Scheduler) GetLastEventTimeFloored(ctx chasm.Context) time.Time { + return s.getLastEventTime(ctx) +} + +// AdvanceLastEventTime exposes the high-water-mark write performed by the +// Generator tick. +func (s *Scheduler) AdvanceLastEventTime(ctx chasm.MutableContext) time.Time { + return s.advanceLastEventTime(ctx) +} + +// IdleDeadline exposes the deadline the idle task is armed against and that +// SchedulerIdleTaskHandler.Validate recomputes. +func (s *Scheduler) IdleDeadline(ctx chasm.Context, idleTime time.Duration) time.Time { + return s.idleDeadline(ctx, idleTime) +} + // ApplyCompletedRetention exposes applyCompletedRetention for tests. func (i *Invoker) ApplyCompletedRetention() { i.applyCompletedRetention() @@ -66,10 +90,12 @@ func (i *Invoker) RecordExecuteResult( completed []*schedulespb.BufferedStart, retryable []*schedulespb.BufferedStart, ) (newlyStarted, droppedDuplicates int, startOnlyActions []*schedulespb.BufferedStart) { - return i.recordExecuteResult(ctx, &executeResult{ + newlyStarted, droppedDuplicates, latestStartTime, startOnlyActions := i.recordExecuteResult(ctx, &executeResult{ CompletedStarts: completed, RetryableStarts: retryable, }) + i.Scheduler.Get(ctx).advanceLastEventTimeTo(latestStartTime) + return newlyStarted, droppedDuplicates, startOnlyActions } func (s *Scheduler) RecordStartOnlyActions( diff --git a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go index 534e3cff7cc..c1a61058df4 100644 --- a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go +++ b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go @@ -54,6 +54,21 @@ type SchedulerState struct { // Surfaced as the ScheduleIdleCloseTime search attribute for stuck-schedule // detection. IdleCloseTime *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=idle_close_time,json=idleCloseTime,proto3" json:"idle_close_time,omitempty"` + // Monotonic high water mark for the most recent "event" on this schedule + // (create, update, or action start). The idle deadline is anchored on this + // value. + // + // This exists because the value cannot be safely recomputed on demand: + // recentActions() is derived from the Invoker's BufferedStarts, which + // completed-action retention truncates to the last `recentActionCount` + // entries. Evicting the start that held the largest StartTime drags a + // recomputed last-event-time - and therefore the idle deadline - backwards, + // closing the schedule earlier than intended. + // + // Unset on schedules created before this field was introduced; callers must + // fall back to the recomputed value, which is what makes the rollout a no-op + // until the first Generator tick advances it. + LastEventTime *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=last_event_time,json=lastEventTime,proto3" json:"last_event_time,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -158,6 +173,13 @@ func (x *SchedulerState) GetIdleCloseTime() *timestamppb.Timestamp { return nil } +func (x *SchedulerState) GetLastEventTime() *timestamppb.Timestamp { + if x != nil { + return x.LastEventTime + } + return nil +} + // WorkflowMigrationState tracks the state of an in-progress V2-to-V1 migration. type WorkflowMigrationState struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -726,7 +748,7 @@ var File_temporal_server_chasm_lib_scheduler_proto_v1_message_proto protoreflect const file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_rawDesc = "" + "\n" + - ":temporal/server/chasm/lib/scheduler/proto/v1/message.proto\x12,temporal.server.chasm.lib.scheduler.proto.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a&temporal/api/schedule/v1/message.proto\x1a-temporal/server/api/schedule/v1/message.proto\"\x82\x04\n" + + ":temporal/server/chasm/lib/scheduler/proto/v1/message.proto\x12,temporal.server.chasm.lib.scheduler.proto.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a%temporal/api/failure/v1/message.proto\x1a&temporal/api/schedule/v1/message.proto\x1a-temporal/server/api/schedule/v1/message.proto\"\xc6\x04\n" + "\x0eSchedulerState\x12>\n" + "\bschedule\x18\x02 \x01(\v2\".temporal.api.schedule.v1.ScheduleR\bschedule\x12:\n" + "\x04info\x18\x03 \x01(\v2&.temporal.api.schedule.v1.ScheduleInfoR\x04info\x12\x1c\n" + @@ -739,7 +761,8 @@ const file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_rawDesc = "\bsentinel\x18\n" + " \x01(\bR\bsentinel\x12s\n" + "\x12workflow_migration\x18\v \x01(\v2D.temporal.server.chasm.lib.scheduler.proto.v1.WorkflowMigrationStateR\x11workflowMigration\x12B\n" + - "\x0fidle_close_time\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\ridleCloseTime\"z\n" + + "\x0fidle_close_time\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\ridleCloseTime\x12B\n" + + "\x0flast_event_time\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\rlastEventTime\"z\n" + "\x16WorkflowMigrationState\x120\n" + "\x14pre_migration_paused\x18\x01 \x01(\bR\x12preMigrationPaused\x12.\n" + "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\"\xa8\x01\n" + @@ -828,34 +851,35 @@ var file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_depIdxs = [] 13, // 1: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState.info:type_name -> temporal.api.schedule.v1.ScheduleInfo 1, // 2: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState.workflow_migration:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.WorkflowMigrationState 14, // 3: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState.idle_close_time:type_name -> google.protobuf.Timestamp - 14, // 4: temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState.last_processed_time:type_name -> google.protobuf.Timestamp - 14, // 5: temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState.future_action_times:type_name -> google.protobuf.Timestamp - 15, // 6: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.buffered_starts:type_name -> temporal.server.api.schedule.v1.BufferedStart - 16, // 7: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.cancel_workflows:type_name -> temporal.api.common.v1.WorkflowExecution - 16, // 8: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.terminate_workflows:type_name -> temporal.api.common.v1.WorkflowExecution - 14, // 9: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.last_processed_time:type_name -> google.protobuf.Timestamp - 17, // 10: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.backfill_request:type_name -> temporal.api.schedule.v1.BackfillRequest - 18, // 11: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.trigger_request:type_name -> temporal.api.schedule.v1.TriggerImmediatelyRequest - 14, // 12: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.last_processed_time:type_name -> google.protobuf.Timestamp - 19, // 13: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.success:type_name -> temporal.api.common.v1.Payload - 20, // 14: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.failure:type_name -> temporal.api.failure.v1.Failure - 0, // 15: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.scheduler_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState - 2, // 16: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.generator_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState - 3, // 17: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.invoker_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.InvokerState - 9, // 18: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.backfillers:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry - 5, // 19: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.last_completion_result:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult - 10, // 20: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.search_attributes:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry - 11, // 21: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.memo:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry - 8, // 22: temporal.server.chasm.lib.scheduler.proto.v1.EventLog.events:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.Event - 14, // 23: temporal.server.chasm.lib.scheduler.proto.v1.Event.time:type_name -> google.protobuf.Timestamp - 4, // 24: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry.value:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState - 19, // 25: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload - 19, // 26: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry.value:type_name -> temporal.api.common.v1.Payload - 27, // [27:27] is the sub-list for method output_type - 27, // [27:27] is the sub-list for method input_type - 27, // [27:27] is the sub-list for extension type_name - 27, // [27:27] is the sub-list for extension extendee - 0, // [0:27] is the sub-list for field type_name + 14, // 4: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState.last_event_time:type_name -> google.protobuf.Timestamp + 14, // 5: temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState.last_processed_time:type_name -> google.protobuf.Timestamp + 14, // 6: temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState.future_action_times:type_name -> google.protobuf.Timestamp + 15, // 7: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.buffered_starts:type_name -> temporal.server.api.schedule.v1.BufferedStart + 16, // 8: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.cancel_workflows:type_name -> temporal.api.common.v1.WorkflowExecution + 16, // 9: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.terminate_workflows:type_name -> temporal.api.common.v1.WorkflowExecution + 14, // 10: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.last_processed_time:type_name -> google.protobuf.Timestamp + 17, // 11: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.backfill_request:type_name -> temporal.api.schedule.v1.BackfillRequest + 18, // 12: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.trigger_request:type_name -> temporal.api.schedule.v1.TriggerImmediatelyRequest + 14, // 13: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.last_processed_time:type_name -> google.protobuf.Timestamp + 19, // 14: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.success:type_name -> temporal.api.common.v1.Payload + 20, // 15: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.failure:type_name -> temporal.api.failure.v1.Failure + 0, // 16: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.scheduler_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState + 2, // 17: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.generator_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState + 3, // 18: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.invoker_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.InvokerState + 9, // 19: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.backfillers:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry + 5, // 20: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.last_completion_result:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult + 10, // 21: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.search_attributes:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry + 11, // 22: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.memo:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry + 8, // 23: temporal.server.chasm.lib.scheduler.proto.v1.EventLog.events:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.Event + 14, // 24: temporal.server.chasm.lib.scheduler.proto.v1.Event.time:type_name -> google.protobuf.Timestamp + 4, // 25: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry.value:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState + 19, // 26: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload + 19, // 27: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry.value:type_name -> temporal.api.common.v1.Payload + 28, // [28:28] is the sub-list for method output_type + 28, // [28:28] is the sub-list for method input_type + 28, // [28:28] is the sub-list for extension type_name + 28, // [28:28] is the sub-list for extension extendee + 0, // [0:28] is the sub-list for field type_name } func init() { file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_init() } diff --git a/chasm/lib/scheduler/generator_tasks.go b/chasm/lib/scheduler/generator_tasks.go index 153bb039950..21993f02d26 100644 --- a/chasm/lib/scheduler/generator_tasks.go +++ b/chasm/lib/scheduler/generator_tasks.go @@ -139,55 +139,77 @@ func (g *GeneratorTaskHandler) Execute( invoker.EnqueueBufferedStarts(ctx, result.BufferedStarts) } + scheduler.advanceLastEventTime(ctx) + // Write the new high water mark and future action times. generator.LastProcessedTime = timestamppb.New(result.LastActionTime) generator.UpdateFutureActionTimes(ctx, g.specBuilder) + g.rearmTasks(ctx, generator, scheduler, metricsHandler, tweakables.IdleTime, result.NextWakeupTime) + return nil +} + +func (g *GeneratorTaskHandler) rearmTasks( + ctx chasm.MutableContext, + generator *Generator, + sched *Scheduler, + metricsHandler metrics.Handler, + idleTimeTotal time.Duration, + nextWakeupTime time.Time, +) { // Schedule the next timer task. Three outcomes are possible: // - isIdle: the schedule is done; arm the idle task to close it. - // - NextWakeupTime is set: arm the next generator tick. + // - nextWakeupTime is set: arm the next generator tick. // - Neither: Hold open without a task. This requires both that // isHeldOpen is true (paused or backfill pending) AND that no spec // wakeup is available, e.g. a paused manual-only schedule. IdleTime=0 // also lands here. An external trigger (Patch.Unpause, Update, or a // BackfillerTask's completion-time Generate call) revives us. - idleTimeTotal := tweakables.IdleTime - idleExpiration, isIdle := scheduler.getIdleExpiration(ctx, idleTimeTotal, result.NextWakeupTime) + idleExpiration, isIdle := sched.getIdleExpiration(ctx, idleTimeTotal, nextWakeupTime) if isIdle { // Schedule is complete, no need for another buffer task. We keep the schedule's // backing mutable state explicitly open for the idle period, during which the // customer can describe/modify/restart the schedule. // // Once the idle timer expires, we close the component. + + // An idle task armed at exactly this deadline is already pending: arming a + // second one would leave a permanent duplicate in the component's PureTasks, + // because SchedulerIdleTaskHandler.Validate only invalidates tasks whose + // deadline has moved *later*. + if sched.GetIdleCloseTime().AsTime().Equal(idleExpiration) { + metricsHandler.Counter(metrics.ScheduleIdleTask.Name()). + Record(1, metrics.OutcomeTag(outcomeSkipped), metrics.ReasonTag(idleAlreadyArmed)) + return + } + generator.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("scheduled idle task for %s", idleExpiration.Format(time.RFC3339))) - ctx.AddTask(scheduler, chasm.TaskAttributes{ + ctx.AddTask(sched, chasm.TaskAttributes{ ScheduledTime: idleExpiration, }, &schedulerpb.SchedulerIdleTask{ IdleTimeTotal: durationpb.New(idleTimeTotal), }) // Record the idle-close deadline so it can be surfaced as the // ScheduleIdleCloseTime search attribute for stuck-schedule detection. - scheduler.IdleCloseTime = timestamppb.New(idleExpiration) + sched.IdleCloseTime = timestamppb.New(idleExpiration) - return nil + return } // Not idle: the schedule has work again (or is being held open), so it's // no longer pending an idle close. - scheduler.IdleCloseTime = nil + sched.IdleCloseTime = nil - if !result.NextWakeupTime.IsZero() { + if !nextWakeupTime.IsZero() { // Keep the generator task perpetually scheduled. When paused, the next // fire will simply advance the HWM without appending actions (handled in // ProcessTimeRange). - generator.scheduleTask(ctx, result.NextWakeupTime) + generator.scheduleTask(ctx, nextWakeupTime) } else { // Hold open without a task: see the comment block above. metricsHandler.Counter(metrics.SchedulerGeneratorLoopCompleted.Name()).Record(1) } - - return nil } func (g *GeneratorTaskHandler) logSchedule(ctx chasm.MutableContext, logger log.Logger, msg string, generator *Generator, sched *Scheduler) { diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 4a3467399f1..74078774aac 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -153,6 +153,7 @@ type testEnv struct { type testEnvConfig struct { specProcessor scheduler.SpecProcessor withMockEngine bool + schedule *schedulepb.Schedule } // testEnvOption is a functional option for configuring testEnv. @@ -174,6 +175,25 @@ func withMockEngine() testEnvOption { } } +// withSchedule overrides defaultSchedule(), for tests that need a spec other +// than the package default 1-minute interval (e.g. an exhausted spec, which is +// what drives the Generator into its idle branch). +func withSchedule(schedule *schedulepb.Schedule) testEnvOption { + return func(c *testEnvConfig) { + c.schedule = schedule + } +} + +// expiredSchedule returns a schedule whose spec has already ended, so the +// Generator finds no next wakeup and takes its idle branch. This is the shape +// of a real schedule that has run to the end of its subscription window. +func expiredSchedule(now time.Time) *schedulepb.Schedule { + schedule := defaultSchedule() + schedule.Spec.StartTime = timestamppb.New(now.Add(-2 * time.Hour)) + schedule.Spec.EndTime = timestamppb.New(now.Add(-1 * time.Hour)) + return schedule +} + // newRealSpecProcessor creates a real SpecProcessor for tests. func newRealSpecProcessor(ctrl *gomock.Controller, logger log.Logger) scheduler.SpecProcessor { mockMetrics := metrics.NewMockHandler(ctrl) @@ -372,6 +392,11 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { now := time.Now() timeSource.Update(now) + schedule := config.schedule + if schedule == nil { + schedule = defaultSchedule() + } + tv := testvars.New(t) nodeBackend := &chasm.MockNodeBackend{ HandleNextTransitionCount: func() int64 { return 2 }, @@ -389,7 +414,7 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { node := chasm.NewEmptyTree(registry, timeSource, nodeBackend, nodePathEncoder, logger, metrics.NoopMetricsHandler) ctx := chasm.NewMutableContext(context.Background(), node) - sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, defaultSchedule(), nil) + sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) if err != nil { t.Fatalf("failed to create scheduler: %v", err) } diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index d26669fb172..4d6a106d4dd 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -158,11 +158,12 @@ func (e *executeResult) Append(o executeResult) executeResult { // completed InvokerExecuteTask. It returns the number of *new* actions recorded // (starts that transitioned from "no RunId" to "has RunId" in this call), the // number of completed results that were dropped because they were previously -// recorded, and starts that do not remain active while awaiting completion. +// recorded, the latest newly-started action time, and starts that do not remain +// active while awaiting completion. func (i *Invoker) recordExecuteResult( ctx chasm.MutableContext, result *executeResult, -) (newlyStarted, droppedDuplicates int, startOnlyActions []*schedulespb.BufferedStart) { +) (newlyStarted, droppedDuplicates int, latestStartTime time.Time, startOnlyActions []*schedulespb.BufferedStart) { completed := make(map[string]*schedulespb.BufferedStart) // request ID -> BufferedStart with RunId/StartTime failed := make(map[string]bool) // request ID -> is present retryable := make(map[string]*schedulespb.BufferedStart) // request ID -> *BufferedStart @@ -221,6 +222,7 @@ func (i *Invoker) recordExecuteResult( } if completedStart, ok := completed[start.RequestId]; ok { newlyStarted++ + latestStartTime = util.MaxTime(latestStartTime, completedStart.GetStartTime().AsTime()) if !internal.TracksCompletionResult(start.GetOverlapPolicy()) { startOnlyActions = append(startOnlyActions, completedStart) startedUntracked[start.RequestId] = struct{}{} @@ -248,7 +250,7 @@ func (i *Invoker) recordExecuteResult( retriedStarts)) i.addTasks(ctx) - return newlyStarted, droppedDuplicates, startOnlyActions + return newlyStarted, droppedDuplicates, latestStartTime, startOnlyActions } // runningWorkflowID returns the workflow ID associated with the given diff --git a/chasm/lib/scheduler/invoker_execute_task_test.go b/chasm/lib/scheduler/invoker_execute_task_test.go index f6e9f8663d3..9d363864b4a 100644 --- a/chasm/lib/scheduler/invoker_execute_task_test.go +++ b/chasm/lib/scheduler/invoker_execute_task_test.go @@ -865,6 +865,7 @@ func TestExecuteTask_RecordResultIdempotentOnRace(t *testing.T) { RunId: "loser-run", StartTime: loserStartTime, }} + lastEventTime := env.Scheduler.LastEventTime.AsTime() newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, loser, nil) require.Equal(t, 0, newlyStarted, "duplicate RunId-set start must not be counted") @@ -874,6 +875,8 @@ func TestExecuteTask_RecordResultIdempotentOnRace(t *testing.T) { require.Equal(t, winning, live.RunId, "live RunId must not be stomped") require.Equal(t, startTime.AsTime(), live.StartTime.AsTime(), "live StartTime must not be stomped") require.True(t, live.HasCallback, "live HasCallback must not be cleared") + require.Equal(t, lastEventTime, env.Scheduler.LastEventTime.AsTime(), + "duplicate result must not advance the last-event high water mark") // First-mover case: a CompletedStart for a fresh RequestId increments // newlyStarted and writes through to the live entry. @@ -900,6 +903,83 @@ func TestExecuteTask_RecordResultIdempotentOnRace(t *testing.T) { require.True(t, freshlyStarted.HasCallback, "first-time RunId assignment must set HasCallback") } +func TestRecordExecuteResult_AdvancesLastEventTimeBeforeRetention(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + invoker := env.Scheduler.Invoker.Get(ctx) + base := env.TimeSource.Now().Add(time.Hour) + + var completedStarts []*schedulespb.BufferedStart + for idx := range scheduler.RecentActionCount + 1 { + startTime := base.Add(time.Duration(idx) * time.Second) + bufferedStart := &schedulespb.BufferedStart{ + NominalTime: timestamppb.New(startTime), + ActualTime: timestamppb.New(startTime), + DesiredTime: timestamppb.New(startTime), + RequestId: fmt.Sprintf("req-%d", idx), + WorkflowId: fmt.Sprintf("wf-%d", idx), + Attempt: 1, + } + invoker.BufferedStarts = append(invoker.BufferedStarts, bufferedStart) + completedStarts = append(completedStarts, &schedulespb.BufferedStart{ + RequestId: fmt.Sprintf("req-%d", idx), + RunId: fmt.Sprintf("run-%d", idx), + StartTime: timestamppb.New(startTime), + }) + } + + newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, completedStarts, nil) + require.Equal(t, scheduler.RecentActionCount+1, newlyStarted) + require.Zero(t, droppedDuplicates) + require.Empty(t, startOnlyActions) + + latestStartTime := base.Add(scheduler.RecentActionCount * time.Second) + require.Equal(t, latestStartTime.UTC(), env.Scheduler.LastEventTime.AsTime()) + + for idx, start := range invoker.BufferedStarts { + start.Completed = &schedulespb.CompletedResult{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + // The latest-started workflow closes first and is evicted by retention. + CloseTime: timestamppb.New(base.Add(time.Duration(scheduler.RecentActionCount-idx) * time.Second)), + } + } + invoker.ApplyCompletedRetention() + + require.Len(t, invoker.BufferedStarts, scheduler.RecentActionCount) + require.NotEqual(t, latestStartTime.UTC(), invoker.BufferedStarts[len(invoker.BufferedStarts)-1].GetStartTime().AsTime()) + require.Equal(t, latestStartTime.UTC(), env.Scheduler.LastEventTime.AsTime(), + "retention must not erase a start time captured before the Generator runs") +} + +func TestRecordExecuteResult_AdvancesLastEventTimeForStartOnlyAction(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + invoker := env.Scheduler.Invoker.Get(ctx) + startTime := env.TimeSource.Now().Add(time.Hour) + invoker.BufferedStarts = []*schedulespb.BufferedStart{{ + NominalTime: timestamppb.New(startTime), + ActualTime: timestamppb.New(startTime), + DesiredTime: timestamppb.New(startTime), + RequestId: "req", + WorkflowId: "wf", + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + Attempt: 1, + }} + + newlyStarted, droppedDuplicates, startOnlyActions := invoker.RecordExecuteResult(ctx, []*schedulespb.BufferedStart{{ + RequestId: "req", + RunId: "run", + StartTime: timestamppb.New(startTime), + }}, nil) + + require.Equal(t, 1, newlyStarted) + require.Zero(t, droppedDuplicates) + require.Len(t, startOnlyActions, 1) + require.Empty(t, invoker.BufferedStarts) + require.Equal(t, startTime.UTC(), env.Scheduler.LastEventTime.AsTime(), + "start-only action must advance the mark before leaving BufferedStarts") +} + // A RetryableStart for a request whose live BufferedStart already has RunId // set must not bump Attempt or set BackoffTime - the same RunId guard that // protects the completed branch must also protect the retryable branch, diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index 2f06a07d668..70396f7bf1a 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -244,7 +244,8 @@ func (h *InvokerExecuteTaskHandler) Execute( s := i.Scheduler.Get(ctx) // Use newlyStarted (not len(result.CompletedStarts)) so a concurrent // ExecuteTask's duplicate StartWorkflow can't inflate ActionCount. - newlyStarted, droppedDuplicates, startOnlyActions := i.recordExecuteResult(ctx, &result) + newlyStarted, droppedDuplicates, latestStartTime, startOnlyActions := i.recordExecuteResult(ctx, &result) + s.advanceLastEventTimeTo(latestStartTime) s.recordStartOnlyActions(ctx, startOnlyActions) s.recordActionResult(&schedulerActionResult{actionCount: int64(newlyStarted)}) if droppedDuplicates > 0 { diff --git a/chasm/lib/scheduler/proto/v1/message.proto b/chasm/lib/scheduler/proto/v1/message.proto index 6cb75c6a7db..d52771e20bd 100644 --- a/chasm/lib/scheduler/proto/v1/message.proto +++ b/chasm/lib/scheduler/proto/v1/message.proto @@ -42,6 +42,22 @@ message SchedulerState { // Surfaced as the ScheduleIdleCloseTime search attribute for stuck-schedule // detection. google.protobuf.Timestamp idle_close_time = 12; + + // Monotonic high water mark for the most recent "event" on this schedule + // (create, update, or action start). The idle deadline is anchored on this + // value. + // + // This exists because the value cannot be safely recomputed on demand: + // recentActions() is derived from the Invoker's BufferedStarts, which + // completed-action retention truncates to the last `recentActionCount` + // entries. Evicting the start that held the largest StartTime drags a + // recomputed last-event-time - and therefore the idle deadline - backwards, + // closing the schedule earlier than intended. + // + // Unset on schedules created before this field was introduced; callers must + // fall back to the recomputed value, which is what makes the rollout a no-op + // until the first Generator tick advances it. + google.protobuf.Timestamp last_event_time = 13; } // WorkflowMigrationState tracks the state of an in-progress V2-to-V1 migration. diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index d11e8f10af0..a607c2918cd 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -511,11 +511,16 @@ func (s *Scheduler) updateConflictToken() { s.ConflictToken++ } -// getLastEventTime returns the time of the last "event" to happen to the schedule. -// An event here is the schedule getting created or updated, or an action. This -// value is used for calculating the retention time (how long an idle schedule -// lives after becoming idle). -func (s *Scheduler) getLastEventTime(ctx chasm.Context) time.Time { +// computeLastEventTime derives the last "event" time from currently-observable +// state: schedule create/update, and the start time of any action still held in +// the Invoker's buffer. +// +// The result is NOT monotonic. recentActions() reads BufferedStarts, which +// applyCompletedRetention truncates to the last recentActionCount completed +// entries (ordered by CloseTime). When the start holding the largest StartTime +// is evicted - which happens whenever completion order differs from start order +// - this value moves backwards. Callers must go through getLastEventTime. +func (s *Scheduler) computeLastEventTime(ctx chasm.Context) time.Time { latest := util.MaxTime( s.Info.GetCreateTime().AsTime(), s.Info.GetUpdateTime().AsTime(), @@ -529,6 +534,34 @@ func (s *Scheduler) getLastEventTime(ctx chasm.Context) time.Time { return latest } +// getLastEventTime returns the time of the last "event" to happen to the schedule. +// An event here is the schedule getting created or updated, or an action. This +// value is used for calculating the retention time (how long an idle schedule +// lives after becoming idle). +// +// Read-only: floors the recomputed value at the persisted high water mark so the +// idle deadline can never regress. LastEventTime is nil on schedules created +// before it was introduced, in which case this degrades exactly to the old +// recompute-only behaviour until advanceLastEventTime writes it. +func (s *Scheduler) getLastEventTime(ctx chasm.Context) time.Time { + return util.MaxTime( + s.computeLastEventTime(ctx), + s.GetLastEventTime().AsTime(), + ) +} + +func (s *Scheduler) advanceLastEventTime(ctx chasm.MutableContext) time.Time { + latest := s.getLastEventTime(ctx) + s.advanceLastEventTimeTo(latest) + return latest +} + +func (s *Scheduler) advanceLastEventTimeTo(latest time.Time) { + if latest.After(s.GetLastEventTime().AsTime()) { + s.LastEventTime = timestamppb.New(latest) + } +} + // isHeldOpen reports whether the schedule must stay open regardless of having // no current work: // - Paused: customer expects to resume. diff --git a/chasm/lib/scheduler/scheduler_idle_deadline_test.go b/chasm/lib/scheduler/scheduler_idle_deadline_test.go new file mode 100644 index 00000000000..04eb674f1ab --- /dev/null +++ b/chasm/lib/scheduler/scheduler_idle_deadline_test.go @@ -0,0 +1,439 @@ +package scheduler_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const idleTestIdleTime = 7 * 24 * time.Hour + +// bufferedStartSeed describes a completed action to seed into the Invoker. +type bufferedStartSeed struct { + requestID string + startTime time.Time + closeTime time.Time +} + +// seedCompletedStarts replaces the Invoker's buffer with completed starts built +// from seeds. Every start carries a RunId, since recentActions() skips starts +// that never started. +func seedCompletedStarts(t *testing.T, sched *scheduler.Scheduler, ctx chasm.Context, seeds []bufferedStartSeed) { + t.Helper() + starts := make([]*schedulespb.BufferedStart, 0, len(seeds)) + for i, seed := range seeds { + starts = append(starts, &schedulespb.BufferedStart{ + NominalTime: timestamppb.New(seed.startTime), + ActualTime: timestamppb.New(seed.startTime), + DesiredTime: timestamppb.New(seed.startTime), + RequestId: seed.requestID, + WorkflowId: fmt.Sprintf("wf-%d", i), + RunId: fmt.Sprintf("run-%d", i), + StartTime: timestamppb.New(seed.startTime), + HasCallback: true, + Completed: &schedulespb.CompletedResult{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(seed.closeTime), + }, + }) + } + sched.Invoker.Get(ctx).BufferedStarts = starts +} + +// burstSeeds reproduces the production shape: `count` actions that all started +// within the same instant, plus one that started marginally later but closed +// first, so retention evicts precisely the start holding the maximum StartTime. +// +// Returned in close-time order, oldest first, which is the order +// applyCompletedRetention evicts in. +func burstSeeds(base time.Time, count int, skew time.Duration) []bufferedStartSeed { + seeds := make([]bufferedStartSeed, 0, count) + // The straggler: latest StartTime, oldest CloseTime -> first to be evicted. + seeds = append(seeds, bufferedStartSeed{ + requestID: "req-straggler", + startTime: base.Add(skew), + closeTime: base.Add(time.Hour), + }) + for i := 1; i < count; i++ { + seeds = append(seeds, bufferedStartSeed{ + requestID: fmt.Sprintf("req-%d", i), + startTime: base, + closeTime: base.Add(time.Hour + time.Duration(i)*time.Second), + }) + } + return seeds +} + +// countIdlePureTasks counts logical SchedulerIdleTask entries persisted on the +// root component. This is the number that grew to 43 in production; the physical +// timer count hides it, because CHASM materializes only the first pure task of +// each (component, scheduled time) group. +func countIdlePureTasks(t *testing.T, env *testEnv) int { + t.Helper() + idleTaskID, ok := env.Registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + require.True(t, ok, "idle task must be registered") + + rootPath, err := chasm.DefaultPathEncoder.Encode(nil, []string{}) + require.NoError(t, err) + + root, ok := env.Node.Snapshot(nil).Nodes[rootPath] + require.True(t, ok, "root node must be present in snapshot") + + count := 0 + for _, task := range root.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() == idleTaskID { + count++ + } + } + return count +} + +// idlePureTaskTimes returns the scheduled times of every persisted idle task, in +// persisted order (which CHASM keeps sorted by scheduled time). +func idlePureTaskTimes(t *testing.T, env *testEnv) []time.Time { + t.Helper() + idleTaskID, ok := env.Registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + require.True(t, ok) + + rootPath, err := chasm.DefaultPathEncoder.Encode(nil, []string{}) + require.NoError(t, err) + root, ok := env.Node.Snapshot(nil).Nodes[rootPath] + require.True(t, ok) + + var times []time.Time + for _, task := range root.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() == idleTaskID { + times = append(times, task.GetScheduledTime().AsTime()) + } + } + return times +} + +// runGeneratorTick executes one GeneratorTask in its own transaction, mirroring +// the production tick that HandleNexusCompletion triggers via Generate(). +func runGeneratorTick(t *testing.T, env *testEnv, handler *scheduler.GeneratorTaskHandler) { + t.Helper() + ctx := env.MutableContext() + sched, err := env.Node.Component(ctx, chasm.ComponentRef{}) + require.NoError(t, err) + typed, ok := sched.(*scheduler.Scheduler) + require.True(t, ok) + + require.NoError(t, handler.Execute( + ctx, typed.Generator.Get(ctx), chasm.TaskAttributes{}, &schedulerpb.GeneratorTask{})) + require.NoError(t, env.CloseTransaction()) +} + +// ----------------------------------------------------------------------------- +// Root cause: the recomputed last-event time is not monotonic. +// ----------------------------------------------------------------------------- + +// Pins the underlying defect independently of the idle machinery: evicting the +// start that held the largest StartTime drags the *recomputed* last-event time +// backwards, while the high-water-mark read does not. +func TestScheduler_LastEventTime_RegressesOnRetentionEviction(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + sched := env.Scheduler + + // Actions start after the schedule was created, so they - not Info - set the + // last-event time. newTestEnv's initial CloseTransaction already ran a + // Generator tick, so the mark is non-nil and sits at creation time. + base := env.TimeSource.Now().Add(time.Hour) + const skew = 1500 * time.Microsecond + // One more completed start than retention keeps, so exactly one is evicted. + seeds := burstSeeds(base, scheduler.RecentActionCount+1, skew) + seedCompletedStarts(t, sched, ctx, seeds) + + before := sched.ComputeLastEventTime(ctx) + require.Equal(t, base.Add(skew).UTC(), before.UTC(), + "the straggler's StartTime should set the pre-eviction value") + + // Publish the high water mark, exactly as the Generator tick does. + require.Equal(t, before.UTC(), sched.AdvanceLastEventTime(ctx).UTC()) + + sched.Invoker.Get(ctx).ApplyCompletedRetention() + require.Len(t, sched.Invoker.Get(ctx).BufferedStarts, scheduler.RecentActionCount) + + after := sched.ComputeLastEventTime(ctx) + require.True(t, after.Before(before), + "recomputed last-event time must be shown regressing: before=%v after=%v", before, after) + require.Equal(t, skew, before.Sub(after), + "the regression should be exactly the evicted start's skew") + + // The monotonic read path is what callers use, and it must hold the line. + require.Equal(t, before.UTC(), sched.GetLastEventTimeFloored(ctx).UTC(), + "high water mark must floor the regressed recompute") + require.Equal(t, before.Add(idleTestIdleTime).UTC(), + sched.IdleDeadline(ctx, idleTestIdleTime).UTC(), + "idle deadline must not regress") +} + +// A schedule persisted before LastEventTime existed has the field unset. The +// read path must then behave exactly as it did before, so the rollout is inert +// until the first Generator tick writes the mark. Rollback safety depends on +// this being symmetric. +func TestScheduler_LastEventTime_UnsetFallsBackToRecompute(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + sched := env.Scheduler + + base := env.TimeSource.Now().Add(time.Hour) + seedCompletedStarts(t, sched, ctx, burstSeeds(base, scheduler.RecentActionCount+1, 1500*time.Microsecond)) + + // Simulate a schedule persisted before LastEventTime existed. newTestEnv's + // initial Generator tick writes the mark, so clear it explicitly rather than + // relying on it being absent. + sched.LastEventTime = nil + require.Nil(t, sched.LastEventTime, "field must start unset") + require.Equal(t, sched.ComputeLastEventTime(ctx).UTC(), sched.GetLastEventTimeFloored(ctx).UTC()) + + // And with the mark unset, eviction still regresses - i.e. this test is + // asserting the old behaviour, not an accidental pass. + before := sched.ComputeLastEventTime(ctx) + sched.Invoker.Get(ctx).ApplyCompletedRetention() + require.True(t, sched.GetLastEventTimeFloored(ctx).Before(before)) +} + +// The mark must never move backwards even if asked to. +func TestScheduler_AdvanceLastEventTime_NeverRegresses(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + sched := env.Scheduler + + base := env.TimeSource.Now().Add(time.Hour) + seedCompletedStarts(t, sched, ctx, burstSeeds(base, scheduler.RecentActionCount+1, time.Minute)) + + high := sched.AdvanceLastEventTime(ctx) + + // Drop every action; the recomputed value collapses back to Info's anchor. + sched.Invoker.Get(ctx).BufferedStarts = nil + require.True(t, sched.ComputeLastEventTime(ctx).Before(high)) + require.Equal(t, high.UTC(), sched.AdvanceLastEventTime(ctx).UTC()) + require.Equal(t, high.UTC(), sched.LastEventTime.AsTime().UTC()) +} + +// ----------------------------------------------------------------------------- +// Accumulation: the Generator must not re-arm an idle task it already armed. +// ----------------------------------------------------------------------------- + +// Repeated ticks with nothing changing must leave exactly one idle task. Before +// the fix this grows by one per tick, because Validate can only invalidate tasks +// whose deadline moved later and a no-op tick moves nothing. +func TestGeneratorTask_IdleTask_NotReArmedWhenDeadlineUnchanged(t *testing.T) { + env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + handler := newGeneratorHandler(env) + + const ticks = 20 + for range ticks { + runGeneratorTick(t, env, handler) + } + + require.Equal(t, 1, countIdlePureTasks(t, env), + "repeated idle ticks must not accumulate idle tasks") + require.NotNil(t, env.Scheduler.IdleCloseTime, "IdleCloseTime must be published") +} + +// The production shape end to end: a burst of same-instant actions, then +// retention evicting the straggler that held the maximum StartTime. Asserts both +// symptoms are gone - a single idle task, and a deadline that never regresses. +func TestGeneratorTask_IdleTask_DoesNotAccumulateAcrossActionBurst(t *testing.T) { + now := time.Now() + env := newTestEnv(t, withSchedule(expiredSchedule(now))) + handler := newGeneratorHandler(env) + + base := env.TimeSource.Now().Add(time.Hour) + const skew = 1500 * time.Microsecond + // 43 actions, matching the production backfill that surfaced this. + seeds := burstSeeds(base, 43, skew) + + var deadlines []time.Time + for i := range seeds { + ctx := env.MutableContext() + sched, err := env.Node.Component(ctx, chasm.ComponentRef{}) + require.NoError(t, err) + typed := sched.(*scheduler.Scheduler) + + // Actions complete one at a time, each one growing the completed set and + // then being trimmed to the retention window - the same sequence + // HandleNexusCompletion -> recordCompletedAction drives in production. + seedCompletedStarts(t, typed, ctx, seeds[:i+1]) + typed.Invoker.Get(ctx).ApplyCompletedRetention() + + require.NoError(t, handler.Execute( + ctx, typed.Generator.Get(ctx), chasm.TaskAttributes{}, &schedulerpb.GeneratorTask{})) + require.NoError(t, env.CloseTransaction()) + + require.NotNil(t, typed.IdleCloseTime) + deadlines = append(deadlines, typed.IdleCloseTime.AsTime()) + } + + for i := 1; i < len(deadlines); i++ { + require.False(t, deadlines[i].Before(deadlines[i-1]), + "idle deadline regressed at action %d: %v -> %v", i, deadlines[i-1], deadlines[i]) + } + + require.Equal(t, 1, countIdlePureTasks(t, env), + "a burst of %d actions must leave one idle task, not one per action", len(seeds)) + require.Equal(t, []time.Time{base.Add(skew).Add(idleTestIdleTime).UTC()}, + normalizeUTC(idlePureTaskTimes(t, env)), + "the surviving task must be armed at the high water mark, not the regressed value") +} + +func normalizeUTC(times []time.Time) []time.Time { + out := make([]time.Time, 0, len(times)) + for _, ts := range times { + out = append(out, ts.UTC()) + } + return out +} + +// The skip must be observable, so an operator can tell "no task armed because +// one already is" apart from "no task armed at all". +func TestGeneratorTask_IdleTask_SkipEmitsMetric(t *testing.T) { + rec := metricstest.NewCaptureHandler() + env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + handler := newGeneratorHandler(env, withGeneratorMetrics(rec)) + + runGeneratorTick(t, env, handler) // arms + capture := rec.StartCapture() + defer rec.StopCapture(capture) + runGeneratorTick(t, env, handler) // skips + + recorded := capture.Snapshot()[metrics.ScheduleIdleTask.Name()] + require.Len(t, recorded, 1) + require.Equal(t, "skipped", recorded[0].Tags["outcome"]) + require.Equal(t, "already_armed", recorded[0].Tags["reason"]) +} + +// ----------------------------------------------------------------------------- +// Stuck-open safety. Skipping the re-arm is only sound if every path that moves +// the deadline still arms a fresh task; otherwise a schedule never closes. +// ----------------------------------------------------------------------------- + +// When the deadline genuinely advances, a new task must be armed and the stale +// one must not survive alongside it. +func TestGeneratorTask_IdleTask_ReArmedWhenDeadlineAdvances(t *testing.T) { + env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + handler := newGeneratorHandler(env) + + runGeneratorTick(t, env, handler) + require.Equal(t, 1, countIdlePureTasks(t, env)) + first := env.Scheduler.IdleCloseTime.AsTime() + + // A real event (here: an update) advances the deadline. + ctx := env.MutableContext() + sched, err := env.Node.Component(ctx, chasm.ComponentRef{}) + require.NoError(t, err) + sched.(*scheduler.Scheduler).Info.UpdateTime = timestamppb.New(env.TimeSource.Now().Add(time.Hour)) + require.NoError(t, env.CloseTransaction()) + + runGeneratorTick(t, env, handler) + + second := env.Scheduler.IdleCloseTime.AsTime() + require.True(t, second.After(first), "deadline must advance: %v -> %v", first, second) + require.Equal(t, 1, countIdlePureTasks(t, env), + "the stale task must be invalidated rather than left beside the new one") + require.Equal(t, []time.Time{second.UTC()}, normalizeUTC(idlePureTaskTimes(t, env))) +} + +// Pausing clears IdleCloseTime, so unpausing must arm again. This is the path +// where a bad skip guard would strand a schedule open forever. +func TestGeneratorTask_IdleTask_ReArmedAfterPauseUnpause(t *testing.T) { + env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + handler := newGeneratorHandler(env) + + runGeneratorTick(t, env, handler) + require.Equal(t, 1, countIdlePureTasks(t, env)) + + setPaused := func(paused bool) { + ctx := env.MutableContext() + sched, err := env.Node.Component(ctx, chasm.ComponentRef{}) + require.NoError(t, err) + typed := sched.(*scheduler.Scheduler) + typed.Schedule.State.Paused = paused + typed.Info.UpdateTime = timestamppb.New(env.TimeSource.Now()) + require.NoError(t, env.CloseTransaction()) + } + + setPaused(true) + runGeneratorTick(t, env, handler) + require.Nil(t, env.Scheduler.IdleCloseTime, "held-open schedules must clear IdleCloseTime") + + env.TimeSource.Update(env.TimeSource.Now().Add(time.Minute)) + setPaused(false) + runGeneratorTick(t, env, handler) + + require.NotNil(t, env.Scheduler.IdleCloseTime, "unpausing must re-arm the idle task") + require.Equal(t, 1, countIdlePureTasks(t, env), + "a schedule with no armed idle task can never close, and a duplicate never drains") +} + +// A live schedule must not arm an idle task at all - the guard must not turn the +// non-idle branch into a skip. +func TestGeneratorTask_IdleTask_NotArmedWhileSpecHasWork(t *testing.T) { + env := newTestEnv(t) + handler := newGeneratorHandler(env) + + runGeneratorTick(t, env, handler) + + require.Equal(t, 0, countIdlePureTasks(t, env)) + require.Nil(t, env.Scheduler.IdleCloseTime) +} + +// ----------------------------------------------------------------------------- +// Validate: the regressed-deadline branch. +// ----------------------------------------------------------------------------- + +// With the high water mark published, a retention eviction can no longer push +// Validate into its "deadline regressed" branch, so the task stays valid and +// fires at the deadline the customer's retention window earned. +func TestIdleTask_Validate_RetentionEvictionDoesNotRegressDeadline(t *testing.T) { + env := newTestEnv(t) + ctx := env.MutableContext() + sched := env.Scheduler + + base := env.TimeSource.Now().Add(time.Hour) + const skew = 1500 * time.Microsecond + seedCompletedStarts(t, sched, ctx, burstSeeds(base, scheduler.RecentActionCount+1, skew)) + + armedAt := sched.AdvanceLastEventTime(ctx).Add(idleTestIdleTime) + sched.Invoker.Get(ctx).ApplyCompletedRetention() + + handler := newIdleHandler(idleTestIdleTime) + valid, err := handler.Validate( + ctx, sched, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: armedAt}}, + &schedulerpb.SchedulerIdleTask{IdleTimeTotal: durationpb.New(idleTestIdleTime)}, + ) + require.NoError(t, err) + require.True(t, valid, "task armed at the high water mark must remain valid after eviction") + require.Equal(t, armedAt.UTC(), sched.IdleDeadline(ctx, idleTestIdleTime).UTC(), + "recomputed deadline must match the armed time exactly, so Validate takes neither shift branch") +} + +// Guard against the snapshot helper silently returning nothing: if the root node +// stopped carrying component attributes, or the idle task's registered FQN +// changed, every count-based assertion above would pass vacuously. +func TestCountIdlePureTasks_SeesArmedTask(t *testing.T) { + // newTestEnv's initial CloseTransaction runs the Generator's immediate task, + // which already arms one for an exhausted spec. + env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + require.Equal(t, 1, countIdlePureTasks(t, env)) + require.Len(t, idlePureTaskTimes(t, env), 1) + + runGeneratorTick(t, env, newGeneratorHandler(env)) + require.Equal(t, 1, countIdlePureTasks(t, env), + "a tick that changes nothing must neither add nor drop the armed task") +} diff --git a/chasm/lib/scheduler/scheduler_idle_tasks_test.go b/chasm/lib/scheduler/scheduler_idle_tasks_test.go index 7af50a8b4bb..bdc6308d2b5 100644 --- a/chasm/lib/scheduler/scheduler_idle_tasks_test.go +++ b/chasm/lib/scheduler/scheduler_idle_tasks_test.go @@ -66,6 +66,7 @@ func runIdleValidateTestCase(t *testing.T, env *testEnv, c *idleValidateTestCase func anchorLastEventTo(sched *scheduler.Scheduler, anchor time.Time) { sched.Info.UpdateTime = timestamppb.New(anchor) sched.Info.CreateTime = timestamppb.New(anchor) + sched.LastEventTime = timestamppb.New(anchor) } func TestIdleTask_Execute(t *testing.T) { diff --git a/chasm/lib/scheduler/scheduler_tasks.go b/chasm/lib/scheduler/scheduler_tasks.go index 3991950b06f..c0195437ee6 100644 --- a/chasm/lib/scheduler/scheduler_tasks.go +++ b/chasm/lib/scheduler/scheduler_tasks.go @@ -67,6 +67,7 @@ const ( idleInvalidatedHeldOpen metrics.ReasonString = "held_open" idleInvalidatedExpirationShift metrics.ReasonString = "expiration_shift" idleInvalidatedClosed metrics.ReasonString = "closed" + idleAlreadyArmed metrics.ReasonString = "already_armed" ) func (r *SchedulerIdleTaskHandler) Validate( @@ -92,9 +93,11 @@ func (r *SchedulerIdleTaskHandler) Validate( return false, nil } - // Deadline moved earlier - shouldn't happen if getLastEventTime is monotonic. - // Fire (closing the schedule is the safe call) but log so a real regression - // surfaces. + // Deadline moved earlier. getLastEventTime floors the recomputed value at the + // persisted LastEventTime mark, so this is now only reachable on a schedule + // whose mark predates that field (nil) - i.e. one that has not ticked since + // the upgrade. Fire anyway (closing the schedule is the safe call) but log, + // so a mark that is failing to hold the line still surfaces. if idleExpiration.Before(taskAttrs.ScheduledTime) { newTaggedLogger(r.baseLogger, scheduler).Warn("idle deadline regressed", tag.Timestamp(idleExpiration), diff --git a/chasm/lib/scheduler/util.go b/chasm/lib/scheduler/util.go index 9a8a8f88919..0cfe4ea23dd 100644 --- a/chasm/lib/scheduler/util.go +++ b/chasm/lib/scheduler/util.go @@ -50,10 +50,13 @@ func newTaggedMetricsHandler(baseHandler metrics.Handler, scheduler *Scheduler) // Outcomes for task-lifecycle counters (e.g. ScheduleIdleTask). Mutually // exclusive: a given task run either fires (Validate=true, Execute ran) or is -// invalidated (Validate=false; reason tag explains why). +// invalidated (Validate=false; reason tag explains why). "skipped" covers the +// arm side rather than a task run: a task that was deliberately not created +// because an equivalent one is already pending. const ( outcomeFired = "fired" outcomeInvalidated = "invalidated" + outcomeSkipped = "skipped" ) // reasonNone is the ReasonTag value emitted on the "fired" outcome. Prometheus From 39357940ed2eda96f090c9c57f7573d2c8be8bf6 Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 26 Aug 2026 11:53:10 -0700 Subject: [PATCH 071/139] fix: test coverage: add invariants for every schedulev2 state transition (#11769) ## Context: We've seen several problems in the SchedulesV2 where we have failed to catch genuine problems where we should re-arm the next timer and therefore the schedule gets 'stuck' (in the sense that it's performed an invalid state transition and now cannot continue. We've done a lot of fancy exploration to try and solve for this and I hope we can land some of these. But in the mean-time I was pretty frustrated that it's still very easy to make these mistakes (there's a lot here I want to explore in testing, but it's very much a source of sharp edges). Concretely, I just asked the LLM to go try and find a low-effort way of catching these and I'm pretty sure this is a success in that it's very dumb: It just checks the state for 'stuck'ness on close. It's very basic, but hopefully a starting-point to address the state-change sharp-edges problem. ### LLM summary: ## How did you test it? - [x] added unit coverage - [ ] functional suite not yet run --------- Co-authored-by: Claude Opus 5 (1M context) --- chasm/chasmtest/task_helpers.go | 2 +- chasm/chasmtest/test_engine.go | 36 ++- chasm/chasmtest/test_engine_test.go | 83 +++++- chasm/lib/scheduler/generator_tasks_test.go | 1 + chasm/lib/scheduler/helper_test.go | 49 +++- chasm/lib/scheduler/invariant_test.go | 270 ++++++++++++++++++ chasm/lib/scheduler/invoker.go | 5 + .../scheduler/scheduler_idle_deadline_test.go | 2 + .../scheduler_stuck_regression_test.go | 87 ++++++ 9 files changed, 521 insertions(+), 14 deletions(-) create mode 100644 chasm/lib/scheduler/invariant_test.go create mode 100644 chasm/lib/scheduler/scheduler_stuck_regression_test.go diff --git a/chasm/chasmtest/task_helpers.go b/chasm/chasmtest/task_helpers.go index d9b198d9992..9cc08de0b86 100644 --- a/chasm/chasmtest/task_helpers.go +++ b/chasm/chasmtest/task_helpers.go @@ -76,7 +76,7 @@ func (e *Engine) FirePureTasks(ref chasm.ComponentRef, referenceTime time.Time) return executed, err } - if err := exec.closeTransaction(); err != nil { + if err := e.closeTransaction(exec); err != nil { return executed, err } return executed, nil diff --git a/chasm/chasmtest/test_engine.go b/chasm/chasmtest/test_engine.go index b0a47953a35..f76dac3d4c0 100644 --- a/chasm/chasmtest/test_engine.go +++ b/chasm/chasmtest/test_engine.go @@ -27,6 +27,11 @@ import ( type ( EngineOption func(*Engine) + // InvariantCheck validates an execution after a successful transition. The + // node is clean, so checks may inspect its snapshot without affecting the + // state being validated. + InvariantCheck func(*testing.T, *chasm.Node, chasm.RootComponent) + // Engine is a lightweight in memory CHASM engine for unit tests. It implements // [chasm.Engine] and supports the full set of conflict and reuse policies, as // well as blocking [PollComponent] with [NotifyExecution], matching the behavior @@ -40,8 +45,9 @@ type ( // currentExecutions maps (namespaceID, businessID) to the latest run (running or closed). currentExecutions map[businessKey]*execution // allExecutions maps (namespaceID, businessID, runID) to any run, for lookups by specific RunID. - allExecutions map[runKey]*execution - notifier *executionNotifier + allExecutions map[runKey]*execution + notifier *executionNotifier + invariantChecks []InvariantCheck } execution struct { @@ -85,6 +91,14 @@ func WithMetricsHandler(handler metrics.Handler) EngineOption { } } +// WithInvariantCheck adds a check that runs after every successful transition. +// Multiple checks run in the order in which they were added. +func WithInvariantCheck(check InvariantCheck) EngineOption { + return func(e *Engine) { + e.invariantChecks = append(e.invariantChecks, check) + } +} + var defaultTransitionOptions = chasm.TransitionOptions{ ReusePolicy: chasm.BusinessIDReusePolicyAllowDuplicate, ConflictPolicy: chasm.BusinessIDConflictPolicyFail, @@ -459,11 +473,11 @@ func (e *Engine) startNew( if err := exec.node.SetRootComponent(root); err != nil { return chasm.StartExecutionResult{}, err } - if err = exec.closeTransaction(); err != nil { + exec.root = root + if err = e.closeTransaction(exec); err != nil { return chasm.StartExecutionResult{}, err } - exec.root = root e.currentExecutions[newBusinessKey(exec.key)] = exec e.allExecutions[newRunKey(exec.key)] = exec @@ -503,11 +517,11 @@ func (e *Engine) startAndUpdateNew( if err := updateFn(mutableCtx, root); err != nil { return chasm.EngineUpdateWithStartExecutionResult{}, err } - if err = exec.closeTransaction(); err != nil { + exec.root = root + if err = e.closeTransaction(exec); err != nil { return chasm.EngineUpdateWithStartExecutionResult{}, err } - exec.root = root e.currentExecutions[newBusinessKey(exec.key)] = exec e.allExecutions[newRunKey(exec.key)] = exec @@ -603,12 +617,16 @@ func (e *Engine) newExecution(key chasm.ExecutionKey) *execution { } } -// closeTransaction closes the execution's transaction and commits its transition count. -func (x *execution) closeTransaction() error { +// closeTransaction closes the execution's transaction, commits its transition +// count, and validates the resulting clean tree. +func (e *Engine) closeTransaction(x *execution) error { if _, err := x.node.CloseTransaction(); err != nil { return err } x.commitTransition() + for _, check := range e.invariantChecks { + check(e.t, x.node, x.root) + } return nil } @@ -660,7 +678,7 @@ func (e *Engine) updateComponentInExecution( return nil, err } - if err = execution.closeTransaction(); err != nil { + if err = e.closeTransaction(execution); err != nil { return nil, err } execution.recordRequestID(requestID) diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index 4c0d9c3cd98..fc46f293bc0 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -41,6 +41,77 @@ func TestTasksArePhysicallyGenerated(t *testing.T) { }) } +func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { + var calls []string + check := func(name string) chasmtest.InvariantCheck { + return func(checkT *testing.T, node *chasm.Node, root chasm.RootComponent) { + checkT.Helper() + require.Same(checkT, t, checkT) + require.IsType(checkT, &tests.PayloadStore{}, root) + require.NotPanics(checkT, func() { node.Snapshot(nil) }) + calls = append(calls, name) + } + } + + e, ref := startStoreWithEngineOptions( + t, + time.Hour, + []chasmtest.EngineOption{ + chasmtest.WithInvariantCheck(check("first")), + chasmtest.WithInvariantCheck(check("second")), + }, + ) + require.Equal(t, []string{"first", "second"}, calls) + + _, _, err := chasm.UpdateComponent(engineContext(e), ref, + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, nil, chasm.WithRequestID("invariant-check")) + require.NoError(t, err) + require.Equal(t, []string{"first", "second", "first", "second"}, calls) + + _, _, err = chasm.UpdateComponent(engineContext(e), ref, + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, nil, chasm.WithRequestID("invariant-check")) + require.ErrorIs(t, err, chasm.ErrRequestIDAlreadyUsed) + require.Equal(t, []string{"first", "second", "first", "second"}, calls) + + transitionErr := errors.New("transition failed") + _, _, err = chasm.UpdateComponent(engineContext(e), ref, + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, transitionErr + }, nil) + require.ErrorIs(t, err, transitionErr) + require.Equal(t, []string{"first", "second", "first", "second"}, calls) + + executed, err := e.FirePureTasks(ref, time.Date(2021, 1, 1, 0, 0, 0, 0, time.UTC)) + require.NoError(t, err) + require.Equal(t, 1, executed) + require.Equal(t, []string{"first", "second", "first", "second", "first", "second"}, calls) + + key := chasm.ExecutionKey{NamespaceID: "test-ns", BusinessID: "update-with-start"} + result, err := chasm.UpdateWithStartExecution( + engineContext(e), + key, + func(mc chasm.MutableContext, _ any) (*tests.PayloadStore, error) { + return tests.NewPayloadStore(mc) + }, + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, + nil, + ) + require.NoError(t, err) + require.True(t, result.Created) + require.Equal(t, []string{ + "first", "second", + "first", "second", + "first", "second", + "first", "second", + }, calls) +} + func TestUpdateComponentDeduplicatesRequestID(t *testing.T) { e, ref := startStore(t, 0) updateCount := 0 @@ -123,6 +194,15 @@ func startStore( t *testing.T, ttl time.Duration, opts ...chasm.TransitionOption, +) (*chasmtest.Engine, chasm.ComponentRef) { + return startStoreWithEngineOptions(t, ttl, nil, opts...) +} + +func startStoreWithEngineOptions( + t *testing.T, + ttl time.Duration, + engineOpts []chasmtest.EngineOption, + opts ...chasm.TransitionOption, ) (*chasmtest.Engine, chasm.ComponentRef) { registry := chasm.NewRegistry(log.NewNoopLogger()) require.NoError(t, registry.Register(&chasm.CoreLibrary{})) @@ -130,7 +210,8 @@ func startStore( ts := clock.NewEventTimeSource() ts.Update(time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC)) - e := chasmtest.NewEngine(t, registry, chasmtest.WithTimeSource(ts)) + engineOpts = append([]chasmtest.EngineOption{chasmtest.WithTimeSource(ts)}, engineOpts...) + e := chasmtest.NewEngine(t, registry, engineOpts...) key := chasm.ExecutionKey{NamespaceID: "test-ns", BusinessID: "store"} result, err := chasm.StartExecution(engineContext(e), key, diff --git a/chasm/lib/scheduler/generator_tasks_test.go b/chasm/lib/scheduler/generator_tasks_test.go index 8ef619ff481..9bb3bed06d5 100644 --- a/chasm/lib/scheduler/generator_tasks_test.go +++ b/chasm/lib/scheduler/generator_tasks_test.go @@ -328,6 +328,7 @@ func TestGeneratorTask_FutureActionTimesRespectLastProcessedTimeWatermark(t *tes require.NoError(t, env.updateScheduler(func(s *scheduler.Scheduler, ctx chasm.MutableContext) error { generator = s.Generator.Get(ctx) generator.LastProcessedTime = timestamppb.New(watermark) + generator.Generate(ctx) return nil })) diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index 74078774aac..4d131d34a5f 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -137,7 +137,7 @@ func newTestLibrary(logger log.Logger, specProcessor scheduler.SpecProcessor) *s // testEnv holds all components needed for scheduler tests. type testEnv struct { - t *testing.T // only used within these setup helpers + t *testing.T Ctrl *gomock.Controller Registry *chasm.Registry Node *chasm.Node @@ -147,6 +147,10 @@ type testEnv struct { SpecProcessor scheduler.SpecProcessor MockEngine *chasm.MockEngine Logger log.Logger + + // allowStuckReason, when non-empty, suppresses the stuckness invariant + // asserted by CloseTransaction. See AllowStuck. + allowStuckReason string } // testEnvConfig holds configuration options for testEnv. @@ -272,6 +276,11 @@ func newTestEngineContextFromConfig( require.NoError(t, registry.Register(&chasm.CoreLibrary{})) require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor))) + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( + func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { + requireValidSchedulerState(t, registry, node, root) + }, + )) engine := chasmtest.NewEngine(t, registry, config.engineOpts...) return engine, chasm.NewEngineContext(context.Background(), engine) } @@ -460,10 +469,44 @@ func (e *testEnv) ReadContext() chasm.Context { return chasm.NewContext(context.Background(), e.Node) } -// CloseTransaction closes the current CHASM transaction. +// CloseTransaction closes the current CHASM transaction and then asserts the +// stuckness invariant: a scheduler that is not in a terminal state must carry +// at least one live logical task, or nothing will ever wake it again. +// +// This runs on every test in the package that closes through testEnv, so the +// existing suite doubles as a stuckness detector at no extra cost. Tests that +// deliberately construct a state with no pending work must opt out explicitly +// via AllowStuck. +// +// The invariant is checked only on a successful close, since Node.Snapshot +// requires a clean tree. func (e *testEnv) CloseTransaction() error { + e.t.Helper() + _, err := e.Node.CloseTransaction() - return err + if err != nil { + return err + } + if e.allowStuckReason == "" { + requireNotStuck(e.t, e.Node, e.Scheduler) + requireIdleCloseTimeBacked(e.t, e.Registry, e.Node, e.Scheduler) + } + return nil +} + +// AllowStuck opts this test out of the stuckness invariant asserted by +// CloseTransaction, for tests that deliberately drive the scheduler into a +// state with no pending work. +// +// reason is mandatory: an unexplained opt-out is indistinguishable from a +// silently tolerated bug, and the set of tests that need one is itself a +// finding worth reviewing. +func (e *testEnv) AllowStuck(reason string) { + e.t.Helper() + if reason == "" { + e.t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") + } + e.allowStuckReason = reason } // HasTask returns true if the given task type was added with the given visibilityTime. diff --git a/chasm/lib/scheduler/invariant_test.go b/chasm/lib/scheduler/invariant_test.go new file mode 100644 index 00000000000..b35b08be0f2 --- /dev/null +++ b/chasm/lib/scheduler/invariant_test.go @@ -0,0 +1,270 @@ +package scheduler_test + +import ( + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// This file implements the "stuckness" invariant for Schedules/V2. +// +// Production has repeatedly produced schedules that are open, unpaused, and +// carry no pending work: they never re-arm and never close. Every confirmed +// instance violated exactly one predicate: +// +// !sched.Closed => the tree carries at least one live logical task. +// +// Note that this is a liveness property, not a transition-validity property. +// The scheduler has no state enum to validate transitions against: +// Scheduler.LifecycleState is derived and two-valued (Completed iff Closed), +// and every sub-component hardcodes LifecycleStateRunning. In each known bug +// every transition taken was legal; the failure was that no transition +// happened at all. +// +// IMPORTANT: nothing here may touch live components. Materializing a chasm +// Field (e.g. sched.Invoker.Get(ctx)) escalates the root node's valueState and +// marks the tree dirty even for a read, which is itself one of the fault +// mechanisms behind the production bugs. So this check reads only: +// - plain Go field access on the embedded *schedulerpb.SchedulerState proto +// - the serialized snapshot returned by Node.Snapshot +// It never constructs a chasm.Context. + +// schedState is a test-only vocabulary for the scheduler's lifecycle. It is +// derived, never persisted: there is deliberately no state enum on the +// component, and adding one is a proto change with V1->V2 migration and +// rollback implications that this check does not require. +type schedState string + +const ( + // stateSentinel is a schedule-ID reservation. Inert by design. + stateSentinel schedState = "sentinel" + // stateMigrating is mid-migration to a workflow-backed (V1) scheduler, + // driven by SchedulerMigrateToWorkflowTask. + stateMigrating schedState = "migrating" + // stateCompleted is terminal: Closed is set. + stateCompleted schedState = "completed" + // stateIdle is non-terminal and quiescent, awaiting its idle timer. + stateIdle schedState = "idle" + // stateRunning is non-terminal with pending work armed. + stateRunning schedState = "running" + // stateStuck is the bug: non-terminal, but nothing will ever wake it. + stateStuck schedState = "stuck" +) + +// liveTaskCount counts logical tasks across every node in the tree. +// +// This deliberately counts *logical* tasks, from ChasmComponentAttributes. +// PureTasks and SideEffectTasks, rather than physical tasks. A physical task +// can exist on the backend while the logical task it points at has been reaped +// or invalidated, so counting physical tasks reports a schedule as armed when +// it is in fact stuck. Side-effect tasks count because some valid states, such +// as migration callback attachment, rely on one before arming pure tasks. +// (chasmtest.Engine.Tasks and testEnv.NodeBackend.TasksByCategory both expose +// physical tasks, and are the wrong source for this check.) +// +// node must be the root and the tree must be clean; Node.Snapshot panics +// otherwise. Both hold immediately after a successful CloseTransaction. +func liveTaskCount(node *chasm.Node) (int, map[string]int) { + byPath := make(map[string]int) + total := 0 + for path, n := range node.Snapshot(nil).Nodes { + attrs := n.GetMetadata().GetComponentAttributes() + taskCount := len(attrs.GetPureTasks()) + len(attrs.GetSideEffectTasks()) + if taskCount == 0 { + continue + } + byPath[path] = taskCount + total += taskCount + } + return total, byPath +} + +// derivedState computes the scheduler's lifecycle state from the tuple that +// actually represents it: (Sentinel, WorkflowMigration, Closed, IdleCloseTime, +// live logical task count). +func derivedState(sched *scheduler.Scheduler, liveTasks int) schedState { + switch { + case sched.GetSentinel(): + return stateSentinel + case sched.GetWorkflowMigration() != nil: + return stateMigrating + case sched.GetClosed(): + return stateCompleted + case liveTasks == 0: + return stateStuck + case sched.GetIdleCloseTime() != nil: + return stateIdle + default: + return stateRunning + } +} + +// idleTaskCount counts armed SchedulerIdleTasks across the tree, resolving the +// task's registered type ID rather than matching on node path (the root +// Scheduler component also carries callback and migration tasks). +func idleTaskCount(registry *chasm.Registry, node *chasm.Node) int { + wantTypeID, ok := registry.TaskIDFor(&schedulerpb.SchedulerIdleTask{}) + if !ok { + return -1 + } + count := 0 + for _, n := range node.Snapshot(nil).Nodes { + for _, task := range n.GetMetadata().GetComponentAttributes().GetPureTasks() { + if task.GetTypeId() == wantTypeID { + count++ + } + } + } + return count +} + +// requireIdleCloseTimeBacked asserts that a declared idle deadline is actually +// backed by an armed idle task. +// +// IdleCloseTime is surfaced as the ScheduleIdleCloseTime search attribute and +// is what the production stuck-schedule scanner reads. When +// SchedulerIdleTaskHandler.Validate drops a task as expiration_shift and +// nothing arms a replacement, IdleCloseTime is left behind: the schedule never +// closes, and the very signal built to detect that reports a close time that +// will never arrive. That combination is always wrong. +// +// Only asserted in this direction. Sentinels legitimately arm an idle task +// without setting IdleCloseTime (see NewSentinel), so an armed task with no +// declared deadline is not a defect. +func requireIdleCloseTimeBacked(t *testing.T, registry *chasm.Registry, node *chasm.Node, sched *scheduler.Scheduler) { + t.Helper() + + if sched.GetSentinel() || sched.GetClosed() || sched.GetIdleCloseTime() == nil { + return + } + total, byPath := liveTaskCount(node) + require.NotZero(t, idleTaskCount(registry, node), + "IdleCloseTime is set to %v but no SchedulerIdleTask is armed, so the schedule will "+ + "never close and ScheduleIdleCloseTime reports a deadline that will never arrive: %s", + sched.GetIdleCloseTime().AsTime(), describeSched(sched, total, byPath)) +} + +// requireNotStuck asserts the stuckness invariant against a clean tree. +func requireNotStuck(t *testing.T, node *chasm.Node, sched *scheduler.Scheduler) { + t.Helper() + + total, byPath := liveTaskCount(node) + state := derivedState(sched, total) + require.NotEqual(t, stateStuck, state, "scheduler is stuck: %s", describeSched(sched, total, byPath)) +} + +func requireValidSchedulerState( + t *testing.T, + registry *chasm.Registry, + node *chasm.Node, + root chasm.RootComponent, +) { + t.Helper() + + sched, ok := root.(*scheduler.Scheduler) + require.Truef(t, ok, "scheduler test engine root has unexpected type %T", root) + requireNotStuck(t, node, sched) + requireIdleCloseTimeBacked(t, registry, node, sched) +} + +// describeSched renders the full state tuple for a failure message, so a +// triage does not require re-running under a debugger. +func describeSched(sched *scheduler.Scheduler, total int, byPath map[string]int) string { + paths := make([]string, 0, len(byPath)) + for p := range byPath { + paths = append(paths, fmt.Sprintf("%s=%d", p, byPath[p])) + } + slices.Sort(paths) + + var b strings.Builder + fmt.Fprintf(&b, "\n scheduleID: %s", sched.GetScheduleId()) + fmt.Fprintf(&b, "\n closed: %v", sched.GetClosed()) + fmt.Fprintf(&b, "\n sentinel: %v", sched.GetSentinel()) + fmt.Fprintf(&b, "\n paused: %v", sched.GetSchedule().GetState().GetPaused()) + fmt.Fprintf(&b, "\n limitedActions: %v", sched.GetSchedule().GetState().GetLimitedActions()) + fmt.Fprintf(&b, "\n remaining: %d", sched.GetSchedule().GetState().GetRemainingActions()) + fmt.Fprintf(&b, "\n conflictToken: %d", sched.GetConflictToken()) + fmt.Fprintf(&b, "\n idleCloseTime: %v", sched.GetIdleCloseTime().AsTime()) + fmt.Fprintf(&b, "\n migration: %v", sched.GetWorkflowMigration() != nil) + fmt.Fprintf(&b, "\n liveTasks: %d [%s]", total, strings.Join(paths, " ")) + return b.String() +} + +// TestDerivedState pins the classification tuple. Without this, a refactor +// that made derivedState never return stateStuck would silently disarm every +// stuckness assertion in the package while leaving the suite green. +func TestDerivedState(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + state *schedulerpb.SchedulerState + liveTasks int + want schedState + }{ + { + name: "open with an armed task is running", + state: &schedulerpb.SchedulerState{}, + liveTasks: 1, + want: stateRunning, + }, + { + name: "open with an armed task and an idle deadline is idle", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 1, + want: stateIdle, + }, + { + name: "open with no armed task is stuck", + state: &schedulerpb.SchedulerState{}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "an idle deadline does not excuse a missing task", + state: &schedulerpb.SchedulerState{IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateStuck, + }, + { + name: "closed with no armed task is terminal, not stuck", + state: &schedulerpb.SchedulerState{Closed: true}, + liveTasks: 0, + want: stateCompleted, + }, + { + name: "sentinel is inert by design", + state: &schedulerpb.SchedulerState{Sentinel: true}, + liveTasks: 0, + want: stateSentinel, + }, + { + name: "mid-migration is excused", + state: &schedulerpb.SchedulerState{WorkflowMigration: &schedulerpb.WorkflowMigrationState{}}, + liveTasks: 0, + want: stateMigrating, + }, + { + name: "closed takes precedence over a stale idle deadline", + state: &schedulerpb.SchedulerState{Closed: true, IdleCloseTime: timestamppb.New(time.Unix(100, 0))}, + liveTasks: 0, + want: stateCompleted, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + sched := &scheduler.Scheduler{SchedulerState: tc.state} + require.Equal(t, tc.want, derivedState(sched, tc.liveTasks)) + }) + } +} diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index 4d6a106d4dd..3580a43d2ae 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -250,6 +250,11 @@ func (i *Invoker) recordExecuteResult( retriedStarts)) i.addTasks(ctx) + + if newlyStarted > 0 { + i.Scheduler.Get(ctx).Generator.Get(ctx).Generate(ctx) + } + return newlyStarted, droppedDuplicates, latestStartTime, startOnlyActions } diff --git a/chasm/lib/scheduler/scheduler_idle_deadline_test.go b/chasm/lib/scheduler/scheduler_idle_deadline_test.go index 04eb674f1ab..535ad2ad6a2 100644 --- a/chasm/lib/scheduler/scheduler_idle_deadline_test.go +++ b/chasm/lib/scheduler/scheduler_idle_deadline_test.go @@ -326,6 +326,7 @@ func TestGeneratorTask_IdleTask_SkipEmitsMetric(t *testing.T) { // one must not survive alongside it. func TestGeneratorTask_IdleTask_ReArmedWhenDeadlineAdvances(t *testing.T) { env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + env.AllowStuck("test commits a direct state mutation before running the generator that re-arms the idle task") handler := newGeneratorHandler(env) runGeneratorTick(t, env, handler) @@ -352,6 +353,7 @@ func TestGeneratorTask_IdleTask_ReArmedWhenDeadlineAdvances(t *testing.T) { // where a bad skip guard would strand a schedule open forever. func TestGeneratorTask_IdleTask_ReArmedAfterPauseUnpause(t *testing.T) { env := newTestEnv(t, withSchedule(expiredSchedule(time.Now()))) + env.AllowStuck("test commits direct pause mutations before running the generator that reconciles idle tasks") handler := newGeneratorHandler(env) runGeneratorTick(t, env, handler) diff --git a/chasm/lib/scheduler/scheduler_stuck_regression_test.go b/chasm/lib/scheduler/scheduler_stuck_regression_test.go new file mode 100644 index 00000000000..c56dfa83ba8 --- /dev/null +++ b/chasm/lib/scheduler/scheduler_stuck_regression_test.go @@ -0,0 +1,87 @@ +package scheduler_test + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// a regression test to ensure idle-tasks are handled correctly +func TestIdleDeadlineRearmedAfterRecordingStart(t *testing.T) { + env := newTestEnv(t) + now := env.TimeSource.Now() + + // Exhaust a limited schedule so the Generator arms the idle task rather + // than another tick: with no actions left useScheduledAction is false, so + // getIdleExpiration reports isIdle. + ctx := env.MutableContext() + sched := env.Scheduler + sched.Schedule.State.LimitedActions = true + sched.Schedule.State.RemainingActions = 0 + sched.ConflictToken++ + + // The final action, readied for execution. Attempt == 1 is what + // recordProcessBufferResult sets before the ExecuteTask runs; it also keeps + // the start out of the Attempt == 0 filter that would send it back through + // ProcessBuffer. + sched.Invoker.Get(ctx).BufferedStarts = append( + sched.Invoker.Get(ctx).BufferedStarts, + &schedulespb.BufferedStart{ + RequestId: "req-final-action", + WorkflowId: "wf-final-action", + Attempt: 1, + ActualTime: timestamppb.New(now), + }, + ) + + sched.Generator.Get(ctx).Generate(ctx) + require.NoError(t, env.CloseTransaction()) + + armedDeadline := sched.GetIdleCloseTime() + require.NotNil(t, armedDeadline, "precondition: exhausted schedule should arm an idle deadline") + require.NotZero(t, idleTaskCount(env.Registry, env.Node), "precondition: an idle task should be armed") + + // Record the start through the real invoker path. + ctx2 := env.MutableContext() + newlyStarted, _, _ := sched.Invoker.Get(ctx2).RecordExecuteResult(ctx2, + []*schedulespb.BufferedStart{{ + RequestId: "req-final-action", + WorkflowId: "wf-final-action", + RunId: "run-final-action", + Attempt: 1, + ActualTime: timestamppb.New(now), + StartTime: timestamppb.New(now.Add(30 * time.Minute)), + }}, + nil, + ) + require.Equal(t, 1, newlyStarted) + require.NoError(t, env.CloseTransaction()) + + // Confirm the deadline really did shift, so a failure below is about the + // missing re-arm and not about a test that set up nothing. + isValid, err := newIdleHandler(scheduler.DefaultTweakables.IdleTime).Validate( + env.ReadContext(), + sched, + chasm.TaskInvocation{TaskAttributes: chasm.TaskAttributes{ScheduledTime: armedDeadline.AsTime()}}, + &schedulerpb.SchedulerIdleTask{ + IdleTimeTotal: durationpb.New(scheduler.DefaultTweakables.IdleTime), + }, + ) + require.NoError(t, err) + require.False(t, isValid, + "precondition: recording the start should shift the deadline past the armed task's time") + + // The schedule must still have a path to closing: a replacement idle task, + // armed from the new deadline. + require.NotZero(t, idleTaskCount(env.Registry, env.Node), + "recording a start invalidated the idle task with no replacement armed, while "+ + "IdleCloseTime still advertises %v: the schedule will never close and the "+ + "stuck-schedule scanner is told otherwise", sched.GetIdleCloseTime().AsTime()) +} From 539790267fd8ea15cc5dd28071439878fa00e34e Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 26 Aug 2026 12:33:55 -0700 Subject: [PATCH 072/139] Fix scheduler migration test activity panic (#11803) ## Summary - mock the normal StartWorkflow and WatchWorkflow activities exercised by the disabled-migration test - mock migration safely and assert that it is never invoked - preserve the active schedule and continue-as-new behavior under test This prevents the test environment from invoking activities through a nil receiver and logging repeated recovered panics. ## Testing - `go test -tags test_dep -count=1 -run "^TestWorkflow$/^TestMigrateDynamicConfigDisabledNoMigration$" -v ./service/worker/scheduler` - `go test -tags test_dep -count=1 ./service/worker/scheduler/...` - `make GOLANGCI_LINT_BASE_REV=origin/main GOLANGCI_LINT_FIX=false LINT_CODE_TARGETS="./service/worker/scheduler" lint-code` --- service/worker/scheduler/workflow_test.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/service/worker/scheduler/workflow_test.go b/service/worker/scheduler/workflow_test.go index af190fdcd67..0207d827109 100644 --- a/service/worker/scheduler/workflow_test.go +++ b/service/worker/scheduler/workflow_test.go @@ -2619,7 +2619,22 @@ func (s *workflowSuite) TestMigrateDynamicConfigDisabledNoMigration() { CurrentTweakablePolicies.EnableCHASMMigration = false defer func() { CurrentTweakablePolicies = prevTweakables }() - // No activity mock registered -- if migration is attempted, the test will fail. + s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + return &schedulespb.StartWorkflowResponse{ + RunId: uuid.NewString(), + RealStartTime: timestamppb.New(s.env.Now()), + }, nil + }) + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.Anything).Maybe().Return( + &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED}, nil) + + migrateCalls := 0 + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrateCalls++ + return nil + }) CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 3 s.env.SetStartTime(baseStartTime) @@ -2643,4 +2658,5 @@ func (s *workflowSuite) TestMigrateDynamicConfigDisabledNoMigration() { // Workflow should CAN normally without attempting migration. s.True(s.env.IsWorkflowCompleted()) s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) + s.Zero(migrateCalls) } From a4a452c00332673eb532708e7485328a6cc20008 Mon Sep 17 00:00:00 2001 From: Shivam <57200924+Shivs11@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:37:44 -0400 Subject: [PATCH 073/139] Add version labels to missing deployment metric (#11799) ## What changed - Add Worker Deployment name and build ID labels to `worker_deployment_version_not_found_during_delete`. - Assert both label values in the existing activity test. ## Why The counter diagnoses stale Deployment workflow references, so it needs to identify the exact missing Worker Deployment Version. ## Testing - `GOWORK=off go test -tags test_dep ./service/worker/workerdeployment -run '^TestDeleteWorkerDeploymentVersion$' -count=1`\n- `GOWORK=off go vet -tags disable_grpc_modules,test_dep ./service/worker/workerdeployment` --- > [!NOTE] > **Low Risk** > Observability-only changes to metric tags and log fields on an already-handled NotFound path; no change to deletion behavior. > > **Overview** > When deleting a worker deployment version hits a **NotFound** from history (stale version workflow), the **`worker_deployment_version_not_found_during_delete`** counter now records **worker deployment name** and **build ID** tags, in addition to namespace, so dashboards can pinpoint which version was missing. > > The same path updates the warning log to use **`versionObj`** for deployment name and build ID, **`namespace.Info().GetName()`** for namespace, and **`args.GetRequestId()`** for the request ID. The activity test now asserts the new metric tag values. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 8776d7fd78592618919314d67d71f2e0318f6f5f. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --- service/worker/workerdeployment/activities.go | 12 +++++++----- service/worker/workerdeployment/activities_test.go | 4 ++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/service/worker/workerdeployment/activities.go b/service/worker/workerdeployment/activities.go index d617b1cba71..84c902019b8 100644 --- a/service/worker/workerdeployment/activities.go +++ b/service/worker/workerdeployment/activities.go @@ -182,15 +182,17 @@ func (a *Activities) DeleteWorkerDeploymentVersion(ctx context.Context, args *de if errors.As(err, ¬FoundErr) { metrics.WorkerDeploymentVersionNotFoundDuringDelete.With(a.MetricsHandler).Record( 1, - metrics.NamespaceTag(a.namespace.Name().String()), + metrics.NamespaceTag(a.namespace.Info().GetName()), + metrics.WorkerDeploymentNameTag(versionObj.GetDeploymentName(), true), + metrics.WorkerDeploymentBuildIDTag(versionObj.GetBuildId(), true), ) activity.GetLogger(ctx).Warn( "version workflow not found during deletion; allowing deployment workflow to remove stale reference", - "namespace", a.namespace.Name().String(), - "deploymentName", args.DeploymentName, - "version", args.Version, + "namespace", a.namespace.Info().GetName(), + "deploymentName", versionObj.GetDeploymentName(), + "version", versionObj.GetBuildId(), "versionWorkflowID", workflowID, - "requestID", args.RequestId, + "requestID", args.GetRequestId(), "error", err, ) return nil diff --git a/service/worker/workerdeployment/activities_test.go b/service/worker/workerdeployment/activities_test.go index 3b6ecbe3c89..9fad6b13565 100644 --- a/service/worker/workerdeployment/activities_test.go +++ b/service/worker/workerdeployment/activities_test.go @@ -78,6 +78,10 @@ func TestDeleteWorkerDeploymentVersion(t *testing.T) { require.Equal(t, int64(1), recordings[0].Value) namespaceTag := metrics.NamespaceTag(tv.NamespaceName().String()) require.Equal(t, namespaceTag.Value, recordings[0].Tags[namespaceTag.Key]) + deploymentTag := metrics.WorkerDeploymentNameTag(tv.DeploymentSeries(), true) + require.Equal(t, deploymentTag.Value, recordings[0].Tags[deploymentTag.Key]) + buildIDTag := metrics.WorkerDeploymentBuildIDTag(tv.BuildID(), true) + require.Equal(t, buildIDTag.Value, recordings[0].Tags[buildIDTag.Key]) } }) } From 9ad49ef57877124670bef37c854db282d0bab8af Mon Sep 17 00:00:00 2001 From: "temporal-cicd[bot]" <138905749+temporal-cicd[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:33:16 -0500 Subject: [PATCH 074/139] Update test shard salt (#11793) Automatically generated by the optimize-test-sharding workflow. Co-authored-by: Temporal Data --- tests/testcore/shard_salt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testcore/shard_salt.txt b/tests/testcore/shard_salt.txt index b1b8624771f..fd110f7d3ff 100644 --- a/tests/testcore/shard_salt.txt +++ b/tests/testcore/shard_salt.txt @@ -1 +1 @@ --salt-1163 +-salt-591 From 38948f8a271e73f06ddbf22bc42c74af33a3c65d Mon Sep 17 00:00:00 2001 From: Alex Mazzeo Date: Wed, 26 Aug 2026 17:17:23 -0700 Subject: [PATCH 075/139] Add failure capability to Nexus cancellation requests (#11808) ## What changed? - Add the Temporal failure response capability header to Nexus cancellation requests in both HSM and CHASM executors. - Respect the `nexusoperation.useNewFailureWireFormat` dynamic config when adding the header to cancellation requests. ## Why? Nexus start requests explicitly advertise support for Temporal failure responses, but cancellation requests relied on the request header map being mutated elsewhere. [#10720](https://github.com/temporalio/temporal/pull/10720) and [#10746](https://github.com/temporalio/temporal/pull/10746) introduced defensive copies of Nexus request headers. That exposed the missing capability header on cancellation requests, causing SDK workers to use the legacy failure wire format for cancel handler failures. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ## Potential risks Cancellation requests now advertise support for Temporal failure responses by default. The existing dynamic config can disable the behavior if needed. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../lib/nexusoperation/cancellation_tasks.go | 8 +++- .../nexusoperation/cancellation_tasks_test.go | 46 +++++++++++++----- chasm/lib/nexusoperation/config.go | 2 +- components/nexusoperations/executors.go | 9 +++- components/nexusoperations/executors_test.go | 48 ++++++++++++++----- tests/nexus_workflow_test.go | 3 ++ 6 files changed, 90 insertions(+), 26 deletions(-) diff --git a/chasm/lib/nexusoperation/cancellation_tasks.go b/chasm/lib/nexusoperation/cancellation_tasks.go index 7eebf0b4f20..d5ede65b03b 100644 --- a/chasm/lib/nexusoperation/cancellation_tasks.go +++ b/chasm/lib/nexusoperation/cancellation_tasks.go @@ -16,6 +16,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/nexus/nexusrpc" queueserrors "go.temporal.io/server/service/history/queues/errors" "go.uber.org/fx" ) @@ -159,8 +160,13 @@ func (h *cancellationInvocationTaskHandler) Execute( if err != nil { return fmt.Errorf("failed to construct invocation: %w", err) } + header := buildRequestHeader(args.headers) + // If this request is handled by a newer server that supports Nexus failure serialization, trigger that behavior. + if h.config.UseNewFailureWireFormat(ns.Name().String()) { + header.Set(nexusrpc.HeaderTemporalNexusFailureSupport, "true") + } startTime := time.Now() // nolint:forbidigo // Time can be used for timing metrics. - callErr := inv.Cancel(callCtx, args, nexus.CancelOperationOptions{Header: nexus.Header(args.headers)}) + callErr := inv.Cancel(callCtx, args, nexus.CancelOperationOptions{Header: header}) failureSource := failureSourceFromContext(callCtx) h.recordCallOutcome(endpoint, cancelCallOutcomeTag(callCtx, callErr), callErr, time.Since(startTime), failureSource, traceCtx) diff --git a/chasm/lib/nexusoperation/cancellation_tasks_test.go b/chasm/lib/nexusoperation/cancellation_tasks_test.go index 694c4fec700..e1e3edfc791 100644 --- a/chasm/lib/nexusoperation/cancellation_tasks_test.go +++ b/chasm/lib/nexusoperation/cancellation_tasks_test.go @@ -305,16 +305,17 @@ func TestCancellationLoadArgs_StandaloneFallsBackToRequestData(t *testing.T) { func TestCancellationInvocationTaskHandler_HTTP(t *testing.T) { cases := []struct { - name string - header map[string]string - onCancelOperation func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error - expectedMetricOutcome string - checkOutcome func(t *testing.T, c *Cancellation) - requestTimeout time.Duration - schedToCloseTimeout time.Duration - startToCloseTimeout time.Duration - destinationDown bool - endpointNotFound bool + name string + header map[string]string + onCancelOperation func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error + expectedMetricOutcome string + checkOutcome func(t *testing.T, c *Cancellation) + requestTimeout time.Duration + schedToCloseTimeout time.Duration + startToCloseTimeout time.Duration + destinationDown bool + endpointNotFound bool + disableNewFailureWireFormat bool }{ { name: "failure", @@ -351,12 +352,34 @@ func TestCancellationInvocationTaskHandler_HTTP(t *testing.T) { }, }, { - name: "success with headers", + name: "success with temporal failure capability and user headers", header: map[string]string{"key": "value"}, onCancelOperation: func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error { if options.Header["key"] != "value" { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, `"key" header is not equal to "value"`) } + if options.Header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) != "true" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "temporal failure capability is not enabled") + } + return nil + }, + expectedMetricOutcome: "successful", + checkOutcome: func(t *testing.T, c *Cancellation) { + require.Equal(t, nexusoperationpb.CANCELLATION_STATUS_SUCCEEDED, c.Status) + require.Nil(t, c.LastAttemptFailure) + }, + }, + { + name: "success without temporal failure capability", + header: map[string]string{"key": "value"}, + disableNewFailureWireFormat: true, + onCancelOperation: func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error { + if options.Header["key"] != "value" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, `"key" header is not equal to "value"`) + } + if options.Header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) != "" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "temporal failure capability is enabled") + } return nil }, expectedMetricOutcome: "successful", @@ -555,6 +578,7 @@ func TestCancellationInvocationTaskHandler_HTTP(t *testing.T) { Header: tc.header, }, endpointReg, clientProvider, metricsHandler, cmp.Or(tc.requestTimeout, time.Hour)) + env.handler.config.UseNewFailureWireFormat = dynamicconfig.GetBoolPropertyFnFilteredByNamespace(!tc.disableNewFailureWireFormat) env.setupReadComponent() env.setupUpdateComponent() diff --git a/chasm/lib/nexusoperation/config.go b/chasm/lib/nexusoperation/config.go index 8d2b6b6c9ec..9bbe3df6927 100644 --- a/chasm/lib/nexusoperation/config.go +++ b/chasm/lib/nexusoperation/config.go @@ -236,7 +236,7 @@ Uses Go's len() function to determine the length.`, var UseNewFailureWireFormat = dynamicconfig.NewNamespaceBoolSetting( "nexusoperation.useNewFailureWireFormat", true, - `Controls whether to use the new failure wire format via an HTTP header that is attached to StartOperation requests. + `Controls whether to use the new failure wire format via an HTTP header that is attached to Nexus operation requests. Added for safety. Defaults to true. Likely to be removed in future server versions.`, ) diff --git a/components/nexusoperations/executors.go b/components/nexusoperations/executors.go index f62d495c1ae..2713c657476 100644 --- a/components/nexusoperations/executors.go +++ b/components/nexusoperations/executors.go @@ -773,8 +773,15 @@ func (e taskExecutor) executeCancelationTask(ctx context.Context, env hsm.Enviro } } + header := nexus.Header(args.headers) + if e.Config.UseNewFailureWireFormat(ns.Name().String()) { + if header == nil { + header = make(nexus.Header, 1) + } + header.Set(nexusrpc.HeaderTemporalNexusFailureSupport, "true") + } startTime = time.Now() - callErr = handle.Cancel(callCtx, nexus.CancelOperationOptions{Header: nexus.Header(args.headers)}) + callErr = handle.Cancel(callCtx, nexus.CancelOperationOptions{Header: header}) } failureSource := failureSourceFromContext(callCtx) methodTag := metrics.NexusMethodTag("CancelOperation") diff --git a/components/nexusoperations/executors_test.go b/components/nexusoperations/executors_test.go index b1ca73b23d3..b5faee1b0a6 100644 --- a/components/nexusoperations/executors_test.go +++ b/components/nexusoperations/executors_test.go @@ -973,16 +973,17 @@ func TestProcessStartToCloseTimeoutTask(t *testing.T) { func TestProcessCancelationTask(t *testing.T) { cases := []struct { - name string - endpointNotFound bool - onCancelOperation func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error - expectedMetricOutcome string - checkOutcome func(t *testing.T, op nexusoperations.Cancelation) - requestTimeout time.Duration - schedToCloseTimeout time.Duration - startToCloseTimeout time.Duration - destinationDown bool - header map[string]string + name string + endpointNotFound bool + onCancelOperation func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error + expectedMetricOutcome string + checkOutcome func(t *testing.T, op nexusoperations.Cancelation) + requestTimeout time.Duration + schedToCloseTimeout time.Duration + startToCloseTimeout time.Duration + destinationDown bool + header map[string]string + disableNewFailureWireFormat bool }{ { name: "failure", @@ -1017,7 +1018,7 @@ func TestProcessCancelationTask(t *testing.T) { }, }, { - name: "success with headers", + name: "success with temporal failure capability and user headers", requestTimeout: time.Hour, destinationDown: false, header: map[string]string{"key": "value"}, @@ -1025,6 +1026,29 @@ func TestProcessCancelationTask(t *testing.T) { if options.Header["key"] != "value" { return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, `"key" header is not equal to "value"`) } + if options.Header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) != "true" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "temporal failure capability is not enabled") + } + return nil + }, + expectedMetricOutcome: "successful", + checkOutcome: func(t *testing.T, c nexusoperations.Cancelation) { + require.Equal(t, enumspb.NEXUS_OPERATION_CANCELLATION_STATE_SUCCEEDED, c.State()) + require.Nil(t, c.LastAttemptFailure.GetApplicationFailureInfo()) + }, + }, + { + name: "success without temporal failure capability", + requestTimeout: time.Hour, + header: map[string]string{"key": "value"}, + disableNewFailureWireFormat: true, + onCancelOperation: func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error { + if options.Header["key"] != "value" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, `"key" header is not equal to "value"`) + } + if options.Header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) != "" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "temporal failure capability is enabled") + } return nil }, expectedMetricOutcome: "successful", @@ -1182,7 +1206,7 @@ func TestProcessCancelationTask(t *testing.T) { RequestTimeout: dynamicconfig.GetDurationPropertyFnFilteredByDestination(tc.requestTimeout), MinRequestTimeout: dynamicconfig.GetDurationPropertyFnFilteredByNamespace(time.Millisecond), RecordCancelRequestCompletionEvents: dynamicconfig.GetBoolPropertyFn(true), - UseNewFailureWireFormat: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(true), + UseNewFailureWireFormat: dynamicconfig.GetBoolPropertyFnFilteredByNamespace(!tc.disableNewFailureWireFormat), RetryPolicy: func() backoff.RetryPolicy { return backoff.NewExponentialRetryPolicy(time.Second) }, diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index fab97d1b34f..72a016951b9 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -96,6 +96,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationCancelation(chasmEnabled bool return &nexus.HandlerStartOperationResultAsync{OperationToken: "test"}, nil }, OnCancelOperation: func(ctx context.Context, service, operation, token string, options nexus.CancelOperationOptions) error { + if options.Header.Get(nexusrpc.HeaderTemporalNexusFailureSupport) != "true" { + return errors.New("expected Temporal failure response capability header") + } if !firstCancelSeen { // Fail cancel request once to test NexusOperationCancelRequestFailed event is recorded and request is retried. firstCancelSeen = true From 4d0afa1cb71f26a112d95b66def21f28720ee0a9 Mon Sep 17 00:00:00 2001 From: michaely520 Date: Thu, 27 Aug 2026 08:07:00 -0700 Subject: [PATCH 076/139] Add exhaustive natural XDC buffered event coverage (#11789) ## Summary - add transition-history XDC E2E coverage for every buffered event that current production code can emit - create buffered events naturally through workflow/activity APIs, workflow commands, transfer and timer tasks, child callbacks, Nexus callbacks, and conflict reapplication - hold a workflow task, fail over, let its timeout flush the old active cluster's buffer, and then release replication to exercise conflict resolution - prove every flushed event is persisted with a real event ID on the non-current losing history branch - verify cherry-pickable inputs are reapplied to the winning branch and branch-dependent outcomes are skipped - reproduce the Nexus #10986 conflict shape: reapply a shared-operation completion while skipping an operation that exists only on the losing branch - keep all test infrastructure in `buffered_events_replication_helpers_test.go`; no production or testcore code is changed ## Coverage The scenarios naturally cover all 30 production-reachable buffered event types: - activity started, completed, failed, timed out, and canceled - timer fired - workflow cancel requested and signaled - workflow options updated, paused, and unpaused - external signal/cancel success and failure callbacks - child start failure, child started, and all five child terminal outcomes - update admitted through real conflict reapplication - Nexus started, completed, failed, canceled, timed out, and both cancel-request outcomes Four values in the buffered-event set cannot be naturally buffered on current `main` and are intentionally not fabricated: - `WORKFLOW_EXECUTION_UPDATE_REJECTED`, `WORKFLOW_PROPERTIES_MODIFIED_EXTERNALLY`, and `ACTIVITY_PROPERTIES_MODIFIED_EXTERNALLY` have no production emitter - `WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED` has an emitter, but its production precondition rejects workflows with a pending workflow task ## What the tests prove - expected events first exist in mutable-state buffering with `BufferedEventID` - failover creates a winning branch before the old active cluster's replication is released - workflow-task timeout flushes the old active cluster's buffer - every expected event is found on the non-current losing branch with a positive, non-buffered event ID - a naturally buffered marker signal proves the losing batch passed through conflict reapplication - signals and updates that are eligible for reapplication reach the winner - activity, timer, child, external-command-result, Nexus cancel-request-result, and losing-only Nexus events remain on the losing branch when the winner lacks the state needed to apply them - shared Nexus operation outcomes are reapplied by scheduled event ID, while losing-only operations are skipped - current histories converge after the relevant replication tasks are released ## Determinism - replication is intercepted and blocked only for the workflow under test - the next workflow task is explicitly created and polled before callbacks are released - Nexus handlers use response barriers released only after the held workflow task is confirmed - losing branches are read directly from persistence and identified by their expected events - assertions poll observable conditions instead of relying on fixed sleeps - the pause feature flag is overridden only for the test that exercises pause/unpause - scenarios skip when transition history is disabled; Nexus conflict scenarios also skip for the CHASM implementation ## Validation - repository `gci` formatting and `git diff --check` - `GOLANGCI_LINT_FIX=false GOLANGCI_LINT_BASE_REV=HEAD~ make lint-code` - XDC package compilation with `test_dep` - focused transition-history E2E runs for mixed inputs/update reapplication, activity outcomes, child outcomes, external workflow outcomes, and Nexus outcomes - CI PostgreSQL XDC, unit, integration, mixed-brain, formatting, and all linter checks pass --- ...uffered_events_replication_helpers_test.go | 1325 +++++++++++++++++ tests/xdc/buffered_events_replication_test.go | 451 ++++++ .../buffered_nexus_events_replication_test.go | 384 +++++ 3 files changed, 2160 insertions(+) create mode 100644 tests/xdc/buffered_events_replication_helpers_test.go create mode 100644 tests/xdc/buffered_events_replication_test.go create mode 100644 tests/xdc/buffered_nexus_events_replication_test.go diff --git a/tests/xdc/buffered_events_replication_helpers_test.go b/tests/xdc/buffered_events_replication_helpers_test.go new file mode 100644 index 00000000000..bf69cbd21f2 --- /dev/null +++ b/tests/xdc/buffered_events_replication_helpers_test.go @@ -0,0 +1,1325 @@ +package xdc + +import ( + "context" + "errors" + "time" + + "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + historypb "go.temporal.io/api/history/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/api/operatorservice/v1" + protocolpb "go.temporal.io/api/protocol/v1" + replicationpb "go.temporal.io/api/replication/v1" + "go.temporal.io/api/serviceerror" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + updatepb "go.temporal.io/api/update/v1" + workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + sdkclient "go.temporal.io/sdk/client" + "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/api/historyservice/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common" + "go.temporal.io/server/common/dynamicconfig" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/persistence" + serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/protoutils" + "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" +) + +type blockedReplicationTask struct { + execute func() error + result chan error +} + +type bufferedEventsNamespace struct { + Name string + ID string +} + +type startBufferedEventsWorkflowArgs struct { + Namespace string + WorkflowID string + TaskQueue *taskqueuepb.TaskQueue +} + +type workflowTaskCompletion struct { + Task *workflowservice.PollWorkflowTaskQueueResponse + Commands []*commandpb.Command +} + +type bufferedEventExpectation struct { + Namespace string + Execution *commonpb.WorkflowExecution + EventTypes []enumspb.EventType +} + +type workflowSignal struct { + Namespace string + Execution *commonpb.WorkflowExecution + SignalName string +} + +type activeClusterSignal struct { + Namespace bufferedEventsNamespace + Execution *commonpb.WorkflowExecution + SignalName string +} + +type workflowCancellation struct { + Namespace string + Execution *commonpb.WorkflowExecution +} + +type workflowTermination struct { + Namespace string + Execution *commonpb.WorkflowExecution + Reason string +} + +type naturallyBufferedConflict struct { + Namespace bufferedEventsNamespace + Execution *commonpb.WorkflowExecution + ReplicationToOldActive <-chan *blockedReplicationTask + ReplicationToNewActive <-chan *blockedReplicationTask + ExpectedEventTypes []enumspb.EventType + WinnerSignal string +} + +type bufferedInputsExpectation struct { + Namespace bufferedEventsNamespace + Execution *commonpb.WorkflowExecution + UpdateID string + OptionsRequestID string + EventTypes []enumspb.EventType +} + +type bufferedNexusCallback struct { + operation string + url string + token string +} + +type bufferedNexusOperationCompletion struct { + Callback bufferedNexusCallback + Result string +} + +type bufferedNexusLosingBranch struct { + Namespace bufferedEventsNamespace + Execution *commonpb.WorkflowExecution + ScheduledEventID int64 +} + +func (s *xdcBaseSuite) createBufferedEventsNamespace(ctx context.Context) bufferedEventsNamespace { + name := s.createGlobalNamespace() + response, err := s.clusters[0].FrontendClient().DescribeNamespace(ctx, &workflowservice.DescribeNamespaceRequest{ + Namespace: name, + }) + s.Require().NoError(err) + return bufferedEventsNamespace{Name: name, ID: response.NamespaceInfo.Id} +} + +func (s *xdcBaseSuite) enableWorkflowPauseForTest() { + s.T().Helper() + for _, cluster := range s.clusters { + cluster.OverrideDynamicConfig(s.T(), dynamicconfig.WorkflowPauseEnabled, true) + } +} + +func (s *xdcBaseSuite) signalWorkflow(ctx context.Context, signal workflowSignal) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: signal.Namespace, + WorkflowExecution: signal.Execution, + SignalName: signal.SignalName, + RequestId: uuid.NewString(), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) requestWorkflowCancellationEventually(ctx context.Context, target workflowCancellation) { + s.T().Helper() + await.Require(ctx, s.T(), func(t *await.T) { + _, err := s.clusters[0].FrontendClient().DescribeWorkflowExecution(t.Context(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: target.Namespace, + Execution: target.Execution, + }) + require.NoError(t, err) + }, replicationWaitTime, replicationCheckInterval) + _, err := s.clusters[0].FrontendClient().RequestCancelWorkflowExecution(ctx, &workflowservice.RequestCancelWorkflowExecutionRequest{ + Namespace: target.Namespace, + WorkflowExecution: target.Execution, + RequestId: uuid.NewString(), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) terminateWorkflow(ctx context.Context, target workflowTermination) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().TerminateWorkflowExecution(ctx, &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: target.Namespace, + WorkflowExecution: target.Execution, + Reason: target.Reason, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) terminateWorkflowEventually(ctx context.Context, target workflowTermination) { + s.T().Helper() + await.Require(ctx, s.T(), func(t *await.T) { + _, err := s.clusters[0].FrontendClient().TerminateWorkflowExecution(t.Context(), &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: target.Namespace, + WorkflowExecution: target.Execution, + Reason: target.Reason, + Identity: "buffered-events-xdc-test", + }) + require.NoError(t, err) + }, replicationWaitTime, replicationCheckInterval) +} + +// The buffered-event conflict tests intentionally use a fixed two-cluster topology: +// +// 1. Cluster 0 starts active, establishes the common history, and holds the workflow task while +// naturally produced events enter its buffer. +// 2. Replication is blocked in both directions for this workflow. A namespace failover makes +// cluster 1 active, where a signal creates the winning branch. +// 3. Cluster 1 -> cluster 0 replication is released first. Cluster 0 is now passive; applying the +// winner resolves the conflict, failover-closes its outstanding workflow task, and flushes the +// buffer onto the losing branch. +// 4. Cluster 0 -> cluster 1 replication is then released so conflict resolution sees the losing +// branch and reapplies or skips its events against the winner. +// 5. Cluster 1 -> cluster 0 replication is finally released until both current histories converge. +// +// Replication channel names below identify the receiving cluster: replicationToOldActive contains +// tasks executing on cluster 0, and replicationToNewActive contains tasks executing on cluster 1. +func (s *xdcBaseSuite) finishNaturallyBufferedConflict( + ctx context.Context, + conflict naturallyBufferedConflict, +) []*historypb.HistoryEvent { + losingBranchMarker := "losing-branch-marker-" + uuid.NewString() + _, err := s.clusters[0].FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: conflict.Namespace.Name, + WorkflowExecution: conflict.Execution, + SignalName: losingBranchMarker, + RequestId: uuid.NewString(), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + s.Require().True(s.hasBufferedEventType(ctx, 0, conflict.Namespace.Name, conflict.Execution, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED)) + + s.failoverToNewActiveCluster(ctx, conflict.Namespace.Name) + s.writeSignalOnNewActive(ctx, activeClusterSignal{ + Namespace: conflict.Namespace, + Execution: conflict.Execution, + SignalName: conflict.WinnerSignal, + }) + s.releaseReplicationTask(ctx, conflict.ReplicationToOldActive) + s.assertNoBufferedEvents(ctx, 0, conflict.Namespace.Name, conflict.Execution) + losingHistory := s.findNonCurrentHistoryBranch(ctx, conflict.Namespace.Name, conflict.Namespace.ID, conflict.Execution, func(history []*historypb.HistoryEvent) bool { + for _, eventType := range conflict.ExpectedEventTypes { + if findHistoryEvent(history, eventType, nil) == nil { + return false + } + } + return true + }) + for _, eventType := range conflict.ExpectedEventTypes { + event := findHistoryEvent(losingHistory, eventType, nil) + s.Require().NotNil(event, "%s must be written to the losing branch", eventType) + s.Require().Positive(event.EventId) + s.Require().NotEqual(common.BufferedEventID, event.EventId) + } + await.Require(ctx, s.T(), func(t *await.T) { + s.tryReleaseReplicationTask(conflict.ReplicationToNewActive) + require.True( + t, + hasSignalNamed(s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, conflict.Namespace.Name, conflict.Execution), losingBranchMarker), + "the losing branch marker must be reapplied before checking the rest of the batch", + ) + }, replicationWaitTime, replicationCheckInterval) + for attempt := 0; attempt < 10 && !s.bufferedEventsHistoriesEqual(ctx, conflict.Namespace.Name, conflict.Execution); attempt++ { + s.releaseReplicationTask(ctx, conflict.ReplicationToOldActive) + } + return losingHistory +} + +func signalExternalWorkflowCommand(workflowID, runID, signalName string) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_SignalExternalWorkflowExecutionCommandAttributes{ + SignalExternalWorkflowExecutionCommandAttributes: &commandpb.SignalExternalWorkflowExecutionCommandAttributes{ + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: runID}, + SignalName: signalName, + }, + }, + } +} + +func cancelExternalWorkflowCommand(workflowID, runID string) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_RequestCancelExternalWorkflowExecutionCommandAttributes{ + RequestCancelExternalWorkflowExecutionCommandAttributes: &commandpb.RequestCancelExternalWorkflowExecutionCommandAttributes{ + WorkflowId: workflowID, + RunId: runID, + }, + }, + } +} + +func startChildWorkflowCommand(childID string, taskQueue *taskqueuepb.TaskQueue, runTimeout time.Duration) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_START_CHILD_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_StartChildWorkflowExecutionCommandAttributes{ + StartChildWorkflowExecutionCommandAttributes: &commandpb.StartChildWorkflowExecutionCommandAttributes{ + WorkflowId: childID, + WorkflowType: &commonpb.WorkflowType{Name: "buffered-child"}, + TaskQueue: taskQueue, + WorkflowRunTimeout: durationpb.New(runTimeout), + WorkflowTaskTimeout: durationpb.New(30 * time.Second), + }, + }, + } +} + +func (s *xdcBaseSuite) startBufferedEventsWorkflow( + ctx context.Context, + args startBufferedEventsWorkflowArgs, +) *commonpb.WorkflowExecution { + response, err := s.clusters[0].FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: args.Namespace, + WorkflowId: args.WorkflowID, + WorkflowType: &commonpb.WorkflowType{Name: "buffered-events-xdc"}, + TaskQueue: args.TaskQueue, + RequestId: uuid.NewString(), + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(2 * time.Minute), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + return &commonpb.WorkflowExecution{WorkflowId: args.WorkflowID, RunId: response.RunId} +} + +func scheduleActivityCommand(activityID string, taskQueue *taskqueuepb.TaskQueue, startToClose time.Duration) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: activityID, + ActivityType: &commonpb.ActivityType{Name: activityID}, + TaskQueue: taskQueue, + ScheduleToCloseTimeout: durationpb.New(startToClose), + StartToCloseTimeout: durationpb.New(startToClose), + }, + }, + } +} + +func requestCancelActivityCommand(scheduledEventID int64) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_REQUEST_CANCEL_ACTIVITY_TASK, + Attributes: &commandpb.Command_RequestCancelActivityTaskCommandAttributes{ + RequestCancelActivityTaskCommandAttributes: &commandpb.RequestCancelActivityTaskCommandAttributes{ + ScheduledEventId: scheduledEventID, + }, + }, + } +} + +func completeWorkflowCommand() *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + } +} + +func failWorkflowCommand(message string) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_FAIL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_FailWorkflowExecutionCommandAttributes{ + FailWorkflowExecutionCommandAttributes: &commandpb.FailWorkflowExecutionCommandAttributes{ + Failure: &failurepb.Failure{ + Message: message, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{NonRetryable: true}, + }, + }, + }, + }, + } +} + +func cancelWorkflowCommand() *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CANCEL_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CancelWorkflowExecutionCommandAttributes{ + CancelWorkflowExecutionCommandAttributes: &commandpb.CancelWorkflowExecutionCommandAttributes{}, + }, + } +} + +func (s *xdcBaseSuite) pollBufferedActivityTask( + ctx context.Context, + ns string, + taskQueue *taskqueuepb.TaskQueue, +) *workflowservice.PollActivityTaskQueueResponse { + response, err := s.clusters[0].FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ + Namespace: ns, + TaskQueue: taskQueue, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + s.Require().NotEmpty(response.TaskToken) + return response +} + +func (s *xdcBaseSuite) completeActivityTask(ctx context.Context, task *workflowservice.PollActivityTaskQueueResponse) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + TaskToken: task.TaskToken, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) failActivityTask(ctx context.Context, task *workflowservice.PollActivityTaskQueueResponse) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().RespondActivityTaskFailed(ctx, &workflowservice.RespondActivityTaskFailedRequest{ + TaskToken: task.TaskToken, + Identity: "buffered-events-xdc-test", + Failure: &failurepb.Failure{ + Message: "expected activity failure", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{NonRetryable: true}, + }, + }, + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) cancelActivityTask(ctx context.Context, task *workflowservice.PollActivityTaskQueueResponse) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().RespondActivityTaskCanceled(ctx, &workflowservice.RespondActivityTaskCanceledRequest{ + TaskToken: task.TaskToken, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) +} + +func findActivityScheduledEventID(history []*historypb.HistoryEvent, activityID string) int64 { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED && + event.GetActivityTaskScheduledEventAttributes().GetActivityId() == activityID { + return event.EventId + } + } + return common.EmptyEventID +} + +func assertOnlyExpectedBufferedEventsReapplied(t require.TestingT, history []*historypb.HistoryEvent, bufferedTypes []enumspb.EventType) { + expectedCounts := map[enumspb.EventType]int{ + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: 2, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED: 1, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED: 1, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED: 1, + } + for _, eventType := range bufferedTypes { + require.Equal(t, expectedCounts[eventType], countBufferedEventType(history, eventType), eventType.String()) + } +} + +func (s *FunctionalClustersTestSuite) startWorkflowWithPendingActivity( + ctx context.Context, + ns string, +) (*commonpb.WorkflowExecution, *taskqueuepb.TaskQueue) { + s.T().Helper() + workflowID := "buffered-events-xdc-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{Name: "buffered-events-xdc", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + startResponse, err := s.clusters[0].FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: ns, + WorkflowId: workflowID, + WorkflowType: &commonpb.WorkflowType{Name: "buffered-events-xdc"}, + TaskQueue: taskQueue, + RequestId: uuid.NewString(), + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(30 * time.Second), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + execution := &commonpb.WorkflowExecution{WorkflowId: workflowID, RunId: startResponse.RunId} + + workflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + _, err = s.clusters[0].FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: workflowTask.TaskToken, + Identity: "buffered-events-xdc-test", + Commands: []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: "buffered-activity", + ActivityType: &commonpb.ActivityType{Name: "buffered-activity"}, + TaskQueue: taskQueue, + ScheduleToCloseTimeout: durationpb.New(time.Minute), + ScheduleToStartTimeout: durationpb.New(time.Minute), + StartToCloseTimeout: durationpb.New(time.Minute), + }, + }, + }}, + }) + s.Require().NoError(err) + return execution, taskQueue +} + +func (s *FunctionalClustersTestSuite) acceptUpdateAndStartTimer( + ctx context.Context, + ns string, + execution *commonpb.WorkflowExecution, + taskQueue *taskqueuepb.TaskQueue, + updateID string, +) { + s.T().Helper() + sdkClient, err := sdkclient.Dial(sdkclient.Options{ + HostPort: s.clusters[0].Host().FrontendGRPCAddress(), + Namespace: ns, + }) + s.Require().NoError(err) + defer sdkClient.Close() + updateCtx, cancelUpdate := context.WithCancel(ctx) + defer cancelUpdate() + updateResult := make(chan error, 1) + go func() { + _, updateErr := sdkClient.UpdateWorkflow(updateCtx, sdkclient.UpdateWorkflowOptions{ + UpdateID: updateID, + WorkflowID: execution.WorkflowId, + RunID: execution.RunId, + UpdateName: "buffered-update", + Args: []any{"source"}, + WaitForStage: sdkclient.WorkflowUpdateStageAccepted, + }) + updateResult <- updateErr + }() + await.Require(ctx, s.T(), func(t *await.T) { + response, pollErr := sdkClient.WorkflowService().PollWorkflowExecutionUpdate(t.Context(), &workflowservice.PollWorkflowExecutionUpdateRequest{ + Namespace: ns, + UpdateRef: &updatepb.UpdateRef{ + WorkflowExecution: execution, + UpdateId: updateID, + }, + WaitPolicy: &updatepb.WaitPolicy{ + LifecycleStage: enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ADMITTED, + }, + }) + require.NoError(t, pollErr) + require.Equal(t, enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_ADMITTED, response.Stage) + }, 10*time.Second, 20*time.Millisecond) + workflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.Require().Len(workflowTask.Messages, 1) + updateRequestMessage := workflowTask.Messages[0] + s.Require().Equal(updateID, updateRequestMessage.ProtocolInstanceId) + acceptMessageID := "accept-" + uuid.NewString() + _, err = s.clusters[0].FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: workflowTask.TaskToken, + Identity: "buffered-events-xdc-test", + ForceCreateNewWorkflowTask: true, + Commands: []*commandpb.Command{{ + CommandType: enumspb.COMMAND_TYPE_PROTOCOL_MESSAGE, + Attributes: &commandpb.Command_ProtocolMessageCommandAttributes{ + ProtocolMessageCommandAttributes: &commandpb.ProtocolMessageCommandAttributes{MessageId: acceptMessageID}, + }, + }, { + CommandType: enumspb.COMMAND_TYPE_START_TIMER, + Attributes: &commandpb.Command_StartTimerCommandAttributes{ + StartTimerCommandAttributes: &commandpb.StartTimerCommandAttributes{ + TimerId: "losing-branch-timer", + StartToFireTimeout: durationpb.New(100 * time.Millisecond), + }, + }, + }}, + Messages: []*protocolpb.Message{{ + Id: acceptMessageID, + ProtocolInstanceId: updateID, + Body: protoutils.MarshalAny(s.T(), &updatepb.Acceptance{ + AcceptedRequestMessageId: updateRequestMessage.Id, + AcceptedRequestSequencingEventId: updateRequestMessage.GetEventId(), + }), + }}, + }) + s.Require().NoError(err) + s.Require().True(hasUpdateAccepted(s.getWorkflowHistory(ctx, s.T(), 0, ns, execution), updateID)) + s.Require().NoError(<-updateResult) +} + +func (s *FunctionalClustersTestSuite) completeActivityAndBufferExternalEvents( + ctx context.Context, + ns string, + execution *commonpb.WorkflowExecution, + taskQueue *taskqueuepb.TaskQueue, +) string { + s.T().Helper() + activityTask, err := s.clusters[0].FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ + Namespace: ns, + TaskQueue: taskQueue, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + s.Require().NotEmpty(activityTask.TaskToken) + _, err = s.clusters[0].FrontendClient().RespondActivityTaskCompleted(ctx, &workflowservice.RespondActivityTaskCompletedRequest{ + TaskToken: activityTask.TaskToken, + Result: payloads.EncodeString("activity-result"), + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + + _, err = s.clusters[0].FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: execution, + SignalName: "buffered-signal", + Input: payloads.EncodeString("source"), + Identity: "buffered-events-xdc-test", + RequestId: uuid.NewString(), + }) + s.Require().NoError(err) + _, err = s.clusters[0].FrontendClient().RequestCancelWorkflowExecution(ctx, &workflowservice.RequestCancelWorkflowExecutionRequest{ + Namespace: ns, + WorkflowExecution: execution, + Identity: "buffered-events-xdc-test", + RequestId: uuid.NewString(), + }) + s.Require().NoError(err) + + optionsRequestID := uuid.NewString() + attachResponse, err := s.clusters[0].FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + Namespace: ns, + WorkflowId: execution.WorkflowId, + WorkflowType: &commonpb.WorkflowType{Name: "buffered-events-xdc"}, + TaskQueue: taskQueue, + RequestId: optionsRequestID, + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(30 * time.Second), + Identity: "buffered-events-xdc-test", + WorkflowIdConflictPolicy: enumspb.WORKFLOW_ID_CONFLICT_POLICY_USE_EXISTING, + OnConflictOptions: &workflowpb.OnConflictOptions{ + AttachRequestId: true, + }, + }) + s.Require().NoError(err) + s.Require().False(attachResponse.Started) + + _, err = s.clusters[0].FrontendClient().PauseWorkflowExecution(ctx, &workflowservice.PauseWorkflowExecutionRequest{ + Namespace: ns, + WorkflowId: execution.WorkflowId, + RunId: execution.RunId, + Identity: "buffered-events-xdc-test", + Reason: "exercise buffered pause", + RequestId: uuid.NewString(), + }) + s.Require().NoError(err) + _, err = s.clusters[0].FrontendClient().UnpauseWorkflowExecution(ctx, &workflowservice.UnpauseWorkflowExecutionRequest{ + Namespace: ns, + WorkflowId: execution.WorkflowId, + RunId: execution.RunId, + Identity: "buffered-events-xdc-test", + Reason: "exercise buffered unpause", + RequestId: uuid.NewString(), + }) + s.Require().NoError(err) + return optionsRequestID +} + +func (s *xdcBaseSuite) failoverToNewActiveCluster(ctx context.Context, ns string) { + s.T().Helper() + response, err := s.clusters[0].FrontendClient().UpdateNamespace(ctx, &workflowservice.UpdateNamespaceRequest{ + Namespace: ns, + ReplicationConfig: &replicationpb.NamespaceReplicationConfig{ + ActiveClusterName: s.clusters[1].ClusterName(), + }, + }) + s.Require().NoError(err) + s.Require().Equal(int64(2), response.FailoverVersion) + await.Require(ctx, s.T(), func(t *await.T) { + for _, cluster := range s.clusters { + describeResponse, describeErr := cluster.FrontendClient().DescribeNamespace(t.Context(), &workflowservice.DescribeNamespaceRequest{ + Namespace: ns, + }) + require.NoError(t, describeErr) + require.Equal(t, s.clusters[1].ClusterName(), describeResponse.ReplicationConfig.ActiveClusterName) + } + }, replicationWaitTime, replicationCheckInterval) + s.waitForNamespaceCacheRefresh() +} + +func (s *xdcBaseSuite) writeSignalOnNewActive( + ctx context.Context, + signal activeClusterSignal, +) { + s.T().Helper() + request := &historyservice.SignalWorkflowExecutionRequest{ + NamespaceId: signal.Namespace.ID, + SignalRequest: &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: signal.Namespace.Name, + WorkflowExecution: signal.Execution, + SignalName: signal.SignalName, + Identity: "buffered-events-xdc-test", + RequestId: uuid.NewString(), + }, + } + await.Require(ctx, s.T(), func(t *await.T) { + _, err := s.clusters[1].HistoryClient().SignalWorkflowExecution(t.Context(), request) + require.NoError(t, err) + }, replicationWaitTime, replicationCheckInterval) + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, signal.Namespace.Name, signal.Execution) + require.True(t, hasSignalNamed(history, signal.SignalName)) + }, replicationWaitTime, replicationCheckInterval) +} + +func (s *xdcBaseSuite) hasBufferedEventType( + ctx context.Context, + clusterIndex int, + ns string, + execution *commonpb.WorkflowExecution, + eventType enumspb.EventType, +) bool { + response, err := s.clusters[clusterIndex].AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: execution, + Archetype: chasm.WorkflowArchetype, + }) + if err != nil { + return false + } + return findHistoryEvent(response.GetCacheMutableState().GetBufferedEvents(), eventType, nil) != nil +} + +func (s *xdcBaseSuite) pollBufferedEventsWorkflowTask( + ctx context.Context, + clusterIndex int, + ns string, + taskQueue *taskqueuepb.TaskQueue, +) *workflowservice.PollWorkflowTaskQueueResponse { + s.T().Helper() + response, err := s.clusters[clusterIndex].FrontendClient().PollWorkflowTaskQueue(ctx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: ns, + TaskQueue: taskQueue, + Identity: "buffered-events-xdc-test", + }) + s.Require().NoError(err) + return response +} + +func (s *xdcBaseSuite) completeWorkflowTask(ctx context.Context, completion workflowTaskCompletion) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: completion.Task.TaskToken, + Identity: "buffered-events-xdc-test", + Commands: completion.Commands, + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) completeWorkflowTaskAndScheduleNext(ctx context.Context, completion workflowTaskCompletion) { + s.T().Helper() + _, err := s.clusters[0].FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: completion.Task.TaskToken, + Identity: "buffered-events-xdc-test", + Commands: completion.Commands, + ForceCreateNewWorkflowTask: true, + }) + s.Require().NoError(err) +} + +func (s *xdcBaseSuite) completeWorkflowTaskAndReturnNext( + ctx context.Context, + completion workflowTaskCompletion, +) *workflowservice.PollWorkflowTaskQueueResponse { + s.T().Helper() + response, err := s.clusters[0].FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: completion.Task.TaskToken, + Identity: "buffered-events-xdc-test", + Commands: completion.Commands, + ForceCreateNewWorkflowTask: true, + ReturnNewWorkflowTask: true, + }) + s.Require().NoError(err) + s.Require().NotNil(response.GetWorkflowTask()) + return response.GetWorkflowTask() +} + +func (s *xdcBaseSuite) blockReplicationForWorkflow( + clusterIndex int, + workflowID string, +) <-chan *blockedReplicationTask { + s.T().Helper() + // The interceptor runs on the receiving cluster, so clusterIndex identifies the replication + // destination rather than the cluster that generated the task. + tasks := make(chan *blockedReplicationTask, 20) + s.clusters[clusterIndex].InjectHook( + s.T(), + testhooks.NewHook(testhooks.HistoryReplicationTaskInterceptor, func( + task *replicationspb.ReplicationTask, + execute func() error, + ) error { + if workflowIDFromReplicationTask(task) != workflowID { + return execute() + } + blockedTask := &blockedReplicationTask{ + execute: execute, + result: make(chan error, 1), + } + tasks <- blockedTask + return <-blockedTask.result + }), + testhooks.GlobalScope, + ) + return tasks +} + +func (s *xdcBaseSuite) releaseReplicationTask( + ctx context.Context, + tasks <-chan *blockedReplicationTask, +) { + s.T().Helper() + select { + case task := <-tasks: + s.executeReplicationTask(task) + case <-ctx.Done(): + s.FailNow("timed out waiting for controlled history replication task", ctx.Err().Error()) + } +} + +func (s *xdcBaseSuite) tryReleaseReplicationTask(tasks <-chan *blockedReplicationTask) bool { + select { + case task := <-tasks: + s.executeReplicationTask(task) + return true + default: + return false + } +} + +func (s *xdcBaseSuite) executeReplicationTask(task *blockedReplicationTask) { + err := task.execute() + task.result <- err + var duplicateError *serviceerror.AlreadyExists + var retryReplicationError *serviceerrors.RetryReplication + s.Require().True( + err == nil || errors.As(err, &duplicateError) || errors.As(err, &retryReplicationError), + "replication task failed: %v", + err, + ) +} + +func workflowIDFromReplicationTask(task *replicationspb.ReplicationTask) string { + if attributes := task.GetSyncVersionedTransitionTaskAttributes(); attributes != nil { + return attributes.WorkflowId + } + if attributes := task.GetSyncWorkflowStateTaskAttributes(); attributes != nil { + return attributes.GetWorkflowState().GetExecutionInfo().GetWorkflowId() + } + if attributes := task.GetSyncHsmAttributes(); attributes != nil { + return attributes.WorkflowId + } + if attributes := task.GetSyncActivityTaskAttributes(); attributes != nil { + return attributes.WorkflowId + } + if attributes := task.GetVerifyVersionedTransitionTaskAttributes(); attributes != nil { + return attributes.WorkflowId + } + if attributes := task.GetBackfillHistoryTaskAttributes(); attributes != nil { + return attributes.WorkflowId + } + if attributes := task.GetHistoryTaskAttributes(); attributes != nil { + return attributes.WorkflowId + } + return "" +} + +func (s *xdcBaseSuite) assertBufferedEventTypes( + ctx context.Context, + expectation bufferedEventExpectation, +) { + s.T().Helper() + await.Require(ctx, s.T(), func(t *await.T) { + response, err := s.clusters[0].AdminClient().DescribeMutableState(t.Context(), &adminservice.DescribeMutableStateRequest{ + Namespace: expectation.Namespace, + Execution: expectation.Execution, + Archetype: chasm.WorkflowArchetype, + }) + require.NoError(t, err) + require.NotNil(t, response.CacheMutableState) + bufferedEvents := response.CacheMutableState.BufferedEvents + actualTypes := make([]enumspb.EventType, 0, len(bufferedEvents)) + for _, event := range bufferedEvents { + require.Equal(t, common.BufferedEventID, event.EventId) + actualTypes = append(actualTypes, event.EventType) + } + require.ElementsMatch(t, expectation.EventTypes, actualTypes) + }, replicationWaitTime, replicationCheckInterval) +} + +func (s *xdcBaseSuite) assertBufferedEventTypesPresent( + ctx context.Context, + expectation bufferedEventExpectation, +) { + s.T().Helper() + await.Require(ctx, s.T(), func(t *await.T) { + response, err := s.clusters[0].AdminClient().DescribeMutableState(t.Context(), &adminservice.DescribeMutableStateRequest{ + Namespace: expectation.Namespace, + Execution: expectation.Execution, + Archetype: chasm.WorkflowArchetype, + }) + require.NoError(t, err) + bufferedEvents := response.GetCacheMutableState().GetBufferedEvents() + for _, event := range bufferedEvents { + require.Equal(t, common.BufferedEventID, event.EventId) + } + for _, expectedType := range expectation.EventTypes { + require.NotNil(t, findHistoryEvent(bufferedEvents, expectedType, nil), "%s must be naturally buffered", expectedType) + } + }, replicationWaitTime, replicationCheckInterval) +} + +func (s *xdcBaseSuite) assertNoBufferedEvents( + ctx context.Context, + clusterIndex int, + ns string, + execution *commonpb.WorkflowExecution, +) { + s.T().Helper() + response, err := s.clusters[clusterIndex].AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: execution, + Archetype: chasm.WorkflowArchetype, + }) + s.Require().NoError(err) + s.Require().Empty(response.GetCacheMutableState().GetBufferedEvents()) + s.Require().Empty(response.GetDatabaseMutableState().GetBufferedEvents()) +} + +func (s *xdcBaseSuite) getWorkflowHistory( + ctx context.Context, + t require.TestingT, + clusterIndex int, + ns string, + execution *commonpb.WorkflowExecution, +) []*historypb.HistoryEvent { + response, err := s.clusters[clusterIndex].FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: ns, + Execution: execution, + }) + require.NoError(t, err) + return response.History.Events +} + +func (s *FunctionalClustersTestSuite) assertBufferedEventsPersistedOnLosingBranch( + ctx context.Context, + expectation bufferedInputsExpectation, +) { + s.T().Helper() + losingHistory := s.findNonCurrentHistoryBranch(ctx, expectation.Namespace.Name, expectation.Namespace.ID, expectation.Execution, func(history []*historypb.HistoryEvent) bool { + return hasSignalNamed(history, "buffered-signal") + }) + s.Require().True(hasUpdateAccepted(losingHistory, expectation.UpdateID)) + s.Require().True(hasWorkflowTaskFailedForFailover(losingHistory)) + s.Require().True(hasOptionsUpdatedRequest(losingHistory, expectation.OptionsRequestID)) + + for _, eventType := range expectation.EventTypes { + event := findHistoryEvent(losingHistory, eventType, nil) + s.Require().NotNil(event, "%s must be written to the losing branch", eventType) + s.Require().NotEqual(common.BufferedEventID, event.EventId) + s.Require().Positive(event.EventId) + } +} + +func (s *xdcBaseSuite) findNonCurrentHistoryBranch( + ctx context.Context, + ns string, + namespaceID string, + execution *commonpb.WorkflowExecution, + matches func([]*historypb.HistoryEvent) bool, +) []*historypb.HistoryEvent { + s.T().Helper() + description, err := s.clusters[0].AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: ns, + Execution: execution, + Archetype: chasm.WorkflowArchetype, + }) + s.Require().NoError(err) + versionHistories := description.GetDatabaseMutableState().GetExecutionInfo().GetVersionHistories() + s.Require().GreaterOrEqual(len(versionHistories.GetHistories()), 2) + + shardID := common.WorkflowIDToHistoryShard(namespaceID, execution.WorkflowId, s.numHistoryShards) + for index, versionHistory := range versionHistories.GetHistories() { + if int32(index) == versionHistories.GetCurrentVersionHistoryIndex() { + continue + } + history := s.readBufferedEventsHistoryBranch(ctx, shardID, versionHistory.GetBranchToken()) + if matches(history) { + return history + } + } + s.FailNow("matching non-current history branch not found") + return nil +} + +func (s *xdcBaseSuite) readBufferedEventsHistoryBranch( + ctx context.Context, + shardID int32, + branchToken []byte, +) []*historypb.HistoryEvent { + s.T().Helper() + request := &persistence.ReadHistoryBranchRequest{ + ShardID: shardID, + BranchToken: branchToken, + MinEventID: common.FirstEventID, + MaxEventID: common.EndEventID, + PageSize: 1000, + } + var events []*historypb.HistoryEvent + for { + response, err := s.clusters[0].ExecutionManager().ReadHistoryBranch(ctx, request) + s.Require().NoError(err) + events = append(events, response.HistoryEvents...) + if len(response.NextPageToken) == 0 { + return events + } + request.NextPageToken = response.NextPageToken + } +} + +func (s *xdcBaseSuite) bufferedEventsHistoriesEqual( + ctx context.Context, + ns string, + execution *commonpb.WorkflowExecution, +) bool { + sourceHistory := s.getWorkflowHistory(ctx, s.T(), 0, ns, execution) + targetHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + if len(targetHistory) != len(sourceHistory) { + return false + } + for index := range targetHistory { + if !proto.Equal(targetHistory[index], sourceHistory[index]) { + return false + } + } + return true +} + +func (s *FunctionalClustersTestSuite) hasReappliedBufferedInputs( + ctx context.Context, + expectation bufferedInputsExpectation, +) bool { + history := s.getWorkflowHistory(ctx, s.T(), 1, expectation.Namespace.Name, expectation.Execution) + hasSourceSignal := false + hasUpdate := false + hasCancel := false + hasOptionsUpdate := false + for _, event := range history { + switch event.EventType { + case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED: + hasSourceSignal = hasSourceSignal || event.GetWorkflowExecutionSignaledEventAttributes().SignalName == "buffered-signal" + case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED: + hasUpdate = hasUpdate || event.GetWorkflowExecutionUpdateAdmittedEventAttributes().GetRequest().GetMeta().GetUpdateId() == expectation.UpdateID + case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED: + hasCancel = true + case enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED: + hasOptionsUpdate = hasOptionsUpdate || event.GetWorkflowExecutionOptionsUpdatedEventAttributes().AttachedRequestId == expectation.OptionsRequestID + default: + } + } + return hasSourceSignal && hasUpdate && hasCancel && hasOptionsUpdate +} + +func countBufferedEventType(history []*historypb.HistoryEvent, eventType enumspb.EventType) int { + count := 0 + for _, event := range history { + if event.EventType == eventType { + count++ + } + } + return count +} + +func hasSignalNamed(history []*historypb.HistoryEvent, signalName string) bool { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED && + event.GetWorkflowExecutionSignaledEventAttributes().SignalName == signalName { + return true + } + } + return false +} + +func hasUpdateAccepted(history []*historypb.HistoryEvent, updateID string) bool { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ACCEPTED && + event.GetWorkflowExecutionUpdateAcceptedEventAttributes().ProtocolInstanceId == updateID { + return true + } + } + return false +} + +func hasWorkflowTaskFailedForFailover(history []*historypb.HistoryEvent) bool { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_TASK_FAILED && + event.GetWorkflowTaskFailedEventAttributes().Cause == enumspb.WORKFLOW_TASK_FAILED_CAUSE_FAILOVER_CLOSE_COMMAND { + return true + } + } + return false +} + +func hasOptionsUpdatedRequest(history []*historypb.HistoryEvent, requestID string) bool { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED && + event.GetWorkflowExecutionOptionsUpdatedEventAttributes().AttachedRequestId == requestID { + return true + } + } + return false +} + +func (s *NexusStateReplicationSuite) createBufferedNexusEndpoint(ctx context.Context, handler nexustest.Handler) string { + s.T().Helper() + listenAddress := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddress, handler) + for _, cluster := range s.clusters { + cluster.OverrideDynamicConfig( + s.T(), + nexusoperations.CallbackURLTemplate, + "http://"+s.clusters[0].Host().FrontendHTTPAddress()+"/namespaces/{{.NamespaceName}}/nexus/callback", + ) + } + endpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + for _, cluster := range s.clusters { + _, err := cluster.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: endpointName, + Target: &nexuspb.EndpointTarget{Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{Url: "http://" + listenAddress}, + }}, + }, + }) + s.Require().NoError(err) + } + return endpointName +} + +func (s *NexusStateReplicationSuite) failNexusOperation(ctx context.Context, callback bufferedNexusCallback) { + client := nexusrpc.NewCompletionHTTPClient(nexusrpc.CompletionHTTPClientOptions{Serializer: commonnexus.PayloadSerializer}) + err := client.CompleteOperation(ctx, callback.url, nexusrpc.CompleteOperationOptions{ + Error: &nexus.OperationError{ + State: nexus.OperationStateFailed, + Cause: &nexus.FailureError{Failure: nexus.Failure{Message: "expected operation failure"}}, + }, + Header: nexus.Header{commonnexus.CallbackTokenHeader: callback.token}, + }) + s.Require().NoError(err) +} + +func (s *NexusStateReplicationSuite) completeBufferedNexusOperation(ctx context.Context, completion bufferedNexusOperationCompletion) { + s.T().Helper() + s.completeNexusOperation(ctx, completion.Result, completion.Callback.url, completion.Callback.token) +} + +func (s *NexusStateReplicationSuite) cancelBufferedNexusOperation(ctx context.Context, callback bufferedNexusCallback) { + s.T().Helper() + s.cancelNexusOperation(ctx, callback.url, callback.token) +} + +func (s *NexusStateReplicationSuite) setupBufferedNexusEndpoint( + ctx context.Context, +) (string, <-chan bufferedNexusCallback, chan struct{}) { + s.T().Helper() + operationCallbacks := make(chan bufferedNexusCallback, 2) + allowLosingOnlyOperationStart := make(chan struct{}) + handler := nexustest.Handler{ + OnStartOperation: func( + _ context.Context, + _, operation string, + _ *nexus.LazyValue, + options nexus.StartOperationOptions, + ) (nexus.HandlerStartOperationResult[any], error) { + operationCallbacks <- bufferedNexusCallback{ + operation: operation, + url: options.CallbackURL, + token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), + } + if operation == "losing-only-operation" { + select { + case <-allowLosingOnlyOperationStart: + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return &nexus.HandlerStartOperationResultAsync{OperationToken: operation}, nil + }, + } + listenAddress := nexustest.AllocListenAddress() + nexustest.NewNexusServer(s.T(), listenAddress, handler) + for _, cluster := range s.clusters { + cluster.OverrideDynamicConfig( + s.T(), + nexusoperations.CallbackURLTemplate, + "http://"+s.clusters[0].Host().FrontendHTTPAddress()+"/namespaces/{{.NamespaceName}}/nexus/callback", + ) + } + + endpointName := testcore.RandomizedNexusEndpoint(s.T().Name()) + for _, cluster := range s.clusters { + _, err := cluster.OperatorClient().CreateNexusEndpoint(ctx, &operatorservice.CreateNexusEndpointRequest{ + Spec: &nexuspb.EndpointSpec{ + Name: endpointName, + Target: &nexuspb.EndpointTarget{Variant: &nexuspb.EndpointTarget_External_{ + External: &nexuspb.EndpointTarget_External{Url: "http://" + listenAddress}, + }}, + }, + }) + s.Require().NoError(err) + } + return endpointName, operationCallbacks, allowLosingOnlyOperationStart +} + +func scheduleBufferedNexusOperationCommand(endpoint, operation string) *commandpb.Command { + return scheduleBufferedNexusOperationCommandWithTimeout(endpoint, operation, time.Minute) +} + +func scheduleBufferedNexusOperationCommandWithTimeout(endpoint, operation string, timeout time.Duration) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_NEXUS_OPERATION, + Attributes: &commandpb.Command_ScheduleNexusOperationCommandAttributes{ + ScheduleNexusOperationCommandAttributes: &commandpb.ScheduleNexusOperationCommandAttributes{ + Endpoint: endpoint, + Service: "service", + Operation: operation, + ScheduleToCloseTimeout: durationpb.New(timeout), + }, + }, + } +} + +func requestCancelNexusOperationCommand(scheduledEventID int64) *commandpb.Command { + return &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_REQUEST_CANCEL_NEXUS_OPERATION, + Attributes: &commandpb.Command_RequestCancelNexusOperationCommandAttributes{ + RequestCancelNexusOperationCommandAttributes: &commandpb.RequestCancelNexusOperationCommandAttributes{ + ScheduledEventId: scheduledEventID, + }, + }, + } +} + +func receiveBufferedNexusCallback( + ctx context.Context, + t require.TestingT, + operationCallbacks <-chan bufferedNexusCallback, + expectedOperation string, +) bufferedNexusCallback { + select { + case operationCallback := <-operationCallbacks: + require.Equal(t, expectedOperation, operationCallback.operation) + return operationCallback + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for Nexus operation callback", expectedOperation) + return bufferedNexusCallback{} + } +} + +func receiveAnyBufferedNexusCallback( + ctx context.Context, + t require.TestingT, + operationCallbacks <-chan bufferedNexusCallback, +) bufferedNexusCallback { + select { + case operationCallback := <-operationCallbacks: + return operationCallback + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for Nexus operation callback") + return bufferedNexusCallback{} + } +} + +func (s *NexusStateReplicationSuite) findBufferedNexusLosingBranch( + ctx context.Context, + branch bufferedNexusLosingBranch, +) []*historypb.HistoryEvent { + s.T().Helper() + history := s.findNonCurrentHistoryBranch(ctx, branch.Namespace.Name, branch.Namespace.ID, branch.Execution, func(history []*historypb.HistoryEvent) bool { + return hasNexusEventForScheduledID(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, branch.ScheduledEventID) + }) + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED || + event.EventType == enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED { + s.Require().NotEqual(common.BufferedEventID, event.EventId) + s.Require().Positive(event.EventId) + } + } + return history +} + +func findNexusScheduledEventID(history []*historypb.HistoryEvent, operation string) int64 { + for _, event := range history { + if event.EventType == enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED && + event.GetNexusOperationScheduledEventAttributes().Operation == operation { + return event.EventId + } + } + return common.EmptyEventID +} + +func hasNexusEventForScheduledID(history []*historypb.HistoryEvent, eventType enumspb.EventType, scheduledEventID int64) bool { + for _, event := range history { + if event.EventType != eventType { + continue + } + var eventScheduledID int64 + switch eventType { + case enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED: + eventScheduledID = event.EventId + case enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED: + eventScheduledID = event.GetNexusOperationStartedEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED: + eventScheduledID = event.GetNexusOperationCompletedEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_FAILED: + eventScheduledID = event.GetNexusOperationFailedEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCELED: + eventScheduledID = event.GetNexusOperationCanceledEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT: + eventScheduledID = event.GetNexusOperationTimedOutEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_COMPLETED: + eventScheduledID = event.GetNexusOperationCancelRequestCompletedEventAttributes().GetScheduledEventId() + case enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_FAILED: + eventScheduledID = event.GetNexusOperationCancelRequestFailedEventAttributes().GetScheduledEventId() + default: + return false + } + if eventScheduledID == scheduledEventID { + return true + } + } + return false +} diff --git a/tests/xdc/buffered_events_replication_test.go b/tests/xdc/buffered_events_replication_test.go new file mode 100644 index 00000000000..491172d3a5f --- /dev/null +++ b/tests/xdc/buffered_events_replication_test.go @@ -0,0 +1,451 @@ +package xdc + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/server/common/testing/await" +) + +// Four values in HistoryBuilder's buffered-event set cannot be naturally +// buffered on current main. EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_REJECTED, +// EVENT_TYPE_WORKFLOW_PROPERTIES_MODIFIED_EXTERNALLY, and +// EVENT_TYPE_ACTIVITY_PROPERTIES_MODIFIED_EXTERNALLY have no production emitter. +// EVENT_TYPE_WORKFLOW_EXECUTION_TIME_SKIPPING_TRANSITIONED has an emitter, but +// its production precondition rejects every workflow with a pending workflow +// task, so it cannot enter the buffer. The tests below cover every other value +// through its production API, transfer task, timer, callback, or reapplier. + +// TestNaturallyBufferedInputsFlushedAndReappliedAfterFailover buffers an activity result, timer, +// cancel request, signal, options update, pause, and unpause behind one workflow task; conflict +// reapplication then buffers an update-admitted event on the winner. It expects external inputs and +// the update to reach the winner while activity, timer, and pause state remain only on the losing branch. +func (s *FunctionalClustersTestSuite) TestNaturallyBufferedInputsFlushedAndReappliedAfterFailover() { + if !s.enableTransitionHistory { + s.T().Skip("buffered event state-based replication requires transition history") + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + s.enableWorkflowPauseForTest() + + // Phase 1: establish identical history with one pending activity. + execution, taskQueue := s.startWorkflowWithPendingActivity(ctx, ns) + workflowID := execution.WorkflowId + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.NotEmpty(t, history) + require.Equal(t, enumspb.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED, history[len(history)-1].EventType) + }, replicationWaitTime, replicationCheckInterval) + + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + + // Phase 2: hold a workflow task and create the losing branch's buffered events. + updateID := "buffered-update-" + uuid.NewString() + s.acceptUpdateAndStartTimer(ctx, ns, execution, taskQueue, updateID) + + heldWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.Require().NotEmpty(heldWorkflowTask.TaskToken) + + optionsRequestID := s.completeActivityAndBufferExternalEvents(ctx, ns, execution, taskQueue) + naturallyBufferedTypes := []enumspb.EventType{ + enumspb.EVENT_TYPE_ACTIVITY_TASK_STARTED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, + enumspb.EVENT_TYPE_TIMER_FIRED, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_PAUSED, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UNPAUSED, + } + bufferedInputs := bufferedInputsExpectation{ + Namespace: namespace, + Execution: execution, + UpdateID: updateID, + OptionsRequestID: optionsRequestID, + EventTypes: naturallyBufferedTypes, + } + s.assertBufferedEventTypes(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: naturallyBufferedTypes, + }) + + // Phase 3: fail over and create a winning branch on the new active cluster. + s.failoverToNewActiveCluster(ctx, ns) + s.writeSignalOnNewActive(ctx, activeClusterSignal{ + Namespace: namespace, + Execution: execution, + SignalName: "winner-signal", + }) + winnerWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 1, ns, taskQueue) + s.Require().NotEmpty(winnerWorkflowTask.TaskToken) + + // Phase 4: resolve the conflict and verify losing-branch storage versus reapplication. + s.releaseReplicationTask(ctx, replicationToOldActive) + s.assertNoBufferedEvents(ctx, 0, ns, execution) + s.assertBufferedEventsPersistedOnLosingBranch(ctx, bufferedInputs) + + // Reapplying the losing update while the winner's workflow task is running + // naturally creates UpdateAdmitted as a buffered event. + for attempt := 0; attempt < 10 && !s.hasBufferedEventType(ctx, 1, ns, execution, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED); attempt++ { + s.releaseReplicationTask(ctx, replicationToNewActive) + } + s.Require().True(s.hasBufferedEventType(ctx, 1, ns, execution, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED)) + await.Require(ctx, s.T(), func(t *await.T) { + require.False(t, s.hasBufferedEventType(t.Context(), 1, ns, execution, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED)) + }, 45*time.Second, replicationCheckInterval) + // The timeout flush and conflict resolution can produce more than one + // state-based task. Release only this workflow's tasks until all reapplied + // inputs reach the active cluster. + for attempt := 0; attempt < 10 && !s.hasReappliedBufferedInputs(ctx, bufferedInputs); attempt++ { + s.releaseReplicationTask(ctx, replicationToNewActive) + } + s.Require().True(s.hasReappliedBufferedInputs(ctx, bufferedInputs)) + for attempt := 0; attempt < 10 && !s.bufferedEventsHistoriesEqual(ctx, ns, execution); attempt++ { + s.releaseReplicationTask(ctx, replicationToOldActive) + } + + await.Require(ctx, s.T(), func(t *await.T) { + sourceHistory := s.getWorkflowHistory(t.Context(), t.AssertionT(), 0, ns, execution) + targetHistory := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.Equal(t, targetHistory, sourceHistory) + }, replicationWaitTime, replicationCheckInterval) + + finalHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + s.Require().Equal(1, countBufferedEventType(finalHistory, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_UPDATE_ADMITTED)) + s.Require().Equal(1, countBufferedEventType(finalHistory, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CANCEL_REQUESTED)) + s.Require().Equal(1, countBufferedEventType(finalHistory, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED)) + s.Require().Equal(2, countBufferedEventType(finalHistory, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED)) + assertOnlyExpectedBufferedEventsReapplied(s.T(), finalHistory, naturallyBufferedTypes) +} + +// TestNaturallyBufferedActivityOutcomesFlushedToLosingBranch buffers started, completed, failed, +// timed-out, and canceled activity outcomes. It expects all outcomes on the losing branch and none +// to be reapplied to the winner, apart from the winner independently producing its own timeout. +func (s *FunctionalClustersTestSuite) TestNaturallyBufferedActivityOutcomesFlushedToLosingBranch() { + if !s.enableTransitionHistory { + s.T().Skip("buffered event state-based replication requires transition history") + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + workflowID := "buffered-activities-xdc-" + uuid.NewString() + workflowQueue := &taskqueuepb.TaskQueue{Name: workflowID + "-workflow"} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: workflowQueue, + }) + firstTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, workflowQueue) + + activityQueues := map[string]*taskqueuepb.TaskQueue{} + var scheduleCommands []*commandpb.Command + for _, activityID := range []string{"started", "completed", "failed", "canceled"} { + activityQueues[activityID] = &taskqueuepb.TaskQueue{Name: workflowID + "-" + activityID} + scheduleCommands = append(scheduleCommands, scheduleActivityCommand(activityID, activityQueues[activityID], time.Minute)) + } + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: firstTask, + Commands: scheduleCommands, + }) + + // Start the activity that will be canceled before establishing the common + // prefix, so its cancellation request can be issued by a workflow command. + canceledTask := s.pollBufferedActivityTask(ctx, ns, activityQueues["canceled"]) + s.signalWorkflow(ctx, workflowSignal{ + Namespace: ns, + Execution: execution, + SignalName: "prepare-activity-cancellation", + }) + cancelCommandTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, workflowQueue) + s.Require().NotNil(cancelCommandTask.History) + canceledScheduledID := findActivityScheduledEventID(cancelCommandTask.History.Events, "canceled") + s.Require().Positive(canceledScheduledID) + s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + Task: cancelCommandTask, + Commands: []*commandpb.Command{ + requestCancelActivityCommand(canceledScheduledID), + scheduleActivityCommand("timed-out", &taskqueuepb.TaskQueue{Name: workflowID + "-unpolled"}, 5*time.Second), + }, + }) + + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.Positive(t, findActivityScheduledEventID(history, "canceled")) + }, replicationWaitTime, replicationCheckInterval) + + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + heldWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, workflowQueue) + s.Require().NotEmpty(heldWorkflowTask.TaskToken) + + startedTask := s.pollBufferedActivityTask(ctx, ns, activityQueues["started"]) + s.Require().NotEmpty(startedTask.TaskToken) + completedTask := s.pollBufferedActivityTask(ctx, ns, activityQueues["completed"]) + s.completeActivityTask(ctx, completedTask) + failedTask := s.pollBufferedActivityTask(ctx, ns, activityQueues["failed"]) + s.failActivityTask(ctx, failedTask) + s.cancelActivityTask(ctx, canceledTask) + + expectedTypes := []enumspb.EventType{ + enumspb.EVENT_TYPE_ACTIVITY_TASK_STARTED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_STARTED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_STARTED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_FAILED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT, + enumspb.EVENT_TYPE_ACTIVITY_TASK_CANCELED, + } + s.assertBufferedEventTypes(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: expectedTypes, + }) + s.finishNaturallyBufferedConflict(ctx, naturallyBufferedConflict{ + Namespace: namespace, + Execution: execution, + ReplicationToOldActive: replicationToOldActive, + ReplicationToNewActive: replicationToNewActive, + ExpectedEventTypes: expectedTypes, + WinnerSignal: "activity-winner-signal", + }) + winningHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + for _, eventType := range []enumspb.EventType{ + enumspb.EVENT_TYPE_ACTIVITY_TASK_STARTED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_COMPLETED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_FAILED, + enumspb.EVENT_TYPE_ACTIVITY_TASK_CANCELED, + } { + s.Require().Zero(countBufferedEventType(winningHistory, eventType), "%s must remain only on the losing branch", eventType) + } + s.Require().Equal( + 1, + countBufferedEventType(winningHistory, enumspb.EVENT_TYPE_ACTIVITY_TASK_TIMED_OUT), + "the winner may produce its own timeout, but must not contain a duplicate reapplied timeout", + ) +} + +// TestNaturallyBufferedChildWorkflowOutcomesFlushedToLosingBranch buffers child start failure, +// started, completed, failed, canceled, timed-out, and terminated callbacks for children created only +// on the losing branch. It expects every callback to be persisted there and skipped on the winner. +func (s *FunctionalClustersTestSuite) TestNaturallyBufferedChildWorkflowOutcomesFlushedToLosingBranch() { + if !s.enableTransitionHistory { + s.T().Skip("buffered event state-based replication requires transition history") + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + workflowID := "buffered-children-xdc-" + uuid.NewString() + parentQueue := &taskqueuepb.TaskQueue{Name: workflowID + "-parent"} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: parentQueue, + }) + firstTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, parentQueue) + + childIDs := map[string]string{} + childQueues := map[string]*taskqueuepb.TaskQueue{} + for _, outcome := range []string{"completed", "failed", "canceled", "timed-out", "terminated", "duplicate"} { + childIDs[outcome] = workflowID + "-" + outcome + childQueues[outcome] = &taskqueuepb.TaskQueue{Name: childIDs[outcome] + "-queue"} + } + duplicateExecution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: childIDs["duplicate"], + TaskQueue: childQueues["duplicate"], + }) + s.Require().NotEmpty(duplicateExecution.RunId) + + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.NotEmpty(t, history) + }, replicationWaitTime, replicationCheckInterval) + + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + var commands []*commandpb.Command + for _, outcome := range []string{"completed", "failed", "canceled", "timed-out", "terminated", "duplicate"} { + runTimeout := time.Minute + if outcome == "timed-out" { + runTimeout = 5 * time.Second + } + commands = append(commands, startChildWorkflowCommand(childIDs[outcome], childQueues[outcome], runTimeout)) + } + s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + Task: firstTask, + Commands: commands, + }) + heldWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, parentQueue) + s.Require().NotEmpty(heldWorkflowTask.TaskToken) + + completedTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, childQueues["completed"]) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: completedTask, + Commands: []*commandpb.Command{completeWorkflowCommand()}, + }) + + failedTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, childQueues["failed"]) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: failedTask, + Commands: []*commandpb.Command{failWorkflowCommand("expected child failure")}, + }) + + s.requestWorkflowCancellationEventually(ctx, workflowCancellation{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: childIDs["canceled"]}, + }) + canceledTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, childQueues["canceled"]) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: canceledTask, + Commands: []*commandpb.Command{cancelWorkflowCommand()}, + }) + + s.terminateWorkflowEventually(ctx, workflowTermination{ + Namespace: ns, + Execution: &commonpb.WorkflowExecution{WorkflowId: childIDs["terminated"]}, + Reason: "expected child termination", + }) + + expectedTypes := []enumspb.EventType{ + enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_FAILED, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_CANCELED, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TIMED_OUT, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_TERMINATED, + } + s.assertBufferedEventTypesPresent(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: expectedTypes, + }) + s.finishNaturallyBufferedConflict(ctx, naturallyBufferedConflict{ + Namespace: namespace, + Execution: execution, + ReplicationToOldActive: replicationToOldActive, + ReplicationToNewActive: replicationToNewActive, + ExpectedEventTypes: expectedTypes, + WinnerSignal: "child-winner-signal", + }) + winningHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + for _, eventType := range expectedTypes { + s.Require().Zero(countBufferedEventType(winningHistory, eventType), "%s has no child initiation on the winning branch and must be skipped", eventType) + } +} + +// TestNaturallyBufferedExternalWorkflowOutcomesFlushedToLosingBranch buffers successful and failed +// signal-external and cancel-external results. It expects every result on the losing branch and none +// on the winner because the corresponding initiated commands do not exist there. +func (s *FunctionalClustersTestSuite) TestNaturallyBufferedExternalWorkflowOutcomesFlushedToLosingBranch() { + if !s.enableTransitionHistory { + s.T().Skip("buffered event state-based replication requires transition history") + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + workflowID := "buffered-external-xdc-" + uuid.NewString() + workflowQueue := &taskqueuepb.TaskQueue{Name: workflowID + "-source"} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: workflowQueue, + }) + firstTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, workflowQueue) + signalTargetID := workflowID + "-signal-target" + signalTargetQueue := &taskqueuepb.TaskQueue{Name: signalTargetID + "-queue"} + signalTargetExecution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: signalTargetID, + TaskQueue: signalTargetQueue, + }) + s.Require().NotEmpty(signalTargetExecution.RunId) + targetTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, signalTargetQueue) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: targetTask, + }) + cancelTargetID := workflowID + "-completed-cancel-target" + cancelTargetExecution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: cancelTargetID, + TaskQueue: &taskqueuepb.TaskQueue{Name: cancelTargetID + "-queue"}, + }) + s.terminateWorkflow(ctx, workflowTermination{ + Namespace: ns, + Execution: cancelTargetExecution, + Reason: "completed target makes external cancellation deterministic", + }) + + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.NotEmpty(t, history) + }, replicationWaitTime, replicationCheckInterval) + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + + missingWorkflowID := workflowID + "-missing" + missingRunID := uuid.NewString() + heldWorkflowTask := s.completeWorkflowTaskAndReturnNext(ctx, workflowTaskCompletion{ + Task: firstTask, + Commands: []*commandpb.Command{ + signalExternalWorkflowCommand(signalTargetID, "", "successful-external-signal"), + signalExternalWorkflowCommand(missingWorkflowID, missingRunID, "failed-external-signal"), + cancelExternalWorkflowCommand(missingWorkflowID, missingRunID), + cancelExternalWorkflowCommand(cancelTargetID, cancelTargetExecution.RunId), + }, + }) + s.Require().NotEmpty(heldWorkflowTask.TaskToken) + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 0, ns, signalTargetExecution) + require.True(t, hasSignalNamed(history, "successful-external-signal")) + }, replicationWaitTime, replicationCheckInterval) + + expectedTypes := []enumspb.EventType{ + enumspb.EVENT_TYPE_REQUEST_CANCEL_EXTERNAL_WORKFLOW_EXECUTION_FAILED, + enumspb.EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_FAILED, + enumspb.EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED, + enumspb.EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_CANCEL_REQUESTED, + } + s.assertBufferedEventTypesPresent(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: expectedTypes, + }) + s.finishNaturallyBufferedConflict(ctx, naturallyBufferedConflict{ + Namespace: namespace, + Execution: execution, + ReplicationToOldActive: replicationToOldActive, + ReplicationToNewActive: replicationToNewActive, + ExpectedEventTypes: expectedTypes, + WinnerSignal: "external-winner-signal", + }) + winningHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + for _, eventType := range expectedTypes { + s.Require().Zero(countBufferedEventType(winningHistory, eventType), "%s must remain only on the losing branch", eventType) + } +} diff --git a/tests/xdc/buffered_nexus_events_replication_test.go b/tests/xdc/buffered_nexus_events_replication_test.go new file mode 100644 index 00000000000..28477ba8f76 --- /dev/null +++ b/tests/xdc/buffered_nexus_events_replication_test.go @@ -0,0 +1,384 @@ +package xdc + +import ( + "context" + "time" + + "github.com/google/uuid" + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commandpb "go.temporal.io/api/command/v1" + enumspb "go.temporal.io/api/enums/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/api/workflowservice/v1" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexustest" + "go.temporal.io/server/common/testing/await" +) + +// TestBufferedNexusEventsReapplySharedOperationAndSkipLosingOnlyOperation buffers completion of an +// operation shared by both branches plus start and completion of an operation created only on the +// losing branch. It expects all three events on the loser, only the shared completion on the winner, +// and the losing-only operation skipped. This covers temporalio/temporal#10986. +func (s *NexusStateReplicationSuite) TestBufferedNexusEventsReapplySharedOperationAndSkipLosingOnlyOperation() { + if !s.enableTransitionHistory || s.chasmEnabled { + s.T().Skip("this conflict-reapplication regression is specific to transition-history HSM Nexus operations") + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + // Phase 1: establish identical history with one shared, started Nexus operation. + endpointName, operationCallbacks, allowLosingOnlyOperationStart := s.setupBufferedNexusEndpoint(ctx) + + workflowID := "buffered-nexus-conflict-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{Name: "buffered-nexus-conflict", Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: taskQueue, + }) + + firstWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: firstWorkflowTask, + Commands: []*commandpb.Command{scheduleBufferedNexusOperationCommand(endpointName, "shared-operation")}, + }) + sharedOperationCallback := receiveBufferedNexusCallback(ctx, s.T(), operationCallbacks, "shared-operation") + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 0, ns, execution) + sharedOperationScheduledEventID := findNexusScheduledEventID(history, "shared-operation") + require.Positive(t, sharedOperationScheduledEventID) + require.True(t, hasNexusEventForScheduledID(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, sharedOperationScheduledEventID)) + }, replicationWaitTime, replicationCheckInterval) + sharedOperationScheduledEventID := findNexusScheduledEventID(s.getWorkflowHistory(ctx, s.T(), 0, ns, execution), "shared-operation") + s.Require().Positive(sharedOperationScheduledEventID) + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + require.True(t, hasNexusEventForScheduledID(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, sharedOperationScheduledEventID)) + }, replicationWaitTime, replicationCheckInterval) + + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + + // Phase 2: hold a workflow task and buffer completions for shared and losing-only operations. + s.signalWorkflow(ctx, workflowSignal{ + Namespace: ns, + Execution: execution, + SignalName: "create-losing-operation", + }) + scheduleLosingOnlyOperationTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + Task: scheduleLosingOnlyOperationTask, + Commands: []*commandpb.Command{scheduleBufferedNexusOperationCommand(endpointName, "losing-only-operation")}, + }) + heldWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.Require().NotEmpty(heldWorkflowTask.TaskToken) + losingOnlyOperationScheduledEventID := findNexusScheduledEventID(heldWorkflowTask.History.Events, "losing-only-operation") + s.Require().Positive(losingOnlyOperationScheduledEventID) + close(allowLosingOnlyOperationStart) + losingOnlyOperationCallback := receiveBufferedNexusCallback(ctx, s.T(), operationCallbacks, "losing-only-operation") + + s.completeBufferedNexusOperation(ctx, bufferedNexusOperationCompletion{ + Callback: sharedOperationCallback, + Result: "shared-result", + }) + s.completeBufferedNexusOperation(ctx, bufferedNexusOperationCompletion{ + Callback: losingOnlyOperationCallback, + Result: "losing-result", + }) + s.assertBufferedEventTypes(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: []enumspb.EventType{ + enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, + enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, + enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, + }, + }) + + // Phase 3: fail over and create a winning branch on the new active cluster. + s.failoverToNewActiveCluster(ctx, ns) + s.writeSignalOnNewActive(ctx, activeClusterSignal{ + Namespace: namespace, + Execution: execution, + SignalName: "nexus-winner-signal", + }) + + // Phase 4: resolve the conflict and verify losing-branch storage versus reapplication. + s.releaseReplicationTask(ctx, replicationToOldActive) + s.assertNoBufferedEvents(ctx, 0, ns, execution) + losingHistory := s.findBufferedNexusLosingBranch(ctx, bufferedNexusLosingBranch{ + Namespace: namespace, + Execution: execution, + ScheduledEventID: losingOnlyOperationScheduledEventID, + }) + s.Require().True(hasNexusEventForScheduledID(losingHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, losingOnlyOperationScheduledEventID)) + s.Require().True(hasNexusEventForScheduledID(losingHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, losingOnlyOperationScheduledEventID)) + s.Require().Equal(2, countBufferedEventType(losingHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED)) + + for range 10 { + targetHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + if hasNexusEventForScheduledID(targetHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, sharedOperationScheduledEventID) { + break + } + s.releaseReplicationTask(ctx, replicationToNewActive) + } + targetHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + s.Require().True(hasNexusEventForScheduledID(targetHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, sharedOperationScheduledEventID)) + s.Require().False(hasNexusEventForScheduledID(targetHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED, losingOnlyOperationScheduledEventID)) + s.Require().False(hasNexusEventForScheduledID(targetHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED, losingOnlyOperationScheduledEventID)) + await.Require(ctx, s.T(), func(t *await.T) { + describeResponse, describeErr := s.clusters[1].FrontendClient().DescribeWorkflowExecution(t.Context(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: ns, + Execution: execution, + }) + require.NoError(t, describeErr) + require.Empty(t, describeResponse.PendingNexusOperations) + }, replicationWaitTime, replicationCheckInterval) + + for attempt := 0; attempt < 10 && !s.bufferedEventsHistoriesEqual(ctx, ns, execution); attempt++ { + s.releaseReplicationTask(ctx, replicationToOldActive) + } + await.Require(ctx, s.T(), func(t *await.T) { + require.True(t, s.bufferedEventsHistoriesEqual(t.Context(), ns, execution)) + }, replicationWaitTime, replicationCheckInterval) +} + +// TestNaturallyBufferedNexusOutcomesFlushedAndReapplied buffers failed, canceled, timed-out, and +// cancel-request-failed outcomes for operations shared by both branches. It expects the terminal operation +// outcomes to be reapplied to the winner and the non-cherry-pickable cancellation result to remain on the loser. +func (s *NexusStateReplicationSuite) TestNaturallyBufferedNexusOutcomesFlushedAndReapplied() { + if !s.enableTransitionHistory || s.chasmEnabled { + s.T().Skip("this conflict-reapplication regression is specific to transition-history HSM Nexus operations") + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + callbacks := make(chan bufferedNexusCallback, 4) + allowCancellationResponse := make(chan struct{}) + handler := nexustest.Handler{ + OnStartOperation: func(_ context.Context, _, operation string, _ *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + callbacks <- bufferedNexusCallback{ + operation: operation, + url: options.CallbackURL, + token: options.CallbackHeader.Get(commonnexus.CallbackTokenHeader), + } + return &nexus.HandlerStartOperationResultAsync{OperationToken: operation}, nil + }, + OnCancelOperation: func(_ context.Context, _, operation, _ string, _ nexus.CancelOperationOptions) error { + <-allowCancellationResponse + if operation == "cancel-failed" { + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "expected cancellation failure") + } + return nil + }, + } + endpointName := s.createBufferedNexusEndpoint(ctx, handler) + + workflowID := "buffered-nexus-outcomes-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{Name: workflowID, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: taskQueue, + }) + firstTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + operations := []string{"failed", "canceled", "timed-out", "cancel-failed"} + commands := make([]*commandpb.Command, 0, len(operations)) + for _, operation := range operations { + timeout := time.Minute + if operation == "timed-out" { + timeout = 5 * time.Second + } + commands = append(commands, scheduleBufferedNexusOperationCommandWithTimeout(endpointName, operation, timeout)) + } + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: firstTask, + Commands: commands, + }) + + operationCallbacks := make(map[string]bufferedNexusCallback, len(operations)) + for range operations { + callback := receiveAnyBufferedNexusCallback(ctx, s.T(), callbacks) + operationCallbacks[callback.operation] = callback + } + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 0, ns, execution) + for _, operation := range operations { + scheduledID := findNexusScheduledEventID(history, operation) + require.Positive(t, scheduledID) + require.True(t, hasNexusEventForScheduledID(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, scheduledID)) + } + }, replicationWaitTime, replicationCheckInterval) + s.waitForClusterSynced() + await.Require(ctx, s.T(), func(t *await.T) { + history := s.getWorkflowHistory(t.Context(), t.AssertionT(), 1, ns, execution) + for _, operation := range operations { + scheduledID := findNexusScheduledEventID(history, operation) + require.Positive(t, scheduledID) + require.True(t, hasNexusEventForScheduledID(history, enumspb.EVENT_TYPE_NEXUS_OPERATION_STARTED, scheduledID)) + } + }, replicationWaitTime, replicationCheckInterval) + await.Require(ctx, s.T(), func(t *await.T) { + response, describeErr := s.clusters[1].FrontendClient().DescribeWorkflowExecution(t.Context(), &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: ns, + Execution: execution, + }) + require.NoError(t, describeErr) + require.Len(t, response.PendingNexusOperations, len(operations)) + }, replicationWaitTime, replicationCheckInterval) + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + + triggerTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + history := triggerTask.History.Events + scheduledIDs := make(map[string]int64, len(operations)) + for _, operation := range operations { + scheduledIDs[operation] = findNexusScheduledEventID(history, operation) + s.Require().Positive(scheduledIDs[operation]) + } + s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + Task: triggerTask, + Commands: []*commandpb.Command{ + requestCancelNexusOperationCommand(findNexusScheduledEventID(history, "cancel-failed")), + }, + }) + heldTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.Require().NotEmpty(heldTask.TaskToken) + close(allowCancellationResponse) + + s.failNexusOperation(ctx, operationCallbacks["failed"]) + s.cancelBufferedNexusOperation(ctx, operationCallbacks["canceled"]) + expectedTypes := []enumspb.EventType{ + enumspb.EVENT_TYPE_NEXUS_OPERATION_FAILED, + enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCELED, + enumspb.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT, + enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_FAILED, + } + s.assertBufferedEventTypesPresent(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: expectedTypes, + }) + + losingHistory := s.finishNaturallyBufferedConflict(ctx, naturallyBufferedConflict{ + Namespace: namespace, + Execution: execution, + ReplicationToOldActive: replicationToOldActive, + ReplicationToNewActive: replicationToNewActive, + ExpectedEventTypes: expectedTypes, + WinnerSignal: "nexus-outcomes-winner-signal", + }) + for _, eventType := range expectedTypes { + s.Require().NotNil(findHistoryEvent(losingHistory, eventType, nil)) + } + winningHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + for operation, eventType := range map[string]enumspb.EventType{ + "failed": enumspb.EVENT_TYPE_NEXUS_OPERATION_FAILED, + "canceled": enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCELED, + "timed-out": enumspb.EVENT_TYPE_NEXUS_OPERATION_TIMED_OUT, + } { + s.Require().True( + hasNexusEventForScheduledID(winningHistory, eventType, scheduledIDs[operation]), + "%s for common operation %q must be reapplied to the winning branch", + eventType, + operation, + ) + } + s.Require().False( + hasNexusEventForScheduledID(winningHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_FAILED, scheduledIDs["cancel-failed"]), + "Nexus cancellation results are not cherry-pickable and must remain only on the losing branch", + ) +} + +// TestNaturallyBufferedNexusCancelRequestCompletedFlushedAndReapplied buffers a successful Nexus +// cancel-request result. It expects the result to be persisted on the losing branch but skipped on the +// winner because cancellation results are not cherry-pickable. +func (s *NexusStateReplicationSuite) TestNaturallyBufferedNexusCancelRequestCompletedFlushedAndReapplied() { + if !s.enableTransitionHistory || s.chasmEnabled { + s.T().Skip("this conflict-reapplication regression is specific to transition-history HSM Nexus operations") + } + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + namespace := s.createBufferedEventsNamespace(ctx) + ns := namespace.Name + + started := make(chan struct{}, 1) + canceled := make(chan struct{}, 1) + allowCancellationResponse := make(chan struct{}) + handler := nexustest.Handler{ + OnStartOperation: func(_ context.Context, _, operation string, _ *nexus.LazyValue, _ nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + started <- struct{}{} + return &nexus.HandlerStartOperationResultAsync{OperationToken: operation}, nil + }, + OnCancelOperation: func(_ context.Context, _, _, _ string, _ nexus.CancelOperationOptions) error { + canceled <- struct{}{} + <-allowCancellationResponse + return nil + }, + } + endpointName := s.createBufferedNexusEndpoint(ctx, handler) + workflowID := "buffered-nexus-cancel-completed-" + uuid.NewString() + taskQueue := &taskqueuepb.TaskQueue{Name: workflowID, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + execution := s.startBufferedEventsWorkflow(ctx, startBufferedEventsWorkflowArgs{ + Namespace: ns, + WorkflowID: workflowID, + TaskQueue: taskQueue, + }) + firstTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.completeWorkflowTask(ctx, workflowTaskCompletion{ + Task: firstTask, + Commands: []*commandpb.Command{scheduleBufferedNexusOperationCommand(endpointName, "cancel-completed")}, + }) + select { + case <-started: + case <-ctx.Done(): + s.FailNow("timed out waiting for Nexus operation to start") + } + triggerTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + scheduledID := findNexusScheduledEventID(triggerTask.History.Events, "cancel-completed") + s.Require().Positive(scheduledID) + s.waitForClusterSynced() + replicationToOldActive := s.blockReplicationForWorkflow(0, workflowID) + replicationToNewActive := s.blockReplicationForWorkflow(1, workflowID) + s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + Task: triggerTask, + Commands: []*commandpb.Command{requestCancelNexusOperationCommand(scheduledID)}, + }) + heldTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, taskQueue) + s.Require().NotEmpty(heldTask.TaskToken) + close(allowCancellationResponse) + select { + case <-canceled: + case <-ctx.Done(): + s.FailNow("timed out waiting for Nexus cancellation handler") + } + expectedTypes := []enumspb.EventType{enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_COMPLETED} + s.assertBufferedEventTypesPresent(ctx, bufferedEventExpectation{ + Namespace: ns, + Execution: execution, + EventTypes: expectedTypes, + }) + + s.finishNaturallyBufferedConflict(ctx, naturallyBufferedConflict{ + Namespace: namespace, + Execution: execution, + ReplicationToOldActive: replicationToOldActive, + ReplicationToNewActive: replicationToNewActive, + ExpectedEventTypes: expectedTypes, + WinnerSignal: "nexus-cancel-completed-winner-signal", + }) + winningHistory := s.getWorkflowHistory(ctx, s.T(), 1, ns, execution) + s.Require().False( + hasNexusEventForScheduledID(winningHistory, enumspb.EVENT_TYPE_NEXUS_OPERATION_CANCEL_REQUEST_COMPLETED, scheduledID), + "Nexus cancellation results are not cherry-pickable and must remain only on the losing branch", + ) +} From c9fd978d45f2f482dbad60d925f66f2ec20f6f20 Mon Sep 17 00:00:00 2001 From: David Porter Date: Thu, 27 Aug 2026 08:26:51 -0700 Subject: [PATCH 077/139] Fix: Schedule scanner false-positives for inactive NS (#11703) ## What This fixes the Schedule invariant scanner's false-positives coming from replication. Due to carelessness it was firing on the passive side because I forgot to filter this out, and for a while during post-replication disconnection, the task processing will cease. Also adds a small check for Described schedules to filter out visibility drift. ## How - Adds a guard for only checking active NS - Adds a describe check for the next fire time, so that visibility-delayed schedules are excluded ## Risks: - That I make a mistake and break the scanner --------- Co-authored-by: Claude Opus 5 (1M context) --- common/metrics/metric_defs.go | 33 ++-- .../scanner/scheduleinvariants/invariants.go | 35 ++++ .../scheduleinvariants/invariants_test.go | 183 +++++++++++++++++- 3 files changed, 227 insertions(+), 24 deletions(-) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index f1b47c5eed2..ab7d3b1b411 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1437,22 +1437,23 @@ var ( HistoryWorkflowExecutionCacheLatency = NewTimerDef("history_workflow_execution_cache_latency") HistoryWorkflowExecutionCacheLockHoldDuration = NewTimerDef("history_workflow_execution_cache_lock_hold_duration") - VisibilityArchiverArchiveNonRetryableErrorCount = NewCounterDef("visibility_archiver_archive_non_retryable_error") - VisibilityArchiverArchiveTransientErrorCount = NewCounterDef("visibility_archiver_archive_transient_error") - VisibilityArchiveSuccessCount = NewCounterDef("visibility_archiver_archive_success") - HistoryScavengerSuccessCount = NewCounterDef("scavenger_success") - HistoryScavengerErrorCount = NewCounterDef("scavenger_errors") - HistoryScavengerSkipCount = NewCounterDef("scavenger_skips") - ScheduleInvariantsScannerOverdueNextActionTimeCount = NewCounterDef("schedule_invariants_scanner_overdue_next_action_time") - ScheduleInvariantsScannerOverdueNextActionTimeCapHitCount = NewCounterDef("schedule_invariants_scanner_overdue_next_action_time_cap_hit") - ScheduleInvariantsScannerStuckOpenCount = NewCounterDef("schedule_invariants_scanner_stuck_open") - ScheduleInvariantsScannerUnknownStateCount = NewCounterDef("schedule_invariants_scanner_unknown_state") - ScheduleInvariantsScannerErrorCount = NewCounterDef("schedule_invariants_scanner_errors") - ExecutionsOutstandingCount = NewGaugeDef("executions_outstanding") - ScavengerValidationRequestsCount = NewCounterDef("scavenger_validation_requests") - ScavengerValidationFailuresCount = NewCounterDef("scavenger_validation_failures") - ScavengerValidationSkipsCount = NewCounterDef("scavenger_validation_skips") - AddSearchAttributesFailuresCount = NewCounterDef("add_search_attributes_failures") + VisibilityArchiverArchiveNonRetryableErrorCount = NewCounterDef("visibility_archiver_archive_non_retryable_error") + VisibilityArchiverArchiveTransientErrorCount = NewCounterDef("visibility_archiver_archive_transient_error") + VisibilityArchiveSuccessCount = NewCounterDef("visibility_archiver_archive_success") + HistoryScavengerSuccessCount = NewCounterDef("scavenger_success") + HistoryScavengerErrorCount = NewCounterDef("scavenger_errors") + HistoryScavengerSkipCount = NewCounterDef("scavenger_skips") + ScheduleInvariantsScannerOverdueNextActionTimeCount = NewCounterDef("schedule_invariants_scanner_overdue_next_action_time") + ScheduleInvariantsScannerOverdueNextActionTimeCapHitCount = NewCounterDef("schedule_invariants_scanner_overdue_next_action_time_cap_hit") + ScheduleInvariantsScannerOverdueNextActionTimeStaleCandidateCount = NewCounterDef("schedule_invariants_scanner_overdue_next_action_time_stale_candidate") + ScheduleInvariantsScannerStuckOpenCount = NewCounterDef("schedule_invariants_scanner_stuck_open") + ScheduleInvariantsScannerUnknownStateCount = NewCounterDef("schedule_invariants_scanner_unknown_state") + ScheduleInvariantsScannerErrorCount = NewCounterDef("schedule_invariants_scanner_errors") + ExecutionsOutstandingCount = NewGaugeDef("executions_outstanding") + ScavengerValidationRequestsCount = NewCounterDef("scavenger_validation_requests") + ScavengerValidationFailuresCount = NewCounterDef("scavenger_validation_failures") + ScavengerValidationSkipsCount = NewCounterDef("scavenger_validation_skips") + AddSearchAttributesFailuresCount = NewCounterDef("add_search_attributes_failures") // Delete Namespace metrics. ReclaimResourcesNamespaceDeleteSuccessCount = NewCounterDef( diff --git a/service/worker/scanner/scheduleinvariants/invariants.go b/service/worker/scanner/scheduleinvariants/invariants.go index efbc8d9d2c7..7dae97dd5f8 100644 --- a/service/worker/scanner/scheduleinvariants/invariants.go +++ b/service/worker/scanner/scheduleinvariants/invariants.go @@ -22,6 +22,7 @@ import ( "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/quotas" "go.temporal.io/server/common/sdk" + "google.golang.org/protobuf/types/known/timestamppb" ) const ( @@ -251,6 +252,10 @@ func (a *Activities) ListAllNamespaces() []string { if ns.State() == enumspb.NAMESPACE_STATE_DELETED { continue } + //nolint:forbidigo // scanner works per namespace, not per workflow. + if !ns.ActiveInCluster(a.currentClusterName) { + continue + } names = append(names, ns.Name().String()) } return names @@ -371,9 +376,39 @@ func (a *Activities) scheduleIsExpectedNotToFire(ctx context.Context, nsName, sc len(desc.GetInfo().GetRunningWorkflows()) > 0 { return true } + + // Confirm the invariant against what Describe just returned: the candidate came from + // a ScheduleNextActionTime index entry, which goes stale on replication or indexing + // lag. + if !a.nextActionTimeIsOverdue(desc.GetInfo().GetFutureActionTimes()) { + a.logger.Info("overdue candidate was not overdue on re-check; visibility entry was stale", + tag.WorkflowNamespace(nsName), tag.ScheduleID(scheduleID)) + metrics.ScheduleInvariantsScannerOverdueNextActionTimeStaleCandidateCount.With( + a.metricsHandler.WithTags(metrics.NamespaceTag(nsName))).Record(1) + return true + } return false } +// nextActionTimeIsOverdue applies the candidate query's predicate to futureActionTimes. +func (a *Activities) nextActionTimeIsOverdue(futureActionTimes []*timestamppb.Timestamp) bool { + var earliest time.Time + var found bool + for _, t := range futureActionTimes { + if t == nil { + continue + } + // Ordered soonest-first, but don't rely on it. + if ts := t.AsTime(); !found || ts.Before(earliest) { + earliest, found = ts, true + } + } + if !found { + return false + } + return earliest.Before(a.timeSource.Now().UTC().Add(-a.opts().OverdueNextActionTimeTolerance)) +} + func (a *Activities) emitCount(metricName, namespaceTagValue string, count int64) { if count <= 0 { return diff --git a/service/worker/scanner/scheduleinvariants/invariants_test.go b/service/worker/scanner/scheduleinvariants/invariants_test.go index 630174121e0..f1d60ecc8af 100644 --- a/service/worker/scanner/scheduleinvariants/invariants_test.go +++ b/service/worker/scanner/scheduleinvariants/invariants_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" @@ -18,6 +19,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/quotas" @@ -25,10 +27,14 @@ import ( "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" "go.temporal.io/server/common/testing/mocksdk" "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" ) const testClusterName = "test-cluster" +// testNow anchors the injected clock; the overdue re-check reads timeSource.Now(). +var testNow = time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) + type testDeps struct { ctrl *gomock.Controller visibilityManager *manager.MockVisibilityManager @@ -37,6 +43,7 @@ type testDeps struct { sdkClient *mocksdk.MockClient frontendClient *workflowservicemock.MockWorkflowServiceClient timeSource *clock.EventTimeSource + metricsHandler metrics.Handler } func newTestDeps(t *testing.T) *testDeps { @@ -50,7 +57,9 @@ func newTestDeps(t *testing.T) *testDeps { sdkClient: mocksdk.NewMockClient(ctrl), frontendClient: workflowservicemock.NewMockWorkflowServiceClient(ctrl), timeSource: clock.NewEventTimeSource(), + metricsHandler: metrics.NoopMetricsHandler, } + d.timeSource.Update(testNow) // The DescribeSchedule path always goes via system client → frontend stub. d.sdkClientFactory.EXPECT().GetSystemClient().Return(d.sdkClient).AnyTimes() d.sdkClient.EXPECT().WorkflowService().Return(d.frontendClient).AnyTimes() @@ -66,7 +75,7 @@ func (d *testDeps) newActivitiesWithParams(params dynamicconfig.ScheduleInvarian rl := quotas.NewDefaultOutgoingRateLimiter(quotas.RateFn(dynamicconfig.GetFloatPropertyFn(10000.0))) return &Activities{ logger: log.NewNoopLogger(), - metricsHandler: metrics.NoopMetricsHandler, + metricsHandler: d.metricsHandler, visibilityManager: d.visibilityManager, namespaceRegistry: d.namespaceRegistry, sdkClientFactory: d.sdkClientFactory, @@ -120,7 +129,9 @@ func TestListAllNamespaces_FiltersInactiveAndDeleted(t *testing.T) { }) names := d.newActivities().ListAllNamespaces() - require.ElementsMatch(t, []string{"ns-1", "ns-2", "ns-3"}, names) + require.ElementsMatch(t, []string{"ns-1", "ns-3"}, names, + "ns-2 is active in another cluster: evaluating its invariants here would read a "+ + "standby replica's stale visibility records") } func TestForEachNamespace_InvokesCallbackWithCount(t *testing.T) { @@ -220,7 +231,25 @@ func TestSchedulesInNamespace_YieldsErrorAndStops(t *testing.T) { require.Error(t, iterErr) } -func describeResp(paused bool, overlap enumspb.ScheduleOverlapPolicy, runningCount int) *workflowservice.DescribeScheduleResponse { +var overdueTolerance = dynamicconfig.DefaultScheduleInvariantsScannerParams.OverdueNextActionTimeTolerance + +// overdueActionTime confirms the invariant; pendingActionTime clears it on re-check. +func overdueActionTime() time.Time { + return testNow.Add(-overdueTolerance).Add(-time.Hour) +} + +func pendingActionTime() time.Time { + return testNow.Add(time.Hour) +} + +// describeResp builds a DescribeSchedule response. Passing no futureActionTimes models +// a schedule with no upcoming action. +func describeResp( + paused bool, + overlap enumspb.ScheduleOverlapPolicy, + runningCount int, + futureActionTimes ...time.Time, +) *workflowservice.DescribeScheduleResponse { resp := &workflowservice.DescribeScheduleResponse{ Schedule: &schedulepb.Schedule{ State: &schedulepb.ScheduleState{Paused: paused}, @@ -231,6 +260,9 @@ func describeResp(paused bool, overlap enumspb.ScheduleOverlapPolicy, runningCou for range runningCount { resp.Info.RunningWorkflows = append(resp.Info.RunningWorkflows, &commonpb.WorkflowExecution{WorkflowId: "running"}) } + for _, t := range futureActionTimes { + resp.Info.FutureActionTimes = append(resp.Info.FutureActionTimes, timestamppb.New(t)) + } return resp } @@ -258,17 +290,17 @@ func TestScheduleIsExpectedNotToFire(t *testing.T) { }, { name: "buffer_one_no_running_workflow", - resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, 0), + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, 0, overdueActionTime()), want: false, }, { - name: "skip_policy_with_running_workflow", - resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 1), + name: "skip_policy_with_running_workflow_still_overdue", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 1, overdueActionTime()), want: false, }, { name: "cancel_other_policy", - resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, 1), + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, 1, overdueActionTime()), want: false, }, { @@ -276,6 +308,69 @@ func TestScheduleIsExpectedNotToFire(t *testing.T) { err: errors.New("describe failed"), want: false, }, + { + // Stale index entry: a standby's frozen record, or a SKIP schedule whose + // action overran while the Generator kept ticking. + name: "next_action_time_still_pending", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 1, pendingActionTime()), + want: true, + }, + { + // Nothing pending can be late. + name: "no_future_action_times", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0), + want: true, + }, + { + name: "next_action_time_exactly_at_threshold", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, testNow.Add(-overdueTolerance)), + want: true, + }, + { + name: "next_action_time_just_past_threshold", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, + testNow.Add(-overdueTolerance).Add(-time.Nanosecond)), + want: false, + }, + { + // The stalled-generator shape: visibility indexes FutureActionTimes[0], + // which is overdue, while the rest of the cached horizon is still future. + // Requiring every entry to be overdue would delay detection by the full + // cache depth. + name: "only_earliest_entry_overdue", + resp: func() *workflowservice.DescribeScheduleResponse { + times := []time.Time{overdueActionTime()} + for i := range 9 { + times = append(times, testNow.Add(time.Duration(i+1)*time.Hour)) + } + return describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, times...) + }(), + want: false, + }, + { + // Ordering isn't guaranteed: the earliest entry decides, wherever it sits. + name: "unordered_earliest_is_overdue", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, + pendingActionTime(), overdueActionTime()), + want: false, + }, + { + // Stale index entry: every cached time is still in the future. + name: "all_entries_pending", + resp: describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, + pendingActionTime(), pendingActionTime().Add(time.Hour)), + want: true, + }, + { + // A nil entry must not read as the zero time, which would look overdue. + name: "nil_entry_among_pending_times", + resp: func() *workflowservice.DescribeScheduleResponse { + r := describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, pendingActionTime()) + r.Info.FutureActionTimes = append([]*timestamppb.Timestamp{nil}, r.Info.FutureActionTimes...) + return r + }(), + want: true, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -314,10 +409,82 @@ func TestRunOverdueScan_FiltersExpectedNotToFireSchedulesAndCountsRest(t *testin }).Return(describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, 1), nil) d.frontendClient.EXPECT().DescribeSchedule(gomock.Any(), &workflowservice.DescribeScheduleRequest{ Namespace: "ns-1", ScheduleId: "sched-actually-overdue", - }).Return(describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0), nil) + }).Return(describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 0, overdueActionTime()), nil) + + rec := metricstest.NewCaptureHandler() + d.metricsHandler = rec + capture := rec.StartCapture() + defer rec.StopCapture(capture) err := d.newActivities().runOverdueScan(context.Background(), "q") require.NoError(t, err) + + snapshot := capture.Snapshot() + anomalies := snapshot[metrics.ScheduleInvariantsScannerOverdueNextActionTimeCount.Name()] + require.Len(t, anomalies, 1) + require.Equal(t, int64(1), anomalies[0].Value, "only sched-actually-overdue should count") + require.Empty(t, snapshot[metrics.ScheduleInvariantsScannerOverdueNextActionTimeStaleCandidateCount.Name()], + "paused and buffer-waiting are exemptions, not stale candidates") +} + +// Asserts by absence: with no expectations registered, any call for ns-passive fails. +func TestRunOverdueScan_SkipsNamespaceActiveInAnotherCluster(t *testing.T) { + d := newTestDeps(t) + + d.namespaceRegistry.EXPECT().GetAllNamespaces().Return([]*namespace.Namespace{ + globalNS("id-passive", "ns-passive", "other-cluster"), + }) + + err := d.newActivities().runOverdueScan(context.Background(), "q") + require.NoError(t, err) +} + +// Same gate for the count-only scanners, which have no confirmation step at all. +func TestRunScan_SkipsNamespaceActiveInAnotherCluster(t *testing.T) { + d := newTestDeps(t) + + d.namespaceRegistry.EXPECT().GetAllNamespaces().Return([]*namespace.Namespace{ + globalNS("id-passive", "ns-passive", "other-cluster"), + localNS("id-local", "ns-local", testClusterName), + }) + // Only the local namespace is queried. + d.namespaceRegistry.EXPECT().GetNamespaceID(namespace.Name("ns-local")).Return(namespace.ID("id-local"), nil) + d.visibilityManager.EXPECT().CountChasmExecutions(gomock.Any(), gomock.Any()). + DoAndReturn(func(_ context.Context, req *visibilityservice.CountChasmExecutionsRequest) (*visibilityservice.CountChasmExecutionsResponse, error) { + require.Equal(t, "ns-local", req.Namespace) + return &visibilityservice.CountChasmExecutionsResponse{Count: 3}, nil + }) + + err := d.newActivities().runScan(context.Background(), "stuck_open", "q", "some_metric") + require.NoError(t, err) +} + +// A stale candidate is not an anomaly, but must still be counted so the suppression +// is observable. +func TestRunOverdueScan_StaleCandidateIsCountedSeparatelyNotAsAnomaly(t *testing.T) { + d := newTestDeps(t) + + d.namespaceRegistry.EXPECT().GetAllNamespaces().Return([]*namespace.Namespace{localNS("id-1", "ns-1", testClusterName)}) + d.namespaceRegistry.EXPECT().GetNamespaceID(namespace.Name("ns-1")).Return(namespace.ID("id-1"), nil) + d.visibilityManager.EXPECT().ListChasmExecutions(gomock.Any(), gomock.Any()).Return(&visibilityservice.ListChasmExecutionsResponse{ + Executions: []*chasmspb.VisibilityExecutionInfo{chasmExec("sched-stale")}, + }, nil) + d.frontendClient.EXPECT().DescribeSchedule(gomock.Any(), gomock.Any()). + Return(describeResp(false, enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, 1, pendingActionTime()), nil) + + rec := metricstest.NewCaptureHandler() + d.metricsHandler = rec + capture := rec.StartCapture() + defer rec.StopCapture(capture) + + require.NoError(t, d.newActivities().runOverdueScan(context.Background(), "q")) + + snapshot := capture.Snapshot() + require.Empty(t, snapshot[metrics.ScheduleInvariantsScannerOverdueNextActionTimeCount.Name()], + "a stale visibility entry is not an anomaly") + stale := snapshot[metrics.ScheduleInvariantsScannerOverdueNextActionTimeStaleCandidateCount.Name()] + require.Len(t, stale, 1) + require.Equal(t, "ns-1", stale[0].Tags["namespace"]) } func TestRunOverdueScan_ContinuesPastPerNamespaceErrors(t *testing.T) { From 15f3532ea131ebf5968e5ed16b432eaf2338057e Mon Sep 17 00:00:00 2001 From: Shivam <57200924+Shivs11@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:38:22 -0400 Subject: [PATCH 078/139] Add Worker Deployment and BuildID labels to (workflow,activity) task completion metrics (#11348) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed - Add `worker_deployment_name` and `worker_build_id` labels to Workflow Task completion and failure metrics. - Add the split labels to Activity success, failure, cancellation, timeout, and completion-latency metrics. - Add the split labels to Workflow Task and Activity schedule-to-start latency and poll-time task-dispatch latency. - Continue emitting the existing combined `worker_version` label on task-dispatch latency for compatibility. - Reuse `metrics.breakdownByBuildID` as the task-queue-scoped cardinality gate for the new labels. ## Which metrics changed Every metric below now emits both `worker_deployment_name` and `worker_build_id` when `metrics.breakdownByBuildID` is enabled and deployment attribution is available. When the gate is disabled or attribution is unavailable—including unversioned tasks and timeouts before a worker starts—both labels remain present with empty values. ### Workflow Task outcomes - `workflow_tasks_completed` - `failed_workflow_tasks` ### Activity outcomes - `activity_success` - `activity_fail` - `activity_task_fail` - `activity_cancel` - `activity_timeout` - `activity_task_timeout` ### Activity latency - `activity_end_to_end_latency` (deprecated; use `activity_start_to_close_latency` instead) - `activity_start_to_close_latency` - `activity_schedule_to_close_latency` ### Task-routing latency - `task_schedule_to_start_latency` - `task_dispatch_latency` (continues to emit the existing combined `worker_version` label as well) ## Why - To improve user-experience for worker-versioning by having more insights - Worker Deployment name and build ID need to be independently filterable. Emitting them separately avoids requiring consumers to parse the combined `worker_version` value. This PR preserves `worker_version` on `task_dispatch_latency` for compatibility and does not remove or deprecate it. ## How each metric was tested The functional tests run real versioned Workflow and Activity workers with `metrics.breakdownByBuildID` enabled. They assert that emitted server metrics contain the actual Worker Deployment name and build ID—not merely that the label keys exist. | Functional scenario | Metrics verified | What is asserted | |---|---|---| | Workflow and Activity task dispatch | `task_schedule_to_start_latency`, `task_dispatch_latency` | Both Workflow and Activity task series contain real deployment/build values when enabled. Disabled and unversioned cases emit empty values. | | Successful Workflow and Activity | `workflow_tasks_completed`, `activity_success`, `activity_end_to_end_latency`, `activity_start_to_close_latency`, `activity_schedule_to_close_latency` | Successful completion series contain real deployment/build values. | | Terminal Activity failure | `activity_task_fail`, `activity_fail`, `activity_end_to_end_latency`, `activity_start_to_close_latency`, `activity_schedule_to_close_latency` | Both the failed attempt and terminal-failure series retain real worker attribution. | | Activity cancellation | `activity_cancel` | The cancellation series is attributed to the worker that started the Activity. | | Terminal Activity timeout | `activity_task_timeout`, `activity_timeout` | Both attempt-level and terminal timeout series retain the started worker’s deployment/build values. | | Failed Workflow Task | `failed_workflow_tasks` | A real versioned Workflow Task is polled and failed; the resulting series contains the poller’s deployment/build values. | ### Local end-to-end validation The PR server was also run locally with the Worker Deployment versioning canary and a `bench-go` workload. Prometheus was scraped directly to verify that: - real `worker_deployment_name` and `worker_build_id` values were emitted; - versioned and unversioned series were both accepted without inconsistent-label errors; - `activity_success`, `activity_task_fail`, `activity_end_to_end_latency`, `activity_start_to_close_latency`, and `activity_schedule_to_close_latency` carried the expected real deployment/build values. The remaining failure, cancellation, timeout, and failed-Workflow-Task paths are covered by the functional tests above. NEW: Also tested each of these 13 metric families in a cloud test cell with sample metrics pasted here: https://grafana.tmprl-internal.cloud/d/shdb6gf/new-dashboard?orgId=1&from=2026-08-27T00:00:00.000Z&to=2026-08-27T23:59:59.000Z&timezone=utc --- > [!NOTE] > **Medium Risk** > Broad metrics surface area across History and Matching with cardinality gated by config; deployment attribution on timeouts and eager starts changes which tag values appear on existing metric names. > > **Overview** > Adds **`worker_deployment_name`** and **`worker_build_id`** to History and Matching metrics so worker versioning can be filtered without parsing the combined **`worker_version`** label. **`metrics.breakdownByBuildID`** (task-queue scoped) controls whether those tags carry real values or empty strings; **`task_dispatch_latency`** still emits **`worker_version`** for compatibility. > > **Workflow tasks:** Completion and failure counters (`workflow_tasks_completed`, `failed_workflow_tasks`) now go through shared helpers that attach versioning behavior plus deployment tags from the poller’s **`DeploymentOptions`**. > > **Workflow activities:** Success, failure, cancel, timeout, and related latency metrics get the same split labels via **`VersioningMetricContext`** on respond paths (from request deployment options), timer-driven timeouts (from **`LastDeploymentVersion`** when an attempt had started), and schedule-to-start latency on **`RecordActivityTaskStarted`**. Eager activities started during WFT completion now record the completing worker’s deployment on the started event. > > **Standalone activities (CHASM):** Completion/timeout/cancel paths use **`completionMetricsHandler`**, which always includes the deployment label keys with empty values until standalone versioning exists—keeping Prometheus label-set parity with workflow-embedded activities. > > **Other:** **`AddActivityTaskStartedEvent`** clears stored deployment when the poller is unversioned; Matching **`task_dispatch_latency`** adds the new tags alongside **`worker_version`**. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 1c60b05f0634d721ddb90877e97bef7f52cfaec2. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --- chasm/lib/activity/activity.go | 6 +- chasm/lib/activity/metrics.go | 18 +- chasm/lib/activity/metrics_test.go | 87 +++++ chasm/lib/activity/operator_commands.go | 2 +- chasm/lib/activity/tasks.go | 8 +- common/dynamicconfig/constants.go | 11 +- .../api/recordactivitytaskstarted/api.go | 14 +- .../api/recordworkflowtaskstarted/api.go | 11 +- .../api/respondactivitytaskcanceled/api.go | 8 +- .../api/respondactivitytaskcompleted/api.go | 6 +- .../api/respondactivitytaskfailed/api.go | 8 +- .../api/respondactivitytaskfailed/api_test.go | 30 ++ .../api/respondworkflowtaskcompleted/api.go | 25 +- .../respondworkflowtaskcompleted/api_test.go | 145 +++++--- .../workflow_task_completed_handler.go | 6 +- .../workflow_task_completed_handler_test.go | 6 +- .../api/respondworkflowtaskfailed/api.go | 23 +- service/history/configs/config.go | 2 + .../timer_queue_active_task_executor.go | 16 +- .../timer_queue_active_task_executor_test.go | 58 ++- service/history/workflow/metrics.go | 98 ++++- service/history/workflow/metrics_test.go | 221 ++++++++++- .../history/workflow/mutable_state_impl.go | 5 +- .../workflow/mutable_state_impl_test.go | 9 +- .../workflow/workflow_task_state_machine.go | 19 +- service/matching/matching_engine.go | 8 +- tests/activity_metrics_parity_test.go | 50 ++- tests/versioning_3_test.go | 40 ++ tests/worker_deployment_version_test.go | 347 ++++++++++++++++++ 29 files changed, 1154 insertions(+), 133 deletions(-) create mode 100644 chasm/lib/activity/metrics_test.go diff --git a/chasm/lib/activity/activity.go b/chasm/lib/activity/activity.go index 04a65799714..9360fbe479a 100644 --- a/chasm/lib/activity/activity.go +++ b/chasm/lib/activity/activity.go @@ -463,7 +463,7 @@ func (a *Activity) HandleCompleted( } baseHandler := a.baseMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCompletedScope) - enrichedHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCompletedScope) + enrichedHandler := a.completionMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCompletedScope) if err := TransitionCompleted.Apply(a, ctx, completeEvent{ req: event.Request, @@ -487,7 +487,7 @@ func (a *Activity) HandleFailed( } baseHandler := a.baseMetricsHandler(ctx, metrics.HistoryRespondActivityTaskFailedScope) - enrichedHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskFailedScope) + enrichedHandler := a.completionMetricsHandler(ctx, metrics.HistoryRespondActivityTaskFailedScope) failedRequest := event.Request.GetFailedRequest() failure := failedRequest.GetFailure() @@ -539,7 +539,7 @@ func (a *Activity) HandleCanceled( return nil, consts.ErrActivityTaskNotCancelRequested } - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) + metricsHandler := a.completionMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) if err := TransitionCanceled.Apply(a, ctx, cancelEvent{ details: event.Request.GetCancelRequest().GetDetails(), diff --git a/chasm/lib/activity/metrics.go b/chasm/lib/activity/metrics.go index 9b979346d4a..84ae751a471 100644 --- a/chasm/lib/activity/metrics.go +++ b/chasm/lib/activity/metrics.go @@ -18,14 +18,14 @@ func (a *Activity) taskScheduleToStartMetricsHandler(ctx chasm.Context) metrics. namespaceEntry := ctx.NamespaceEntry() namespaceName := namespaceEntry.Name().String() taskQueue := a.GetTaskQueue().GetName() - return metrics.GetPerTaskQueuePartitionTypeScope( + return withUnversionedWorkerDeploymentMetricTags(metrics.GetPerTaskQueuePartitionTypeScope( a.baseMetricsHandler(ctx, metrics.HistoryRecordActivityTaskStartedScope), namespaceName, tqid.UnsafeTaskQueueFamily(namespaceEntry.ID().String(), taskQueue). TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY). RootPartition(), actCtx.config.BreakdownMetricsByTaskQueue(namespaceName, taskQueue, enumspb.TASK_QUEUE_TYPE_ACTIVITY), - ) + )) } // baseMetricsHandler adds only the operation tag. @@ -53,6 +53,20 @@ func (a *Activity) enrichedMetricsHandler(ctx chasm.Context, operation string) m ) } +func (a *Activity) completionMetricsHandler(ctx chasm.Context, operation string) metrics.Handler { + return withUnversionedWorkerDeploymentMetricTags(a.enrichedMetricsHandler(ctx, operation)) +} + +func withUnversionedWorkerDeploymentMetricTags(handler metrics.Handler) metrics.Handler { + // TODO: Populate these labels when Standalone Activities support Worker Versioning. + // Until then, preserve label-key parity with Workflow Activity metrics so Prometheus does + // not reject samples emitted under the same metric names. + return handler.WithTags( + metrics.WorkerDeploymentNameTag("", false), + metrics.WorkerDeploymentBuildIDTag("", false), + ) +} + func (a *Activity) emitOnAttemptTimedOutMetrics(metricsHandler metrics.Handler, timeoutType enumspb.TimeoutType) { timeoutTag := metrics.StringTag("timeout_type", timeoutType.String()) metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) diff --git a/chasm/lib/activity/metrics_test.go b/chasm/lib/activity/metrics_test.go new file mode 100644 index 00000000000..e2ca367c5c2 --- /dev/null +++ b/chasm/lib/activity/metrics_test.go @@ -0,0 +1,87 @@ +package activity + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestCompletionMetricsWorkerDeploymentLabelKeyParity(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleMetricsHandler: func() metrics.Handler { return metricsHandler }, + HandleNamespaceEntry: testNamespaceEntry, + GoCtx: context.WithValue(context.Background(), ctxKeyActivityContext, &activityContext{ + config: &Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + }, + }), + }, + } + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + ScheduleTime: timestamppb.New(defaultTime), + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + }, + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + StartedTime: timestamppb.New(defaultTime), + }), + } + + baseHandler := activity.baseMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCompletedScope) + completionHandler := activity.completionMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCompletedScope) + activity.emitOnCompletedMetrics(ctx, baseHandler, completionHandler, nil, true) + activity.emitOnFailedMetrics(ctx, baseHandler, completionHandler, &failurepb.Failure{}) + activity.emitOnCanceledMetrics(ctx, completionHandler, activitypb.ACTIVITY_EXECUTION_STATUS_STARTED) + activity.emitOnTimedOutMetrics( + completionHandler, + enumspb.TIMEOUT_TYPE_START_TO_CLOSE, + ) + + expectedTags := []metrics.Tag{ + metrics.WorkerDeploymentNameTag("", false), + metrics.WorkerDeploymentBuildIDTag("", false), + } + for _, metricName := range []string{ + metrics.ActivitySuccess.Name(), + metrics.ActivityFail.Name(), + metrics.ActivityTaskFail.Name(), + metrics.ActivityCancel.Name(), + metrics.ActivityTaskTimeout.Name(), + metrics.ActivityTimeout.Name(), + metrics.ActivityStartToCloseLatency.Name(), + metrics.ActivityScheduleToCloseLatency.Name(), + } { + recordings := capture.Snapshot()[metricName] + require.NotEmpty(t, recordings, "expected %s to be emitted", metricName) + for _, recording := range recordings { + for _, expectedTag := range expectedTags { + require.Contains(t, recording.Tags, expectedTag.Key, "%s is missing label %s", metricName, expectedTag.Key) + require.Equal(t, expectedTag.Value, recording.Tags[expectedTag.Key]) + } + } + } + + activity.emitOnTerminatedMetrics(activity.enrichedMetricsHandler(ctx, metrics.ActivityTerminatedScope)) + terminated := capture.Snapshot()[metrics.ActivityTerminate.Name()] + require.Len(t, terminated, 1) + for _, expectedTag := range expectedTags { + require.NotContains(t, terminated[0].Tags, expectedTag.Key) + } +} diff --git a/chasm/lib/activity/operator_commands.go b/chasm/lib/activity/operator_commands.go index 62b66a310e7..d9c480b19f1 100644 --- a/chasm/lib/activity/operator_commands.go +++ b/chasm/lib/activity/operator_commands.go @@ -268,7 +268,7 @@ func (a *Activity) handleCancellationRequested(ctx chasm.MutableContext, request // Transition to Canceled if no attempt in progress; otherwise wait for worker response. if !hasAttemptInProgress { - metricsHandler := a.enrichedMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) + metricsHandler := a.completionMetricsHandler(ctx, metrics.HistoryRespondActivityTaskCanceledScope) err := TransitionCanceled.Apply(a, ctx, cancelEvent{ metricsHandler: metricsHandler, fromStatus: originalStatus, diff --git a/chasm/lib/activity/tasks.go b/chasm/lib/activity/tasks.go index de7658d8c32..b7e87887b02 100644 --- a/chasm/lib/activity/tasks.go +++ b/chasm/lib/activity/tasks.go @@ -102,7 +102,7 @@ func (h *scheduleToStartTimeoutTaskHandler) Execute( _ chasm.TaskAttributes, _ *activitypb.ScheduleToStartTimeoutTask, ) error { - metricsHandler := activity.enrichedMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) + metricsHandler := activity.completionMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) event := timeoutEvent{ timeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, @@ -146,7 +146,7 @@ func (h *scheduleToCloseTimeoutTaskHandler) Execute( _ chasm.TaskAttributes, _ *activitypb.ScheduleToCloseTimeoutTask, ) error { - metricsHandler := activity.enrichedMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) + metricsHandler := activity.completionMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) retryState := enumspb.RETRY_STATE_TIMEOUT if activity.GetStatus() == activitypb.ACTIVITY_EXECUTION_STATUS_CANCEL_REQUESTED { retryState = enumspb.RETRY_STATE_CANCEL_REQUESTED @@ -196,7 +196,7 @@ func (h *startToCloseTimeoutTaskHandler) Execute( return err } - metricsHandler := activity.enrichedMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) + metricsHandler := activity.completionMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) if retryState == enumspb.RETRY_STATE_IN_PROGRESS { activity.emitOnAttemptTimedOutMetrics(metricsHandler, enumspb.TIMEOUT_TYPE_START_TO_CLOSE) @@ -276,7 +276,7 @@ func (h *heartbeatTimeoutTaskHandler) Execute( return err } - metricsHandler := activity.enrichedMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) + metricsHandler := activity.completionMetricsHandler(ctx, metrics.TimerActiveTaskActivityTimeoutScope) if retryState == enumspb.RETRY_STATE_IN_PROGRESS { activity.emitOnAttemptTimedOutMetrics(metricsHandler, enumspb.TIMEOUT_TYPE_HEARTBEAT) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 26ba8ec802b..2ec47d89ded 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1379,11 +1379,12 @@ observability stack. Disabling this option will disable all the per-Task Queue g MetricsBreakdownByBuildID = NewTaskQueueBoolSetting( "metrics.breakdownByBuildID", true, - `MetricsBreakdownByBuildID determines if the 'worker_version' tag in Matching metrics should -contain the actual Worker Deployment Version or a generic "__versioned__" value. Regardless of this config, the tag value for unversioned -queues will be "__unversioned__". Disable this option if the version cardinality is too high for your -observability stack. Disabling this option will disable all the per-Task Queue gauges such as backlog lag, count, and age -for VERSIONED queues.`, + `MetricsBreakdownByBuildID determines if Worker Deployment tags in Matching and History metrics should +contain actual deployment and build ID values. When disabled, the deployment and build ID tags are empty, and the +'worker_version' tag in metrics contains a generic "__versioned__" value. Regardless of this config, the +'worker_version' tag value for unversioned task queues is "__unversioned__". Disable this option if the version cardinality +is too high for your observability stack. Disabling this option will disable all the per-Task Queue gauges such as +backlog lag, count, and age for VERSIONED queues.`, ) MatchingForwarderMaxOutstandingPolls = NewTaskQueueIntSetting( "matching.forwarderMaxOutstandingPolls", diff --git a/service/history/api/recordactivitytaskstarted/api.go b/service/history/api/recordactivitytaskstarted/api.go index 997e7ca26d2..a40fc1b66da 100644 --- a/service/history/api/recordactivitytaskstarted/api.go +++ b/service/history/api/recordactivitytaskstarted/api.go @@ -268,6 +268,12 @@ func recordActivityTaskStarted( } scheduleToStartLatency := ai.GetStartedTime().AsTime().Sub(ai.GetScheduledTime().AsTime()) + config := shardContext.GetConfig() + breakdownMetricsByBuildID := config.BreakdownMetricsByBuildID( + namespaceName, + ai.GetTaskQueue(), + enumspb.TASK_QUEUE_TYPE_ACTIVITY, + ) metrics.TaskScheduleToStartLatency.With( metrics.GetPerTaskQueuePartitionTypeScope( taggedMetrics, @@ -275,11 +281,15 @@ func recordActivityTaskStarted( // passing the root partition all the time as we don't care about partition ID in this metric tqid.UnsafeTaskQueueFamily(namespaceEntry.ID().String(), ai.GetTaskQueue()).TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY).RootPartition(), - shardContext.GetConfig().BreakdownMetricsByTaskQueue(namespaceName, + config.BreakdownMetricsByTaskQueue(namespaceName, ai.GetTaskQueue(), enumspb.TASK_QUEUE_TYPE_ACTIVITY), ), - ).Record(scheduleToStartLatency) + ).Record( + scheduleToStartLatency, + metrics.WorkerDeploymentNameTag(pollerDeployment.GetSeriesName(), breakdownMetricsByBuildID), + metrics.WorkerDeploymentBuildIDTag(pollerDeployment.GetBuildId(), breakdownMetricsByBuildID), + ) response.Clock = clock diff --git a/service/history/api/recordworkflowtaskstarted/api.go b/service/history/api/recordworkflowtaskstarted/api.go index baee830603c..e88a3c6c9c3 100644 --- a/service/history/api/recordworkflowtaskstarted/api.go +++ b/service/history/api/recordworkflowtaskstarted/api.go @@ -208,6 +208,11 @@ func Invoke( workflowScheduleToStartLatency := workflowTask.StartedTime.Sub(workflowTask.ScheduledTime) namespaceName := namespaceEntry.Name() tqPartition := tqid.UnsafePartitionFromProto(workflowTask.TaskQueue, req.GetNamespaceId(), enumspb.TASK_QUEUE_TYPE_WORKFLOW) + breakdownMetricsByBuildID := config.BreakdownMetricsByBuildID( + namespaceName.String(), + tqPartition.TaskQueue().Name(), + enumspb.TASK_QUEUE_TYPE_WORKFLOW, + ) metrics.TaskScheduleToStartLatency.With( metrics.GetPerTaskQueuePartitionTypeScope( metricsScope, @@ -215,7 +220,11 @@ func Invoke( tqPartition, config.BreakdownMetricsByTaskQueue(namespaceName.String(), tqPartition.TaskQueue().Name(), enumspb.TASK_QUEUE_TYPE_WORKFLOW), ), - ).Record(workflowScheduleToStartLatency) + ).Record( + workflowScheduleToStartLatency, + metrics.WorkerDeploymentNameTag(pollerDeployment.GetSeriesName(), breakdownMetricsByBuildID), + metrics.WorkerDeploymentBuildIDTag(pollerDeployment.GetBuildId(), breakdownMetricsByBuildID), + ) resp, err = CreateRecordWorkflowTaskStartedResponseWithRawHistory( ctx, diff --git a/service/history/api/respondactivitytaskcanceled/api.go b/service/history/api/respondactivitytaskcanceled/api.go index 3d0fabf8e0f..0b4a1b5b836 100644 --- a/service/history/api/respondactivitytaskcanceled/api.go +++ b/service/history/api/respondactivitytaskcanceled/api.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -121,11 +122,14 @@ func Invoke( AttemptStartedTime: attemptStartedTime, FirstScheduledTime: firstScheduledTime, Closed: true, + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: versioningBehavior, + DeploymentVersion: worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions()), + }, }, metrics.OperationTag(metrics.HistoryRespondActivityTaskCanceledScope), metrics.WorkflowTypeTag(workflowTypeName), - metrics.ActivityTypeTag(token.ActivityType), - metrics.VersioningBehaviorTag(versioningBehavior)) + metrics.ActivityTypeTag(token.ActivityType)) } return &historyservice.RespondActivityTaskCanceledResponse{}, err } diff --git a/service/history/api/respondactivitytaskcompleted/api.go b/service/history/api/respondactivitytaskcompleted/api.go index 2e5b52892fb..ab09e6d5868 100644 --- a/service/history/api/respondactivitytaskcompleted/api.go +++ b/service/history/api/respondactivitytaskcompleted/api.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -139,11 +140,14 @@ func Invoke( FirstScheduledTime: firstScheduledTime, Status: workflow.ActivityStatusSucceeded, Closed: true, + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: versioningBehavior, + DeploymentVersion: worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions()), + }, }, metrics.OperationTag(metrics.HistoryRespondActivityTaskCompletedScope), metrics.WorkflowTypeTag(workflowTypeName), metrics.ActivityTypeTag(token.ActivityType), - metrics.VersioningBehaviorTag(versioningBehavior), ) } return &historyservice.RespondActivityTaskCompletedResponse{}, err diff --git a/service/history/api/respondactivitytaskfailed/api.go b/service/history/api/respondactivitytaskfailed/api.go index 2108eed3dd9..e6572041450 100644 --- a/service/history/api/respondactivitytaskfailed/api.go +++ b/service/history/api/respondactivitytaskfailed/api.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -133,6 +134,10 @@ func Invoke( FirstScheduledTime: firstScheduledTime, Status: workflow.ActivityStatusFailed, Closed: closed, + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: versioningBehavior, + DeploymentVersion: worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions()), + }, } workflow.RecordActivityCompletionMetrics(shard, @@ -141,8 +146,7 @@ func Invoke( completionMetrics, metrics.OperationTag(metrics.HistoryRespondActivityTaskFailedScope), metrics.WorkflowTypeTag(workflowTypeName), - metrics.ActivityTypeTag(token.ActivityType), - metrics.VersioningBehaviorTag(versioningBehavior)) + metrics.ActivityTypeTag(token.ActivityType)) } return &historyservice.RespondActivityTaskFailedResponse{}, err } diff --git a/service/history/api/respondactivitytaskfailed/api_test.go b/service/history/api/respondactivitytaskfailed/api_test.go index 8b889c15b45..c088b15fe98 100644 --- a/service/history/api/respondactivitytaskfailed/api_test.go +++ b/service/history/api/respondactivitytaskfailed/api_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" + deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" "go.temporal.io/api/workflowservice/v1" @@ -95,6 +96,7 @@ type UsecaseConfig struct { tokenAttempt int32 isCacheStale bool includeHeartbeat bool + deploymentOptions *deploymentpb.WorkerDeploymentOptions } func (s *workflowSuite) Test_NormalFlowShouldRescheduleActivity_UpdatesWorkflowExecutionAsActive() { @@ -107,6 +109,11 @@ func (s *workflowSuite) Test_NormalFlowShouldRescheduleActivity_UpdatesWorkflowE expectRetryActivity: true, isCacheStale: false, retryActivityState: enumspb.RETRY_STATE_IN_PROGRESS, + deploymentOptions: &deploymentpb.WorkerDeploymentOptions{ + WorkerVersioningMode: enumspb.WORKER_VERSIONING_MODE_VERSIONED, + DeploymentName: "deployment", + BuildId: "build-id", + }, }) request := s.newRespondActivityTaskFailedRequest(uc) s.setupStubs(uc) @@ -438,6 +445,7 @@ func (s *workflowSuite) newRespondActivityTaskFailedRequest(uc UsecaseConfig) *h Namespace: uc.namespaceId.String(), TaskToken: taskToken, LastHeartbeatDetails: hbDetails, + DeploymentOptions: uc.deploymentOptions, }, } return request @@ -477,11 +485,14 @@ func (s *workflowSuite) setupShardContext(registry namespace.Registry) *historyi func (s *workflowSuite) expectTransientFailureMetricsRecorded(uc UsecaseConfig, shardContext *historyi.MockShardContext) { timer := metrics.NewMockTimerIface(s.controller) counter := metrics.NewMockCounterIface(s.controller) + deploymentNameTag, buildIDTag := expectedWorkerDeploymentMetricTags(uc.deploymentOptions) tags := []metrics.Tag{ metrics.OperationTag(metrics.HistoryRespondActivityTaskFailedScope), metrics.WorkflowTypeTag(uc.wfType.Name), metrics.ActivityTypeTag(uc.activityType), metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), + deploymentNameTag, + buildIDTag, metrics.NamespaceTag(uc.namespaceName.String()), metrics.UnsafeTaskQueueTag(uc.taskQueueId), } @@ -507,11 +518,14 @@ func (s *workflowSuite) expectTransientFailureMetricsRecordedWithStartedTime( startToCloseTimer := metrics.NewMockTimerIface(s.controller) e2eTimer := metrics.NewMockTimerIface(s.controller) counter := metrics.NewMockCounterIface(s.controller) + deploymentNameTag, buildIDTag := expectedWorkerDeploymentMetricTags(uc.deploymentOptions) tags := []metrics.Tag{ metrics.OperationTag(metrics.HistoryRespondActivityTaskFailedScope), metrics.WorkflowTypeTag(uc.wfType.Name), metrics.ActivityTypeTag(uc.activityType), metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), + deploymentNameTag, + buildIDTag, metrics.NamespaceTag(uc.namespaceName.String()), metrics.UnsafeTaskQueueTag(uc.taskQueueId), } @@ -542,11 +556,14 @@ func (s *workflowSuite) expectTransientFailureMetricsRecordedWithStartedTime( func (s *workflowSuite) expectTerminalFailureMetricsRecorded(uc UsecaseConfig, shardContext *historyi.MockShardContext) { timer := metrics.NewMockTimerIface(s.controller) counter := metrics.NewMockCounterIface(s.controller) + deploymentNameTag, buildIDTag := expectedWorkerDeploymentMetricTags(uc.deploymentOptions) tags := []metrics.Tag{ metrics.OperationTag(metrics.HistoryRespondActivityTaskFailedScope), metrics.WorkflowTypeTag(uc.wfType.Name), metrics.ActivityTypeTag(uc.activityType), metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), + deploymentNameTag, + buildIDTag, metrics.NamespaceTag(uc.namespaceName.String()), metrics.UnsafeTaskQueueTag(uc.taskQueueId), } @@ -565,6 +582,19 @@ func (s *workflowSuite) expectTerminalFailureMetricsRecorded(uc UsecaseConfig, s shardContext.EXPECT().GetMetricsHandler().Return(metricsHandler).AnyTimes() } +func expectedWorkerDeploymentMetricTags( + options *deploymentpb.WorkerDeploymentOptions, +) (deploymentNameTag metrics.Tag, buildIDTag metrics.Tag) { + deploymentName := "" + buildID := "" + if options.GetWorkerVersioningMode() == enumspb.WORKER_VERSIONING_MODE_VERSIONED { + deploymentName = options.GetDeploymentName() + buildID = options.GetBuildId() + } + return metrics.WorkerDeploymentNameTag(deploymentName, true), + metrics.WorkerDeploymentBuildIDTag(buildID, true) +} + func (s *workflowSuite) expectCounterRecorded(shardContext *historyi.MockShardContext) *historyi.MockShardContext { counter := metrics.NewMockCounterIface(s.controller) counter.EXPECT().Record(int64(1), metrics.OperationTag(metrics.HistoryRespondActivityTaskFailedScope)) diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index 945867c1c7e..a569e0ecc26 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -219,7 +219,8 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( } behavior := request.GetVersioningBehavior() - deployment := worker_versioning.DeploymentFromDeploymentVersion(worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions())) + wftDeploymentVersion := worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions()) + deployment := worker_versioning.DeploymentFromDeploymentVersion(wftDeploymentVersion) //nolint:staticcheck // SA1019 deprecated Deployment will clean up later if behavior != enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED && request.GetDeployment() == nil && (request.GetDeploymentOptions() == nil || request.GetDeploymentOptions().GetWorkerVersioningMode() != enumspb.WORKER_VERSIONING_MODE_VERSIONED) { @@ -421,6 +422,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( request.GetIdentity(), request.GetWorkerControlTaskQueue(), completedEvent.GetEventId(), // If completedEvent is nil, then GetEventId() returns 0 and this value shouldn't be used in workflowTaskHandler. + deployment, ms, updateRegistry, &effects, @@ -491,13 +493,20 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( // registryClearedErr, which may lead to continuous retries of UpdateWorkflowExecution API. updateRegistry.Abort(update.AbortReasonWorkflowTaskFailed) - metrics.FailedWorkflowTasksCounter.With(handler.metricsHandler).Record( - 1, - metrics.OperationTag(metrics.HistoryRespondWorkflowTaskCompletedScope), - metrics.NamespaceTag(namespaceEntry.Name().String()), - metrics.VersioningBehaviorTag(ms.GetEffectiveVersioningBehavior()), - metrics.FailureTag(wtFailedCause.failedCause.String()), - metrics.FirstAttemptTag(currentWorkflowTask.Attempt), + workflow.RecordWorkflowTaskFailedMetrics( + handler.config, + handler.metricsHandler, + namespaceEntry.Name(), + ms.GetExecutionInfo().GetTaskQueue(), + metrics.HistoryRespondWorkflowTaskCompletedScope, + wtFailedCause.failedCause.String(), + workflow.WorkflowTaskCompletionMetrics{ + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: ms.GetEffectiveVersioningBehavior(), + DeploymentVersion: wftDeploymentVersion, + }, + Attempt: currentWorkflowTask.Attempt, + }, ) handler.logger.Info("Failing the workflow task.", tag.Value(wtFailedCause.Message()), diff --git a/service/history/api/respondworkflowtaskcompleted/api_test.go b/service/history/api/respondworkflowtaskcompleted/api_test.go index bc859ea9c95..5aa650e091c 100644 --- a/service/history/api/respondworkflowtaskcompleted/api_test.go +++ b/service/history/api/respondworkflowtaskcompleted/api_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" + "github.com/uber-go/tally/v4" commandpb "go.temporal.io/api/command/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" @@ -25,6 +26,7 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/contextutil" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/effect" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" @@ -140,52 +142,11 @@ func (s *WorkflowTaskCompletedHandlerSuite) TearDownTest() { func (s *WorkflowTaskCompletedHandlerSuite) TestUpdateWorkflow() { s.Run("Accept Complete", func() { - tv := testvars.New(s.T()) - tv = tv.WithRunID(tv.Any().RunID()) - s.mockNamespaceCache.EXPECT().GetNamespaceByID(tv.NamespaceID()).Return(tv.Namespace(), nil).AnyTimes() - wfContext := s.createStartedWorkflow(tv) - writtenHistoryCh := s.captureWrittenHistory(1) - - _, err := wfContext.LoadMutableState(context.Background(), s.workflowTaskCompletedHandler.shardContext) - s.NoError(err) - - updRequestMsg, upd, serializedTaskToken := s.createSentUpdate(tv, wfContext) - s.NotNil(upd) - - // Use context with metadata to verify it's set via transaction close - ctx := contextutil.WithMetadataContext(context.Background()) - - _, err = s.workflowTaskCompletedHandler.Invoke(ctx, &historyservice.RespondWorkflowTaskCompletedRequest{ - NamespaceId: tv.NamespaceID().String(), - CompleteRequest: &workflowservice.RespondWorkflowTaskCompletedRequest{ - TaskToken: serializedTaskToken, - Commands: s.UpdateAcceptCompleteCommands(tv), - Messages: s.UpdateAcceptCompleteMessages(tv, updRequestMsg), - Identity: tv.Any().String(), - }, - }) - s.NoError(err) - - updStatus, err := upd.WaitLifecycleStage(context.Background(), enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED, time.Duration(0)) - s.NoError(err) - s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED.String(), updStatus.Stage.String()) - s.ProtoEqual(payloads.EncodeString("success-result-of-"+tv.UpdateID()), updStatus.Outcome.GetSuccess()) - - s.EqualHistoryEvents(` - 2 WorkflowTaskScheduled // Speculative WFT events are persisted on WFT completion. - 3 WorkflowTaskStarted // Speculative WFT events are persisted on WFT completion. - 4 WorkflowTaskCompleted - 5 WorkflowExecutionUpdateAccepted - 6 WorkflowExecutionUpdateCompleted`, <-writtenHistoryCh) - - // VERIFY: Context metadata set via transaction close (normal case WITH commands) - contextWorkflowType, ok := contextutil.ContextMetadataGet(ctx, contextutil.MetadataKeyWorkflowType) - s.True(ok, "context workflow type MUST be set in normal case with commands") - s.Equal(tv.WorkflowType().GetName(), contextWorkflowType) + s.testAcceptCompleteWithWorkerDeploymentMetrics(true) + }) - contextTaskQueue, ok := contextutil.ContextMetadataGet(ctx, contextutil.MetadataKeyWorkflowTaskQueue) - s.True(ok, "context task queue MUST be set in normal case with commands") - s.Equal(tv.TaskQueue().GetName(), contextTaskQueue) + s.Run("Accept Complete without build ID breakdown", func() { + s.testAcceptCompleteWithWorkerDeploymentMetrics(false) }) s.Run("Reject", func() { @@ -638,6 +599,62 @@ func (s *WorkflowTaskCompletedHandlerSuite) TestHandleBufferedQueries() { }) } +func (s *WorkflowTaskCompletedHandlerSuite) testAcceptCompleteWithWorkerDeploymentMetrics( + breakdownMetricsByBuildID bool, +) { + s.workflowTaskCompletedHandler.config.BreakdownMetricsByBuildID = + dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(breakdownMetricsByBuildID) + + tv := testvars.New(s.T()) + tv = tv.WithRunID(tv.Any().RunID()) + s.mockNamespaceCache.EXPECT().GetNamespaceByID(tv.NamespaceID()).Return(tv.Namespace(), nil).AnyTimes() + wfContext := s.createStartedWorkflow(tv) + writtenHistoryCh := s.captureWrittenHistory(1) + + _, err := wfContext.LoadMutableState(context.Background(), s.workflowTaskCompletedHandler.shardContext) + s.NoError(err) + + updRequestMsg, upd, serializedTaskToken := s.createSentUpdate(tv, wfContext) + s.NotNil(upd) + + // Use context with metadata to verify it's set via transaction close + ctx := contextutil.WithMetadataContext(context.Background()) + + _, err = s.workflowTaskCompletedHandler.Invoke(ctx, &historyservice.RespondWorkflowTaskCompletedRequest{ + NamespaceId: tv.NamespaceID().String(), + CompleteRequest: &workflowservice.RespondWorkflowTaskCompletedRequest{ + TaskToken: serializedTaskToken, + Commands: s.UpdateAcceptCompleteCommands(tv), + Messages: s.UpdateAcceptCompleteMessages(tv, updRequestMsg), + Identity: tv.Any().String(), + DeploymentOptions: tv.WorkerDeploymentOptions(true), + }, + }) + s.NoError(err) + + updStatus, err := upd.WaitLifecycleStage(context.Background(), enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_UNSPECIFIED, time.Duration(0)) + s.NoError(err) + s.Equal(enumspb.UPDATE_WORKFLOW_EXECUTION_LIFECYCLE_STAGE_COMPLETED.String(), updStatus.Stage.String()) + s.ProtoEqual(payloads.EncodeString("success-result-of-"+tv.UpdateID()), updStatus.Outcome.GetSuccess()) + + s.EqualHistoryEvents(` + 2 WorkflowTaskScheduled // Speculative WFT events are persisted on WFT completion. + 3 WorkflowTaskStarted // Speculative WFT events are persisted on WFT completion. + 4 WorkflowTaskCompleted + 5 WorkflowExecutionUpdateAccepted + 6 WorkflowExecutionUpdateCompleted`, <-writtenHistoryCh) + s.requireCounterHasWorkerDeploymentTags(metrics.WorkflowTasksCompleted.Name(), tv, breakdownMetricsByBuildID) + + // VERIFY: Context metadata set via transaction close (normal case WITH commands) + contextWorkflowType, ok := contextutil.ContextMetadataGet(ctx, contextutil.MetadataKeyWorkflowType) + s.True(ok, "context workflow type MUST be set in normal case with commands") + s.Equal(tv.WorkflowType().GetName(), contextWorkflowType) + + contextTaskQueue, ok := contextutil.ContextMetadataGet(ctx, contextutil.MetadataKeyWorkflowTaskQueue) + s.True(ok, "context task queue MUST be set in normal case with commands") + s.Equal(tv.TaskQueue().GetName(), contextTaskQueue) +} + func (s *WorkflowTaskCompletedHandlerSuite) captureWrittenHistory(n int) <-chan []*historypb.HistoryEvent { writtenHistoryCh := make(chan []*historypb.HistoryEvent, n) s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, request *persistence.UpdateWorkflowExecutionRequest) (*persistence.UpdateWorkflowExecutionResponse, error) { @@ -659,6 +676,32 @@ func (s *WorkflowTaskCompletedHandlerSuite) captureWrittenHistory(n int) <-chan return writtenHistoryCh } +func (s *WorkflowTaskCompletedHandlerSuite) requireCounterHasWorkerDeploymentTags( + metricName string, + tv *testvars.TestVars, + breakdownMetricsByBuildID bool, +) { + expectedDeploymentName := "" + expectedBuildID := "" + if breakdownMetricsByBuildID { + expectedDeploymentName = tv.DeploymentSeries() + expectedBuildID = tv.BuildID() + } + + counters := s.mockShard.Resource.MetricsScope.(tally.TestScope).Snapshot().Counters() + for _, counter := range counters { + if counter.Name() != "test."+metricName { + continue + } + tags := counter.Tags() + if tags["worker_deployment_name"] == expectedDeploymentName && + tags["worker_build_id"] == expectedBuildID { + return + } + } + s.Failf("metric not found", "%s does not have the expected Worker Deployment tags", metricName) +} + func (s *WorkflowTaskCompletedHandlerSuite) createStartedWorkflow(tv *testvars.TestVars) historyi.WorkflowContext { ms := workflow.TestLocalMutableState(s.workflowTaskCompletedHandler.shardContext, s.mockEventsCache, tv.Namespace(), tv.WorkflowID(), tv.RunID(), log.NewTestLogger()) @@ -1204,11 +1247,12 @@ func (s *WorkflowTaskCompletedHandlerSuite) TestWorkflowTaskCompletionPagination _, err := s.workflowTaskCompletedHandler.Invoke(context.Background(), &historyservice.RespondWorkflowTaskCompletedRequest{ NamespaceId: tv.NamespaceID().String(), CompleteRequest: &workflowservice.RespondWorkflowTaskCompletedRequest{ - TaskToken: serializedTaskToken, - Commands: []*commandpb.Command{{CommandType: enumspb.COMMAND_TYPE_RECORD_MARKER}}, - IntermediatePage: true, - PageNumber: 0, - Identity: tv.Any().String(), + TaskToken: serializedTaskToken, + Commands: []*commandpb.Command{{CommandType: enumspb.COMMAND_TYPE_RECORD_MARKER}}, + IntermediatePage: true, + PageNumber: 0, + Identity: tv.Any().String(), + DeploymentOptions: tv.WorkerDeploymentOptions(true), }, }) s.Error(err) @@ -1218,6 +1262,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) TestWorkflowTaskCompletionPagination s.EqualHistoryEvents(` 4 WorkflowTaskFailed 5 WorkflowExecutionTerminated`, <-writtenHistoryCh) + s.requireCounterHasWorkerDeploymentTags(metrics.FailedWorkflowTasksCounter.Name(), tv, true) }) s.Run("Pagination request rejected when pagination is disabled", func() { diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go index f0925d013c3..4e8da517eae 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler.go @@ -12,6 +12,7 @@ import ( "github.com/google/uuid" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" + deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" failurepb "go.temporal.io/api/failure/v1" historypb "go.temporal.io/api/history/v1" @@ -57,6 +58,7 @@ type ( identity string workerControlTaskQueue string workflowTaskCompletedID int64 + workflowTaskDeployment *deploymentpb.Deployment // internal state hasBufferedEventsOrMessages bool @@ -111,6 +113,7 @@ func newWorkflowTaskCompletedHandler( identity string, workerControlTaskQueue string, workflowTaskCompletedID int64, + workflowTaskDeployment *deploymentpb.Deployment, mutableState historyi.MutableState, updateRegistry update.Registry, effects effect.Controller, @@ -132,6 +135,7 @@ func newWorkflowTaskCompletedHandler( identity: identity, workerControlTaskQueue: workerControlTaskQueue, workflowTaskCompletedID: workflowTaskCompletedID, + workflowTaskDeployment: workflowTaskDeployment, // internal state hasBufferedEventsOrMessages: hasBufferedEventsOrMessages, @@ -603,7 +607,7 @@ func (handler *workflowTaskCompletedHandler) handlePostCommandEagerExecuteActivi uuid.NewString(), handler.identity, stamp, - nil, + handler.workflowTaskDeployment, nil, handler.workerControlTaskQueue, // Eager: activity runs on the same worker that completed the WFT. shardClock, diff --git a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go index 4ae28ff3298..501cf4dc51e 100644 --- a/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go +++ b/service/history/api/respondworkflowtaskcompleted/workflow_task_completed_handler_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" + deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" protocolpb "go.temporal.io/api/protocol/v1" @@ -114,6 +115,7 @@ func TestCommandProtocolMessage(t *testing.T) { t.Name(), // identity "", // workerControlTaskQueue 123, // workflowTaskCompletedID + nil, // workflowTaskDeployment out.ms, out.updates, &effects, @@ -630,12 +632,13 @@ func TestHandlePostCommandEagerExecuteActivity(t *testing.T) { } expectedClock := &clockspb.VectorClock{ClusterId: 1, ShardId: 1, Clock: 42} + expectedDeployment := &deploymentpb.Deployment{SeriesName: "test-deployment", BuildId: "test-build-id"} ms.EXPECT().IsWorkflowExecutionRunning().Return(true) ms.EXPECT().GetActivityByActivityID(activityID).Return(ai, true) ms.EXPECT().GetAssignedBuildId().Return("") ms.EXPECT().AddActivityTaskStartedEvent( - ai, scheduledEventID, gomock.Any(), "test-identity", gomock.Any(), nil, nil, "/_sys/worker-commands/test-ns/key1", expectedClock, + ai, scheduledEventID, gomock.Any(), "test-identity", gomock.Any(), expectedDeployment, nil, "/_sys/worker-commands/test-ns/key1", expectedClock, ).Return(&historypb.HistoryEvent{EventId: 7}, nil) ms.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ NamespaceId: "test-namespace-id", @@ -656,6 +659,7 @@ func TestHandlePostCommandEagerExecuteActivity(t *testing.T) { handler := &workflowTaskCompletedHandler{ identity: "test-identity", workerControlTaskQueue: "/_sys/worker-commands/test-ns/key1", + workflowTaskDeployment: expectedDeployment, mutableState: ms, shard: shardCtx, tokenSerializer: tasktoken.NewSerializer(), diff --git a/service/history/api/respondworkflowtaskfailed/api.go b/service/history/api/respondworkflowtaskfailed/api.go index 410d8e1758f..0fd90ce24a8 100644 --- a/service/history/api/respondworkflowtaskfailed/api.go +++ b/service/history/api/respondworkflowtaskfailed/api.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -76,13 +77,21 @@ func Invoke( }, nil } - metrics.FailedWorkflowTasksCounter.With(shardContext.GetMetricsHandler()).Record( - 1, - metrics.OperationTag(metrics.HistoryRespondWorkflowTaskFailedScope), - metrics.NamespaceTag(namespaceEntry.Name().String()), - metrics.VersioningBehaviorTag(mutableState.GetEffectiveVersioningBehavior()), - metrics.FailureTag(request.GetCause().String()), - metrics.FirstAttemptTag(workflowTask.Attempt), + wftDeploymentVersion := worker_versioning.DeploymentVersionFromOptions(request.GetDeploymentOptions()) + workflow.RecordWorkflowTaskFailedMetrics( + shardContext.GetConfig(), + shardContext.GetMetricsHandler(), + namespaceEntry.Name(), + mutableState.GetExecutionInfo().GetTaskQueue(), + metrics.HistoryRespondWorkflowTaskFailedScope, + request.GetCause().String(), + workflow.WorkflowTaskCompletionMetrics{ + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: mutableState.GetEffectiveVersioningBehavior(), + DeploymentVersion: wftDeploymentVersion, + }, + Attempt: workflowTask.Attempt, + }, ) if request.GetCause() == enumspb.WORKFLOW_TASK_FAILED_CAUSE_GRPC_MESSAGE_TOO_LARGE { diff --git a/service/history/configs/config.go b/service/history/configs/config.go index e59e3989aa9..b07d5fdbd2b 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -443,6 +443,7 @@ type Config struct { HealthRPCErrorRatio dynamicconfig.FloatPropertyFn HealthHistoryInitializationTime dynamicconfig.DurationPropertyFn BreakdownMetricsByTaskQueue dynamicconfig.BoolPropertyFnWithTaskQueueFilter + BreakdownMetricsByBuildID dynamicconfig.BoolPropertyFnWithTaskQueueFilter LogAllReqErrors dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -854,6 +855,7 @@ func NewConfig( HealthHistoryInitializationTime: dynamicconfig.HealthHistoryInitializationTime.Get(dc), BreakdownMetricsByTaskQueue: dynamicconfig.MetricsBreakdownByTaskQueue.Get(dc), + BreakdownMetricsByBuildID: dynamicconfig.MetricsBreakdownByBuildID.Get(dc), LogAllReqErrors: dynamicconfig.LogAllReqErrors.Get(dc), diff --git a/service/history/timer_queue_active_task_executor.go b/service/history/timer_queue_active_task_executor.go index 4b908773f6a..e6fad3009bc 100644 --- a/service/history/timer_queue_active_task_executor.go +++ b/service/history/timer_queue_active_task_executor.go @@ -10,6 +10,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" + deploymentspb "go.temporal.io/server/api/deployment/v1" enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" @@ -304,6 +305,7 @@ func (t *timerQueueActiveTaskExecutor) processSingleActivityTimeoutTask( failureMsg := fmt.Sprintf(common.FailureReasonActivityTimeout, timerSequenceID.TimerType.String()) timeoutFailure := failure.NewTimeoutFailure(failureMsg, timerSequenceID.TimerType) + activityAttemptStarted := ai.GetStartedEventId() != common.EmptyEventID retryState, err := mutableState.RetryActivity(ai, timeoutFailure) if err != nil { return result, nil @@ -320,6 +322,13 @@ func (t *timerQueueActiveTaskExecutor) processSingleActivityTimeoutTask( ) } + var deploymentVersion *deploymentspb.WorkerDeploymentVersion + if activityAttemptStarted && ai.GetLastDeploymentVersion() != nil { + deploymentVersion = &deploymentspb.WorkerDeploymentVersion{ + DeploymentName: ai.GetLastDeploymentVersion().GetDeploymentName(), + BuildId: ai.GetLastDeploymentVersion().GetBuildId(), + } + } workflow.RecordActivityCompletionMetrics( t.shardContext, mutableState.GetNamespaceEntry().Name(), @@ -330,11 +339,14 @@ func (t *timerQueueActiveTaskExecutor) processSingleActivityTimeoutTask( FirstScheduledTime: timestamp.TimeValue(ai.FirstScheduledTime), Closed: retryState != enumspb.RETRY_STATE_IN_PROGRESS, TimerType: timerSequenceID.TimerType, + VersioningInfo: workflow.VersioningMetricContext{ + Behavior: mutableState.GetEffectiveVersioningBehavior(), + DeploymentVersion: deploymentVersion, + }, }, metrics.OperationTag(metrics.TimerActiveTaskActivityTimeoutScope), metrics.WorkflowTypeTag(mutableState.GetWorkflowType().GetName()), - metrics.ActivityTypeTag(ai.ActivityType.GetName()), - metrics.VersioningBehaviorTag(mutableState.GetEffectiveVersioningBehavior())) + metrics.ActivityTypeTag(ai.ActivityType.GetName())) if retryState == enumspb.RETRY_STATE_IN_PROGRESS { // TODO uncommment once RETRY_STATE_PAUSED is supported diff --git a/service/history/timer_queue_active_task_executor_test.go b/service/history/timer_queue_active_task_executor_test.go index 393e7fd82fc..5dc5a930cfc 100644 --- a/service/history/timer_queue_active_task_executor_test.go +++ b/service/history/timer_queue_active_task_executor_test.go @@ -9,7 +9,9 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" + deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" enumsspb "go.temporal.io/server/api/enums/v1" @@ -22,9 +24,11 @@ import ( "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" @@ -2551,9 +2555,12 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask expectRetryActivity bool retryState enumspb.RetryState retryError error + clearStartedStateOnRetry bool expectAddTimedTask bool expectedUpdateMutableState bool expectedScheduleWorkflowTask bool + expectedDeploymentName string + expectedBuildID string }{ { name: "Retry Policy Not Set", @@ -2561,8 +2568,13 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask Attempt: 1, }, ai: &persistencespb.ActivityInfo{ - Attempt: 1, - Stamp: 1, + Attempt: 1, + Stamp: 1, + StartedEventId: common.TransientEventID, + LastDeploymentVersion: &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "deployment", + BuildId: "build-id", + }, }, expectRetryActivity: true, retryState: enumspb.RETRY_STATE_RETRY_POLICY_NOT_SET, @@ -2570,15 +2582,22 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask expectAddTimedTask: true, expectedUpdateMutableState: true, expectedScheduleWorkflowTask: true, + expectedDeploymentName: "deployment", + expectedBuildID: "build-id", }, { name: "Retry State Timeout", timerSequenceID: workflow.TimerSequenceID{ - Attempt: 1, + Attempt: 1, + TimerType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, }, ai: &persistencespb.ActivityInfo{ Attempt: 1, Stamp: 1, + LastDeploymentVersion: &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "previous-deployment", + BuildId: "previous-build", + }, }, expectRetryActivity: true, retryState: enumspb.RETRY_STATE_TIMEOUT, @@ -2590,17 +2609,26 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask { name: "Retry State In Progress", timerSequenceID: workflow.TimerSequenceID{ - Attempt: 1, + Attempt: 1, + TimerType: enumspb.TIMEOUT_TYPE_START_TO_CLOSE, }, ai: &persistencespb.ActivityInfo{ - Attempt: 1, + Attempt: 1, + StartedEventId: common.TransientEventID, + LastDeploymentVersion: &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "deployment", + BuildId: "build-id", + }, }, expectRetryActivity: true, retryState: enumspb.RETRY_STATE_IN_PROGRESS, retryError: nil, + clearStartedStateOnRetry: true, expectAddTimedTask: false, expectedUpdateMutableState: true, expectedScheduleWorkflowTask: false, + expectedDeploymentName: "deployment", + expectedBuildID: "build-id", }, { name: "Attempt dont match", @@ -2617,11 +2645,23 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask }, } info := &persistencespb.WorkflowExecutionInfo{} + s.config.BreakdownMetricsByBuildID = dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true) for _, tc := range testCases { s.Run(tc.name, func() { + handler := metricstest.NewCaptureHandler() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + s.mockShard.SetMetricsHandler(handler) + if tc.expectRetryActivity { - ms.EXPECT().RetryActivity(gomock.Any(), gomock.Any()).Return(tc.retryState, tc.retryError) + retryCall := ms.EXPECT().RetryActivity(gomock.Any(), gomock.Any()) + if tc.clearStartedStateOnRetry { + retryCall.Do(func(ai *persistencespb.ActivityInfo, _ *failurepb.Failure) { + workflow.ClearActivityStartedState(ai) + }) + } + retryCall.Return(tc.retryState, tc.retryError) ms.EXPECT().GetWorkflowType().Return(&commonpb.WorkflowType{Name: "test-workflow-type"}).AnyTimes() } @@ -2638,6 +2678,12 @@ func (s *timerQueueActiveTaskExecutorSuite) TestProcessSingleActivityTimeoutTask s.NoError(err) s.Equal(tc.expectedScheduleWorkflowTask, result.shouldScheduleWorkflowTask, "scheduleWorkflowTask") s.Equal(tc.expectedUpdateMutableState, result.shouldUpdateMutableState, "updateMutableState") + if tc.expectRetryActivity && tc.retryError == nil { + recordings := capture.Snapshot()[metrics.ActivityTaskTimeout.Name()] + s.Require().Len(recordings, 1) + s.Equal(tc.expectedDeploymentName, recordings[0].Tags["worker_deployment_name"]) + s.Equal(tc.expectedBuildID, recordings[0].Tags["worker_build_id"]) + } }) } } diff --git a/service/history/workflow/metrics.go b/service/history/workflow/metrics.go index 01f33dda204..6b44ad18882 100644 --- a/service/history/workflow/metrics.go +++ b/service/history/workflow/metrics.go @@ -5,6 +5,7 @@ import ( "time" enumspb "go.temporal.io/api/enums/v1" + deploymentspb "go.temporal.io/server/api/deployment/v1" enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/metrics" @@ -161,6 +162,82 @@ func GetPerTaskQueueFamilyScope( ) } +type VersioningMetricContext struct { + Behavior enumspb.VersioningBehavior + DeploymentVersion *deploymentspb.WorkerDeploymentVersion +} + +type WorkflowTaskCompletionMetrics struct { + VersioningInfo VersioningMetricContext + Attempt int32 +} + +func RecordWorkflowTaskCompletedMetrics( + config *configs.Config, + handler metrics.Handler, + namespaceName namespace.Name, + taskQueue string, + completion WorkflowTaskCompletionMetrics, +) { + metrics.WorkflowTasksCompleted.With(handler).Record( + 1, + workflowTaskCompletionMetricTags(config, namespaceName, taskQueue, completion)..., + ) +} + +func RecordWorkflowTaskFailedMetrics( + config *configs.Config, + handler metrics.Handler, + namespaceName namespace.Name, + taskQueue string, + operation string, + failure string, + completion WorkflowTaskCompletionMetrics, +) { + tags := workflowTaskCompletionMetricTags(config, namespaceName, taskQueue, completion) + tags = append(tags, metrics.OperationTag(operation), metrics.FailureTag(failure)) + metrics.FailedWorkflowTasksCounter.With(handler).Record(1, tags...) +} + +func workflowTaskCompletionMetricTags( + config *configs.Config, + namespaceName namespace.Name, + taskQueue string, + completion WorkflowTaskCompletionMetrics, +) []metrics.Tag { + tags := []metrics.Tag{ + metrics.NamespaceTag(namespaceName.String()), + metrics.FirstAttemptTag(completion.Attempt), + } + return append(tags, versioningMetricTags( + config, + namespaceName, + taskQueue, + enumspb.TASK_QUEUE_TYPE_WORKFLOW, + completion.VersioningInfo, + )...) +} + +func versioningMetricTags( + config *configs.Config, + namespaceName namespace.Name, + taskQueue string, + taskQueueType enumspb.TaskQueueType, + versioning VersioningMetricContext, +) []metrics.Tag { + breakdownMetricsByBuildID := config.BreakdownMetricsByBuildID( + namespaceName.String(), + taskQueue, + taskQueueType, + ) + + return []metrics.Tag{ + metrics.VersioningBehaviorTag(versioning.Behavior), + metrics.WorkerDeploymentNameTag(versioning.DeploymentVersion.GetDeploymentName(), breakdownMetricsByBuildID), + metrics.WorkerDeploymentBuildIDTag(versioning.DeploymentVersion.GetBuildId(), breakdownMetricsByBuildID), + } +} + type ActivityExecutionStatus int const ( @@ -181,7 +258,8 @@ type ActivityCompletionMetrics struct { // Closed is true if no more attempts will be made to execute the activity. Closed bool // TimerType is the type of timer that caused the activity execution to timeout. - TimerType enumspb.TimeoutType + TimerType enumspb.TimeoutType + VersioningInfo VersioningMetricContext } func RecordActivityCompletionMetrics( @@ -191,11 +269,19 @@ func RecordActivityCompletionMetrics( completion ActivityCompletionMetrics, tags ...metrics.Tag, ) { + config := shard.GetConfig() + tags = append(tags, versioningMetricTags( + config, + namespaceName, + taskQueue, + enumspb.TASK_QUEUE_TYPE_ACTIVITY, + completion.VersioningInfo, + )...) metricsHandler := GetPerTaskQueueFamilyScope( shard.GetMetricsHandler(), namespaceName, taskQueue, - shard.GetConfig(), + config, tags..., ) @@ -226,9 +312,13 @@ func RecordActivityCompletionMetrics( case ActivityStatusSucceeded: metrics.ActivitySuccess.With(metricsHandler).Record(1) case ActivityStatusTimeout: - metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, metrics.StringTag("timeout_type", completion.TimerType.String())) + timeoutTag := metrics.StringTag( + "timeout_type", + completion.TimerType.String(), + ) + metrics.ActivityTaskTimeout.With(metricsHandler).Record(1, timeoutTag) if completion.Closed { - metrics.ActivityTimeout.With(metricsHandler).Record(1, metrics.StringTag("timeout_type", completion.TimerType.String())) + metrics.ActivityTimeout.With(metricsHandler).Record(1, timeoutTag) } default: // Do nothing diff --git a/service/history/workflow/metrics_test.go b/service/history/workflow/metrics_test.go index db84bc81cfb..0a37431679d 100644 --- a/service/history/workflow/metrics_test.go +++ b/service/history/workflow/metrics_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" + deploymentspb "go.temporal.io/server/api/deployment/v1" "go.temporal.io/server/chasm" chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/common/clock" @@ -21,6 +22,212 @@ import ( "google.golang.org/protobuf/types/known/timestamppb" ) +func TestRecordWorkflowTaskMetrics(t *testing.T) { + for _, tc := range []struct { + name string + breakdownMetricsByBuildID bool + expectedDeploymentNameTag string + expectedBuildIDTag string + }{ + { + name: "breakdown enabled", + breakdownMetricsByBuildID: true, + expectedDeploymentNameTag: "deployment", + expectedBuildIDTag: "build-id", + }, + { + name: "breakdown disabled", + breakdownMetricsByBuildID: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + handler := metricstest.NewCaptureHandler() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + + config := &configs.Config{ + BreakdownMetricsByBuildID: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(tc.breakdownMetricsByBuildID), + } + completion := WorkflowTaskCompletionMetrics{ + VersioningInfo: VersioningMetricContext{ + Behavior: enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE, + DeploymentVersion: &deploymentspb.WorkerDeploymentVersion{ + DeploymentName: "deployment", + BuildId: "build-id", + }, + }, + Attempt: 1, + } + + RecordWorkflowTaskCompletedMetrics( + config, + handler, + namespace.Name("test-namespace"), + "test-task-queue", + completion, + ) + RecordWorkflowTaskFailedMetrics( + config, + handler, + namespace.Name("test-namespace"), + "test-task-queue", + metrics.HistoryRespondWorkflowTaskFailedScope, + enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE.String(), + completion, + ) + + snapshot := capture.Snapshot() + for _, metricName := range []string{ + metrics.WorkflowTasksCompleted.Name(), + metrics.FailedWorkflowTasksCounter.Name(), + } { + recordings := snapshot[metricName] + require.Len(t, recordings, 1) + require.Equal(t, tc.expectedDeploymentNameTag, recordings[0].Tags["worker_deployment_name"]) + require.Equal(t, tc.expectedBuildIDTag, recordings[0].Tags["worker_build_id"]) + } + + failedRecording := snapshot[metrics.FailedWorkflowTasksCounter.Name()][0] + require.Equal(t, metrics.HistoryRespondWorkflowTaskFailedScope, failedRecording.Tags[metrics.OperationTagName]) + require.Equal( + t, + enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE.String(), + failedRecording.Tags[metrics.FailureTagName], + ) + }) + } +} + +func TestRecordActivityCompletionMetrics_WorkerDeploymentTags(t *testing.T) { + for _, tc := range []struct { + name string + breakdownMetricsByBuildID bool + expectedDeploymentNameTag string + expectedBuildIDTag string + }{ + { + name: "breakdown enabled", + breakdownMetricsByBuildID: true, + expectedDeploymentNameTag: "deployment", + expectedBuildIDTag: "build-id", + }, + { + name: "breakdown disabled", + breakdownMetricsByBuildID: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + controller := gomock.NewController(t) + handler := metricstest.NewCaptureHandler() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + + config := &configs.Config{ + BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + BreakdownMetricsByBuildID: func(namespace string, taskQueue string, taskQueueType enumspb.TaskQueueType) bool { + require.Equal(t, "test-namespace", namespace) + require.Equal(t, "test-task-queue", taskQueue) + require.Equal(t, enumspb.TASK_QUEUE_TYPE_ACTIVITY, taskQueueType) + return tc.breakdownMetricsByBuildID + }, + } + shard := historyi.NewMockShardContext(controller) + shard.EXPECT().GetConfig().Return(config).Times(4) + shard.EXPECT().GetMetricsHandler().Return(handler).Times(4) + shard.EXPECT().GetTimeSource().Return(clock.NewRealTimeSource()).Times(4) + + completion := ActivityCompletionMetrics{ + Status: ActivityStatusFailed, + Closed: true, + VersioningInfo: VersioningMetricContext{ + Behavior: enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE, + DeploymentVersion: &deploymentspb.WorkerDeploymentVersion{ + DeploymentName: "deployment", + BuildId: "build-id", + }, + }, + AttemptStartedTime: time.Now().Add(-time.Minute), + FirstScheduledTime: time.Now().Add(-2 * time.Minute), + } + RecordActivityCompletionMetrics( + shard, + namespace.Name("test-namespace"), + "test-task-queue", + completion, + ) + completion.Status = ActivityStatusSucceeded + RecordActivityCompletionMetrics( + shard, + namespace.Name("test-namespace"), + "test-task-queue", + completion, + ) + completion.Status = ActivityStatusCanceled + RecordActivityCompletionMetrics( + shard, + namespace.Name("test-namespace"), + "test-task-queue", + completion, + ) + completion.Status = ActivityStatusTimeout + completion.TimerType = enumspb.TIMEOUT_TYPE_HEARTBEAT + RecordActivityCompletionMetrics( + shard, + namespace.Name("test-namespace"), + "test-task-queue", + completion, + ) + + snapshot := capture.Snapshot() + for _, metricName := range []string{ + metrics.ActivityTaskFail.Name(), + metrics.ActivityFail.Name(), + metrics.ActivitySuccess.Name(), + metrics.ActivityCancel.Name(), + metrics.ActivityTaskTimeout.Name(), + metrics.ActivityTimeout.Name(), + } { + recordings := snapshot[metricName] + require.Len(t, recordings, 1) + require.Equal(t, tc.expectedDeploymentNameTag, recordings[0].Tags["worker_deployment_name"]) + require.Equal(t, tc.expectedBuildIDTag, recordings[0].Tags["worker_build_id"]) + require.Equal( + t, + enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE.String(), + recordings[0].Tags["versioning_behavior"], + ) + } + for metricName, expectedCount := range map[string]int{ + metrics.ActivityE2ELatency.Name(): 3, + metrics.ActivityStartToCloseLatency.Name(): 3, + metrics.ActivityScheduleToCloseLatency.Name(): 4, + } { + recordings := snapshot[metricName] + require.Len(t, recordings, expectedCount) + for _, recording := range recordings { + require.Equal(t, tc.expectedDeploymentNameTag, recording.Tags["worker_deployment_name"]) + require.Equal(t, tc.expectedBuildIDTag, recording.Tags["worker_build_id"]) + require.Equal( + t, + enumspb.VERSIONING_BEHAVIOR_AUTO_UPGRADE.String(), + recording.Tags["versioning_behavior"], + ) + } + } + require.Equal( + t, + enumspb.TIMEOUT_TYPE_HEARTBEAT.String(), + snapshot[metrics.ActivityTaskTimeout.Name()][0].Tags["timeout_type"], + ) + require.Equal( + t, + enumspb.TIMEOUT_TYPE_HEARTBEAT.String(), + snapshot[metrics.ActivityTimeout.Name()][0].Tags["timeout_type"], + ) + }) + } +} + func TestEmitWorkflowCompletionStats_WorkflowDuration(t *testing.T) { logger := log.NewTestLogger() testHandler, _ := metricstest.NewHandler(logger, metrics.ClientConfig{}) @@ -205,7 +412,10 @@ func TestRecordActivityCompletionMetrics_TimeoutWithStartedTimeSkipsLatency(t *t expectActivityMetricsScope(handler, metrics.TimerActiveTaskActivityTimeoutScope) timeoutCounter := metrics.NewMockCounterIface(controller) - timeoutCounter.EXPECT().Record(int64(1), metrics.StringTag("timeout_type", enumspb.TIMEOUT_TYPE_START_TO_CLOSE.String())).Times(1) + timeoutCounter.EXPECT().Record( + int64(1), + metrics.StringTag("timeout_type", enumspb.TIMEOUT_TYPE_START_TO_CLOSE.String()), + ).Times(1) handler.EXPECT().Counter(metrics.ActivityTaskTimeout.Name()).Return(timeoutCounter) RecordActivityCompletionMetrics( @@ -228,7 +438,10 @@ func TestRecordActivityCompletionMetrics_TimeoutWithMissingStartedTimeSkipsLaten expectActivityMetricsScope(handler, metrics.TimerActiveTaskActivityTimeoutScope) timeoutCounter := metrics.NewMockCounterIface(controller) - timeoutCounter.EXPECT().Record(int64(1), metrics.StringTag("timeout_type", enumspb.TIMEOUT_TYPE_HEARTBEAT.String())).Times(1) + timeoutCounter.EXPECT().Record( + int64(1), + metrics.StringTag("timeout_type", enumspb.TIMEOUT_TYPE_HEARTBEAT.String()), + ).Times(1) handler.EXPECT().Counter(metrics.ActivityTaskTimeout.Name()).Return(timeoutCounter) RecordActivityCompletionMetrics( @@ -251,6 +464,7 @@ func newActivityMetricsTestShard( shard.EXPECT().GetMetricsHandler().Return(handler).Times(1) shard.EXPECT().GetConfig().Return(&configs.Config{ BreakdownMetricsByTaskQueue: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), + BreakdownMetricsByBuildID: dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true), }).Times(1) shard.EXPECT().GetTimeSource().Return(clock.NewRealTimeSource()).Times(1) return shard @@ -261,6 +475,9 @@ func expectActivityMetricsScope(handler *metrics.MockHandler, operation string) metrics.OperationTag(operation), metrics.WorkflowTypeTag("test-workflow"), metrics.ActivityTypeTag("test-activity"), + metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), + metrics.WorkerDeploymentNameTag("", true), + metrics.WorkerDeploymentBuildIDTag("", true), metrics.NamespaceTag("test-namespace"), metrics.UnsafeTaskQueueTag("test-task-queue"), } diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 5c73fd1bb93..201299bea8a 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -4479,7 +4479,10 @@ func (ms *MutableStateImpl) AddActivityTaskStartedEvent( } ms.approximateSize -= ai.Size() - if deployment != nil { + if deployment == nil { + ai.LastWorkerDeploymentVersion = "" + ai.LastDeploymentVersion = nil + } else { ai.LastWorkerDeploymentVersion = worker_versioning.WorkerDeploymentVersionToStringV31(worker_versioning.DeploymentVersionFromDeployment(deployment)) ai.LastDeploymentVersion = worker_versioning.ExternalWorkerDeploymentVersionFromDeployment(deployment) } diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index f005f34a354..0143a8d7083 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -7197,7 +7197,7 @@ func (s *mutableStateSuite) TestSetContextMetadata_ActivityNotFound() { s.False(ok) } -func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTaskQueue() { +func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTaskQueueAndClearsUnversionedDeployment() { s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() // Setup workflow execution @@ -7250,6 +7250,11 @@ func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTa ) s.NoError(err) s.Empty(activityInfo.WorkerControlTaskQueue, "WorkerControlTaskQueue should be empty before activity starts") + activityInfo.LastWorkerDeploymentVersion = "previous-deployment:previous-build" + activityInfo.LastDeploymentVersion = &deploymentpb.WorkerDeploymentVersion{ + DeploymentName: "previous-deployment", + BuildId: "previous-build", + } // Start activity with workerControlTaskQueue expectedWorkerControlTaskQueue := "test-control-queue" @@ -7270,6 +7275,8 @@ func (s *mutableStateSuite) TestAddActivityTaskStartedEventStoresWorkerControlTa updatedActivityInfo, ok := s.mutableState.GetActivityInfo(activityInfo.ScheduledEventId) s.True(ok) s.Equal(expectedWorkerControlTaskQueue, updatedActivityInfo.WorkerControlTaskQueue) + s.Empty(updatedActivityInfo.LastWorkerDeploymentVersion) + s.Nil(updatedActivityInfo.LastDeploymentVersion) } func (s *mutableStateSuite) TestAddActivityTaskStartedEventApproximateSize() { diff --git a/service/history/workflow/workflow_task_state_machine.go b/service/history/workflow/workflow_task_state_machine.go index 85c6e529074..72123c71bfa 100644 --- a/service/history/workflow/workflow_task_state_machine.go +++ b/service/history/workflow/workflow_task_state_machine.go @@ -822,8 +822,9 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskCompletedEvent( vb = enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED } + wftDeploymentVersion := worker_versioning.DeploymentVersionFromOptions(request.DeploymentOptions) //nolint:staticcheck // SA1019 deprecated Deployment will clean up later - wftDeployment := worker_versioning.DeploymentOrVersion(request.Deployment, worker_versioning.DeploymentVersionFromOptions(request.DeploymentOptions)) + wftDeployment := worker_versioning.DeploymentOrVersion(request.Deployment, wftDeploymentVersion) // Now write the completed event event := m.ms.hBuilder.AddWorkflowTaskCompletedEvent( @@ -864,10 +865,18 @@ func (m *workflowTaskStateMachine) AddWorkflowTaskCompletedEvent( tag.BuildId(oneTimeTarget.GetBuildId())) } - metrics.WorkflowTasksCompleted.With(m.metricsHandler).Record(1, - metrics.NamespaceTag(m.ms.GetNamespaceEntry().Name().String()), - metrics.VersioningBehaviorTag(vb), - metrics.FirstAttemptTag(workflowTask.Attempt), + RecordWorkflowTaskCompletedMetrics( + m.ms.config, + m.metricsHandler, + m.ms.GetNamespaceEntry().Name(), + m.ms.GetExecutionInfo().GetTaskQueue(), + WorkflowTaskCompletionMetrics{ + VersioningInfo: VersioningMetricContext{ + Behavior: vb, + DeploymentVersion: wftDeploymentVersion, + }, + Attempt: workflowTask.Attempt, + }, ) numConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute := m.ms.config.NumConsecutiveWorkflowTaskProblemsToTriggerSearchAttribute(m.ms.GetNamespaceEntry().Name().String()) diff --git a/service/matching/matching_engine.go b/service/matching/matching_engine.go index 6cd023eef83..049da260866 100644 --- a/service/matching/matching_engine.go +++ b/service/matching/matching_engine.go @@ -3114,7 +3114,9 @@ func (e *matchingEngineImpl) emitTaskDispatchLatency( } // else ignore the error and use the current partition } - workerVersion := worker_versioning.WorkerDeploymentVersionToStringV32(worker_versioning.DeploymentVersionFromOptions(pollMetadata.deploymentOptions)) + deploymentVersion := worker_versioning.DeploymentVersionFromOptions(pollMetadata.deploymentOptions) + workerVersion := worker_versioning.WorkerDeploymentVersionToStringV32(deploymentVersion) + breakdownMetricsByBuildID := e.config.BreakdownMetricsByBuildID(namespaceName, tqName, taskType) handler := metrics.GetPerTaskQueuePartitionIDScope( e.metricsHandler, @@ -3129,7 +3131,9 @@ func (e *matchingEngineImpl) emitTaskDispatchLatency( metrics.TaskSourceTag(task.source), metrics.ForwardedTag(task.isForwarded()), metrics.MatchingTaskPriorityTag(task.getPriority().GetPriorityKey()), - metrics.WorkerVersionTag(workerVersion, e.config.BreakdownMetricsByBuildID(namespaceName, tqName, taskType)), + metrics.WorkerVersionTag(workerVersion, breakdownMetricsByBuildID), + metrics.WorkerDeploymentNameTag(deploymentVersion.GetDeploymentName(), breakdownMetricsByBuildID), + metrics.WorkerDeploymentBuildIDTag(deploymentVersion.GetBuildId(), breakdownMetricsByBuildID), ) } diff --git a/tests/activity_metrics_parity_test.go b/tests/activity_metrics_parity_test.go index 055178f24c3..f695ffc5ff8 100644 --- a/tests/activity_metrics_parity_test.go +++ b/tests/activity_metrics_parity_test.go @@ -72,12 +72,14 @@ func (s *activityParityTestSuite) TestScheduleToStartMetric() { taskQueue = result.taskQueue } expectedTags := map[string]string{ - "namespace": result.namespace, - "operation": metrics.HistoryRecordActivityTaskStartedScope, - "partition": "__normal__", - "service_name": string(primitives.HistoryService), - "task_type": enumspb.TASK_QUEUE_TYPE_ACTIVITY.String(), - "taskqueue": taskQueue, + "namespace": result.namespace, + "operation": metrics.HistoryRecordActivityTaskStartedScope, + "partition": "__normal__", + "service_name": string(primitives.HistoryService), + "task_type": enumspb.TASK_QUEUE_TYPE_ACTIVITY.String(), + "taskqueue": taskQueue, + "worker_build_id": "", + "worker_deployment_name": "", } for _, recording := range result.recordings { require.Equal(t, expectedTags, recording.Tags, "%s metric tags must match the activity-task start scope", implementation) @@ -105,12 +107,13 @@ func (s *activityParityTestSuite) TestScheduleToStartMetric() { func (s *activityParityTestSuite) TestMetrics() { type activityMetric struct { - name string - compared bool - counter bool - baseHandler bool - legacyWFAHandler bool - recordingTagKeys []string + name string + compared bool + counter bool + baseHandler bool + legacyWFAHandler bool + workerDeploymentTags bool + recordingTagKeys []string } type scenario struct { name string @@ -130,21 +133,25 @@ func (s *activityParityTestSuite) TestMetrics() { "versioning_behavior", "workflowType", } + workerDeploymentTagKeys := []string{ + "worker_build_id", + "worker_deployment_name", + } legacyWFATagKeys := []string{ "activity_targeting_method", "namespace", "service_name", } catalog := []activityMetric{ - {name: metrics.ActivitySuccess.Name(), compared: true, counter: true}, - {name: metrics.ActivityFail.Name(), compared: true, counter: true}, - {name: metrics.ActivityTaskFail.Name(), compared: true, counter: true}, - {name: metrics.ActivityCancel.Name(), compared: true, counter: true}, + {name: metrics.ActivitySuccess.Name(), compared: true, counter: true, workerDeploymentTags: true}, + {name: metrics.ActivityFail.Name(), compared: true, counter: true, workerDeploymentTags: true}, + {name: metrics.ActivityTaskFail.Name(), compared: true, counter: true, workerDeploymentTags: true}, + {name: metrics.ActivityCancel.Name(), compared: true, counter: true, workerDeploymentTags: true}, {name: metrics.ActivityTerminate.Name()}, - {name: metrics.ActivityTimeout.Name(), compared: true, counter: true, recordingTagKeys: []string{"timeout_type"}}, - {name: metrics.ActivityTaskTimeout.Name(), compared: true, counter: true, recordingTagKeys: []string{"timeout_type"}}, - {name: metrics.ActivityStartToCloseLatency.Name(), compared: true}, - {name: metrics.ActivityScheduleToCloseLatency.Name(), compared: true}, + {name: metrics.ActivityTimeout.Name(), compared: true, counter: true, workerDeploymentTags: true, recordingTagKeys: []string{"timeout_type"}}, + {name: metrics.ActivityTaskTimeout.Name(), compared: true, counter: true, workerDeploymentTags: true, recordingTagKeys: []string{"timeout_type"}}, + {name: metrics.ActivityStartToCloseLatency.Name(), compared: true, workerDeploymentTags: true}, + {name: metrics.ActivityScheduleToCloseLatency.Name(), compared: true, workerDeploymentTags: true}, {name: metrics.ActivityPause.Name(), compared: true, counter: true, legacyWFAHandler: true}, {name: metrics.ActivityUnpause.Name(), compared: true, counter: true, legacyWFAHandler: true}, {name: metrics.ActivityReset.Name(), compared: true, counter: true, legacyWFAHandler: true}, @@ -304,6 +311,9 @@ func (s *activityParityTestSuite) TestMetrics() { } else { if !metric.baseHandler && len(wfa[metric.name]) > 0 { expectedTagKeys := append([]string{}, perActivityTagKeys...) + if metric.workerDeploymentTags { + expectedTagKeys = append(expectedTagKeys, workerDeploymentTagKeys...) + } expectedTagKeys = append(expectedTagKeys, metric.recordingTagKeys...) sort.Strings(expectedTagKeys) require.Equal(t, expectedTagKeys, wfaTagKeys, diff --git a/tests/versioning_3_test.go b/tests/versioning_3_test.go index 8a3f5c8affb..4c38c7b1018 100644 --- a/tests/versioning_3_test.go +++ b/tests/versioning_3_test.go @@ -33,6 +33,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" @@ -1245,6 +1246,45 @@ func (s *Versioning3Suite) TestEagerActivity() { env.verifyWorkflowVersioning(s, tv, vbUnpinned, tv.Deployment(), nil, nil) } +func (s *Versioning3Suite) TestEagerActivityTimeoutMetricTags() { + env := s.setupEnv( + testcore.WithDynamicConfig(dynamicconfig.EnableActivityEagerExecution, true), + testcore.WithDynamicConfig(dynamicconfig.MetricsBreakdownByBuildID, true), + ) + tv := env.Tv() + + env.updateTaskQueueDeploymentDataWithRoutingConfig(s, tv, &deploymentpb.RoutingConfig{ + CurrentDeploymentVersion: worker_versioning.ExternalWorkerDeploymentVersionFromStringV31(tv.DeploymentVersionString()), + CurrentVersionChangedTime: timestamp.TimePtr(time.Now()), + RevisionNumber: 1, + }, map[string]*deploymentspb.WorkerDeploymentVersionData{tv.DeploymentVersion().GetBuildId(): { + Status: enumspb.WORKER_DEPLOYMENT_VERSION_STATUS_CURRENT, + }}, []string{}, tqTypeWf, tqTypeAct) + env.startWorkflow(s, tv, nil) + capture := env.StartNamespaceMetricCapture() + + _, resp := env.pollWftAndHandle(s, tv, false, nil, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + s.NotNil(task) + response := env.respondWftWithActivities(tv, tv, true, vbUnpinned, "eager-timeout") + activity := response.Commands[0].GetScheduleActivityTaskCommandAttributes() + activity.RequestEagerExecution = true + activity.StartToCloseTimeout = durationpb.New(200 * time.Millisecond) + activity.RetryPolicy = &commonpb.RetryPolicy{MaximumAttempts: 1} + return response, nil + }) + s.NotEmpty(resp.GetActivityTasks()) + + requireWorkerDeploymentMetricTags( + s, + capture, + tv, + metrics.ActivityTaskTimeout.Name(), + metrics.ActivityTimeout.Name(), + metrics.ActivityScheduleToCloseLatency.Name(), + ) +} + func (s *Versioning3Suite) TestTransitionFromActivity_Sticky() { s.testTransitionFromActivity(true) } diff --git a/tests/worker_deployment_version_test.go b/tests/worker_deployment_version_test.go index d154aec94de..f9a81afbe9e 100644 --- a/tests/worker_deployment_version_test.go +++ b/tests/worker_deployment_version_test.go @@ -22,13 +22,17 @@ import ( workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" computeprovider "go.temporal.io/auto-scaled-workers/wci/workflow/compute_provider" + "go.temporal.io/sdk/activity" sdkclient "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" "go.temporal.io/sdk/worker" "go.temporal.io/sdk/workflow" deploymentspb "go.temporal.io/server/api/deployment/v1" "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/testhooks" @@ -146,6 +150,22 @@ func (s *DeploymentVersionSuite) updateMetadata(env *testcore.TestEnv, tv *testv func (s *DeploymentVersionSuite) startVersionWorkflow(ctx context.Context, env *testcore.TestEnv, tv *testvars.TestVars) { go s.pollFromDeployment(ctx, env, tv) + s.waitForVersionWorkflow(ctx, env, tv) +} + +func (s *DeploymentVersionSuite) startVersionWorkflowAndStopPoll(ctx context.Context, env *testcore.TestEnv, tv *testvars.TestVars) { + pollCtx, cancelPoll := context.WithCancel(ctx) + pollDone := make(chan struct{}) + go func() { + defer close(pollDone) + s.pollFromDeployment(pollCtx, env, tv) + }() + s.waitForVersionWorkflow(ctx, env, tv) + cancelPoll() + await.Rcv(s.T(), pollDone) +} + +func (s *DeploymentVersionSuite) waitForVersionWorkflow(ctx context.Context, env *testcore.TestEnv, tv *testvars.TestVars) { s.EventuallyWithT(func(t *assert.CollectT) { a := assert.New(t) resp, err := s.describeVersion(env, tv) @@ -517,6 +537,333 @@ func (s *DeploymentVersionSuite) startUnpinnedWorkflow(ctx context.Context, env return s.startVersionedWorkflow(ctx, env, tv, workflow.VersioningBehaviorAutoUpgrade) } +func (s *DeploymentVersionSuite) TestWorkerDeploymentLatencyMetricTags() { + for _, tc := range []struct { + name string + versioned bool + breakdownMetricsByBuildID bool + }{ + { + name: "versioned with breakdown enabled", + versioned: true, + breakdownMetricsByBuildID: true, + }, + { + name: "versioned with breakdown disabled", + versioned: true, + breakdownMetricsByBuildID: false, + }, + { + name: "unversioned with breakdown enabled", + breakdownMetricsByBuildID: true, + }, + } { + s.Run(tc.name, func(s *DeploymentVersionSuite) { + env := s.newTestEnv( + testcore.WithDynamicConfig( + dynamicconfig.MetricsBreakdownByBuildID, + tc.breakdownMetricsByBuildID, + ), + ) + tv := env.Tv() + + if tc.versioned { + s.startVersionWorkflowAndStopPoll(s.Context(), env, tv) + s.NoError(s.setCurrent(env, tv, false)) + } + + capture := env.StartNamespaceMetricCapture() + activity := func(context.Context) error { + return nil + } + wf := func(ctx workflow.Context) error { + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: time.Minute, + }) + return workflow.ExecuteActivity(ctx, activity).Get(ctx, nil) + } + workerOptions := worker.Options{ + Identity: tv.WorkerIdentity(), + DisableEagerActivities: true, + } + if tc.versioned { + workerOptions.DeploymentOptions = worker.DeploymentOptions{ + Version: tv.SDKDeploymentVersion(), + UseVersioning: true, + } + } + w := worker.New(env.SdkClient(), tv.TaskQueue().GetName(), workerOptions) + if tc.versioned { + w.RegisterWorkflowWithOptions(wf, workflow.RegisterOptions{ + VersioningBehavior: workflow.VersioningBehaviorAutoUpgrade, + }) + } else { + w.RegisterWorkflow(wf) + } + w.RegisterActivity(activity) + s.NoError(w.Start()) + defer w.Stop() + + run, err := env.SdkClient().ExecuteWorkflow( + s.Context(), + sdkclient.StartWorkflowOptions{ + ID: tv.WorkflowID(), + TaskQueue: tv.TaskQueue().GetName(), + }, + wf, + ) + s.NoError(err) + s.NoError(run.Get(s.Context(), nil)) + + expectedDeploymentName := "" + expectedBuildID := "" + if tc.versioned && tc.breakdownMetricsByBuildID { + expectedDeploymentName = tv.DeploymentSeries() + expectedBuildID = tv.BuildID() + } + + for _, metricName := range []string{ + metrics.TaskScheduleToStartLatency.Name(), + metrics.TaskDispatchLatencyPerTaskQueue.Name(), + } { + seenTaskTypes := make(map[string]bool) + for _, recording := range capture.Metric(metricName) { + if recording.Tags["taskqueue"] != tv.TaskQueue().GetName() { + continue + } + taskType := recording.Tags["task_type"] + if taskType != enumspb.TASK_QUEUE_TYPE_WORKFLOW.String() && + taskType != enumspb.TASK_QUEUE_TYPE_ACTIVITY.String() { + continue + } + s.Equal(expectedDeploymentName, recording.Tags["worker_deployment_name"]) + s.Equal(expectedBuildID, recording.Tags["worker_build_id"]) + seenTaskTypes[taskType] = true + } + s.True(seenTaskTypes[enumspb.TASK_QUEUE_TYPE_WORKFLOW.String()]) + s.True(seenTaskTypes[enumspb.TASK_QUEUE_TYPE_ACTIVITY.String()]) + } + }) + } +} + +func (s *DeploymentVersionSuite) TestWorkerDeploymentActivityOutcomeMetricTags() { + for _, tc := range []struct { + name string + outcome string + expectError bool + expectedMetrics []string + }{ + { + name: "success", + outcome: "success", + expectedMetrics: []string{ + metrics.WorkflowTasksCompleted.Name(), + metrics.ActivitySuccess.Name(), + metrics.ActivityE2ELatency.Name(), + metrics.ActivityStartToCloseLatency.Name(), + metrics.ActivityScheduleToCloseLatency.Name(), + }, + }, + { + name: "terminal failure", + outcome: "fail", + expectError: true, + expectedMetrics: []string{ + metrics.ActivityTaskFail.Name(), + metrics.ActivityFail.Name(), + metrics.ActivityE2ELatency.Name(), + metrics.ActivityStartToCloseLatency.Name(), + metrics.ActivityScheduleToCloseLatency.Name(), + }, + }, + { + name: "cancellation", + outcome: "cancel", + expectError: true, + expectedMetrics: []string{metrics.ActivityCancel.Name()}, + }, + { + name: "terminal timeout", + outcome: "timeout", + expectError: true, + expectedMetrics: []string{ + metrics.ActivityTaskTimeout.Name(), + metrics.ActivityTimeout.Name(), + }, + }, + } { + s.Run(tc.name, func(s *DeploymentVersionSuite) { + env, tv, capture := s.newWorkerDeploymentMetricTestEnv() + releaseTimeoutActivity := make(chan struct{}) + w := worker.New(env.SdkClient(), tv.TaskQueue().GetName(), worker.Options{ + DeploymentOptions: worker.DeploymentOptions{ + Version: tv.SDKDeploymentVersion(), + UseVersioning: true, + }, + Identity: tv.WorkerIdentity(), + DisableEagerActivities: true, + }) + activityStarted := make(chan struct{}, 1) + activityFn := func(ctx context.Context) error { + activityStarted <- struct{}{} + switch tc.outcome { + case "success": + return nil + case "fail": + return errors.New("intentional activity failure") //nolint:err113 + case "cancel": + heartbeatTicker := time.NewTicker(10 * time.Millisecond) + defer heartbeatTicker.Stop() + for { + activity.RecordHeartbeat(ctx) + select { + case <-ctx.Done(): + return ctx.Err() + case <-heartbeatTicker.C: + } + } + case "timeout": + <-releaseTimeoutActivity + return nil + default: + return fmt.Errorf("unknown activity outcome %q", tc.outcome) + } + } + workflowFn := func(ctx workflow.Context) error { + startToCloseTimeout := time.Minute + if tc.outcome == "timeout" { + startToCloseTimeout = 200 * time.Millisecond + } + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: startToCloseTimeout, + HeartbeatTimeout: time.Second, + WaitForCancellation: true, + RetryPolicy: &temporal.RetryPolicy{ + MaximumAttempts: 1, + }, + }) + return workflow.ExecuteActivity(ctx, "worker-deployment-metric-activity").Get(ctx, nil) + } + + w.RegisterWorkflowWithOptions(workflowFn, workflow.RegisterOptions{ + Name: "worker-deployment-activity-outcome-metric-workflow", + VersioningBehavior: workflow.VersioningBehaviorAutoUpgrade, + }) + w.RegisterActivityWithOptions(activityFn, activity.RegisterOptions{ + Name: "worker-deployment-metric-activity", + }) + s.NoError(w.Start()) + defer w.Stop() + defer close(releaseTimeoutActivity) + + run, err := env.SdkClient().ExecuteWorkflow( + s.Context(), + sdkclient.StartWorkflowOptions{ + ID: tv.WorkflowID(), + TaskQueue: tv.TaskQueue().GetName(), + }, + "worker-deployment-activity-outcome-metric-workflow", + ) + s.NoError(err) + if tc.outcome == "cancel" { + await.Rcv(s.T(), activityStarted) + s.NoError(env.SdkClient().CancelWorkflow(s.Context(), run.GetID(), run.GetRunID())) + } + if tc.expectError { + s.Error(run.Get(s.Context(), nil)) + } else { + s.NoError(run.Get(s.Context(), nil)) + } + + requireWorkerDeploymentMetricTags(s, capture, tv, tc.expectedMetrics...) + }) + } +} + +func (s *DeploymentVersionSuite) TestWorkerDeploymentFailedWorkflowTaskMetricTags() { + env, tv, capture := s.newWorkerDeploymentMetricTestEnv() + run, err := env.SdkClient().ExecuteWorkflow( + s.Context(), + sdkclient.StartWorkflowOptions{ + ID: tv.WorkflowID(), + TaskQueue: tv.TaskQueue().GetName(), + VersioningOverride: &sdkclient.PinnedVersioningOverride{ + Version: tv.SDKDeploymentVersion(), + }, + }, + "worker-deployment-failed-workflow-task-metric-workflow", + ) + s.NoError(err) + defer func() { + _ = env.SdkClient().TerminateWorkflow(s.Context(), run.GetID(), run.GetRunID(), "test cleanup") + }() + + pollCtx, cancelPoll := context.WithTimeout(s.Context(), 10*time.Second) + defer cancelPoll() + task, err := env.FrontendClient().PollWorkflowTaskQueue(pollCtx, &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: tv.TaskQueue(), + Identity: tv.WorkerIdentity(), + DeploymentOptions: tv.WorkerDeploymentOptions(true), + }) + s.NoError(err) + s.NotEmpty(task.GetTaskToken()) + _, err = env.FrontendClient().RespondWorkflowTaskFailed(s.Context(), &workflowservice.RespondWorkflowTaskFailedRequest{ + Namespace: env.Namespace().String(), + TaskToken: task.GetTaskToken(), + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_WORKFLOW_WORKER_UNHANDLED_FAILURE, + Identity: tv.WorkerIdentity(), + DeploymentOptions: tv.WorkerDeploymentOptions(true), + }) + s.NoError(err) + + requireWorkerDeploymentMetricTags(s, capture, tv, metrics.FailedWorkflowTasksCounter.Name()) +} + +func (s *DeploymentVersionSuite) newWorkerDeploymentMetricTestEnv() ( + *testcore.TestEnv, + *testvars.TestVars, + *testcore.NamespaceMetricCapture, +) { + env := s.newTestEnv( + testcore.WithDynamicConfig(dynamicconfig.MetricsBreakdownByBuildID, true), + testcore.WithDynamicConfig(dynamicconfig.MetricsBreakdownByTaskQueue, true), + ) + tv := env.Tv() + s.startVersionWorkflowAndStopPoll(s.Context(), env, tv) + s.NoError(s.setCurrent(env, tv, false)) + capture := env.StartNamespaceMetricCapture() + return env, tv, capture +} + +func requireWorkerDeploymentMetricTags( + s parallelsuite.Scope, + capture *testcore.NamespaceMetricCapture, + tv *testvars.TestVars, + metricNames ...string, +) { + for _, metricName := range metricNames { + await.Require(s.Context(), s.TB(), func(t *await.T) { + r := t.Require() + recordings := capture.CollectMetric(metricName, func(recording *metricstest.CapturedRecording) bool { + taskQueue, hasTaskQueue := recording.Tags["taskqueue"] + return !hasTaskQueue || taskQueue == tv.TaskQueue().GetName() + }) + r.NotEmpty(recordings, "expected %s in namespace %s", metricName, tv.NamespaceName()) + hasExpectedTags := false + for _, recording := range recordings { + if recording.Tags["worker_deployment_name"] == tv.DeploymentSeries() && + recording.Tags["worker_build_id"] == tv.BuildID() { + hasExpectedTags = true + break + } + } + r.True(hasExpectedTags, "expected %s with deployment %q and build ID %q", metricName, tv.DeploymentSeries(), tv.BuildID()) + }, 5*time.Second, 50*time.Millisecond) + } +} + func (s *DeploymentVersionSuite) TestVersionIgnoresDrainageSignalWhenCurrentOrRamping() { env := s.newTestEnv() tv1 := env.Tv().WithBuildIDNumber(1) From 34c2032e0143c871bf06015c70011bf01ef82d5a Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Thu, 27 Aug 2026 12:58:23 -0700 Subject: [PATCH 079/139] Capture panics in Visibility query converter (#11800) ## What changed? Capture any panics in Visibility query converter. ## Why? Visibility query converter is complex, and at times might make assumptions that might not hold (due to bugs in the store query converter implementation for example). Capturing at top level, and returning an error instead. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../store/elasticsearch/visibility_store.go | 10 +- .../visibility/store/query/converter.go | 16 +- .../visibility/store/query/converter_test.go | 222 ++++++------------ .../store/query/interceptors_test.go | 6 + .../visibility/store/query/nil_converter.go | 6 + .../store/query/nil_converter_test.go | 12 +- .../visibility/store/sql/visibility_store.go | 10 +- .../store/sql/visibility_store_test.go | 8 +- service/frontend/workflow_handler.go | 27 ++- service/worker/scheduler/query.go | 10 + service/worker/scheduler/query_test.go | 11 +- 11 files changed, 180 insertions(+), 158 deletions(-) diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store.go b/common/persistence/visibility/store/elasticsearch/visibility_store.go index 50e3f3404ef..5a56408befe 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store.go @@ -763,8 +763,14 @@ func (s *VisibilityStore) convertQuery( return nil, err } - c := query.NewQueryConverter(&queryConverter{}, namespaceName, saTypeMap, saMapper). - WithChasmMapper(chasmMapper). + c := query.NewQueryConverter( + &queryConverter{}, + namespaceName, + saTypeMap, + saMapper, + s.metricsHandler, + s.logger, + ).WithChasmMapper(chasmMapper). WithArchetypeID(archetypeID) queryParams, err := c.Convert(queryString) diff --git a/common/persistence/visibility/store/query/converter.go b/common/persistence/visibility/store/query/converter.go index 53362a3e24b..33c957eea62 100644 --- a/common/persistence/visibility/store/query/converter.go +++ b/common/persistence/visibility/store/query/converter.go @@ -13,6 +13,8 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" @@ -63,6 +65,9 @@ type ( archetypeID chasm.ArchetypeID seenNamespaceDivision bool + + metricsHandler metrics.Handler + logger log.Logger } QueryConverterOptionFunc[ExprT any] func(*QueryConverter[ExprT]) @@ -134,6 +139,8 @@ func NewQueryConverter[ExprT any]( namespaceName namespace.Name, saTypeMap searchattribute.NameTypeMap, saMapper searchattribute.Mapper, + metricsHandler metrics.Handler, + logger log.Logger, ) *QueryConverter[ExprT] { c := &QueryConverter[ExprT]{ storeQC: storeQC, @@ -144,6 +151,9 @@ func NewQueryConverter[ExprT any]( saMapper: saMapper, seenNamespaceDivision: false, + + metricsHandler: metricsHandler, + logger: logger, } return c } @@ -178,7 +188,11 @@ func (c *QueryConverter[ExprT]) SeenNamespaceDivision() bool { func (c *QueryConverter[ExprT]) Convert( queryString string, -) (*QueryParams[ExprT], error) { +) (_ *QueryParams[ExprT], retError error) { + // Convert function may throw unexpected panics (eg: bugs). + // Capture them here to replace with an error. + defer metrics.CapturePanic(c.logger, c.metricsHandler, &retError) + queryParams, err := c.convertWhereString(queryString) if err != nil { return nil, err diff --git a/common/persistence/visibility/store/query/converter_test.go b/common/persistence/visibility/store/query/converter_test.go index 7d787ec0aef..5b6c8aa6671 100644 --- a/common/persistence/visibility/store/query/converter_test.go +++ b/common/persistence/visibility/store/query/converter_test.go @@ -11,6 +11,8 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" @@ -19,7 +21,6 @@ import ( const ( testNamespaceName = namespace.Name("test-namespace") - testNamespaceID = namespace.ID("test-namespace-id") ) func TestWithSearchAttributeInterceptor(t *testing.T) { @@ -29,31 +30,18 @@ func TestWithSearchAttributeInterceptor(t *testing.T) { storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) // QueryConverter without explicit SearchAttributeInterceptor sets the nop interceptor. - c := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + c := newTestQueryConverter(storeQCMock) r.Equal(nopSearchAttributeInterceptor, c.saInterceptor) // Setting nil interceptor sets the nop interceptor. - c = NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ).WithSearchAttributeInterceptor(nil) + c = newTestQueryConverter(storeQCMock). + WithSearchAttributeInterceptor(nil) r.Equal(nopSearchAttributeInterceptor, c.saInterceptor) // Setting non-nil interceptor i := &testSearchAttributeInterceptor{} - c = NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ).WithSearchAttributeInterceptor(i) + c = newTestQueryConverter(storeQCMock). + WithSearchAttributeInterceptor(i) r.Equal(i, c.saInterceptor) } @@ -247,12 +235,7 @@ func TestQueryConverter_Convert(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -381,12 +364,7 @@ func TestQueryConverter_ConvertWhereString(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -554,12 +532,7 @@ func TestQueryConverter_ConvertSelectStmt(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -824,12 +797,7 @@ func TestQueryConverter_ConvertWhereExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -914,12 +882,7 @@ func TestQueryConverter_ConvertParenExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -994,12 +957,7 @@ func TestQueryConverter_ConvertNotExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -1103,12 +1061,7 @@ func TestQueryConverter_ConvertAndExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -1212,12 +1165,7 @@ func TestQueryConverter_ConvertOrExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -1334,12 +1282,7 @@ func TestQueryConverter_ConvertComparisonExprStoreQueryConverterCalled(t *testin r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) storeQCMock.EXPECT().GetDatetimeFormat().Return(time.RFC3339Nano).AnyTimes() input := &sqlparser.ComparisonExpr{ @@ -1427,12 +1370,7 @@ func TestQueryConverter_ConvertComparisonExprFail(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) inExpr := parseWhereString(tc.in).(*sqlparser.ComparisonExpr) _, err := queryConverter.convertComparisonExpr(inExpr) @@ -1522,12 +1460,7 @@ func TestQueryConverter_ConvertRangeCond(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -1659,12 +1592,7 @@ func TestQueryConverter_ConvertIsExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) if tc.setupMocks != nil { tc.setupMocks(storeQCMock) @@ -1764,12 +1692,7 @@ func TestQueryConverter_ConvertColName(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) out, err := queryConverter.convertColName(tc.in) if tc.err != "" { @@ -1927,6 +1850,8 @@ func TestQueryConverter_ResolveSearchAttributeAlias(t *testing.T) { &searchattribute.TestMapper{ WithCustomScheduleID: tc.withCustomScheduleID, }, + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), ) if tc.useNoopMapper { @@ -2065,12 +1990,7 @@ func TestQueryConverter_ParseValueExpr(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) out, err := queryConverter.parseValueExpr(tc.expr, tc.alias, tc.field, tc.saType) if tc.err != "" { @@ -2188,12 +2108,7 @@ func TestQueryConverter_ParseSQLVal(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) storeQCMock.EXPECT().GetDatetimeFormat().Return(time.RFC3339Nano).AnyTimes() out, err := queryConverter.parseSQLVal(tc.expr, tc.saName, tc.saFieldName, tc.saType) @@ -2405,12 +2320,7 @@ func TestQueryConverter_ValidateValueType(t *testing.T) { r := require.New(t) ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) storeQCMock.EXPECT().GetDatetimeFormat().Return(time.RFC3339Nano).AnyTimes() out, err := queryConverter.validateValueType(tc.saName, tc.saType, tc.value) @@ -2566,12 +2476,7 @@ func TestQueryConverter_WithChasmMapper(t *testing.T) { }, ) - c := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + c := newTestQueryConverter(storeQCMock) r.Nil(c.chasmMapper) c = c.WithChasmMapper(chasmMapper) @@ -2587,12 +2492,7 @@ func TestQueryConverter_WithArchetypeID(t *testing.T) { ctrl := gomock.NewController(t) storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - c := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + c := newTestQueryConverter(storeQCMock) r.Equal(chasm.UnspecifiedArchetypeID, c.archetypeID) c = c.WithArchetypeID(123) @@ -2610,12 +2510,8 @@ func TestQueryConverter_TemporalSystemExecutionStatus(t *testing.T) { // Test that TemporalSystemExecutionStatus maps to ExecutionStatus only for SchedulerArchetypeID t.Run("with SchedulerArchetypeID", func(t *testing.T) { r := require.New(t) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ).WithArchetypeID(chasm.SchedulerArchetypeID) + queryConverter := newTestQueryConverter(storeQCMock). + WithArchetypeID(chasm.SchedulerArchetypeID) in := &sqlparser.ColName{ Name: sqlparser.NewColIdent("TemporalSystemExecutionStatus"), @@ -2631,12 +2527,7 @@ func TestQueryConverter_TemporalSystemExecutionStatus(t *testing.T) { t.Run("without SchedulerArchetypeID", func(t *testing.T) { r := require.New(t) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ) + queryConverter := newTestQueryConverter(storeQCMock) in := &sqlparser.ColName{ Name: sqlparser.NewColIdent("TemporalSystemExecutionStatus"), @@ -2663,12 +2554,8 @@ func TestQueryConverter_ResolveSearchAttributeAlias_WithChasmMapper(t *testing.T }, ) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{}, - ).WithChasmMapper(chasmMapper) + queryConverter := newTestQueryConverter(storeQCMock). + WithChasmMapper(chasmMapper) testCases := []struct { name string @@ -2724,3 +2611,48 @@ func TestQueryConverter_ResolveSearchAttributeAlias_WithChasmMapper(t *testing.T }) } } + +func TestQueryConverter_CapturePanic(t *testing.T) { + t.Parallel() + r := require.New(t) + ctrl := gomock.NewController(t) + metricsHandlerMock := metrics.NewMockHandler(ctrl) + loggerMock := log.NewMockLogger(ctrl) + storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) + queryConverter := newTestQueryConverter(storeQCMock) + queryConverter.metricsHandler = metricsHandlerMock + queryConverter.logger = loggerMock + + keywordCol := NewSAColumn( + "AliasForKeyword01", + "Keyword01", + enumspb.INDEXED_VALUE_TYPE_KEYWORD, + ) + + counterMock := metrics.NewMockCounterIface(ctrl) + counterMock.EXPECT().Record(int64(1)) + metricsHandlerMock.EXPECT().Counter(metrics.ServicePanic.Name()).Return(counterMock) + loggerMock.EXPECT().Error("Panic is captured", gomock.Any(), gomock.Any()).Return() + storeQCMock.EXPECT().ConvertKeywordComparisonExpr(sqlparser.EqualStr, keywordCol, "foo"). + DoAndReturn( + func(operator string, col *SAColumn, value any) (sqlparser.Expr, error) { + panic("random") + }, + ) + out, err := queryConverter.Convert("AliasForKeyword01 = 'foo'") + r.ErrorContains(err, "panic: random") + r.Nil(out) +} + +func newTestQueryConverter( + storeQC StoreQueryConverter[sqlparser.Expr], +) *QueryConverter[sqlparser.Expr] { + return NewQueryConverter( + storeQC, + testNamespaceName, + searchattribute.TestNameTypeMap(), + &searchattribute.TestMapper{}, + nil, // metricsHandler + log.NewNoopLogger(), + ) +} diff --git a/common/persistence/visibility/store/query/interceptors_test.go b/common/persistence/visibility/store/query/interceptors_test.go index 1abe08adb4c..6b524a3c411 100644 --- a/common/persistence/visibility/store/query/interceptors_test.go +++ b/common/persistence/visibility/store/query/interceptors_test.go @@ -5,7 +5,10 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/searchattribute" + "go.uber.org/mock/gomock" ) type testSearchAttributeInterceptor struct { @@ -24,12 +27,15 @@ func (t *testSearchAttributeInterceptor) Intercept(col *SAColumn) error { func TestSearchAttributeInterceptor(t *testing.T) { t.Parallel() + ctrl := gomock.NewController(t) interceptor := &testSearchAttributeInterceptor{} c := NewNilQueryConverter( "", searchattribute.TestNameTypeMap(), &searchattribute.TestMapper{}, + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), ).WithSearchAttributeInterceptor(interceptor) _, err := c.Convert("ExecutionStatus='Running' order by StartTime") diff --git a/common/persistence/visibility/store/query/nil_converter.go b/common/persistence/visibility/store/query/nil_converter.go index a13f8b475f4..e4391c21950 100644 --- a/common/persistence/visibility/store/query/nil_converter.go +++ b/common/persistence/visibility/store/query/nil_converter.go @@ -1,6 +1,8 @@ package query import ( + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" ) @@ -83,11 +85,15 @@ func NewNilQueryConverter( namespaceName namespace.Name, saTypeMap searchattribute.NameTypeMap, saMapper searchattribute.Mapper, + metricsHandler metrics.Handler, + logger log.Logger, ) NilQueryConverter { return NewQueryConverter( &nilStoreQueryConverter{}, namespaceName, saTypeMap, saMapper, + metricsHandler, + logger, ) } diff --git a/common/persistence/visibility/store/query/nil_converter_test.go b/common/persistence/visibility/store/query/nil_converter_test.go index 9c81b97b3be..5b536c79826 100644 --- a/common/persistence/visibility/store/query/nil_converter_test.go +++ b/common/persistence/visibility/store/query/nil_converter_test.go @@ -4,7 +4,10 @@ import ( "testing" "github.com/stretchr/testify/require" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/searchattribute" + "go.uber.org/mock/gomock" ) func TestNilStoreQueryConverter_GetDatetimeFormat(t *testing.T) { @@ -95,6 +98,13 @@ func TestNilStoreQueryConverter_ConvertIsExpr(t *testing.T) { func TestNewNilQueryConverter(t *testing.T) { t.Parallel() - c := NewNilQueryConverter("", searchattribute.TestNameTypeMap(), nil) + ctrl := gomock.NewController(t) + c := NewNilQueryConverter( + "", + searchattribute.TestNameTypeMap(), + nil, // saMapper + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), + ) require.Equal(t, &nilStoreQueryConverter{}, c.storeQC) } diff --git a/common/persistence/visibility/store/sql/visibility_store.go b/common/persistence/visibility/store/sql/visibility_store.go index 1a9a0d80ec1..7dc1a8b591e 100644 --- a/common/persistence/visibility/store/sql/visibility_store.go +++ b/common/persistence/visibility/store/sql/visibility_store.go @@ -275,6 +275,8 @@ func (s *VisibilityStore) countChasmExecutions( saMapper, mapper, request.ArchetypeId, + s.metricsHandler, + s.logger, ) if err != nil { if converterErr, ok := errors.AsType[*query.ConverterError](err); ok { @@ -385,6 +387,8 @@ func (s *VisibilityStore) listExecutionsInternal( saMapper, request.ChasmMapper, request.ArchetypeID, + s.metricsHandler, + s.logger, ) if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be @@ -617,6 +621,8 @@ func (s *VisibilityStore) countWorkflowExecutions( saMapper, nil, chasm.UnspecifiedArchetypeID, + s.metricsHandler, + s.logger, ) if err != nil { // Convert ConverterError to InvalidArgument and pass through all other errors (which should be @@ -929,8 +935,10 @@ func buildQueryParams( saMapper searchattribute.Mapper, chasmMapper *chasm.VisibilitySearchAttributesMapper, archetypeID chasm.ArchetypeID, + metricsHandler metrics.Handler, + logger log.Logger, ) (*query.QueryParams[sqlparser.Expr], error) { - c := query.NewQueryConverter(sqlQC, namespaceName, saTypeMap, saMapper). + c := query.NewQueryConverter(sqlQC, namespaceName, saTypeMap, saMapper, metricsHandler, logger). WithChasmMapper(chasmMapper). WithArchetypeID(archetypeID) diff --git a/common/persistence/visibility/store/sql/visibility_store_test.go b/common/persistence/visibility/store/sql/visibility_store_test.go index 42208748a0e..b3714181c6b 100644 --- a/common/persistence/visibility/store/sql/visibility_store_test.go +++ b/common/persistence/visibility/store/sql/visibility_store_test.go @@ -7,10 +7,13 @@ import ( "github.com/stretchr/testify/require" "github.com/temporalio/sqlparser" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/sql/sqlplugin/mysql" "go.temporal.io/server/common/persistence/sql/sqlplugin/postgresql" "go.temporal.io/server/common/persistence/sql/sqlplugin/sqlite" "go.temporal.io/server/common/searchattribute" + "go.uber.org/mock/gomock" ) var pluginNames = []string{ @@ -65,6 +68,7 @@ func TestBuildQueryParams(t *testing.T) { tcName := fmt.Sprintf("%s/%s", pluginName, tc.name) t.Run(tcName, func(t *testing.T) { r := require.New(t) + ctrl := gomock.NewController(t) sqlQC, err := NewSQLQueryConverter(pluginName) r.NoError(err) @@ -75,8 +79,10 @@ func TestBuildQueryParams(t *testing.T) { sqlQC, searchattribute.TestNameTypeMap(), &searchattribute.TestMapper{}, - nil, + nil, // chasmMapper chasm.UnspecifiedArchetypeID, + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), ) if tc.err != "" { r.Error(err) diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index ea04af7495c..2a1230a3c04 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -5176,17 +5176,23 @@ func (wh *WorkflowHandler) ListSchedules( return nil, errListNotAllowed } + metricsHandler := wh.metricsScope(ctx).WithTags(metrics.HeaderCallsiteTag("ListSchedules")) chasmEnabled := wh.chasmSchedulerEnabled(ctx, namespaceName.String()) - query, err := wh.prepareSchedulerQuery(chasmEnabled, request.Query, namespaceName) + schedulerQuery, err := wh.prepareSchedulerQuery( + chasmEnabled, + request.Query, + namespaceName, + metricsHandler, + ) if err != nil { return nil, err } if chasmEnabled { // CHASM ListSchedules will include schedules created in the V1/workflow stack. - return wh.listSchedulesChasm(ctx, request, namespaceName, namespaceID, query) + return wh.listSchedulesChasm(ctx, request, namespaceName, namespaceID, schedulerQuery) } - return wh.listSchedulesWorkflow(ctx, request, namespaceName, namespaceID, query) + return wh.listSchedulesWorkflow(ctx, request, namespaceName, namespaceID, schedulerQuery) } // prepareSchedulerQuery validates a scheduler RPC's query argument, and wraps it @@ -5195,6 +5201,7 @@ func (wh *WorkflowHandler) prepareSchedulerQuery( chasmEnabled bool, query string, namespaceName namespace.Name, + metricsHandler metrics.Handler, ) (string, error) { // Use different base queries based on code path: // - CHASM path uses TemporalSystemExecutionStatus (translated via archetype ID) @@ -5229,6 +5236,8 @@ func (wh *WorkflowHandler) prepareSchedulerQuery( chasmMapper, wh.config.VisibilityEnableUnifiedQueryConverter, query, + metricsHandler, + wh.logger, ); err != nil { return "", err } @@ -5365,17 +5374,23 @@ func (wh *WorkflowHandler) CountSchedules( return nil, errListNotAllowed } + metricsHandler := wh.metricsScope(ctx).WithTags(metrics.HeaderCallsiteTag("CountSchedules")) chasmEnabled := wh.chasmSchedulerEnabled(ctx, namespaceName.String()) - query, err := wh.prepareSchedulerQuery(chasmEnabled, request.Query, namespaceName) + schedulerQuery, err := wh.prepareSchedulerQuery( + chasmEnabled, + request.Query, + namespaceName, + metricsHandler, + ) if err != nil { return nil, err } // Route to CHASM or V1 based on config (same pattern as ListSchedules) if chasmEnabled { - return wh.countSchedulesChasm(ctx, namespaceID, namespaceName, query) + return wh.countSchedulesChasm(ctx, namespaceID, namespaceName, schedulerQuery) } - return wh.countSchedulesWorkflow(ctx, namespaceID, namespaceName, query) + return wh.countSchedulesWorkflow(ctx, namespaceID, namespaceName, schedulerQuery) } // countSchedulesChasm counts schedules using CHASM APIs diff --git a/service/worker/scheduler/query.go b/service/worker/scheduler/query.go index 25671d0cb3f..5b69872aeff 100644 --- a/service/worker/scheduler/query.go +++ b/service/worker/scheduler/query.go @@ -7,6 +7,8 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/visibility/store/elasticsearch" "go.temporal.io/server/common/persistence/visibility/store/query" @@ -72,6 +74,8 @@ func ValidateVisibilityQuery( chasmMapper *chasm.VisibilitySearchAttributesMapper, enableUnifiedQueryConverter dynamicconfig.BoolPropertyFn, queryString string, + metricsHandler metrics.Handler, + logger log.Logger, ) error { var fields []string var err error @@ -82,6 +86,8 @@ func ValidateVisibilityQuery( saMapperProvider, chasmMapper, queryString, + metricsHandler, + logger, ) } else { fields, err = getQueryFieldsLegacy(namespaceName, saNameType, saMapperProvider, chasmMapper, queryString) @@ -105,6 +111,8 @@ func getQueryFields( saMapperProvider searchattribute.MapperProvider, chasmMapper *chasm.VisibilitySearchAttributesMapper, queryString string, + metricsHandler metrics.Handler, + logger log.Logger, ) ([]string, error) { saMapper, err := saMapperProvider.GetMapper(namespaceName) if err != nil { @@ -115,6 +123,8 @@ func getQueryFields( namespaceName, saNameType, saMapper, + metricsHandler, + logger, ).WithChasmMapper(chasmMapper).WithSearchAttributeInterceptor(saInterceptor) _, err = queryConverter.Convert(queryString) if err != nil { diff --git a/service/worker/scheduler/query_test.go b/service/worker/scheduler/query_test.go index 936990c7e81..8d951391d93 100644 --- a/service/worker/scheduler/query_test.go +++ b/service/worker/scheduler/query_test.go @@ -7,10 +7,13 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/visibility/store/query" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" + "go.uber.org/mock/gomock" ) const ( @@ -256,12 +259,15 @@ func TestGetQueryFields(t *testing.T) { tc.name, func(t *testing.T) { s := require.New(t) + ctrl := gomock.NewController(t) fields, err := getQueryFields( testNamespace, searchattribute.TestNameTypeMap(), searchattribute.NewTestMapperProvider(&searchattribute.TestMapper{}), - nil, + nil, // chasmMapper tc.input, + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), ) if tc.expectedErrMsg == "" { s.NoError(err) @@ -337,6 +343,7 @@ func TestValidateVisibilityQuery(t *testing.T) { tc.name, func(t *testing.T) { s := require.New(t) + ctrl := gomock.NewController(t) err := ValidateVisibilityQuery( testNamespace, searchattribute.TestNameTypeMap(), @@ -344,6 +351,8 @@ func TestValidateVisibilityQuery(t *testing.T) { nil, dynamicconfig.GetBoolPropertyFn(true), tc.input, + metrics.NewMockHandler(ctrl), + log.NewNoopLogger(), ) if tc.expectedErrMsg == "" { s.NoError(err) From 1863fbe665ca34a4e49ae8e381adc49798b65e93 Mon Sep 17 00:00:00 2001 From: Qian Chen Date: Thu, 27 Aug 2026 13:19:48 -0700 Subject: [PATCH 080/139] Tag replicator errors by replication task type (#11760) ## What changed? `replicator_errors` now records the existing `replication_task_type` metric dimension when a namespace replication processor exhausts its task-application retries. This distinguishes values such as `NamespaceTask` and `TaskQueueUserData` without introducing a new metric. A unit test verifies that the failure counter carries the task type from the failed replication task. ## Why? This is a prerequisite for splitting task queue user data (`TaskQueueUserData`) failures from namespace metadata replication (`NamespaceTask`) alerts. Today both task kinds feed `replicator_errors` under `NamespaceReplicationTask` and are indistinguishable, so TQUD failures can be reported as namespace replication poison pills. This PR only adds the server metric label. Alert definitions will be updated separately after the label is available in deployed server versions. ## How did you test it? - [x] built - [x] covered by existing tests - [x] added unit tests - [ ] added new functional test(s) Manual two-cluster XDC E2E using `development-cluster-a.yaml` and `development-cluster-b.yaml`: - A namespace replication conflict exported `replicationTaskType="NamespaceTask"`. - A TQUD apply failure exported `replicationTaskType="TaskQueueUserData"`. --- .../worker/replicator/replication_message_processor.go | 2 +- .../replicator/replication_message_processor_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/service/worker/replicator/replication_message_processor.go b/service/worker/replicator/replication_message_processor.go index f521909d9f7..b3f2384d368 100644 --- a/service/worker/replicator/replication_message_processor.go +++ b/service/worker/replicator/replication_message_processor.go @@ -218,7 +218,7 @@ func (p *replicationMessageProcessor) handleReplicationTasks() { }, policy, isTransientRetryableError) if err != nil { - metrics.ReplicatorFailures.With(p.metricsHandler).Record(1) + metrics.ReplicatorFailures.With(p.metricsHandler).Record(1, metrics.ReplicationTaskTypeTag(task.TaskType)) p.logger.Error("Failed to apply replication tasks", tag.Error(err)) dlqErr := backoff.ThrottleRetry(func() error { diff --git a/service/worker/replicator/replication_message_processor_test.go b/service/worker/replicator/replication_message_processor_test.go index 55bdaf68696..26f546cc985 100644 --- a/service/worker/replicator/replication_message_processor_test.go +++ b/service/worker/replicator/replication_message_processor_test.go @@ -21,6 +21,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/membership" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace/nsreplication" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/wideevents" @@ -153,6 +154,9 @@ func TestHandleNamespaceReplicationTaskCountsRetries(t *testing.T) { func TestHandleNamespaceReplicationTaskEmitsDLQed(t *testing.T) { p, task, executor, queue, eventLogger := newReplicationEventTestProcessor(t, true, 1) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + p.metricsHandler = metricsHandler executor.EXPECT().Execute(gomock.Any(), task.GetNamespaceTaskAttributes()).Return(serviceerror.NewInvalidArgument("bad task")) queue.EXPECT().PublishToDLQ(gomock.Any(), task).Return(nil) @@ -162,6 +166,10 @@ func TestHandleNamespaceReplicationTaskEmitsDLQed(t *testing.T) { require.InDelta(t, float64(1), dlqed["attempt_count"], 0) require.Equal(t, "bad task", dlqed["error"]) require.NotContains(t, dlqed, "persistence_request") + recordings := capture.Snapshot()[metrics.ReplicatorFailures.Name()] + require.Len(t, recordings, 1) + taskTypeTag := metrics.ReplicationTaskTypeTag(task.TaskType) + require.Equal(t, taskTypeTag.Value, recordings[0].Tags[taskTypeTag.Key]) } func TestHandleNamespaceReplicationTaskEventsDisabled(t *testing.T) { From c7cd6e263ec2b96821067ade20869fdedc5e3d1c Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 27 Aug 2026 13:58:40 -0700 Subject: [PATCH 081/139] Stop overriding the global OTEL error handler (#11551) ## What changed? Stop installing Temporal-specific process-global OTEL error handlers. OTEL errors now use the default ie stderr. ## Why? The process-global OTEL handler retained the Temporal server logger after shutdown; ie it leaked. --- temporal/fx.go | 22 ++-------------------- tests/leakcheck/leak_test.go | 1 - 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/temporal/fx.go b/temporal/fx.go index dd7c9331afd..221b731335a 100644 --- a/temporal/fx.go +++ b/temporal/fx.go @@ -6,11 +6,9 @@ import ( "fmt" "maps" "os" - "sync/atomic" "time" "github.com/google/uuid" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" otellog "go.opentelemetry.io/otel/log" lognoop "go.opentelemetry.io/otel/log/noop" @@ -966,7 +964,6 @@ func verifyPersistenceCompatibleVersion( type SpanExporterInputs struct { fx.In Lifecycyle fx.Lifecycle - Logger log.Logger Config *config.Config `optional:"true"` } @@ -977,13 +974,6 @@ type SpanExporterInputs struct { // - []go.opentelemetry.io/otel/sdk/trace.SpanExporter var TraceExportModule = fx.Options( fx.Provide(func(inputs SpanExporterInputs) ([]otelsdktrace.SpanExporter, error) { - var tracingReady atomic.Bool - otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) { - if tracingReady.Load() { // ignore errors during startup - inputs.Logger.Warn("OTEL error", tag.Error(err), tag.ServiceErrorType(err)) - } - })) - // (1) Exporters from config. exportersByType := map[telemetry.SpanExporterType]otelsdktrace.SpanExporter{} if inputs.Config != nil { @@ -1010,12 +1000,8 @@ var TraceExportModule = fx.Options( // Configure exporters' lifecycle hooks. inputs.Lifecycyle.Append(fx.Hook{ - OnStart: func(ctx context.Context) error { - err = startAll(exporters)(ctx) - tracingReady.Store(true) - return err - }, - OnStop: shutdownAll(exporters), + OnStart: startAll(exporters), + OnStop: shutdownAll(exporters), }) return exporters, nil }), @@ -1086,10 +1072,6 @@ var ServiceTracingModule = fx.Options( tp := otelsdktrace.NewTracerProvider(opts...) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { - otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) { - // ignore errors during shutdown - })) - shutdownCtx, cancel := context.WithTimeout(context.Background(), 1*time.Second) defer cancel() diff --git a/tests/leakcheck/leak_test.go b/tests/leakcheck/leak_test.go index 9b8bf02d305..439a9f7a6e1 100644 --- a/tests/leakcheck/leak_test.go +++ b/tests/leakcheck/leak_test.go @@ -29,7 +29,6 @@ var goleakOpts = []goleak.Option{ var objectLeakOpts = []objectleak.Option{ objectleak.WithPruneType("google.golang.org/protobuf/internal/impl.*"), - objectleak.WithExpected("FunctionalTestBase.Logger*"), objectleak.WithExpected("FunctionalTestBase.testCluster.host*"), objectleak.WithExpected("FunctionalTestBase.testCluster.testBase*"), } From 994deca4efbdeb7795565425f43d698265863500 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Thu, 27 Aug 2026 14:14:22 -0700 Subject: [PATCH 082/139] Fix buffered child XDC test race (#11824) ## What changed? - Atomically return and hold the next parent workflow task after starting child workflows. - Release pending intercepted replication tasks during test cleanup. - Ensure each blocked replication task executes at most once. ## Why? This is a follow-up to [#11789](https://github.com/temporalio/temporal/pull/11789). The test separately scheduled and polled the next workflow task, leaving a race where a duplicate child outcome could be recorded before the parent workflow task started. When the assertion failed, blocked replication tasks were not released, potentially stalling subsequent XDC tests until CI timed out. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) Ran the affected test with the race detector 10 times: ```bash go test -tags test_dep -race -count=10 ./tests/xdc \ -run '^TestFuncClustersTestSuite$/^EnableTransitionHistory$/^TestNaturallyBufferedChildWorkflowOutcomesFlushedToLosingBranch$' \ -args -persistenceType=sql -persistenceDriver=sqlite --- ...uffered_events_replication_helpers_test.go | 38 +++++++++++++++---- tests/xdc/buffered_events_replication_test.go | 3 +- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/tests/xdc/buffered_events_replication_helpers_test.go b/tests/xdc/buffered_events_replication_helpers_test.go index bf69cbd21f2..1a21145756e 100644 --- a/tests/xdc/buffered_events_replication_helpers_test.go +++ b/tests/xdc/buffered_events_replication_helpers_test.go @@ -3,6 +3,7 @@ package xdc import ( "context" "errors" + "sync" "time" "github.com/google/uuid" @@ -46,7 +47,16 @@ import ( type blockedReplicationTask struct { execute func() error - result chan error + once sync.Once + done chan struct{} + err error +} + +func (t *blockedReplicationTask) run() { + t.once.Do(func() { + t.err = t.execute() + close(t.done) + }) } type bufferedEventsNamespace struct { @@ -776,6 +786,7 @@ func (s *xdcBaseSuite) blockReplicationForWorkflow( // The interceptor runs on the receiving cluster, so clusterIndex identifies the replication // destination rather than the cluster that generated the task. tasks := make(chan *blockedReplicationTask, 20) + stopped := make(chan struct{}) s.clusters[clusterIndex].InjectHook( s.T(), testhooks.NewHook(testhooks.HistoryReplicationTaskInterceptor, func( @@ -787,13 +798,25 @@ func (s *xdcBaseSuite) blockReplicationForWorkflow( } blockedTask := &blockedReplicationTask{ execute: execute, - result: make(chan error, 1), + done: make(chan struct{}), } - tasks <- blockedTask - return <-blockedTask.result + select { + case tasks <- blockedTask: + case <-stopped: + return execute() + } + select { + case <-blockedTask.done: + case <-stopped: + blockedTask.run() + } + return blockedTask.err }), testhooks.GlobalScope, ) + s.T().Cleanup(func() { + close(stopped) + }) return tasks } @@ -821,14 +844,13 @@ func (s *xdcBaseSuite) tryReleaseReplicationTask(tasks <-chan *blockedReplicatio } func (s *xdcBaseSuite) executeReplicationTask(task *blockedReplicationTask) { - err := task.execute() - task.result <- err + task.run() var duplicateError *serviceerror.AlreadyExists var retryReplicationError *serviceerrors.RetryReplication s.Require().True( - err == nil || errors.As(err, &duplicateError) || errors.As(err, &retryReplicationError), + task.err == nil || errors.As(task.err, &duplicateError) || errors.As(task.err, &retryReplicationError), "replication task failed: %v", - err, + task.err, ) } diff --git a/tests/xdc/buffered_events_replication_test.go b/tests/xdc/buffered_events_replication_test.go index 491172d3a5f..2178b7ce1be 100644 --- a/tests/xdc/buffered_events_replication_test.go +++ b/tests/xdc/buffered_events_replication_test.go @@ -292,11 +292,10 @@ func (s *FunctionalClustersTestSuite) TestNaturallyBufferedChildWorkflowOutcomes } commands = append(commands, startChildWorkflowCommand(childIDs[outcome], childQueues[outcome], runTimeout)) } - s.completeWorkflowTaskAndScheduleNext(ctx, workflowTaskCompletion{ + heldWorkflowTask := s.completeWorkflowTaskAndReturnNext(ctx, workflowTaskCompletion{ Task: firstTask, Commands: commands, }) - heldWorkflowTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, parentQueue) s.Require().NotEmpty(heldWorkflowTask.TaskToken) completedTask := s.pollBufferedEventsWorkflowTask(ctx, 0, ns, childQueues["completed"]) From c2e2215ea33a6af3c3dd7a8976c4ae81eefeaa10 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Thu, 27 Aug 2026 15:00:09 -0700 Subject: [PATCH 083/139] Resend child workflow async when missing on passive (#11705) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? - Added an opt-in recovery path for a child workflow missing from a standby cluster. - After the existing resend delay, VerifyFirstWorkflowTaskScheduled asynchronously fetches the child state from the active cluster, applies it locally, and verifies it again. - Added deduplication, per-shard concurrency limits, metrics, namespace checks, and transition-history gating. Corrected the discard-time source check to verify the child workflow rather than the parent. - Moved the reusable in-flight resend tracker into the shared workflowresend package. - Updated the existing XDC parent-child test to assert that the missing child and its first workflow task are restored. ## Why? Cross-shard replication may deliver the parent’s ChildWorkflowExecutionStarted event before the child workflow reaches the standby cluster. Previously, verification repeatedly returned NotFound and eventually discarded the standby task, leaving the child missing. This adds the child-side symmetric recovery behavior to the parent resend implemented in #11424 . ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) ### Rollout `history.enableChildWorkflowResend` default to `false`. ### Known - Enabling this feature introduces additional cross-cluster state-sync traffic. It is disabled by default and protected by deduplication and a per-shard concurrency limit. - Regular replication may race with state sync; duplicate application is treated as success. - The resend delay and replication timeout should remain below the standby task discard delay so recovery has time to complete. --- api/historyservice/v1/request_response.pb.go | 13 +- common/dynamicconfig/constants.go | 26 +- common/metrics/metric_defs.go | 18 +- .../historyservice/v1/request_response.proto | 1 + .../api.go | 253 +++---- .../in_flight_resends.go | 44 -- .../in_flight_resends_test.go | 73 -- .../verifyfirstworkflowtaskscheduled/api.go | 174 ++++- .../api_test.go | 101 ++- .../history/api/workflowresend/scheduler.go | 214 ++++++ .../api/workflowresend/scheduler_test.go | 365 ++++++++++ .../api/workflowresend/sync_workflow_state.go | 119 +++ .../sync_workflow_state_test.go | 324 ++++++++ service/history/configs/config.go | 6 +- service/history/fx.go | 42 ++ service/history/history_engine.go | 44 +- service/history/history_engine2_test.go | 689 +++++++++++++++++- service/history/history_engine_factory.go | 3 + service/history/history_engine_test.go | 16 + service/history/ndc_standby_task_util.go | 4 + .../transfer_queue_standby_task_executor.go | 79 +- ...ansfer_queue_standby_task_executor_test.go | 88 ++- tests/xdc/parent_child_harness.go | 203 +++++- tests/xdc/parent_child_test.go | 125 ++-- 24 files changed, 2640 insertions(+), 384 deletions(-) delete mode 100644 service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends.go delete mode 100644 service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends_test.go create mode 100644 service/history/api/workflowresend/scheduler.go create mode 100644 service/history/api/workflowresend/scheduler_test.go create mode 100644 service/history/api/workflowresend/sync_workflow_state.go create mode 100644 service/history/api/workflowresend/sync_workflow_state_test.go diff --git a/api/historyservice/v1/request_response.pb.go b/api/historyservice/v1/request_response.pb.go index e91eb145872..a7f13f19b10 100644 --- a/api/historyservice/v1/request_response.pb.go +++ b/api/historyservice/v1/request_response.pb.go @@ -3718,6 +3718,7 @@ type VerifyFirstWorkflowTaskScheduledRequest struct { NamespaceId string `protobuf:"bytes,1,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` WorkflowExecution *v14.WorkflowExecution `protobuf:"bytes,2,opt,name=workflow_execution,json=workflowExecution,proto3" json:"workflow_execution,omitempty"` Clock *v18.VectorClock `protobuf:"bytes,3,opt,name=clock,proto3" json:"clock,omitempty"` + ResendChild bool `protobuf:"varint,4,opt,name=resend_child,json=resendChild,proto3" json:"resend_child,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -3773,6 +3774,13 @@ func (x *VerifyFirstWorkflowTaskScheduledRequest) GetClock() *v18.VectorClock { return nil } +func (x *VerifyFirstWorkflowTaskScheduledRequest) GetResendChild() bool { + if x != nil { + return x.ResendChild + } + return false +} + type VerifyFirstWorkflowTaskScheduledResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -11018,11 +11026,12 @@ const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc "\vchild_clock\x18\x04 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\n" + "childClock\x12L\n" + "\fparent_clock\x18\x05 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\vparentClock:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"\x1e\n" + - "\x1cScheduleWorkflowTaskResponse\"\x8d\x02\n" + + "\x1cScheduleWorkflowTaskResponse\"\xb0\x02\n" + "'VerifyFirstWorkflowTaskScheduledRequest\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12X\n" + "\x12workflow_execution\x18\x02 \x01(\v2).temporal.api.common.v1.WorkflowExecutionR\x11workflowExecution\x12?\n" + - "\x05clock\x18\x03 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\x05clock:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"*\n" + + "\x05clock\x18\x03 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\x05clock\x12!\n" + + "\fresend_child\x18\x04 \x01(\bR\vresendChild:$\x92\xc4\x03 *\x1eworkflow_execution.workflow_id\"*\n" + "(VerifyFirstWorkflowTaskScheduledResponse\"\xd4\x04\n" + "$RecordChildExecutionCompletedRequest\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12T\n" + diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 2ec47d89ded..72548ca6a1d 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -2796,15 +2796,8 @@ the number of children greater than or equal to this threshold`, ReplicationTaskApplyTimeout = NewGlobalDurationSetting( "history.ReplicationTaskApplyTimeout", 20*time.Second, - `ReplicationTaskApplyTimeout is the context timeout for replication task apply, and for the -standby CloseExecutionTask's child-to-parent completion verification`, - ) - ParentWorkflowResendMaxInFlight = NewGlobalIntSetting( - "history.parentWorkflowResendMaxInFlight", - 8, - `ParentWorkflowResendMaxInFlight caps how many parent workflow resends a shard may run -concurrently when EnableAsyncParentWorkflowResend is on. Attempts beyond the cap are dropped; the -verifying task retries. This bounds the goroutines this path can create per shard.`, + `ReplicationTaskApplyTimeout is the context timeout for replication task apply, and for +standby parent-child verification resends`, ) EnableAsyncParentWorkflowResend = NewGlobalBoolSetting( "history.enableAsyncParentWorkflowResend", @@ -2812,6 +2805,21 @@ verifying task retries. This bounds the goroutines this path can create per shar `EnableAsyncParentWorkflowResend controls whether the standby child-to-parent completion verification resends the parent workflow in the background rather than inline, so the verifying task is not held for the duration of the cross-cluster sync.`, + ) + EnableChildWorkflowResend = NewGlobalBoolSetting( + "history.enableChildWorkflowResend", + false, + `EnableChildWorkflowResend controls whether standby parent-to-child first workflow task +verification may resend a missing child workflow in the background from the active cluster. When +disabled, verification remains local-only. StandbyTaskMissingEventsResendDelay plus +ReplicationTaskApplyTimeout should remain below StandbyTaskMissingEventsDiscardDelay.`, + ) + WorkflowResendHostMaxInFlight = NewGlobalIntSetting( + "history.workflowResendHostMaxInFlight", + 16, + `WorkflowResendHostMaxInFlight caps the total number of asynchronous parent and child workflow +resends that may run concurrently on a history host. Values less than one reject all asynchronous +workflow resends.`, ) ReplicationTaskFetcherParallelism = NewGlobalIntSetting( "history.ReplicationTaskFetcherParallelism", diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index ab7d3b1b411..3b845ef2358 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1137,12 +1137,24 @@ var ( ParentWorkflowResendAttempts = NewCounterDef("parent_workflow_resend_attempts") // ParentWorkflowResendSkipped counts attempts that found a resend for the same parent in flight. ParentWorkflowResendSkipped = NewCounterDef("parent_workflow_resend_skipped") - // ParentWorkflowResendFailures counts failed resends. Async resends report failure nowhere else. + // ParentWorkflowResendFailures counts unexpected parent resend failures. ParentWorkflowResendFailures = NewCounterDef("parent_workflow_resend_failures") - // ParentWorkflowResendLimited counts resends dropped because the shard was at its in-flight cap. + // ParentWorkflowResendLimited counts parent resends rejected at the host-level concurrency limit. ParentWorkflowResendLimited = NewCounterDef("parent_workflow_resend_limited") - // ParentWorkflowResendLatency measures a resend: cross-cluster state fetch plus local apply. + // ParentWorkflowResendLatency measures a parent resend and subsequent verification. ParentWorkflowResendLatency = NewTimerDef("parent_workflow_resend_latency") + // ChildWorkflowResendAttempts counts child resends started by standby first-task verification. + ChildWorkflowResendAttempts = NewCounterDef("child_workflow_resend_attempts") + // ChildWorkflowResendSkipped counts attempts that found a resend for the same child in flight. + ChildWorkflowResendSkipped = NewCounterDef("child_workflow_resend_skipped") + // ChildWorkflowResendFailures counts unexpected child resend failures. + ChildWorkflowResendFailures = NewCounterDef("child_workflow_resend_failures") + // ChildWorkflowResendLimited counts child resends rejected at the host-level concurrency limit. + ChildWorkflowResendLimited = NewCounterDef("child_workflow_resend_limited") + // ChildWorkflowResendLatency measures a child resend and subsequent verification. + ChildWorkflowResendLatency = NewTimerDef("child_workflow_resend_latency") + // WorkflowResendSchedulerAtCapacity counts host-level workflow resends rejected at the concurrency limit. + WorkflowResendSchedulerAtCapacity = NewCounterDef("workflow_resend_scheduler_at_capacity") // ReplicationOrphanedHistoryBranch tracks cases where history branch cleanup was skipped on error // to avoid deleting successfully written history. These orphaned branches will be cleaned up by GC. ReplicationOrphanedHistoryBranch = NewCounterDef("replication_orphaned_history_branch") diff --git a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto index 86aa8bdaf60..de813adcf43 100644 --- a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto +++ b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto @@ -593,6 +593,7 @@ message VerifyFirstWorkflowTaskScheduledRequest { string namespace_id = 1; temporal.api.common.v1.WorkflowExecution workflow_execution = 2; temporal.server.api.clock.v1.VectorClock clock = 3; + bool resend_child = 4; } message VerifyFirstWorkflowTaskScheduledResponse {} diff --git a/service/history/api/verifychildworkflowcompletionrecorded/api.go b/service/history/api/verifychildworkflowcompletionrecorded/api.go index 5a1b59da0c0..0fc8ffa668c 100644 --- a/service/history/api/verifychildworkflowcompletionrecorded/api.go +++ b/service/history/api/verifychildworkflowcompletionrecorded/api.go @@ -3,21 +3,18 @@ package verifychildworkflowcompletionrecorded import ( "context" "errors" + "fmt" "maps" "time" - commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" - "go.temporal.io/server/api/adminservice/v1" enumsspb "go.temporal.io/server/api/enums/v1" historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" - "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/locks" - "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -27,6 +24,7 @@ import ( "go.temporal.io/server/common/util" "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" ) @@ -101,7 +99,7 @@ func Invoke( request *historyservice.VerifyChildExecutionCompletionRecordedRequest, workflowConsistencyChecker api.WorkflowConsistencyChecker, shardContext historyi.ShardContext, - inFlightResends *InFlightResends, + resendScheduler workflowresend.Scheduler, ) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { namespaceID := namespace.ID(request.GetNamespaceId()) if err := api.ValidateNamespaceUUID(namespaceID); err != nil { @@ -122,51 +120,68 @@ func Invoke( metricsHandler := shardContext.GetMetricsHandler() emitLifecycle := shardContext.GetConfig().EmitReplicationLifecycleEvents() - asyncResend := shardContext.GetConfig().EnableAsyncParentWorkflowResend() - // The measured resend, run either inline or in the background. - resend := func(ctx context.Context) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { + resend := func(ctx context.Context) error { metrics.ParentWorkflowResendAttempts.With(metricsHandler).Record(1) startTime := time.Now().UTC() - resp, err := resendParentAndVerify(ctx, request, workflowConsistencyChecker, shardContext, namespaceID, versionedTransition, versionHistories, errVerify, parentWorkflowState, emitLifecycle) + err := resendParentAndVerify( + ctx, + request, + workflowConsistencyChecker, + shardContext, + namespaceID, + versionedTransition, + versionHistories, + errVerify, + parentWorkflowState, + emitLifecycle, + ) metrics.ParentWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime)) - if err != nil { - recordResendFailure(shardContext, metricsHandler, request, err) + if err != nil && !isExpectedResendError(err) { + metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1) } - return resp, err + logResendFailure(shardContext, request, err) + return err } - if !asyncResend { - return resend(ctx) + if resendScheduler == nil || !shardContext.GetConfig().EnableAsyncParentWorkflowResend() { + if err := resend(ctx); err != nil { + return nil, err + } + return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil } // The resend can take minutes while the calling standby task's deadline is short, so run it in // the background and let that task retry until the parent lands. // - // At most one resend per parent, and at most ParentWorkflowResendMaxInFlight per shard: callers - // retry while an earlier resend runs, so without these a stale parent, or a namespace with many - // of them, would spawn goroutines without bound. + // The host scheduler deduplicates by workflow and applies an aggregate limit across shards. parentKey := definition.NewWorkflowKey(request.NamespaceId, request.ParentExecution.WorkflowId, request.ParentExecution.RunId) - maxInFlight := shardContext.GetConfig().ParentWorkflowResendMaxInFlight() - claimed, atCapacity := inFlightResends.tryClaim(parentKey, maxInFlight) - if atCapacity { - metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1) - if emitLifecycle { - details := parentResendEventDetails(errVerify) - details["max_in_flight"] = maxInFlight - emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, wideevents.ParentChildOutcomeLimited, nil, details) - } - shardContext.GetLogger().Warn("Dropped parent workflow resend, shard is at its in-flight limit", - tag.WorkflowNamespaceID(request.GetNamespaceId()), - tag.NewStringTag("parent-workflow-id", request.ParentExecution.GetWorkflowId()), - tag.NewStringTag("parent-run-id", request.ParentExecution.GetRunId()), - tag.NewStringTag("child-workflow-id", request.ChildExecution.GetWorkflowId()), - tag.NewStringTag("child-run-id", request.ChildExecution.GetRunId()), - tag.NewInt("max-in-flight", shardContext.GetConfig().ParentWorkflowResendMaxInFlight()), - ) - return nil, errVerify - } - if !claimed { + + // The context is detached from the request, which gRPC cancels when this handler returns, and + // rooted at the shard lifecycle so the work stops with the shard. + resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx) + submitResult := resendScheduler.TrySubmit( + resendCtx, + parentKey, + shardContext.GetConfig().ReplicationTaskApplyTimeout(), + func(ctx context.Context) { + if emitLifecycle { + emitParentResendLifecycleEvent( + shardContext, + request, + parentWorkflowState, + wideevents.ParentChildOutcomeScheduled, + nil, + parentResendEventDetails(errVerify), + ) + } + _ = resend(ctx) + }, + ) + switch submitResult { + case workflowresend.SubmitResultAccepted: + // Accepted work records its attempt when execution starts. + case workflowresend.SubmitResultDuplicate: metrics.ParentWorkflowResendSkipped.With(metricsHandler).Record(1) if emitLifecycle { emitParentResendLifecycleEvent( @@ -178,48 +193,38 @@ func Invoke( parentResendEventDetails(errVerify), ) } - return nil, errVerify - } - if emitLifecycle { - emitParentResendLifecycleEvent( - shardContext, - request, - parentWorkflowState, - wideevents.ParentChildOutcomeScheduled, - nil, - parentResendEventDetails(errVerify), - ) + case workflowresend.SubmitResultAtCapacity: + metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1) + if emitLifecycle { + details := parentResendEventDetails(errVerify) + details["max_in_flight"] = shardContext.GetConfig().WorkflowResendHostMaxInFlight() + emitParentResendLifecycleEvent( + shardContext, + request, + parentWorkflowState, + wideevents.ParentChildOutcomeLimited, + nil, + details, + ) + } + default: + metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1) } - - // The context is detached from the request, which gRPC cancels when this handler returns, and - // rooted at the shard lifecycle so the work stops with the shard. - resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx) - resendCtx, cancel := context.WithTimeout(resendCtx, shardContext.GetConfig().ReplicationTaskApplyTimeout()) - go func() { - defer cancel() - defer inFlightResends.release(parentKey) - defer func() { - var panicErr error - log.CapturePanic(shardContext.GetLogger(), &panicErr) - if panicErr != nil { - metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1) - } - }() - _, _ = resend(resendCtx) - }() + // The submission result is intentionally used only for metrics. Preserve the verification error + // so the durable standby task retries regardless of the admission outcome. return nil, errVerify } -// recordResendFailure reports a failed parent resend. On the asynchronous path no caller receives -// the error, so these are its only signals. -func recordResendFailure( +func logResendFailure( shardContext historyi.ShardContext, - metricsHandler metrics.Handler, request *historyservice.VerifyChildExecutionCompletionRecordedRequest, err error, ) { - metrics.ParentWorkflowResendFailures.With(metricsHandler).Record(1) - shardContext.GetLogger().Error("Failed to resend parent workflow for child completion verification", + if isExpectedResendError(err) { + return + } + shardContext.GetThrottledLogger().Error( + "Failed to resend parent workflow for child completion verification", tag.WorkflowNamespaceID(request.GetNamespaceId()), tag.NewStringTag("parent-workflow-id", request.ParentExecution.GetWorkflowId()), tag.NewStringTag("parent-run-id", request.ParentExecution.GetRunId()), @@ -229,6 +234,18 @@ func recordResendFailure( ) } +func isExpectedResendError(err error) bool { + if err == nil || common.IsContextCanceledErr(err) { + return true + } + var notFoundErr *serviceerror.NotFound + var workflowNotReadyErr *serviceerror.WorkflowNotReady + var namespaceNotFoundErr *serviceerror.NamespaceNotFound + return errors.As(err, ¬FoundErr) || + errors.As(err, &workflowNotReadyErr) || + errors.As(err, &namespaceNotFoundErr) +} + // resendParentAndVerify pulls the parent workflow's state from the source cluster, applies it, and // re-checks the child's completion. Separate from Invoke so the async path can run it detached. func resendParentAndVerify( @@ -242,11 +259,7 @@ func resendParentAndVerify( errVerify error, parentWorkflowState string, emitLifecycle bool, -) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { - // Resend parent workflow from source cluster - - clusterMetadata := shardContext.GetClusterMetadata() - targetClusterInfo := clusterMetadata.GetAllClusterInfo()[clusterMetadata.GetCurrentClusterName()] +) error { activeClusterName := "" emitResult := func(outcome string, eventErr error, stage string) { if !emitLifecycle { @@ -261,80 +274,48 @@ func resendParentAndVerify( } emitParentResendLifecycleEvent(shardContext, request, parentWorkflowState, outcome, eventErr, details) } - - namespaceEntry, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespaceID) - if err != nil { - emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_namespace") - return nil, err - } - - activeClusterName = namespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: request.ParentExecution.WorkflowId}) - if activeClusterName == clusterMetadata.GetCurrentClusterName() { - err = errors.New("namespace becomes active when processing task as standby") - emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_source_cluster") - return nil, err - } - - remoteAdminClient, err := shardContext.GetRemoteAdminClient(activeClusterName) - if err != nil { - emitResult(wideevents.ParentChildOutcomeFailed, err, "resolve_remote_client") - return nil, err - } - emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state") - - resp, err := remoteAdminClient.SyncWorkflowState(ctx, &adminservice.SyncWorkflowStateRequest{ - NamespaceId: request.NamespaceId, - Execution: &commonpb.WorkflowExecution{ - WorkflowId: request.ParentExecution.WorkflowId, - RunId: request.ParentExecution.RunId, + result, err := workflowresend.SyncWorkflowStateFromSource( + ctx, + shardContext, + namespaceID, + request.ParentExecution, + versionedTransition, + versionHistories, + func(sourceCluster string) { + activeClusterName = sourceCluster + emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state") }, - ArchetypeId: chasm.WorkflowArchetypeID, - VersionedTransition: versionedTransition, - VersionHistories: versionHistories, - TargetClusterId: int32(targetClusterInfo.InitialFailoverVersion), - }) - + ) if err != nil { - if common.IsNotFoundError(err) { - // parent workflow is not found on source cluster, - // we can return empty response to indicate that verification is done - // TODO: add parent workflow to workflowNotFoundCache - emitResult(wideevents.ParentChildOutcomeSourceNotFound, err, "sync_workflow_state") - return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil - } - if _, ok := errors.AsType[*serviceerror.FailedPrecondition](err); ok { - // Unable to perform sync state. Transition history maybe disabled in source cluster. - emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state") - return nil, errVerify - } emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state") - return nil, err + return err } - - engine, err := shardContext.GetEngine(ctx) - if err != nil { - emitResult(wideevents.ParentChildOutcomeFailed, err, "get_engine") - return nil, err - } - err = engine.ReplicateVersionedTransition(ctx, chasm.WorkflowArchetypeID, resp.VersionedTransitionArtifact, activeClusterName) - if err != nil { - if !errors.Is(err, consts.ErrDuplicate) { - emitResult(wideevents.ParentChildOutcomeFailed, err, "replicate_versioned_transition") - return nil, err - } + switch result { + case workflowresend.SyncWorkflowStateResultSourceNotFound: + // TODO: add parent workflow to workflowNotFoundCache + emitResult( + wideevents.ParentChildOutcomeSourceNotFound, + serviceerror.NewNotFound("parent workflow not found on source cluster"), + "sync_workflow_state", + ) + return nil + case workflowresend.SyncWorkflowStateResultSkipped: + return errVerify + case workflowresend.SyncWorkflowStateResultApplied: + default: + return fmt.Errorf("unknown workflow state sync result: %d", result) } - // Verify child execution again after resending parent workflow _, _, observedParentWorkflowState, err := verifyChildExecution(ctx, workflowConsistencyChecker, request) if observedParentWorkflowState != "" { parentWorkflowState = observedParentWorkflowState } if err != nil { emitResult(wideevents.ParentChildOutcomeFailed, err, "verify_after_resend") - return nil, err + return err } emitResult(wideevents.ParentChildOutcomeSucceeded, nil, "") - return &historyservice.VerifyChildExecutionCompletionRecordedResponse{}, nil + return nil } func parentResendEventDetails(initialError error) map[string]any { diff --git a/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends.go b/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends.go deleted file mode 100644 index d97f0c5a009..00000000000 --- a/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends.go +++ /dev/null @@ -1,44 +0,0 @@ -package verifychildworkflowcompletionrecorded - -import ( - "sync" - - "go.temporal.io/server/common/definition" -) - -// InFlightResends tracks the parent workflows a shard is currently resending, so concurrent -// verification attempts for the same parent do not each pull its state from the source cluster. A -// resend clones the parent's full mutable state, and callers retry while one is still running. -// -// It also caps how many resends a shard runs at once, bounding the goroutines this path creates. -// -// The zero value is ready to use; hold it by pointer, never copy it. -type InFlightResends struct { - mu sync.Mutex - keys map[definition.WorkflowKey]struct{} -} - -// tryClaim reserves key for the caller. It reports claimed=false when a resend for the same parent -// is already running, or atCapacity=true when the shard already has maxInFlight resends. A caller -// that claims the key must release it when the resend finishes. -func (r *InFlightResends) tryClaim(key definition.WorkflowKey, maxInFlight int) (claimed bool, atCapacity bool) { - r.mu.Lock() - defer r.mu.Unlock() - if _, ok := r.keys[key]; ok { - return false, false - } - if len(r.keys) >= maxInFlight { - return false, true - } - if r.keys == nil { - r.keys = make(map[definition.WorkflowKey]struct{}) - } - r.keys[key] = struct{}{} - return true, false -} - -func (r *InFlightResends) release(key definition.WorkflowKey) { - r.mu.Lock() - defer r.mu.Unlock() - delete(r.keys, key) -} diff --git a/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends_test.go b/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends_test.go deleted file mode 100644 index 7eea09b59d5..00000000000 --- a/service/history/api/verifychildworkflowcompletionrecorded/in_flight_resends_test.go +++ /dev/null @@ -1,73 +0,0 @@ -package verifychildworkflowcompletionrecorded - -import ( - "testing" - - "github.com/stretchr/testify/require" - "go.temporal.io/server/common/definition" -) - -func key(workflowID string) definition.WorkflowKey { - return definition.NewWorkflowKey("ns", workflowID, "run") -} - -func TestInFlightResends_ZeroValueIsUsable(t *testing.T) { - var r InFlightResends // no constructor - claimed, atCapacity := r.tryClaim(key("a"), 8) - require.True(t, claimed) - require.False(t, atCapacity) -} - -func TestInFlightResends_DedupesSameParent(t *testing.T) { - var r InFlightResends - - claimed, atCapacity := r.tryClaim(key("a"), 8) - require.True(t, claimed) - require.False(t, atCapacity) - - // Same parent while the first is still held. - claimed, atCapacity = r.tryClaim(key("a"), 8) - require.False(t, claimed) - require.False(t, atCapacity, "a duplicate is not a capacity problem") - - // A different parent is unaffected. - claimed, _ = r.tryClaim(key("b"), 8) - require.True(t, claimed) - - // Releasing lets the parent be claimed again. - r.release(key("a")) - claimed, _ = r.tryClaim(key("a"), 8) - require.True(t, claimed) -} - -func TestInFlightResends_EnforcesMaxInFlight(t *testing.T) { - var r InFlightResends - - claimed, _ := r.tryClaim(key("a"), 2) - require.True(t, claimed) - claimed, _ = r.tryClaim(key("b"), 2) - require.True(t, claimed) - - // Third distinct parent exceeds the cap. - claimed, atCapacity := r.tryClaim(key("c"), 2) - require.False(t, claimed) - require.True(t, atCapacity) - - // A duplicate of an already-held parent still reports dedup, not capacity. - claimed, atCapacity = r.tryClaim(key("a"), 2) - require.False(t, claimed) - require.False(t, atCapacity) - - // Freeing a slot admits the previously rejected parent. - r.release(key("b")) - claimed, atCapacity = r.tryClaim(key("c"), 2) - require.True(t, claimed) - require.False(t, atCapacity) -} - -func TestInFlightResends_ZeroMaxRejectsEverything(t *testing.T) { - var r InFlightResends - claimed, atCapacity := r.tryClaim(key("a"), 0) - require.False(t, claimed) - require.True(t, atCapacity) -} diff --git a/service/history/api/verifyfirstworkflowtaskscheduled/api.go b/service/history/api/verifyfirstworkflowtaskscheduled/api.go index aa57d9a5edf..7757d6a029d 100644 --- a/service/history/api/verifyfirstworkflowtaskscheduled/api.go +++ b/service/history/api/verifyfirstworkflowtaskscheduled/api.go @@ -2,26 +2,121 @@ package verifyfirstworkflowtaskscheduled import ( "context" + "errors" + "fmt" + "time" + "go.temporal.io/api/serviceerror" enumsspb "go.temporal.io/server/api/enums/v1" + historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/locks" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/persistence/transitionhistory" + "go.temporal.io/server/common/persistence/versionhistory" + "go.temporal.io/server/common/rpc" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" ) func Invoke( ctx context.Context, req *historyservice.VerifyFirstWorkflowTaskScheduledRequest, workflowConsistencyChecker api.WorkflowConsistencyChecker, -) (retError error) { + shardContext historyi.ShardContext, + resendScheduler workflowresend.Scheduler, +) error { namespaceID := namespace.ID(req.GetNamespaceId()) if err := api.ValidateNamespaceUUID(namespaceID); err != nil { return err } + versionedTransition, versionHistories, firstWorkflowTaskMissing, errVerify := verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker) + if errVerify == nil { + return nil + } + switch errVerify.(type) { + case *serviceerror.NotFound: + case *serviceerror.WorkflowNotReady: + if !firstWorkflowTaskMissing { + return errVerify + } + default: + return errVerify + } + if !req.GetResendChild() || !shardContext.GetConfig().EnableChildWorkflowResend() { + return errVerify + } + + metricsHandler := shardContext.GetMetricsHandler() + resend := func(ctx context.Context) error { + metrics.ChildWorkflowResendAttempts.With(metricsHandler).Record(1) + startTime := time.Now().UTC() + err := resendChildAndVerify( + ctx, + req, + workflowConsistencyChecker, + shardContext, + namespaceID, + versionedTransition, + versionHistories, + errVerify, + ) + metrics.ChildWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime)) + if err != nil && !isExpectedResendError(err) { + metrics.ChildWorkflowResendFailures.With(metricsHandler).Record(1) + } + logResendFailure(shardContext, req, err) + return err + } + + if resendScheduler == nil { + return resend(ctx) + } + + childKey := definition.NewWorkflowKey(req.NamespaceId, req.WorkflowExecution.WorkflowId, req.WorkflowExecution.RunId) + resendCtx := rpc.CopyContextValues(shardContext.GetLifecycleContext(), ctx) + submitResult := resendScheduler.TrySubmit( + resendCtx, + childKey, + shardContext.GetConfig().ReplicationTaskApplyTimeout(), + func(ctx context.Context) { + _ = resend(ctx) + }, + ) + switch submitResult { + case workflowresend.SubmitResultAccepted: + // Accepted work records its attempt when execution starts. + case workflowresend.SubmitResultDuplicate: + metrics.ChildWorkflowResendSkipped.With(metricsHandler).Record(1) + case workflowresend.SubmitResultAtCapacity: + metrics.ChildWorkflowResendLimited.With(metricsHandler).Record(1) + default: + // SubmitResultFailed and unknown values are admission failures. + metrics.ChildWorkflowResendFailures.With(metricsHandler).Record(1) + } + // The submission result is intentionally used only for metrics. Preserve the verification error + // so the durable standby task retries regardless of the admission outcome. + return errVerify +} + +func verifyFirstWorkflowTaskScheduled( + ctx context.Context, + req *historyservice.VerifyFirstWorkflowTaskScheduledRequest, + workflowConsistencyChecker api.WorkflowConsistencyChecker, +) ( + versionedTransition *persistencespb.VersionedTransition, + versionHistories *historyspb.VersionHistories, + firstWorkflowTaskMissing bool, + retError error, +) { workflowLease, err := workflowConsistencyChecker.GetWorkflowLease( ctx, req.Clock, @@ -33,19 +128,88 @@ func Invoke( locks.PriorityLow, ) if err != nil { - return err + return nil, nil, false, err } defer func() { workflowLease.GetReleaseFn()(retError) }() mutableState := workflowLease.GetMutableState() if !mutableState.IsWorkflowExecutionRunning() && mutableState.GetExecutionState().State != enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE { - return nil + return nil, nil, false, nil } if !mutableState.HadOrHasWorkflowTask() { - return consts.ErrWorkflowNotReady + executionInfo := mutableState.GetExecutionInfo() + versionedTransition = transitionhistory.CopyVersionedTransition( + transitionhistory.LastVersionedTransition(executionInfo.TransitionHistory), + ) + versionHistories = versionhistory.CopyVersionHistories(executionInfo.VersionHistories) + return versionedTransition, versionHistories, true, consts.ErrWorkflowNotReady + } + + return nil, nil, false, nil +} + +func logResendFailure( + shardContext historyi.ShardContext, + req *historyservice.VerifyFirstWorkflowTaskScheduledRequest, + err error, +) { + if isExpectedResendError(err) { + return + } + shardContext.GetThrottledLogger().Error( + "Failed to resend child workflow for first workflow task verification", + tag.WorkflowNamespaceID(req.GetNamespaceId()), + tag.WorkflowID(req.WorkflowExecution.GetWorkflowId()), + tag.WorkflowRunID(req.WorkflowExecution.GetRunId()), + tag.Error(err), + ) +} + +func isExpectedResendError(err error) bool { + if err == nil || common.IsContextCanceledErr(err) { + return true } + var notFoundErr *serviceerror.NotFound + var workflowNotReadyErr *serviceerror.WorkflowNotReady + var namespaceNotFoundErr *serviceerror.NamespaceNotFound + return errors.As(err, ¬FoundErr) || + errors.As(err, &workflowNotReadyErr) || + errors.As(err, &namespaceNotFoundErr) +} - return nil +func resendChildAndVerify( + ctx context.Context, + req *historyservice.VerifyFirstWorkflowTaskScheduledRequest, + workflowConsistencyChecker api.WorkflowConsistencyChecker, + shardContext historyi.ShardContext, + namespaceID namespace.ID, + versionedTransition *persistencespb.VersionedTransition, + versionHistories *historyspb.VersionHistories, + errVerify error, +) error { + result, err := workflowresend.SyncWorkflowStateFromSource( + ctx, + shardContext, + namespaceID, + req.WorkflowExecution, + versionedTransition, + versionHistories, + nil, + ) + if err != nil { + return err + } + switch result { + case workflowresend.SyncWorkflowStateResultSourceNotFound: + return nil + case workflowresend.SyncWorkflowStateResultSkipped: + return errVerify + case workflowresend.SyncWorkflowStateResultApplied: + _, _, _, err = verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker) + return err + default: + return fmt.Errorf("unknown workflow state sync result: %d", result) + } } diff --git a/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go b/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go index 5a452b4b1b4..548b145b6d3 100644 --- a/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go +++ b/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go @@ -20,12 +20,15 @@ import ( workflowspb "go.temporal.io/server/api/workflow/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/definition" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" historyi "go.temporal.io/server/service/history/interfaces" @@ -49,6 +52,7 @@ type ( mockExecutionMgr *persistence.MockExecutionManager shardContext *shard.ContextTest workflowConsistencyChecker api.WorkflowConsistencyChecker + resendScheduler *workflowresend.BoundedWorkflowScheduler logger log.Logger } @@ -63,6 +67,12 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) SetupTest() { s.controller = gomock.NewController(s.T()) config := tests.NewDynamicConfig() + config.WorkflowResendHostMaxInFlight = func() int { return 1 } + s.resendScheduler = workflowresend.NewBoundedWorkflowScheduler( + config.WorkflowResendHostMaxInFlight, + log.NewNoopLogger(), + metrics.NoopMetricsHandler, + ) s.shardContext = shard.NewTestContext( s.controller, &persistencespb.ShardInfo{ @@ -95,24 +105,95 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) SetupTest() { } func (s *VerifyFirstWorkflowTaskScheduledSuite) TearDownTest() { + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() s.controller.Finish() } -func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowNotFound() { +func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowNotFound_ResendDisabled() { request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ NamespaceId: tests.NamespaceID.String(), WorkflowExecution: &commonpb.WorkflowExecution{ WorkflowId: tests.WorkflowID, RunId: tests.RunID, }, + ResendChild: true, } s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) - err := Invoke(context.Background(), request, s.workflowConsistencyChecker) + err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) s.IsType(&serviceerror.NotFound{}, err) } +func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_DoesNotResendUnrelatedWorkflowNotReady() { + s.shardContext.GetConfig().EnableChildWorkflowResend = func() bool { return true } + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + workflowNotReadyErr := serviceerror.NewWorkflowNotReady("unrelated workflow state") + workflowConsistencyChecker := api.NewMockWorkflowConsistencyChecker(s.controller) + workflowConsistencyChecker.EXPECT().GetWorkflowLease( + gomock.Any(), + gomock.Any(), + definition.NewWorkflowKey(tests.NamespaceID.String(), tests.WorkflowID, tests.RunID), + gomock.Any(), + ).Return(nil, workflowNotReadyErr) + + err := Invoke(s.T().Context(), request, workflowConsistencyChecker, s.shardContext, s.resendScheduler) + s.Same(workflowNotReadyErr, err) +} + +func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_HostAtCapacity() { + s.shardContext.GetConfig().EnableChildWorkflowResend = func() bool { return true } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.shardContext.SetMetricsHandler(metricsHandler) + + started := make(chan struct{}) + release := make(chan struct{}) + defer close(release) + s.Require().Equal(workflowresend.SubmitResultAccepted, s.resendScheduler.TrySubmit( + s.T().Context(), + definition.NewWorkflowKey("blocker namespace", "blocker workflow", "blocker run"), + time.Minute, + func(ctx context.Context) { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + }, + )) + waitCtx, cancel := context.WithTimeout(s.T().Context(), 5*time.Second) + defer cancel() + select { + case <-started: + case <-waitCtx.Done(): + s.T().Fatal("timed out waiting for host scheduler worker") + } + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + s.Require().Len(capture.Snapshot()[metrics.ChildWorkflowResendLimited.Name()], 1) +} + func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskScheduled_WorkflowCompleted() { request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ NamespaceId: tests.NamespaceID.String(), @@ -137,11 +218,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched ) s.NoError(err) - wfMs := workflow.TestCloneToProto(context.Background(), ms) + wfMs := workflow.TestCloneToProto(s.T().Context(), ms) gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs} s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil) - err = Invoke(context.Background(), request, s.workflowConsistencyChecker) + err = Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) s.NoError(err) } @@ -170,11 +251,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched ) s.NoError(err) - wfMs := workflow.TestCloneToProto(context.Background(), ms) + wfMs := workflow.TestCloneToProto(s.T().Context(), ms) gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs} s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil) - err = Invoke(context.Background(), request, s.workflowConsistencyChecker) + err = Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) s.IsType(&serviceerror.WorkflowNotReady{}, err) } @@ -197,11 +278,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched 25*time.Second, 20*time.Second, 200*time.Second, nil, "identity") _, _ = ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) - wfMs := workflow.TestCloneToProto(context.Background(), ms) + wfMs := workflow.TestCloneToProto(s.T().Context(), ms) gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs} s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil) - err := Invoke(context.Background(), request, s.workflowConsistencyChecker) + err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) s.NoError(err) } @@ -249,11 +330,11 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) TestVerifyFirstWorkflowTaskSched &workflowservice.RespondWorkflowTaskCompletedRequest{Identity: "some random identity"}, defaultWorkflowTaskCompletionLimits) ms.FlushBufferedEvents() - wfMs := workflow.TestCloneToProto(context.Background(), ms) + wfMs := workflow.TestCloneToProto(s.T().Context(), ms) gwmsResponse := &persistence.GetWorkflowExecutionResponse{State: wfMs} s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(gwmsResponse, nil) - err := Invoke(context.Background(), request, s.workflowConsistencyChecker) + err := Invoke(s.T().Context(), request, s.workflowConsistencyChecker, s.shardContext, s.resendScheduler) s.NoError(err) } diff --git a/service/history/api/workflowresend/scheduler.go b/service/history/api/workflowresend/scheduler.go new file mode 100644 index 00000000000..d6a165aa77f --- /dev/null +++ b/service/history/api/workflowresend/scheduler.go @@ -0,0 +1,214 @@ +package workflowresend + +import ( + "context" + "sync" + "time" + + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/metrics" + ctasks "go.temporal.io/server/common/tasks" +) + +const OperationName = "WorkflowResend" + +// SubmitResult describes the outcome of scheduler admission. +type SubmitResult int + +const ( + SubmitResultFailed SubmitResult = 0 + SubmitResultAccepted SubmitResult = 1 + SubmitResultDuplicate SubmitResult = 2 + SubmitResultAtCapacity SubmitResult = 3 +) + +// Scheduler runs workflow resend jobs asynchronously. +type Scheduler interface { + // TrySubmit reports whether the job was accepted, deduplicated, rejected at capacity, or failed + // during submission. + TrySubmit( + ctx context.Context, + key definition.WorkflowKey, + timeout time.Duration, + run func(context.Context), + ) SubmitResult +} + +// BoundedWorkflowScheduler deduplicates workflow resends and bounds their concurrency. +type BoundedWorkflowScheduler struct { + pool *ctasks.DynamicWorkerPoolScheduler + + logger log.Logger + metricsHandler metrics.Handler + + mu sync.Mutex + inFlight map[definition.WorkflowKey]struct{} +} + +var _ Scheduler = (*BoundedWorkflowScheduler)(nil) + +// NewBoundedWorkflowScheduler creates a bounded workflow scheduler with no task buffer. +func NewBoundedWorkflowScheduler( + maxConcurrency dynamicconfig.IntPropertyFn, + logger log.Logger, + metricsHandler metrics.Handler, +) *BoundedWorkflowScheduler { + limiter := boundedWorkflowSchedulerLimiter{ + maxConcurrency: maxConcurrency, + logger: logger, + } + return &BoundedWorkflowScheduler{ + pool: ctasks.NewDynamicWorkerPoolScheduler( + limiter, + metrics.NoopMetricsHandler, + ), + logger: logger, + metricsHandler: metricsHandler, + inFlight: make(map[definition.WorkflowKey]struct{}), + } +} + +func (s *BoundedWorkflowScheduler) TrySubmit( + ctx context.Context, + key definition.WorkflowKey, + timeout time.Duration, + run func(context.Context), +) (result SubmitResult) { + var panicErr error + defer log.CapturePanic(log.With(s.logger, workflowKeyTags(key)...), &panicErr) + return s.trySubmit(ctx, key, timeout, run) +} + +func (s *BoundedWorkflowScheduler) trySubmit( + ctx context.Context, + key definition.WorkflowKey, + timeout time.Duration, + run func(context.Context), +) SubmitResult { + if !s.tryClaim(key) { + return SubmitResultDuplicate + } + claimed := true + var runnable *resendRunnable + defer func() { + if runnable != nil { + runnable.Abort() + } else if claimed { + s.release(key) + } + }() + + // The timeout starts at admission; this scheduler intentionally has no task buffer. + jobCtx, cancel := context.WithTimeout(ctx, timeout) + runnable = &resendRunnable{ + ctx: jobCtx, + run: run, + logger: log.With(s.logger, workflowKeyTags(key)...), + cleanup: func() { + cancel() + s.release(key) + }, + } + if !s.pool.TrySubmit(runnable) { + metrics.WorkflowResendSchedulerAtCapacity.With(s.metricsHandler).Record(1) + return SubmitResultAtCapacity + } + runnable = nil + claimed = false + return SubmitResultAccepted +} + +func (s *BoundedWorkflowScheduler) tryClaim(key definition.WorkflowKey) bool { + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.inFlight[key]; ok { + return false + } + s.inFlight[key] = struct{}{} + return true +} + +func (s *BoundedWorkflowScheduler) release(key definition.WorkflowKey) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.inFlight, key) +} + +// InitiateShutdown cancels running jobs and aborts jobs waiting to run. +func (s *BoundedWorkflowScheduler) InitiateShutdown() { + s.pool.InitiateShutdown() +} + +// WaitShutdown waits for all scheduler goroutines to exit. +func (s *BoundedWorkflowScheduler) WaitShutdown() { + s.pool.WaitShutdown() +} + +type boundedWorkflowSchedulerLimiter struct { + maxConcurrency dynamicconfig.IntPropertyFn + logger log.Logger +} + +func (l boundedWorkflowSchedulerLimiter) Concurrency() (concurrency int) { + var panicErr error + defer func() { + if panicErr != nil { + concurrency = 0 + } + }() + defer log.CapturePanic(l.logger, &panicErr) + return l.maxConcurrency() +} + +func (boundedWorkflowSchedulerLimiter) BufferSize() int { + return 0 +} + +type resendRunnable struct { + ctx context.Context + run func(context.Context) + logger log.Logger + cleanup func() + once sync.Once +} + +func (r *resendRunnable) Run(shutdownCtx context.Context) { + var panicErr error + defer log.CapturePanic(r.logger, &panicErr) + defer r.clean() + + runCtx, cancel := context.WithCancel(r.ctx) + stopShutdownCancellation := context.AfterFunc(shutdownCtx, cancel) + defer func() { + stopShutdownCancellation() + cancel() + }() + if shutdownCtx.Err() != nil || runCtx.Err() != nil { + return + } + + r.run(runCtx) +} + +func (r *resendRunnable) Abort() { + r.clean() +} + +func (r *resendRunnable) clean() { + r.once.Do(func() { + if r.cleanup != nil { + r.cleanup() + } + }) +} + +func workflowKeyTags(key definition.WorkflowKey) []tag.Tag { + return []tag.Tag{ + tag.WorkflowNamespaceID(key.NamespaceID), + tag.WorkflowID(key.WorkflowID), + tag.WorkflowRunID(key.RunID), + } +} diff --git a/service/history/api/workflowresend/scheduler_test.go b/service/history/api/workflowresend/scheduler_test.go new file mode 100644 index 00000000000..670d449c51b --- /dev/null +++ b/service/history/api/workflowresend/scheduler_test.go @@ -0,0 +1,365 @@ +package workflowresend + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/testing/await" +) + +const testJobTimeout = time.Minute + +type panicDeadlineContext struct { + context.Context +} + +func (panicDeadlineContext) Deadline() (time.Time, bool) { + panic("test panic") +} + +func testWorkflowKey(workflowID string) definition.WorkflowKey { + return definition.NewWorkflowKey("namespace", workflowID, "run") +} + +func TestSubmitResultZeroValueFailsSafe(t *testing.T) { + var result SubmitResult + require.Equal(t, SubmitResultFailed, result) +} + +func TestBoundedWorkflowSchedulerDeduplicatesAndReleasesWorkflow(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + scheduler := NewBoundedWorkflowScheduler(func() int { return 2 }, log.NewNoopLogger(), metricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + started := make(chan struct{}) + release := make(chan struct{}) + finished := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(ctx context.Context) { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + close(finished) + }, + )) + requireSignal(t, started) + + var duplicateRuns atomic.Int32 + require.Equal(t, SubmitResultDuplicate, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(context.Context) { duplicateRuns.Add(1) }, + )) + require.Equal(t, SubmitResultDuplicate, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(context.Context) { duplicateRuns.Add(1) }, + )) + require.Zero(t, duplicateRuns.Load()) + require.Empty(t, capture.Snapshot()[metrics.WorkflowResendSchedulerAtCapacity.Name()]) + + differentWorkflowRan := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("b"), + testJobTimeout, + func(context.Context) { close(differentWorkflowRan) }, + )) + requireSignal(t, differentWorkflowRan) + + close(release) + requireSignal(t, finished) + resubmittedWorkflowRan := make(chan struct{}) + await.RequireTrue(t, func() bool { + return scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + ) == SubmitResultAccepted + }, 5*time.Second, 10*time.Millisecond) + requireSignal(t, resubmittedWorkflowRan) + require.Zero(t, duplicateRuns.Load()) +} + +func TestBoundedWorkflowSchedulerSharedConcurrencyRejectsAndReleasesWorkflow(t *testing.T) { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + started := make(chan struct{}) + release := make(chan struct{}) + finished := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(ctx context.Context) { + close(started) + select { + case <-release: + case <-ctx.Done(): + } + close(finished) + }, + )) + requireSignal(t, started) + + var rejectedRuns atomic.Int32 + require.Equal(t, SubmitResultAtCapacity, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("b"), + testJobTimeout, + func(context.Context) { rejectedRuns.Add(1) }, + )) + require.Zero(t, rejectedRuns.Load()) + require.Len(t, capture.Snapshot()[metrics.WorkflowResendSchedulerAtCapacity.Name()], 1) + require.Empty(t, capture.Snapshot()[metrics.DynamicWorkerPoolSchedulerRejectedTasks.Name()]) + + close(release) + requireSignal(t, finished) + resubmittedWorkflowRan := make(chan struct{}) + await.RequireTrue(t, func() bool { + return scheduler.TrySubmit( + t.Context(), + testWorkflowKey("b"), + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + ) == SubmitResultAccepted + }, 5*time.Second, 10*time.Millisecond) + requireSignal(t, resubmittedWorkflowRan) +} + +func TestBoundedWorkflowSchedulerShutdownCancelsAndCleansUp(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + + started := make(chan struct{}) + finished := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(ctx context.Context) { + close(started) + <-ctx.Done() + close(finished) + }, + )) + requireSignal(t, started) + + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + requireSignal(t, finished) + require.Empty(t, scheduler.inFlight) +} + +func TestBoundedWorkflowSchedulerCallerCancellationReachesRunningJob(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + jobCtx, cancelJob := context.WithCancel(t.Context()) + started := make(chan struct{}) + finished := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + jobCtx, + testWorkflowKey("a"), + testJobTimeout, + func(ctx context.Context) { + close(started) + <-ctx.Done() + close(finished) + }, + )) + requireSignal(t, started) + + cancelJob() + requireSignal(t, finished) +} + +func TestBoundedWorkflowSchedulerAbortCleansUp(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + scheduler.InitiateShutdown() + + var runs atomic.Int32 + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(context.Context) { runs.Add(1) }, + )) + scheduler.WaitShutdown() + + require.Zero(t, runs.Load()) + require.Empty(t, scheduler.inFlight) +} + +func TestBoundedWorkflowSchedulerTimeoutReleasesWorkflow(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + finished := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + 10*time.Millisecond, + func(ctx context.Context) { + <-ctx.Done() + close(finished) + }, + )) + requireSignal(t, finished) + + resubmittedWorkflowRan := make(chan struct{}) + await.RequireTrue(t, func() bool { + return scheduler.TrySubmit( + t.Context(), + testWorkflowKey("a"), + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + ) == SubmitResultAccepted + }, 5*time.Second, 10*time.Millisecond) + requireSignal(t, resubmittedWorkflowRan) +} + +func TestBoundedWorkflowSchedulerTaskPanicFailsOpen(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + key := testWorkflowKey("a") + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + key, + testJobTimeout, + func(context.Context) { panic("test panic") }, + )) + + resubmittedWorkflowRan := make(chan struct{}) + await.RequireTrue(t, func() bool { + return scheduler.TrySubmit( + t.Context(), + key, + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + ) == SubmitResultAccepted + }, 5*time.Second, 10*time.Millisecond) + requireSignal(t, resubmittedWorkflowRan) +} + +func TestBoundedWorkflowSchedulerLimitProviderPanicFailsOpen(t *testing.T) { + var panicLimit atomic.Bool + panicLimit.Store(true) + scheduler := NewBoundedWorkflowScheduler(func() int { + if panicLimit.Swap(false) { + panic("test panic") + } + return 1 + }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + key := testWorkflowKey("a") + require.Equal(t, SubmitResultAtCapacity, scheduler.TrySubmit( + t.Context(), + key, + testJobTimeout, + func(context.Context) { require.FailNow(t, "rejected job ran") }, + )) + + resubmittedWorkflowRan := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + key, + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + )) + requireSignal(t, resubmittedWorkflowRan) +} + +func TestBoundedWorkflowSchedulerSubmitPanicFailsOpen(t *testing.T) { + scheduler := NewBoundedWorkflowScheduler(func() int { return 1 }, log.NewNoopLogger(), metrics.NoopMetricsHandler) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + key := testWorkflowKey("a") + require.Equal(t, SubmitResultFailed, scheduler.TrySubmit( + panicDeadlineContext{Context: t.Context()}, + key, + testJobTimeout, + func(context.Context) { require.FailNow(t, "failed job ran") }, + )) + + resubmittedWorkflowRan := make(chan struct{}) + require.Equal(t, SubmitResultAccepted, scheduler.TrySubmit( + t.Context(), + key, + testJobTimeout, + func(context.Context) { close(resubmittedWorkflowRan) }, + )) + requireSignal(t, resubmittedWorkflowRan) +} + +func TestResendRunnableDoesNotRunAfterShutdown(t *testing.T) { + shutdownCtx, cancel := context.WithCancel(t.Context()) + cancel() + + var runs atomic.Int32 + var cleanups atomic.Int32 + runnable := &resendRunnable{ + ctx: t.Context(), + run: func(context.Context) { runs.Add(1) }, + logger: log.NewNoopLogger(), + cleanup: func() { cleanups.Add(1) }, + } + runnable.Run(shutdownCtx) + + require.Zero(t, runs.Load()) + require.Equal(t, int32(1), cleanups.Load()) +} + +func requireSignal(t *testing.T, signal <-chan struct{}) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + select { + case <-signal: + case <-ctx.Done(): + require.FailNow(t, "timed out waiting for signal") + } +} diff --git a/service/history/api/workflowresend/sync_workflow_state.go b/service/history/api/workflowresend/sync_workflow_state.go new file mode 100644 index 00000000000..b43c5efc32c --- /dev/null +++ b/service/history/api/workflowresend/sync_workflow_state.go @@ -0,0 +1,119 @@ +package workflowresend + +import ( + "context" + "errors" + "fmt" + + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/adminservice/v1" + historyspb "go.temporal.io/server/api/history/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" +) + +// SyncWorkflowStateResult describes the outcome of pulling and applying workflow state. +type SyncWorkflowStateResult int + +const ( + // Keep the zero value fail-safe for callers that do not initialize a result. + SyncWorkflowStateResultSkipped SyncWorkflowStateResult = iota + SyncWorkflowStateResultApplied + SyncWorkflowStateResultSourceNotFound +) + +// SyncWorkflowStateFromSource pulls workflow state from the namespace's active cluster and applies +// it locally if the namespace routing state remains unchanged for the duration of the RPC. +func SyncWorkflowStateFromSource( + ctx context.Context, + shardContext historyi.ShardContext, + namespaceID namespace.ID, + execution *commonpb.WorkflowExecution, + versionedTransition *persistencespb.VersionedTransition, + versionHistories *historyspb.VersionHistories, + onSourceResolved func(string), +) (SyncWorkflowStateResult, error) { + clusterMetadata := shardContext.GetClusterMetadata() + currentClusterName := clusterMetadata.GetCurrentClusterName() + namespaceRegistry := shardContext.GetNamespaceRegistry() + namespaceEntry, err := namespaceRegistry.GetNamespaceByID(namespaceID) + if err != nil { + return SyncWorkflowStateResultSkipped, err + } + if !namespaceEntry.IsOnCluster(currentClusterName) { + return SyncWorkflowStateResultSkipped, nil + } + + routingKey := namespace.RoutingKey{ID: execution.GetWorkflowId()} + activeClusterName := namespaceEntry.ActiveClusterName(routingKey) + if activeClusterName == currentClusterName { + return SyncWorkflowStateResultSkipped, nil + } + + targetClusterInfo, ok := clusterMetadata.GetAllClusterInfo()[currentClusterName] + if !ok { + return SyncWorkflowStateResultSkipped, fmt.Errorf("current cluster %q is missing from cluster metadata", currentClusterName) + } + remoteAdminClient, err := shardContext.GetRemoteAdminClient(activeClusterName) + if err != nil { + return SyncWorkflowStateResultSkipped, err + } + if onSourceResolved != nil { + onSourceResolved(activeClusterName) + } + + resp, err := remoteAdminClient.SyncWorkflowState(ctx, &adminservice.SyncWorkflowStateRequest{ + NamespaceId: namespaceID.String(), + Execution: execution, + ArchetypeId: chasm.WorkflowArchetypeID, + VersionedTransition: versionedTransition, + VersionHistories: versionHistories, + TargetClusterId: int32(targetClusterInfo.InitialFailoverVersion), + }) + if err != nil { + if common.IsNotFoundError(err) { + return SyncWorkflowStateResultSourceNotFound, nil + } + var failedPreconditionErr *serviceerror.FailedPrecondition + if errors.As(err, &failedPreconditionErr) { + return SyncWorkflowStateResultSkipped, nil + } + return SyncWorkflowStateResultSkipped, err + } + if resp == nil || resp.VersionedTransitionArtifact == nil { + return SyncWorkflowStateResultSkipped, serviceerror.NewInternal("SyncWorkflowState returned an empty artifact") + } + + namespaceEntry, err = namespaceRegistry.GetNamespaceByID(namespaceID) + if err != nil { + var namespaceNotFoundErr *serviceerror.NamespaceNotFound + if errors.As(err, &namespaceNotFoundErr) { + return SyncWorkflowStateResultSkipped, nil + } + return SyncWorkflowStateResultSkipped, err + } + if !namespaceEntry.IsOnCluster(currentClusterName) || + namespaceEntry.ActiveClusterName(routingKey) != activeClusterName { + return SyncWorkflowStateResultSkipped, nil + } + + engine, err := shardContext.GetEngine(ctx) + if err != nil { + return SyncWorkflowStateResultSkipped, err + } + if err := engine.ReplicateVersionedTransition( + ctx, + chasm.WorkflowArchetypeID, + resp.VersionedTransitionArtifact, + activeClusterName, + ); err != nil && !errors.Is(err, consts.ErrDuplicate) { + return SyncWorkflowStateResultSkipped, err + } + + return SyncWorkflowStateResultApplied, nil +} diff --git a/service/history/api/workflowresend/sync_workflow_state_test.go b/service/history/api/workflowresend/sync_workflow_state_test.go new file mode 100644 index 00000000000..bce54e874b5 --- /dev/null +++ b/service/history/api/workflowresend/sync_workflow_state_test.go @@ -0,0 +1,324 @@ +package workflowresend + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/api/adminservicemock/v1" + historyspb "go.temporal.io/server/api/history/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/protomock" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" + "go.uber.org/mock/gomock" +) + +const ( + syncTestNamespaceID = namespace.ID("sync-namespace-id") + syncTestWorkflowID = "sync-workflow-id" + syncTestRunID = "sync-run-id" + syncTestCurrentCluster = "sync-cluster-b" + syncTestSourceCluster = "sync-cluster-a" + syncTestAlternativeSourceCluster = "sync-cluster-c" + syncTestCurrentFailoverVersion = int64(22) +) + +type syncWorkflowStateFixture struct { + shard *historyi.MockShardContext + registry *namespace.MockRegistry + cluster *cluster.MockMetadata + remoteClient *adminservicemock.MockAdminServiceClient + engine *historyi.MockEngine + execution *commonpb.WorkflowExecution + transition *persistencespb.VersionedTransition + versionHistory *historyspb.VersionHistories +} + +func TestSyncWorkflowStateResultZeroValueSkips(t *testing.T) { + var result SyncWorkflowStateResult + require.Equal(t, SyncWorkflowStateResultSkipped, result) +} + +func TestSyncWorkflowStateFromSource_AppliesArtifact(t *testing.T) { + tests := []struct { + name string + applyErr error + }{ + {name: "applied"}, + {name: "duplicate is applied", applyErr: consts.ErrDuplicate}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + artifact := &replicationspb.VersionedTransitionArtifact{} + f.expectEligibleNamespace() + f.remoteClient.EXPECT().SyncWorkflowState( + gomock.Any(), + protomock.Eq(f.expectedRequest()), + ).Return(&adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: artifact, + }, nil) + f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return( + syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster, syncTestCurrentCluster), + nil, + ) + f.shard.EXPECT().GetEngine(gomock.Any()).Return(f.engine, nil) + f.engine.EXPECT().ReplicateVersionedTransition( + gomock.Any(), + chasm.WorkflowArchetypeID, + artifact, + syncTestSourceCluster, + ).Return(test.applyErr) + + result, err := f.sync(t.Context()) + + require.NoError(t, err) + require.Equal(t, SyncWorkflowStateResultApplied, result) + }) + } +} + +func TestSyncWorkflowStateFromSource_SourceResponse(t *testing.T) { + tests := []struct { + name string + rpcErr error + result SyncWorkflowStateResult + }{ + { + name: "source workflow not found", + rpcErr: serviceerror.NewNotFound("workflow not found"), + result: SyncWorkflowStateResultSourceNotFound, + }, + { + name: "transition history unsupported", + rpcErr: serviceerror.NewFailedPrecondition("transition history disabled"), + result: SyncWorkflowStateResultSkipped, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + f.expectEligibleNamespace() + f.remoteClient.EXPECT().SyncWorkflowState( + gomock.Any(), + protomock.Eq(f.expectedRequest()), + ).Return(nil, test.rpcErr) + + result, err := f.sync(t.Context()) + + require.NoError(t, err) + require.Equal(t, test.result, result) + }) + } +} + +func TestSyncWorkflowStateFromSource_SkipsWhenNamespaceBecomesIneligible(t *testing.T) { + t.Run("removed from current cluster before request", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + f.expectNamespaceLookup( + syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster), + nil, + ) + + result, err := f.sync(t.Context()) + + require.NoError(t, err) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + + t.Run("active on current cluster before request", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + f.expectNamespaceLookup( + syncTestNamespace(syncTestCurrentCluster, syncTestSourceCluster, syncTestCurrentCluster), + nil, + ) + + result, err := f.sync(t.Context()) + + require.NoError(t, err) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + + t.Run("namespace deleted before request", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + f.expectNamespaceLookup(nil, serviceerror.NewNamespaceNotFound(syncTestNamespaceID.String())) + + result, err := f.sync(t.Context()) + + require.ErrorAs(t, err, new(*serviceerror.NamespaceNotFound)) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + + postRequestStates := []struct { + name string + entry *namespace.Namespace + lookupErr error + }{ + { + name: "namespace removed from current cluster during request", + entry: syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster), + }, + { + name: "namespace becomes active locally during request", + entry: syncTestNamespace(syncTestCurrentCluster, syncTestSourceCluster, syncTestCurrentCluster), + }, + { + name: "source changes during request", + entry: syncTestNamespace(syncTestAlternativeSourceCluster, syncTestAlternativeSourceCluster, syncTestCurrentCluster), + }, + { + name: "namespace deleted during request", + lookupErr: serviceerror.NewNamespaceNotFound(syncTestNamespaceID.String()), + }, + } + for _, postRequestState := range postRequestStates { + t.Run(postRequestState.name, func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + f.expectEligibleNamespace() + f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return( + &adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{}, + }, + nil, + ) + f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return( + postRequestState.entry, + postRequestState.lookupErr, + ) + + result, err := f.sync(t.Context()) + + require.NoError(t, err) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + } +} + +func TestSyncWorkflowStateFromSource_PropagatesErrors(t *testing.T) { + t.Run("initial namespace lookup", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + testErr := errors.New("namespace lookup failed") + f.expectNamespaceLookup(nil, testErr) + + result, err := f.sync(t.Context()) + + require.ErrorIs(t, err, testErr) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + + t.Run("sync request", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + testErr := errors.New("sync request failed") + f.expectEligibleNamespace() + f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return(nil, testErr) + + result, err := f.sync(t.Context()) + + require.ErrorIs(t, err, testErr) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) + + t.Run("refreshed namespace lookup", func(t *testing.T) { + f := newSyncWorkflowStateFixture(t) + testErr := errors.New("namespace refresh failed") + f.expectEligibleNamespace() + f.remoteClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Return( + &adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{}, + }, + nil, + ) + f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(nil, testErr) + + result, err := f.sync(t.Context()) + + require.ErrorIs(t, err, testErr) + require.Equal(t, SyncWorkflowStateResultSkipped, result) + }) +} + +func newSyncWorkflowStateFixture(t *testing.T) *syncWorkflowStateFixture { + controller := gomock.NewController(t) + return &syncWorkflowStateFixture{ + shard: historyi.NewMockShardContext(controller), + registry: namespace.NewMockRegistry(controller), + cluster: cluster.NewMockMetadata(controller), + remoteClient: adminservicemock.NewMockAdminServiceClient(controller), + engine: historyi.NewMockEngine(controller), + execution: &commonpb.WorkflowExecution{ + WorkflowId: syncTestWorkflowID, + RunId: syncTestRunID, + }, + transition: &persistencespb.VersionedTransition{ + NamespaceFailoverVersion: 11, + TransitionCount: 12, + }, + versionHistory: &historyspb.VersionHistories{}, + } +} + +func (f *syncWorkflowStateFixture) expectNamespaceLookup(entry *namespace.Namespace, err error) { + f.shard.EXPECT().GetClusterMetadata().Return(f.cluster) + f.cluster.EXPECT().GetCurrentClusterName().Return(syncTestCurrentCluster) + f.shard.EXPECT().GetNamespaceRegistry().Return(f.registry) + f.registry.EXPECT().GetNamespaceByID(syncTestNamespaceID).Return(entry, err) +} + +func (f *syncWorkflowStateFixture) expectEligibleNamespace() { + f.expectNamespaceLookup( + syncTestNamespace(syncTestSourceCluster, syncTestSourceCluster, syncTestCurrentCluster), + nil, + ) + f.cluster.EXPECT().GetAllClusterInfo().Return(map[string]cluster.ClusterInformation{ + syncTestCurrentCluster: { + InitialFailoverVersion: syncTestCurrentFailoverVersion, + }, + }) + f.shard.EXPECT().GetRemoteAdminClient(syncTestSourceCluster).Return(f.remoteClient, nil) +} + +func (f *syncWorkflowStateFixture) expectedRequest() *adminservice.SyncWorkflowStateRequest { + return &adminservice.SyncWorkflowStateRequest{ + NamespaceId: syncTestNamespaceID.String(), + Execution: f.execution, + ArchetypeId: chasm.WorkflowArchetypeID, + VersionedTransition: f.transition, + VersionHistories: f.versionHistory, + TargetClusterId: int32(syncTestCurrentFailoverVersion), + } +} + +func (f *syncWorkflowStateFixture) sync(ctx context.Context) (SyncWorkflowStateResult, error) { + return SyncWorkflowStateFromSource( + ctx, + f.shard, + syncTestNamespaceID, + f.execution, + f.transition, + f.versionHistory, + nil, + ) +} + +func syncTestNamespace(activeCluster string, clusters ...string) *namespace.Namespace { + return namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: syncTestNamespaceID.String(), Name: "sync-test-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: activeCluster, + Clusters: clusters, + }, + 1, + ) +} diff --git a/service/history/configs/config.go b/service/history/configs/config.go index b07d5fdbd2b..2e197e0e43c 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -288,7 +288,8 @@ type Config struct { // The following is used by the new RPC replication stack ReplicationTaskApplyTimeout dynamicconfig.DurationPropertyFn EnableAsyncParentWorkflowResend dynamicconfig.BoolPropertyFn - ParentWorkflowResendMaxInFlight dynamicconfig.IntPropertyFn + EnableChildWorkflowResend dynamicconfig.BoolPropertyFn + WorkflowResendHostMaxInFlight dynamicconfig.IntPropertyFn ReplicationTaskFetcherParallelism dynamicconfig.IntPropertyFn ReplicationTaskFetcherAggregationInterval dynamicconfig.DurationPropertyFn ReplicationTaskFetcherTimerJitterCoefficient dynamicconfig.FloatPropertyFn @@ -731,7 +732,8 @@ func NewConfig( ReplicationTaskApplyTimeout: dynamicconfig.ReplicationTaskApplyTimeout.Get(dc), EnableAsyncParentWorkflowResend: dynamicconfig.EnableAsyncParentWorkflowResend.Get(dc), - ParentWorkflowResendMaxInFlight: dynamicconfig.ParentWorkflowResendMaxInFlight.Get(dc), + EnableChildWorkflowResend: dynamicconfig.EnableChildWorkflowResend.Get(dc), + WorkflowResendHostMaxInFlight: dynamicconfig.WorkflowResendHostMaxInFlight.Get(dc), ReplicationTaskFetcherParallelism: dynamicconfig.ReplicationTaskFetcherParallelism.Get(dc), ReplicationTaskFetcherAggregationInterval: dynamicconfig.ReplicationTaskFetcherAggregationInterval.Get(dc), ReplicationTaskFetcherTimerJitterCoefficient: dynamicconfig.ReplicationTaskFetcherTimerJitterCoefficient.Get(dc), diff --git a/service/history/fx.go b/service/history/fx.go index ce20b90be3a..54d0432c51a 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -42,6 +42,7 @@ import ( hsmnexusworkflow "go.temporal.io/server/components/nexusoperations/workflow" "go.temporal.io/server/service" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/archival" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -92,6 +93,7 @@ var Module = fx.Options( service.PersistenceLazyLoadedServiceResolverModule, fx.Provide(ServiceResolverProvider), fx.Provide(EventNotifierProvider), + fx.Provide(WorkflowResendSchedulerProvider), fx.Provide(HistoryEngineFactoryProvider), fx.Provide(HandlerProvider), fx.Provide(HistoryServiceServerProvider), @@ -488,6 +490,46 @@ func ServiceLifetimeHooks(lc fx.Lifecycle, svc *Service) { lc.Append(fx.StartStopHook(svc.Start, svc.Stop)) } +func WorkflowResendSchedulerProvider( + lc fx.Lifecycle, + serviceConfig *configs.Config, + metricsHandler metrics.Handler, + logger log.ThrottledLogger, +) workflowresend.Scheduler { + schedulerLogger := log.With( + logger, + tag.ComponentTaskScheduler, + tag.ScopeHost, + tag.Operation(workflowresend.OperationName), + ) + workflowResendScheduler := workflowresend.NewBoundedWorkflowScheduler( + serviceConfig.WorkflowResendHostMaxInFlight, + schedulerLogger, + metricsHandler, + ) + lc.Append(fx.Hook{ + OnStop: func(ctx context.Context) error { + workflowResendScheduler.InitiateShutdown() + + shutdownCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + stopped := make(chan struct{}) + go func() { + workflowResendScheduler.WaitShutdown() + close(stopped) + }() + select { + case <-stopped: + return nil + case <-shutdownCtx.Done(): + schedulerLogger.Warn("Workflow resend scheduler timed out during shutdown", tag.Error(shutdownCtx.Err())) + return shutdownCtx.Err() + } + }, + }) + return workflowResendScheduler +} + func ReplicationProgressCacheProvider( serviceConfig *configs.Config, logger log.Logger, diff --git a/service/history/history_engine.go b/service/history/history_engine.go index b36ce959e26..aab1aa4b67b 100644 --- a/service/history/history_engine.go +++ b/service/history/history_engine.go @@ -85,6 +85,7 @@ import ( "go.temporal.io/server/service/history/api/updateworkflowoptions" "go.temporal.io/server/service/history/api/verifychildworkflowcompletionrecorded" "go.temporal.io/server/service/history/api/verifyfirstworkflowtaskscheduled" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/circuitbreakerpool" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -103,6 +104,13 @@ import ( ) type ( + engineOptions struct { + workflowResendScheduler workflowresend.Scheduler + } + + // EngineOption adds optional history engine dependencies without adding required parameters. + EngineOption func(*engineOptions) + historyEngineImpl struct { status int32 currentClusterName string @@ -137,7 +145,7 @@ type ( workflowDeleteManager deletemanager.DeleteManager serializer serialization.Serializer workflowConsistencyChecker api.WorkflowConsistencyChecker - parentResends verifychildworkflowcompletionrecorded.InFlightResends + workflowResendScheduler workflowresend.Scheduler chasmEngine chasm.Engine versionChecker headers.VersionChecker versionCache worker_versioning.VersionMembershipAndReactivationStatusCache @@ -155,6 +163,21 @@ type ( } ) +// WithWorkflowResendScheduler configures the host-level workflow resend scheduler. +func WithWorkflowResendScheduler(scheduler workflowresend.Scheduler) EngineOption { + return func(options *engineOptions) { + options.workflowResendScheduler = scheduler + } +} + +func applyEngineOptions(options []EngineOption) engineOptions { + var result engineOptions + for _, option := range options { + option(&result) + } + return result +} + // NewEngineWithShardContext creates an instance of history engine func NewEngineWithShardContext( shard historyi.ShardContext, @@ -185,7 +208,9 @@ func NewEngineWithShardContext( persistenceRateLimiter quotas.RequestRateLimiter, testHooks testhooks.TestHooks, chasmEngine chasm.Engine, + options ...EngineOption, ) historyi.Engine { + engineOptions := applyEngineOptions(options) currentClusterName := shard.GetClusterMetadata().GetCurrentClusterName() logger := shard.GetLogger() @@ -220,6 +245,7 @@ func NewEngineWithShardContext( eventNotifier: eventNotifier, fastForwardNotifier: notification.NewTimeSkippingFastForwardNotifier(), config: config, + workflowResendScheduler: engineOptions.workflowResendScheduler, sdkClientFactory: sdkClientFactory, matchingClient: matchingClient, rawMatchingClient: rawMatchingClient, @@ -560,7 +586,13 @@ func (e *historyEngineImpl) VerifyFirstWorkflowTaskScheduled( ctx context.Context, request *historyservice.VerifyFirstWorkflowTaskScheduledRequest, ) (retError error) { - return verifyfirstworkflowtaskscheduled.Invoke(ctx, request, e.workflowConsistencyChecker) + return verifyfirstworkflowtaskscheduled.Invoke( + ctx, + request, + e.workflowConsistencyChecker, + e.shardContext, + e.workflowResendScheduler, + ) } // RecordWorkflowTaskStarted starts a workflow task @@ -743,7 +775,13 @@ func (e *historyEngineImpl) VerifyChildExecutionCompletionRecorded( ctx context.Context, req *historyservice.VerifyChildExecutionCompletionRecordedRequest, ) (*historyservice.VerifyChildExecutionCompletionRecordedResponse, error) { - return verifychildworkflowcompletionrecorded.Invoke(ctx, req, e.workflowConsistencyChecker, e.shardContext, &e.parentResends) + return verifychildworkflowcompletionrecorded.Invoke( + ctx, + req, + e.workflowConsistencyChecker, + e.shardContext, + e.workflowResendScheduler, + ) } func (e *historyEngineImpl) ReplicateEventsV2( diff --git a/service/history/history_engine2_test.go b/service/history/history_engine2_test.go index de14cf86c76..eeef5f471c6 100644 --- a/service/history/history_engine2_test.go +++ b/service/history/history_engine2_test.go @@ -37,6 +37,7 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" @@ -49,11 +50,13 @@ import ( serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/protomock" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/common/util" "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" @@ -97,6 +100,7 @@ type ( workflowCache wcache.Cache historyEngine *historyEngineImpl mockExecutionMgr *persistence.MockExecutionManager + resendScheduler *workflowresend.BoundedWorkflowScheduler config *configs.Config logger *log.MockLogger @@ -147,6 +151,16 @@ func (s *engine2Suite) SetupTest() { s.config = tests.NewDynamicConfig() s.parentChildEventCapture = &parentChildEventCapture{} + resendScheduler := workflowresend.NewBoundedWorkflowScheduler( + func() int { return s.config.WorkflowResendHostMaxInFlight() }, + log.NewNoopLogger(), + metrics.NoopMetricsHandler, + ) + s.T().Cleanup(func() { + resendScheduler.InitiateShutdown() + resendScheduler.WaitShutdown() + }) + s.resendScheduler = resendScheduler mockShard := shard.NewTestContext( s.controller, &persistencespb.ShardInfo{ @@ -232,6 +246,7 @@ func (s *engine2Suite) SetupTest() { log.NewNoopLogger(), ), workflowConsistencyChecker: api.NewWorkflowConsistencyChecker(mockShard, s.workflowCache), + workflowResendScheduler: s.resendScheduler, persistenceVisibilityMgr: s.mockVisibilityManager, nDCWorkflowStateReplicator: s.mockWorkflowStateReplicator, workerDeploymentClient: noopWorkerDeploymentClient{}, @@ -249,6 +264,8 @@ func (s *engine2Suite) SetupSubTest() { } func (s *engine2Suite) TearDownTest() { + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() s.controller.Finish() s.mockShard.StopForTest() } @@ -2665,6 +2682,388 @@ func (s *engine2Suite) TestRecordChildExecutionCompleted_MissingChildStartedEven } } +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildAsync() { + s.config.EnableChildWorkflowResend = func() bool { return true } + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + + syncRequest := &adminservice.SyncWorkflowStateRequest{ + NamespaceId: request.NamespaceId, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: request.WorkflowExecution.WorkflowId, + RunId: request.WorkflowExecution.RunId, + }, + TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion), + ArchetypeId: chasm.WorkflowArchetypeID, + } + syncResponse := &adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{ + StateAttributes: &replicationspb.VersionedTransitionArtifact_SyncWorkflowStateSnapshotAttributes{ + SyncWorkflowStateSnapshotAttributes: &replicationspb.SyncWorkflowStateSnapshotAttributes{ + State: &persistencespb.WorkflowMutableState{}, + }, + }, + }, + } + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), syncRequest).Return(syncResponse, nil) + s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition( + gomock.Any(), + chasm.WorkflowArchetypeID, + syncResponse.VersionedTransitionArtifact, + cluster.TestCurrentClusterName, + ).Return(nil) + + ms := workflow.TestGlobalMutableState( + s.historyEngine.shardContext, + s.mockEventsCache, + log.NewTestLogger(), + tests.Version, + tests.WorkflowID, + tests.RunID, + ) + addWorkflowExecutionStartedEvent( + ms, + request.WorkflowExecution, + "wType", + "testTaskQueue", + payloads.EncodeString("input"), + 25*time.Second, + 20*time.Second, + 200*time.Second, + "identity", + ) + _, err := ms.AddWorkflowTaskScheduledEvent(false, enumsspb.WORKFLOW_TASK_TYPE_NORMAL) + s.NoError(err) + + resendVerified := make(chan struct{}) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *persistence.GetWorkflowExecutionRequest) (*persistence.GetWorkflowExecutionResponse, error) { + close(resendVerified) + return &persistence.GetWorkflowExecutionResponse{State: workflow.TestCloneToProto(s.T().Context(), ms)}, nil + }, + ) + + err = s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + var notFound *serviceerror.NotFound + s.ErrorAs(err, ¬Found) + + select { + case <-resendVerified: + case <-time.After(10 * time.Second): + s.Fail("background child resend was not re-verified") + } +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_NilSchedulerResendsSynchronously() { + s.config.EnableChildWorkflowResend = func() bool { return true } + scheduler := s.historyEngine.workflowResendScheduler + s.historyEngine.workflowResendScheduler = nil + s.T().Cleanup(func() { + s.historyEngine.workflowResendScheduler = scheduler + }) + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). + Return(nil, serviceerror.NewUnavailable("source cluster unavailable")) + + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.Unavailable)) +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendsChildWhenWorkflowNotReady() { + s.config.EnableChildWorkflowResend = func() bool { return true } + scheduler := s.historyEngine.workflowResendScheduler + s.historyEngine.workflowResendScheduler = nil + s.T().Cleanup(func() { + s.historyEngine.workflowResendScheduler = scheduler + }) + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + + ms := workflow.TestGlobalMutableState( + s.historyEngine.shardContext, + s.mockEventsCache, + log.NewTestLogger(), + tests.Version, + tests.WorkflowID, + tests.RunID, + ) + addWorkflowExecutionStartedEvent( + ms, + request.WorkflowExecution, + "wType", + "testTaskQueue", + payloads.EncodeString("input"), + 25*time.Second, + 20*time.Second, + 200*time.Second, + "identity", + ) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return( + &persistence.GetWorkflowExecutionResponse{State: workflow.TestCloneToProto(s.T().Context(), ms)}, + nil, + ) + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). + Return(nil, serviceerror.NewUnavailable("source cluster unavailable")) + + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.Unavailable)) +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_SkipsResendForRemovedNamespace() { + s.config.EnableChildWorkflowResend = func() bool { return true } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + namespaceID := namespace.ID(uuid.NewString()) + initialNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + removedNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: []string{cluster.TestAlternativeClusterName}, + }, + tests.Version, + ) + resendChecked := make(chan struct{}) + gomock.InOrder( + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(initialNamespaceEntry, nil), + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn( + func(namespace.ID) (*namespace.Namespace, error) { + close(resendChecked) + return removedNamespaceEntry, nil + }, + ), + ) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: namespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + var notFound *serviceerror.NotFound + s.ErrorAs(err, ¬Found) + select { + case <-resendChecked: + case <-time.After(10 * time.Second): + s.Fail("background child resend did not check namespace membership") + } + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() + metricSnapshot := capture.Snapshot() + s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendAttempts.Name()], 1) + s.Require().Empty(metricSnapshot[metrics.ChildWorkflowResendFailures.Name()]) +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_SkipsApplyWhenActiveClusterChanges() { + s.config.EnableChildWorkflowResend = func() bool { return true } + + namespaceID := namespace.ID(uuid.NewString()) + initialNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "failover-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestCurrentClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + failedOverNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "failover-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + failedOver := make(chan struct{}) + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn( + func(namespace.ID) (*namespace.Namespace, error) { + select { + case <-failedOver: + return failedOverNamespaceEntry, nil + default: + return initialNamespaceEntry, nil + } + }, + ).AnyTimes() + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: namespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + + syncResponse := &adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{}, + } + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), protomock.Eq(&adminservice.SyncWorkflowStateRequest{ + NamespaceId: request.NamespaceId, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: request.WorkflowExecution.WorkflowId, + RunId: request.WorkflowExecution.RunId, + }, + TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion), + ArchetypeId: chasm.WorkflowArchetypeID, + })).DoAndReturn( + func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { + close(failedOver) + return syncResponse, nil + }, + ) + + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + var notFound *serviceerror.NotFound + s.ErrorAs(err, ¬Found) + select { + case <-failedOver: + case <-time.After(10 * time.Second): + s.Fail("background child resend did not reach the source cluster") + } +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildDeduped() { + s.config.EnableChildWorkflowResend = func() bool { return true } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}).AnyTimes() + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + + entered := make(chan struct{}) + release := make(chan struct{}) + finished := make(chan struct{}) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, _ *adminservice.SyncWorkflowStateRequest, _ ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { + close(entered) + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } + close(finished) + return nil, serviceerror.NewUnavailable("source cluster unavailable") + }).Times(1) + + ctx := metrics.AddMetricsContext(s.T().Context()) + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(ctx, request) + var notFound *serviceerror.NotFound + s.ErrorAs(err, ¬Found) + + select { + case <-entered: + case <-time.After(10 * time.Second): + s.Fail("first child resend did not reach the source cluster") + } + + err = s.historyEngine.VerifyFirstWorkflowTaskScheduled(ctx, request) + s.ErrorAs(err, ¬Found) + + close(release) + select { + case <-finished: + case <-time.After(10 * time.Second): + s.Fail("background child resend did not finish") + } + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() + metricSnapshot := capture.Snapshot() + s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendAttempts.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendSkipped.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendFailures.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendLatency.Name()], 1) +} + func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowNotExist() { request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ @@ -2692,6 +3091,10 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() s.config.EnableAsyncParentWorkflowResend = func() bool { return false } s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) capture := s.parentChildEventCapture + metricsHandler := metricstest.NewCaptureHandler() + metricsCapture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(metricsCapture) + s.mockShard.SetMetricsHandler(metricsHandler) request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: tests.ParentNamespaceID.String(), @@ -2836,6 +3239,70 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParent() s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(sourceNotFoundRecord)["outcome"].AsString()) sourceNotFoundDetails := wideEventDetails(sourceNotFoundRecord) s.Equal(util.ErrorType(serviceerror.NewNotFound("")), sourceNotFoundDetails["error_type"]) + metricSnapshot := metricsCapture.Snapshot() + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 2) + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendLatency.Name()], 2) + s.Require().Empty(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()]) +} + +func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentInlinePreservesNamespaceNotFound() { + s.config.EnableAsyncParentWorkflowResend = func() bool { return false } + + namespaceID := namespace.ID(uuid.NewString()) + namespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "missing-parent-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + namespaceNotFoundErr := serviceerror.NewNamespaceNotFound(namespaceID.String()) + guardChecked := make(chan struct{}) + gomock.InOrder( + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(namespaceEntry, nil), + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn( + func(namespace.ID) (*namespace.Namespace, error) { + close(guardChecked) + return nil, namespaceNotFoundErr + }, + ), + ) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Times(0) + s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Times(0) + + request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: namespaceID.String(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ChildExecution: &commonpb.WorkflowExecution{ + WorkflowId: "child workflowId", + RunId: "child runId", + }, + ParentInitiatedId: 123, + ParentInitiatedVersion: 100, + ResendParent: true, + } + + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded( + metrics.AddMetricsContext(s.T().Context()), + request, + ) + s.Require().Same(namespaceNotFoundErr, err) + select { + case <-guardChecked: + default: + s.Fail("inline parent resend did not reach the namespace guard") + } } // Async resend: the RPC returns the verification error immediately and the pull runs in the @@ -2907,7 +3374,7 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAs func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLimited() { s.config.EnableAsyncParentWorkflowResend = func() bool { return true } - s.config.ParentWorkflowResendMaxInFlight = func() int { return 0 } + s.config.WorkflowResendHostMaxInFlight = func() int { return 0 } s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) capture := s.parentChildEventCapture @@ -2936,12 +3403,217 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLi s.InDelta(0, details["max_in_flight"], 0) } +func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_SkipsResendForRemovedNamespace() { + s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + namespaceID := namespace.ID(uuid.NewString()) + initialNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-parent-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + removedNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "removed-parent-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: []string{cluster.TestAlternativeClusterName}, + }, + tests.Version, + ) + guardChecked := make(chan struct{}) + gomock.InOrder( + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).Return(initialNamespaceEntry, nil), + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn( + func(namespace.ID) (*namespace.Namespace, error) { + close(guardChecked) + return removedNamespaceEntry, nil + }, + ), + ) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()).Times(0) + s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Times(0) + + request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: namespaceID.String(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ChildExecution: &commonpb.WorkflowExecution{ + WorkflowId: "child workflowId", + RunId: "child runId", + }, + ParentInitiatedId: 123, + ParentInitiatedVersion: 100, + ResendParent: true, + } + + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded( + metrics.AddMetricsContext(s.T().Context()), + request, + ) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + + select { + case <-guardChecked: + case <-time.After(10 * time.Second): + s.Fail("background parent resend did not check namespace membership") + } + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() + metricSnapshot := capture.Snapshot() + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 1) + s.Require().Empty(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()]) +} + +func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentAsyncSkipsApplyAfterFailover() { + s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + + namespaceID := namespace.ID(uuid.NewString()) + initialNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "parent-resend-failover-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestCurrentClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + failedOverNamespaceEntry := namespace.NewGlobalNamespaceForTest( + &persistencespb.NamespaceInfo{Id: namespaceID.String(), Name: "parent-resend-failover-namespace"}, + &persistencespb.NamespaceConfig{}, + &persistencespb.NamespaceReplicationConfig{ + ActiveClusterName: cluster.TestAlternativeClusterName, + Clusters: cluster.TestAllClusterNames, + }, + tests.Version, + ) + failedOver := make(chan struct{}) + secondGuardLookup := make(chan struct{}, 1) + s.mockNamespaceCache.EXPECT().GetNamespaceByID(namespaceID).DoAndReturn( + func(namespace.ID) (*namespace.Namespace, error) { + select { + case <-failedOver: + select { + case secondGuardLookup <- struct{}{}: + default: + } + return failedOverNamespaceEntry, nil + default: + return initialNamespaceEntry, nil + } + }, + ).AnyTimes() + + request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: namespaceID.String(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ChildExecution: &commonpb.WorkflowExecution{ + WorkflowId: "child workflowId", + RunId: "child runId", + }, + ParentInitiatedId: 123, + ParentInitiatedVersion: 100, + ResendParent: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + + syncResponse := &adminservice.SyncWorkflowStateResponse{ + VersionedTransitionArtifact: &replicationspb.VersionedTransitionArtifact{}, + } + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), protomock.Eq(&adminservice.SyncWorkflowStateRequest{ + NamespaceId: request.NamespaceId, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: request.ParentExecution.WorkflowId, + RunId: request.ParentExecution.RunId, + }, + TargetClusterId: int32(cluster.TestAlternativeClusterInitialFailoverVersion), + ArchetypeId: chasm.WorkflowArchetypeID, + })).DoAndReturn( + func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { + close(failedOver) + return syncResponse, nil + }, + ) + s.mockWorkflowStateReplicator.EXPECT().ReplicateVersionedTransition( + gomock.Any(), + gomock.Any(), + gomock.Any(), + gomock.Any(), + ).Times(0) + + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + + select { + case <-secondGuardLookup: + case <-time.After(10 * time.Second): + s.Fail("background parent resend did not recheck namespace state after SyncWorkflowState") + } +} + +func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_HostAtCapacity() { + s.config.EnableAsyncParentWorkflowResend = func() bool { return true } + s.config.WorkflowResendHostMaxInFlight = func() int { return 0 } + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ + NamespaceId: tests.ParentNamespaceID.String(), + ParentExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ChildExecution: &commonpb.WorkflowExecution{ + WorkflowId: "child workflowId", + RunId: "child runId", + }, + ResendParent: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + _, err := s.historyEngine.VerifyChildExecutionCompletionRecorded(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + s.Require().Len(capture.Snapshot()[metrics.ParentWorkflowResendLimited.Name()], 1) +} + // TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped asserts that a second attempt for // the same parent does not start a concurrent resend while the first is still running. func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDeduped() { s.config.EnableAsyncParentWorkflowResend = func() bool { return true } s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) capture := s.parentChildEventCapture + metricsHandler := metricstest.NewCaptureHandler() + metricsCapture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(metricsCapture) + s.mockShard.SetMetricsHandler(metricsHandler) request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ NamespaceId: tests.ParentNamespaceID.String(), @@ -2977,9 +3649,13 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe } }() s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). - DoAndReturn(func(context.Context, *adminservice.SyncWorkflowStateRequest, ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { + DoAndReturn(func(ctx context.Context, _ *adminservice.SyncWorkflowStateRequest, _ ...grpc.CallOption) (*adminservice.SyncWorkflowStateResponse, error) { close(entered) - <-release + select { + case <-release: + case <-ctx.Done(): + return nil, ctx.Err() + } return nil, serviceerror.NewUnavailable("source cluster unavailable") }).Times(1) @@ -3015,6 +3691,13 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentDe string(wideevents.ParentChildOutcomeDeduplicated), string(wideevents.ParentChildOutcomeFailed), }, parentChildOutcomes(capture)) + s.resendScheduler.InitiateShutdown() + s.resendScheduler.WaitShutdown() + metricSnapshot := metricsCapture.Snapshot() + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendAttempts.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendSkipped.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendFailures.Name()], 1) + s.Require().Len(metricSnapshot[metrics.ParentWorkflowResendLatency.Name()], 1) } func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowClosed() { diff --git a/service/history/history_engine_factory.go b/service/history/history_engine_factory.go index 25904c0668d..a2dd26c2324 100644 --- a/service/history/history_engine_factory.go +++ b/service/history/history_engine_factory.go @@ -13,6 +13,7 @@ import ( "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/circuitbreakerpool" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/events" @@ -55,6 +56,7 @@ type ( VersionMembershipCache worker_versioning.VersionMembershipAndReactivationStatusCache WorkerDeploymentClient workerdeployment.Client RoutingInfoCache worker_versioning.RoutingInfoCache + WorkflowResendScheduler workflowresend.Scheduler `optional:"true"` } historyEngineFactory struct { @@ -94,5 +96,6 @@ func (f *historyEngineFactory) CreateEngine( f.PersistenceRateLimiter, f.TestHooks, f.ChasmEngine, + WithWorkflowResendScheduler(f.WorkflowResendScheduler), ) } diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index ee0ad9e73e0..c8b2ff25e0e 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -61,6 +61,7 @@ import ( "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/getworkflowexecutionrawhistoryv2" + "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" @@ -84,6 +85,21 @@ const ( esIndexName = "" ) +func TestWithWorkflowResendScheduler(t *testing.T) { + scheduler := workflowresend.NewBoundedWorkflowScheduler( + func() int { return 1 }, + log.NewNoopLogger(), + metrics.NoopMetricsHandler, + ) + t.Cleanup(func() { + scheduler.InitiateShutdown() + scheduler.WaitShutdown() + }) + + options := applyEngineOptions([]EngineOption{WithWorkflowResendScheduler(scheduler)}) + require.Same(t, scheduler, options.workflowResendScheduler) +} + type ( engineSuite struct { suite.Suite diff --git a/service/history/ndc_standby_task_util.go b/service/history/ndc_standby_task_util.go index a97e44e8237..cf118181231 100644 --- a/service/history/ndc_standby_task_util.go +++ b/service/history/ndc_standby_task_util.go @@ -157,6 +157,10 @@ type ( parentWorkflowKey *definition.WorkflowKey } + startChildExecutionPostActionInfo struct { + childWorkflowKey *definition.WorkflowKey + } + workflowTaskPostActionInfo struct { workflowTaskScheduleToStartTimeout time.Duration taskqueue *taskqueuepb.TaskQueue diff --git a/service/history/transfer_queue_standby_task_executor.go b/service/history/transfer_queue_standby_task_executor.go index e8c309d382b..87dbf2b3524 100644 --- a/service/history/transfer_queue_standby_task_executor.go +++ b/service/history/transfer_queue_standby_task_executor.go @@ -30,6 +30,7 @@ import ( const ( recordChildCompletionVerificationFailedMsg = "Failed to verify child execution completion recorded" + verifyFirstWorkflowTaskScheduledFailedMsg = "Failed to verify first workflow task scheduled" ) type ( @@ -331,7 +332,7 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution( taskTime := transferTask.GetVisibilityTime() localVerificationTime := taskTime.Add(t.config.MaxLocalParentWorkflowVerificationDuration()) - resendParent := now.After(localVerificationTime) && mutableState.IsTransitionHistoryEnabled() && mutableState.CurrentVersionedTransition() != nil + resendParent := now.After(localVerificationTime) // Copy needed values from executionInfo before releasing mutable state parentNamespaceID := executionInfo.ParentNamespaceId @@ -344,6 +345,18 @@ func (t *transferQueueStandbyTaskExecutor) processCloseExecution( // no need for mutable state anymore, release workflow lock release(nil) + if resendParent { + // Parent and child workflows may use different namespace transition-history settings. + parentNamespaceEntry, err := t.registry.GetNamespaceByID(namespace.ID(parentNamespaceID)) + switch err.(type) { + case nil: + resendParent = t.config.EnableTransitionHistory(parentNamespaceEntry.Name().String()) + case *serviceerror.NamespaceNotFound: + return nil, nil + default: + return nil, err + } + } emitChildCompletionVerificationStarted( t.shardContext, @@ -519,6 +532,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( childStartedWorkflowID := childWorkflowInfo.StartedWorkflowId childStartedRunID := childWorkflowInfo.StartedRunId childClock := childWorkflowInfo.Clock + // no need for mutable state anymore, release workflow lock release(nil) @@ -532,7 +546,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( } if !childStarted { - return &struct{}{}, nil + return &startChildExecutionPostActionInfo{}, nil } if childTargetNamespaceID == "" { @@ -545,6 +559,27 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( } childTargetNamespaceID = targetNamespaceEntry.ID().String() } + childWorkflowKey := definition.NewWorkflowKey( + childTargetNamespaceID, + childStartedWorkflowID, + childStartedRunID, + ) + resendTime := transferTask.GetVisibilityTime().Add( + t.config.StandbyTaskMissingEventsResendDelay(transferTask.GetType()), + ) + resendChild := t.getCurrentTime().After(resendTime) + if resendChild { + // Parent and child workflows may use different namespace transition-history settings. + childNamespaceEntry, err := t.registry.GetNamespaceByID(namespace.ID(childTargetNamespaceID)) + switch err.(type) { + case nil: + resendChild = t.config.EnableTransitionHistory(childNamespaceEntry.Name().String()) + case *serviceerror.NamespaceNotFound: + return nil, nil + default: + return nil, err + } + } _, err = t.historyRawClient.VerifyFirstWorkflowTaskScheduled(ctx, &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ NamespaceId: childTargetNamespaceID, @@ -552,7 +587,8 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( WorkflowId: childStartedWorkflowID, RunId: childStartedRunID, }, - Clock: childClock, + Clock: childClock, + ResendChild: resendChild, }) switch err.(type) { case nil, *serviceerror.NamespaceNotFound, *serviceerror.Unimplemented: @@ -561,13 +597,15 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( case *serviceerror.NotFound, *serviceerror.WorkflowNotReady: // Case 2: Target workflow is not in the desired state. // Return a non-nil pointer as postActionInfo here to indicate that verification is not done yet. - return &struct{}{}, nil + return &startChildExecutionPostActionInfo{ + childWorkflowKey: &childWorkflowKey, + }, nil default: // Case 3: Verification itself failed. // NOTE: Wrapping the error as a verification error to prevent mutable state from being cleared and reloaded upon retry, // which is unnecessary as the error is in the target workflow, not this workflow. return nil, &verificationErr{ - msg: recordChildCompletionVerificationFailedMsg, + msg: verifyFirstWorkflowTaskScheduledFailedMsg, err: err, } } @@ -582,7 +620,7 @@ func (t *transferQueueStandbyTaskExecutor) processStartChildExecution( transferTask, t.getCurrentTime, t.config.StandbyTaskMissingEventsDiscardDelay(transferTask.GetType()), - t.checkExecutionStillExistsOnSourceBeforeDiscard, + t.checkStartChildExecutionStillExistsOnSourceBeforeDiscard, ), ) } @@ -729,6 +767,35 @@ func (t *transferQueueStandbyTaskExecutor) checkExecutionStillExistsOnSourceBefo return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, postActionInfo, logger) } +func (t *transferQueueStandbyTaskExecutor) checkStartChildExecutionStillExistsOnSourceBeforeDiscard( + ctx context.Context, + taskInfo tasks.Task, + postActionInfo any, + logger log.Logger, +) error { + if postActionInfo == nil { + return nil + } + startChildInfo, ok := postActionInfo.(*startChildExecutionPostActionInfo) + if !ok || startChildInfo.childWorkflowKey == nil { + return t.checkExecutionStillExistsOnSourceBeforeDiscard(ctx, taskInfo, postActionInfo, logger) + } + + if !executionExistsOnSource( + ctx, + *startChildInfo.childWorkflowKey, + chasm.WorkflowArchetypeID, + logger, + t.clusterName, + t.clientBean, + t.shardContext.GetNamespaceRegistry(), + t.shardContext.ChasmRegistry(), + ) { + return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, nil, logger) + } + return standbyTransferTaskPostActionTaskDiscarded(ctx, taskInfo, postActionInfo, logger) +} + func (t *transferQueueStandbyTaskExecutor) checkParentWorkflowStillExistOnSourceBeforeDiscard( ctx context.Context, taskInfo tasks.Task, diff --git a/service/history/transfer_queue_standby_task_executor_test.go b/service/history/transfer_queue_standby_task_executor_test.go index 81729905e2b..457e1d4c4fc 100644 --- a/service/history/transfer_queue_standby_task_executor_test.go +++ b/service/history/transfer_queue_standby_task_executor_test.go @@ -24,6 +24,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" workflowspb "go.temporal.io/server/api/workflow/v1" "go.temporal.io/server/chasm" + chasmworkflow "go.temporal.io/server/chasm/lib/workflow" "go.temporal.io/server/client" "go.temporal.io/server/common/archiver" "go.temporal.io/server/common/archiver/provider" @@ -143,6 +144,8 @@ func (s *transferQueueStandbyTaskExecutorSuite) SetupTest() { err := workflow.RegisterStateMachine(reg) s.NoError(err) s.mockShard.SetStateMachineRegistry(reg) + err = s.mockShard.ChasmRegistry().Register(chasmworkflow.NewLibrary(chasmworkflow.NewRegistry())) + s.NoError(err) s.mockShard.SetEventsCacheForTesting(events.NewHostLevelEventsCache( s.mockShard.GetExecutionManager(), @@ -942,6 +945,13 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() { ignoredAttributes := wideEventAttributes(records[3]) s.Equal(string(wideevents.ReplicationApplied), ignoredAttributes["phase"].AsString()) s.Equal(wideevents.ParentChildOutcomeVerified, ignoredAttributes["outcome"].AsString()) + + persistenceMutableState.ExecutionInfo.TransitionHistory = nil + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) + s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, nil) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) } func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCancelExecution_Pending() { @@ -1241,7 +1251,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P event = addWorkflowTaskCompletedEvent(&s.Suite, mutableState, wt.ScheduledEventID, wt.StartedEventID, "some random identity") taskID := s.mustGenerateTaskID() - event, _ = addStartChildWorkflowExecutionInitiatedEvent(mutableState, event.GetEventId(), + event, childInfo := addStartChildWorkflowExecutionInitiatedEvent(mutableState, event.GetEventId(), tests.ChildNamespace, tests.ChildNamespaceID, childWorkflowID, childWorkflowType, childTaskQueueName, nil, 1*time.Second, 1*time.Second, 1*time.Second, enumspb.PARENT_CLOSE_POLICY_ABANDON) now := time.Now().UTC() @@ -1264,7 +1274,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P resp := s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) - s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.fetchHistoryDuration)) + s.mockShard.SetCurrentTime(s.clusterName, now.Add(time.Minute)) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) @@ -1277,41 +1287,99 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P persistenceMutableState = s.createPersistenceMutableState(mutableState, event.GetEventId(), event.GetVersion()) s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil) + expectedVerificationRequest := protomock.Eq(&historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.ChildNamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: childWorkflowID, + RunId: childRunID, + }, + Clock: childInfo.Clock, + }) + expectedVerificationWithResendChildRequest := protomock.Eq(&historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.ChildNamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: childWorkflowID, + RunId: childRunID, + }, + Clock: childInfo.Clock, + ResendChild: true, + }) + + s.mockShard.SetCurrentTime(s.clusterName, now) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.NoError(resp.ExecutionErr) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrWorkflowNotReady) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrWorkflowNotReady) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrWorkflowExecutionNotFound) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrWorkflowExecutionNotFound) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.Unimplemented{}) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, &serviceerror.Unimplemented{}) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.NoError(resp.ExecutionErr) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, consts.ErrResourceExhaustedBusyWorkflow) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, consts.ErrResourceExhaustedBusyWorkflow) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) var verificationErr *verificationErr s.ErrorAs(resp.ExecutionErr, &verificationErr) var resourceExhaustedErr *serviceerror.ResourceExhausted s.ErrorAs(resp.ExecutionErr, &resourceExhaustedErr) + s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.fetchHistoryDuration)) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, consts.ErrWorkflowNotReady) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.Equal(consts.ErrTaskRetry, resp.ExecutionErr) + + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: childWorkflowID, + RunId: childRunID, + } + expectedDescribeChildMutableStateRequest := protomock.Eq(&adminservice.DescribeMutableStateRequest{ + Namespace: tests.ChildNamespace.String(), + Execution: childExecution, + Archetype: chasm.WorkflowArchetype, + SkipForceReload: true, + }) + s.clientBean.EXPECT().GetRemoteAdminClient( + tests.GlobalChildNamespaceEntry.ActiveClusterName(namespace.RoutingKey{ID: childWorkflowID}), + ).Return(s.mockRemoteAdminClient, nil).AnyTimes() + s.mockShard.SetCurrentTime(s.clusterName, now.Add(s.discardDuration)) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.WorkflowNotReady{}) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, &serviceerror.WorkflowNotReady{}) + s.mockRemoteAdminClient.EXPECT().DescribeMutableState(gomock.Any(), expectedDescribeChildMutableStateRequest).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.Equal(consts.ErrTaskDiscarded, resp.ExecutionErr) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, &serviceerror.WorkflowNotReady{}) + s.mockRemoteAdminClient.EXPECT().DescribeMutableState(gomock.Any(), expectedDescribeChildMutableStateRequest).Return(nil, &serviceerror.NotFound{}) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) + randomErr := errors.New("some random error") - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, randomErr) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, randomErr) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.ErrorAs(resp.ExecutionErr, &verificationErr) s.Equal(randomErr, verificationErr.Unwrap()) - s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, nil) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) + + persistenceMutableState.ExecutionInfo.TransitionHistory = nil + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, nil) + resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) + s.NoError(resp.ExecutionErr) + + s.mockShard.GetConfig().EnableTransitionHistory = func(namespaceName string) bool { + return namespaceName != tests.ChildNamespace.String() + } + s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationRequest).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) s.NoError(resp.ExecutionErr) diff --git a/tests/xdc/parent_child_harness.go b/tests/xdc/parent_child_harness.go index 4588e521e50..bc1a930036b 100644 --- a/tests/xdc/parent_child_harness.go +++ b/tests/xdc/parent_child_harness.go @@ -76,13 +76,14 @@ type ( parentTestVars *testvars.TestVars childTestVars *testvars.TestVars - activeClusterIndex int - gates [2]*parentChildReplicationGate - removeHooks []func() - cleanups []func() - delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask - metricCaptures [2]parentChildMetricCapture - trace []string + activeClusterIndex int + gates [2]*parentChildReplicationGate + removeHooks []func() + cleanups []func() + legacyReplicationCleanups []func() + delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask + metricCaptures [2]parentChildMetricCapture + trace []string } parentChildMetricCapture struct { @@ -248,17 +249,32 @@ func useLegacyHistoryReplication() parentChildScenarioStep { name: "use legacy history replication for this scenario", run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { for _, cluster := range runtime.suite.clusters { - runtime.cleanups = append(runtime.cleanups, cluster.OverrideDynamicConfig( + cleanup := cluster.OverrideDynamicConfig( runtime.suite.T(), dynamicconfig.EnableTransitionHistory, false, - )) + ) + runtime.cleanups = append(runtime.cleanups, cleanup) + runtime.legacyReplicationCleanups = append(runtime.legacyReplicationCleanups, cleanup) } return nil }, } } +func useTransitionHistory() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "use transition history for this scenario", + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + for index := len(runtime.legacyReplicationCleanups) - 1; index >= 0; index-- { + runtime.legacyReplicationCleanups[index]() + } + runtime.legacyReplicationCleanups = nil + return nil + }, + } +} + func setLocalParentVerificationGrace( cluster parentChildCluster, duration time.Duration, @@ -326,6 +342,49 @@ func setStandbyTaskDiscardDelay( } } +func enableChildWorkflowResend(cluster parentChildCluster) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("enable child workflow resend on %s", cluster), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.EnableChildWorkflowResend, + true, + )) + return nil + }, + } +} + +func setStandbyTaskResendDelay( + cluster parentChildCluster, + taskType enumsspb.TaskType, + duration time.Duration, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("set standby %s resend delay on %s to %s", taskType, cluster, duration), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + runtime.cleanups = append(runtime.cleanups, runtime.suite.clusters[clusterIndex].OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.StandbyTaskMissingEventsResendDelay, + []dynamicconfig.ConstrainedValue{{ + Constraints: dynamicconfig.Constraints{TaskType: taskType}, + Value: duration, + }}, + )) + return nil + }, + } +} + func startParentWorkflow() parentChildScenarioStep { return parentChildScenarioStep{ name: "start parent workflow on the active cluster", @@ -364,6 +423,18 @@ func applyDelayedReplication( } } +func acknowledgeDelayedReplicationWithoutApplying( + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("acknowledge delayed %s replication to %s without applying", workflow, targetCluster), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.acknowledgeDelayedReplicationWithoutApplying(targetCluster, workflow) + }, + } +} + func acknowledgeReplicationTaskContainingEventWithoutApplying( targetCluster parentChildCluster, workflow parentChildWorkflow, @@ -372,6 +443,18 @@ func acknowledgeReplicationTaskContainingEventWithoutApplying( return replicationTaskStep(ackReplicationTaskWithoutApplying, targetCluster, workflow, eventType) } +func acknowledgeNextReplicationTaskWithoutApplying( + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("acknowledge next %s replication to %s without applying", workflow, targetCluster), + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.acknowledgeNextReplicationTaskWithoutApplying(ctx, targetCluster, workflow) + }, + } +} + func replicationTaskStep( action parentChildReplicationTaskAction, targetCluster parentChildCluster, @@ -404,6 +487,15 @@ func completeChildWorkflowTask() parentChildScenarioStep { } } +func signalChildWorkflow() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "signal the child workflow on the active cluster", + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.signalChildWorkflow(ctx) + }, + } +} + func waitForWorkflowEventOnCluster( cluster parentChildCluster, workflow parentChildWorkflow, @@ -477,18 +569,6 @@ func currentWorkflowHasStatusOnCluster( } } -func workflowIsMissingOnCluster( - cluster parentChildCluster, - workflow parentChildWorkflow, -) parentChildExpectation { - return parentChildExpectation{ - name: fmt.Sprintf("%s is missing on %s", workflow, cluster), - check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { - return runtime.confirmWorkflowMissing(ctx, cluster, workflow) - }, - } -} - func workflowHasEventOnCluster( cluster parentChildCluster, workflow parentChildWorkflow, @@ -651,6 +731,37 @@ func (r *parentChildScenarioRuntime) processReplicationThroughTaskContainingEven } } +func (r *parentChildScenarioRuntime) acknowledgeNextReplicationTaskWithoutApplying( + ctx context.Context, + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) error { + targetClusterIndex := int(targetCluster) + if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { + return fmt.Errorf("unknown parent-child cluster %d", targetCluster) + } + workflowID, err := r.workflowID(workflow) + if err != nil { + return err + } + task, err := r.gates[targetClusterIndex].nextForWorkflow(ctx, workflowID) + if err != nil { + return err + } + events, err := decodeParentChildReplicationEvents(task.task) + if err != nil { + return err + } + r.tracef( + " acknowledge task %d to cluster %d for %s without applying [%s]", + task.task.GetSourceTaskId(), + targetClusterIndex, + workflow, + formatParentChildReplicationTask(task.task, events), + ) + return task.acknowledgeWithoutApplying() +} + func (r *parentChildScenarioRuntime) applyDelayedReplication( targetCluster parentChildCluster, workflow parentChildWorkflow, @@ -688,6 +799,39 @@ func (r *parentChildScenarioRuntime) applyDelayedReplication( return nil } +func (r *parentChildScenarioRuntime) acknowledgeDelayedReplicationWithoutApplying( + targetCluster parentChildCluster, + workflow parentChildWorkflow, +) error { + targetClusterIndex := int(targetCluster) + if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { + return fmt.Errorf("unknown parent-child cluster %d", targetCluster) + } + if _, err := r.workflowID(workflow); err != nil { + return err + } + + lane := parentChildReplicationLane{targetClusterIndex: targetClusterIndex, workflow: workflow} + task, delayed := r.delayedTasks[lane] + if !delayed { + return fmt.Errorf("no delayed %s replication task to %s", workflow, targetCluster) + } + events, err := decodeParentChildReplicationEvents(task.task) + if err != nil { + return err + } + + r.tracef( + " acknowledge delayed task %d to cluster %d for %s without applying [%s]", + task.task.GetSourceTaskId(), + targetClusterIndex, + workflow, + formatParentChildReplicationTask(task.task, events), + ) + delete(r.delayedTasks, lane) + return task.acknowledgeWithoutApplying() +} + func (r *parentChildScenarioRuntime) completeParentWorkflowTaskWithStartChildCommand(ctx context.Context) error { if r.parentRunID == "" { return errors.New("parent workflow is not started") @@ -751,6 +895,23 @@ func (r *parentChildScenarioRuntime) completeChildWorkflowTask(ctx context.Conte return err } +func (r *parentChildScenarioRuntime) signalChildWorkflow(ctx context.Context) error { + if r.childRunID == "" { + return errors.New("child workflow is not started") + } + _, err := r.activeCluster().FrontendClient().SignalWorkflowExecution(ctx, &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: r.namespace, + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: r.childID, + RunId: r.childRunID, + }, + SignalName: "initialize-transition-history", + Identity: r.childTestVars.WorkerIdentity(), + RequestId: uuid.NewString(), + }) + return err +} + func (r *parentChildScenarioRuntime) workflowHistoryOnCluster( ctx context.Context, cluster parentChildCluster, diff --git a/tests/xdc/parent_child_test.go b/tests/xdc/parent_child_test.go index d98c5ade1ed..03f11e5e3dc 100644 --- a/tests/xdc/parent_child_test.go +++ b/tests/xdc/parent_child_test.go @@ -12,7 +12,6 @@ import ( historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" enumsspb "go.temporal.io/server/api/enums/v1" - "go.temporal.io/server/common" "go.temporal.io/server/common/metrics" "go.temporal.io/server/tests/testcore" ) @@ -192,25 +191,33 @@ func (s *parentChildXDCTestSuite) TestReproOrphanedChildAfterForceFailover() { }) } -// TestStandbyVerifiesMissingChild covers the cross-shard ordering where the parent update +// TestStandbyResendsMissingChild covers the cross-shard ordering where the parent update // identifying a started child reaches the passive while the child's WorkflowExecutionStarted // task remains delayed. // -// | parent on target | child on target | active | outcome -// ---------------------------+---------------------------------+-----------------+----------------+--------------------------------------------- -// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix -// child start is delayed | unchanged | does not exist | initial active | child start remains unapplied -// parent child-start arrives | ChildWorkflowExecutionStarted | does not exist | initial active | standby verifies the child's first WFT -// verification fails | child-start relationship exists | does not exist | initial active | VerifyFirstWorkflowTaskScheduled: NotFound -// discard window expires | unchanged | does not exist | initial active | standby StartChild task is discarded +// | parent on target | child on target | active | outcome +// ---------------------------+---------------------------------+-----------------------+----------------+--------------------------------------------- +// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix +// child start is delayed | unchanged | does not exist | initial active | child start remains unapplied +// parent child-start arrives | ChildWorkflowExecutionStarted | does not exist | initial active | standby verifies the child's first WFT +// verification fails | child-start relationship exists | does not exist | initial active | VerifyFirstWorkflowTaskScheduled: NotFound +// child is resent | unchanged | RUNNING, WFT scheduled | initial active | child state is restored from the source // // Event checkpoints select an entire replication task, not an individual event. The delayed child -// start task intentionally remains unapplied while the standby StartChild task is allowed to expire. -func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() { +// start task intentionally remains unapplied while child state is restored through state sync. +func (s *parentChildXDCTestSuite) TestStandbyResendsMissingChild() { s.runParentChildScenario(parentChildScenario{ steps: []parentChildScenarioStep{ - // Track source time without the production standby lag so task expiration is observable quickly. + // Track source time without the production standby lag so the task becomes resend-eligible quickly. setStandbyClusterDelay(initialStandbyCluster, 0), + // Enable the child resend path on the passive cluster. + enableChildWorkflowResend(initialStandbyCluster), + // Skip the normal resend delay so the pending standby StartChild task requests a state sync. + setStandbyTaskResendDelay( + initialStandbyCluster, + enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, + 0, + ), // Create the parent and its first workflow task on the initial active cluster. startParentWorkflow(), // Establish the parent on the passive before replicating its child relationship. @@ -227,36 +234,42 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() { childWorkflow, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, ), + // Confirm that the ordinary child replication task remains unapplied before verification. + confirmWorkflowIsMissingOnCluster(initialStandbyCluster, childWorkflow), // Apply the parent's child-start record, triggering verification of the missing child. applyReplicationThroughTaskContainingEvent( initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, ), - // Observe the exact missing-child error before shortening the task's discard window. + // Observe the original missing-child error returned while the resend runs in the background. waitForHistoryVerificationFailureOnCluster( initialStandbyCluster, historyClientVerifyFirstWorkflowTask, &serviceerror.NotFound{}, ), - // Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded. - setStandbyTaskDiscardDelay( - initialStandbyCluster, - enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, - 0, - ), }, expectations: []parentChildExpectation{ - workflowIsMissingOnCluster(initialStandbyCluster, childWorkflow), - taskWasDiscardedOnCluster( + { + name: "child workflow resend is attempted on the initial standby cluster", + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.requireCapturedMetric( + initialStandbyCluster, + metrics.ChildWorkflowResendAttempts.Name(), + nil, + ) + }, + }, + workflowHasEventOnCluster( initialStandbyCluster, - metrics.TaskTypeTransferStandbyTaskStartChildExecution, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, ), }, }) } -// TestStandbyVerifiesChildWithoutFirstWorkflowTask covers the cross-shard ordering where the +// TestStandbyResendsChildWithoutFirstWorkflowTask covers the cross-shard ordering where the // child's WorkflowExecutionStarted reaches the passive, its first WorkflowTaskScheduled remains // delayed, and the parent update identifying the started child then arrives. // @@ -267,18 +280,25 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesMissingChild() { // child first WFT is delayed | unchanged | unchanged | initial active | WorkflowTaskScheduled remains unapplied // parent child-start arrives | ChildWorkflowExecutionStarted | unchanged | initial active | standby verifies the child's first WFT // verification fails | child-start relationship exists | next event ID 2, no scheduled ID | initial active | VerifyFirstWorkflowTaskScheduled: WorkflowNotReady -// discard window expires | unchanged | unchanged | initial active | standby StartChild task is discarded +// child is resent | unchanged | RUNNING, WFT scheduled | initial active | child state is restored from the source // -// This scenario uses legacy history replication because it fixes each transaction's event range. -// The gate can therefore apply WorkflowExecutionStarted while keeping the separate -// WorkflowTaskScheduled task delayed through the assertions. -func (s *parentChildXDCTestSuite) TestStandbyVerifiesChildWithoutFirstWorkflowTask() { +// The replication gate applies WorkflowExecutionStarted while keeping the separate +// WorkflowTaskScheduled update delayed, so recovery must come from state sync. +func (s *parentChildXDCTestSuite) TestStandbyResendsChildWithoutFirstWorkflowTask() { s.runParentChildScenario(parentChildScenario{ steps: []parentChildScenarioStep{ // Keep child Started and its first WFT in separate event-range replication tasks. useLegacyHistoryReplication(), - // Track source time without the production standby lag so task expiration is observable quickly. + // Track source time without the production standby lag so the task becomes resend-eligible quickly. setStandbyClusterDelay(initialStandbyCluster, 0), + // Enable the child resend path on the passive cluster. + enableChildWorkflowResend(initialStandbyCluster), + // Skip the normal resend delay so the pending standby StartChild task requests a state sync. + setStandbyTaskResendDelay( + initialStandbyCluster, + enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, + 0, + ), // Create the parent and its first workflow task on the initial active cluster. startParentWorkflow(), // Establish the parent on the passive before replicating its child relationship. @@ -301,51 +321,42 @@ func (s *parentChildXDCTestSuite) TestStandbyVerifiesChildWithoutFirstWorkflowTa childWorkflow, enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, ), + // Drop the ordinary WFT replication task so it cannot heal the passive child. + acknowledgeDelayedReplicationWithoutApplying(initialStandbyCluster, childWorkflow), + // State sync uses transition history after the partial child state has been reproduced. + useTransitionHistory(), + // Initialize transition history on the source child while retaining its scheduled WFT. + signalChildWorkflow(), + // Do not let the transition update repair the passive through ordinary replication. + acknowledgeNextReplicationTaskWithoutApplying(initialStandbyCluster, childWorkflow), // Apply the parent's child-start record, triggering verification of that partial child. applyReplicationThroughTaskContainingEvent( initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, ), - // Observe the exact not-ready error before shortening the task's discard window. + // Observe the original not-ready error returned while the resend runs in the background. waitForHistoryVerificationFailureOnCluster( initialStandbyCluster, historyClientVerifyFirstWorkflowTask, &serviceerror.WorkflowNotReady{}, ), - // Expire only the pending standby StartChild task; its next attempt should return ErrTaskDiscarded. - setStandbyTaskDiscardDelay( - initialStandbyCluster, - enumsspb.TASK_TYPE_TRANSFER_START_CHILD_EXECUTION, - 0, - ), }, expectations: []parentChildExpectation{ { - name: "child exists without its first workflow task on the initial standby cluster", - check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { - mutableState, err := runtime.workflowMutableState(ctx, initialStandbyCluster, childWorkflow) - if err != nil { - return err - } - if state := mutableState.GetExecutionState().GetState(); state != enumsspb.WORKFLOW_EXECUTION_STATE_CREATED { - return fmt.Errorf("child state is %s, want %s", state, enumsspb.WORKFLOW_EXECUTION_STATE_CREATED) - } - if status := mutableState.GetExecutionState().GetStatus(); status != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING { - return fmt.Errorf("child status is %s, want %s", status, enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING) - } - if scheduledEventID := mutableState.GetExecutionInfo().GetWorkflowTaskScheduledEventId(); scheduledEventID != common.EmptyEventID { - return fmt.Errorf("child workflow task scheduled event ID is %d, want %d", scheduledEventID, common.EmptyEventID) - } - if nextEventID := mutableState.GetNextEventId(); nextEventID != common.FirstEventID+1 { - return fmt.Errorf("child next event ID is %d, want %d", nextEventID, common.FirstEventID+1) - } - return nil + name: "child workflow resend is attempted on the initial standby cluster", + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.requireCapturedMetric( + initialStandbyCluster, + metrics.ChildWorkflowResendAttempts.Name(), + nil, + ) }, }, - taskWasDiscardedOnCluster( + workflowHasEventOnCluster( initialStandbyCluster, - metrics.TaskTypeTransferStandbyTaskStartChildExecution, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, ), }, }) From 5c2cb12aad354c8fc79490e9dca37adffc82e931 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Thu, 27 Aug 2026 15:00:10 -0700 Subject: [PATCH 084/139] Add wide events for child workflow resends (#11792) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Added wide events for the child workflow resend lifecycle, including: - scheduled - started - succeeded - source not found - failed - deduplicated - limited by the host-level concurrency cap ## Why? Child workflow resends run asynchronously, making missing-child replication issues difficult to diagnose from request logs alone. These events make it possible to trace scheduler admission, remote state synchronization, verification, and terminal outcomes while keeping parent and child resend events consistent. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- Stack created with GitHub Stacks CLI • Give Feedback 💬 --- common/wideevents/replication_events.go | 1 + .../api.go | 86 +++----------- .../verifyfirstworkflowtaskscheduled/api.go | 96 ++++++++++++++- .../history/api/workflowresend/wide_events.go | 81 +++++++++++++ service/history/history_engine2_test.go | 111 +++++++++++++++++- 5 files changed, 301 insertions(+), 74 deletions(-) create mode 100644 service/history/api/workflowresend/wide_events.go diff --git a/common/wideevents/replication_events.go b/common/wideevents/replication_events.go index 3621db16419..6951a2f78d7 100644 --- a/common/wideevents/replication_events.go +++ b/common/wideevents/replication_events.go @@ -111,6 +111,7 @@ const ( ParentChildPhaseVerifyChildCompletion = "verify_child_completion" ParentChildPhaseParentResend = "parent_resend" + ParentChildPhaseChildResend = "child_resend" ParentChildOutcomeNotFound = "not_found" ParentChildOutcomeCompletionMissing = "completion_missing" diff --git a/service/history/api/verifychildworkflowcompletionrecorded/api.go b/service/history/api/verifychildworkflowcompletionrecorded/api.go index 0fc8ffa668c..435aa554ba1 100644 --- a/service/history/api/verifychildworkflowcompletionrecorded/api.go +++ b/service/history/api/verifychildworkflowcompletionrecorded/api.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "maps" "time" "go.temporal.io/api/serviceerror" @@ -196,15 +195,13 @@ func Invoke( case workflowresend.SubmitResultAtCapacity: metrics.ParentWorkflowResendLimited.With(metricsHandler).Record(1) if emitLifecycle { - details := parentResendEventDetails(errVerify) - details["max_in_flight"] = shardContext.GetConfig().WorkflowResendHostMaxInFlight() emitParentResendLifecycleEvent( shardContext, request, parentWorkflowState, wideevents.ParentChildOutcomeLimited, nil, - details, + parentResendEventDetails(errVerify), ) } default: @@ -332,67 +329,22 @@ func emitParentResendLifecycleEvent( err error, details map[string]any, ) { - eventDetails := make(map[string]any, len(details)+11) - maps.Copy(eventDetails, details) - eventDetails["event_type"] = wideevents.ParentChildLifecycleEventType - eventDetails["phase"] = wideevents.ParentChildPhaseParentResend - eventDetails["outcome"] = outcome - eventDetails["local_cluster"] = shardContext.GetClusterMetadata().GetCurrentClusterName() - eventDetails["parent_namespace_id"] = request.GetNamespaceId() - eventDetails["parent_workflow_id"] = request.GetParentExecution().GetWorkflowId() - eventDetails["parent_run_id"] = request.GetParentExecution().GetRunId() - eventDetails["parent_workflow_state"] = parentWorkflowState - eventDetails["child_workflow_id"] = request.GetChildExecution().GetWorkflowId() - eventDetails["child_run_id"] = request.GetChildExecution().GetRunId() - eventDetails["parent_initiated_id"] = request.GetParentInitiatedId() - eventDetails["parent_initiated_version"] = request.GetParentInitiatedVersion() - - namespaceName := "" - if entry, namespaceErr := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(request.GetNamespaceId())); namespaceErr == nil { - namespaceName = entry.Name().String() - } - sourceCluster, ok := eventDetails["source_cluster"].(string) - if !ok { - sourceCluster = "" - } - delete(eventDetails, "source_cluster") - if err != nil { - eventDetails["error"] = err.Error() - eventDetails["error_type"] = util.ErrorType(err) - } - payload := wideevents.ReplicationLifecyclePayload{ - TaskType: wideevents.ReplTaskSyncWorkflowState, - Shard: shardContext.GetShardID(), - Namespace: namespaceName, - NamespaceID: request.GetNamespaceId(), - WorkflowID: request.GetParentExecution().GetWorkflowId(), - RunID: request.GetParentExecution().GetRunId(), - SourceCluster: sourceCluster, - } - switch outcome { - case wideevents.ParentChildOutcomeScheduled, - wideevents.ParentChildOutcomeStarted, - wideevents.ParentChildOutcomeDeduplicated: - eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState - eventDetails["message"] = "Parent workflow resend checkpoint" - payload.Phase = wideevents.ReplicationExecuting - payload.Details = eventDetails - wideevents.Emit(shardContext.GetEventLogger(), payload) - case wideevents.ParentChildOutcomeSucceeded, wideevents.ParentChildOutcomeSourceNotFound: - eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState - eventDetails["message"] = "Parent workflow resend checkpoint" - payload.Phase = wideevents.ReplicationApplied - payload.Outcome = wideevents.ParentChildOutcomeVerified - payload.Details = eventDetails - wideevents.Emit(shardContext.GetEventLogger(), payload) - default: - wideevents.EmitReplicationError( - shardContext.GetEventLogger(), - payload, - wideevents.ReplOperationStandbyVerificationSyncState, - "Parent workflow resend checkpoint", - err, - eventDetails, - ) - } + details["parent_namespace_id"] = request.GetNamespaceId() + details["parent_workflow_id"] = request.GetParentExecution().GetWorkflowId() + details["parent_run_id"] = request.GetParentExecution().GetRunId() + details["parent_workflow_state"] = parentWorkflowState + details["child_workflow_id"] = request.GetChildExecution().GetWorkflowId() + details["child_run_id"] = request.GetChildExecution().GetRunId() + details["parent_initiated_id"] = request.GetParentInitiatedId() + details["parent_initiated_version"] = request.GetParentInitiatedVersion() + workflowresend.EmitLifecycleEvent(workflowresend.LifecycleEvent{ + ShardContext: shardContext, + NamespaceID: namespace.ID(request.GetNamespaceId()), + Execution: request.GetParentExecution(), + ResendPhase: wideevents.ParentChildPhaseParentResend, + Message: "Parent workflow resend checkpoint", + Outcome: outcome, + Err: err, + Details: details, + }) } diff --git a/service/history/api/verifyfirstworkflowtaskscheduled/api.go b/service/history/api/verifyfirstworkflowtaskscheduled/api.go index 7757d6a029d..102ba024bd4 100644 --- a/service/history/api/verifyfirstworkflowtaskscheduled/api.go +++ b/service/history/api/verifyfirstworkflowtaskscheduled/api.go @@ -20,6 +20,8 @@ import ( "go.temporal.io/server/common/persistence/transitionhistory" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/rpc" + "go.temporal.io/server/common/util" + "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/consts" @@ -56,6 +58,7 @@ func Invoke( } metricsHandler := shardContext.GetMetricsHandler() + emitLifecycle := shardContext.GetConfig().EmitReplicationLifecycleEvents() resend := func(ctx context.Context) error { metrics.ChildWorkflowResendAttempts.With(metricsHandler).Record(1) startTime := time.Now().UTC() @@ -68,6 +71,7 @@ func Invoke( versionedTransition, versionHistories, errVerify, + emitLifecycle, ) metrics.ChildWorkflowResendLatency.With(metricsHandler).Record(time.Since(startTime)) if err != nil && !isExpectedResendError(err) { @@ -88,6 +92,15 @@ func Invoke( childKey, shardContext.GetConfig().ReplicationTaskApplyTimeout(), func(ctx context.Context) { + if emitLifecycle { + emitChildResendLifecycleEvent( + shardContext, + req, + wideevents.ParentChildOutcomeScheduled, + nil, + childResendEventDetails(errVerify), + ) + } _ = resend(ctx) }, ) @@ -96,8 +109,26 @@ func Invoke( // Accepted work records its attempt when execution starts. case workflowresend.SubmitResultDuplicate: metrics.ChildWorkflowResendSkipped.With(metricsHandler).Record(1) + if emitLifecycle { + emitChildResendLifecycleEvent( + shardContext, + req, + wideevents.ParentChildOutcomeDeduplicated, + nil, + childResendEventDetails(errVerify), + ) + } case workflowresend.SubmitResultAtCapacity: metrics.ChildWorkflowResendLimited.With(metricsHandler).Record(1) + if emitLifecycle { + emitChildResendLifecycleEvent( + shardContext, + req, + wideevents.ParentChildOutcomeLimited, + nil, + childResendEventDetails(errVerify), + ) + } default: // SubmitResultFailed and unknown values are admission failures. metrics.ChildWorkflowResendFailures.With(metricsHandler).Record(1) @@ -188,7 +219,23 @@ func resendChildAndVerify( versionedTransition *persistencespb.VersionedTransition, versionHistories *historyspb.VersionHistories, errVerify error, + emitLifecycle bool, ) error { + activeClusterName := "" + emitResult := func(outcome string, eventErr error, stage string) { + if !emitLifecycle { + return + } + details := childResendEventDetails(errVerify) + if activeClusterName != "" { + details["source_cluster"] = activeClusterName + } + if stage != "" { + details["stage"] = stage + } + emitChildResendLifecycleEvent(shardContext, req, outcome, eventErr, details) + } + result, err := workflowresend.SyncWorkflowStateFromSource( ctx, shardContext, @@ -196,20 +243,63 @@ func resendChildAndVerify( req.WorkflowExecution, versionedTransition, versionHistories, - nil, + func(sourceCluster string) { + activeClusterName = sourceCluster + emitResult(wideevents.ParentChildOutcomeStarted, nil, "sync_workflow_state") + }, ) if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "sync_workflow_state") return err } switch result { case workflowresend.SyncWorkflowStateResultSourceNotFound: + emitResult( + wideevents.ParentChildOutcomeSourceNotFound, + serviceerror.NewNotFound("child workflow not found on source cluster"), + "sync_workflow_state", + ) return nil case workflowresend.SyncWorkflowStateResultSkipped: return errVerify case workflowresend.SyncWorkflowStateResultApplied: - _, _, _, err = verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker) - return err default: return fmt.Errorf("unknown workflow state sync result: %d", result) } + + _, _, _, err = verifyFirstWorkflowTaskScheduled(ctx, req, workflowConsistencyChecker) + if err != nil { + emitResult(wideevents.ParentChildOutcomeFailed, err, "verify_after_resend") + return err + } + emitResult(wideevents.ParentChildOutcomeSucceeded, nil, "") + return nil +} + +func childResendEventDetails(initialError error) map[string]any { + return map[string]any{ + "initial_error_type": util.ErrorType(initialError), + } +} + +func emitChildResendLifecycleEvent( + shardContext historyi.ShardContext, + request *historyservice.VerifyFirstWorkflowTaskScheduledRequest, + outcome string, + err error, + details map[string]any, +) { + details["child_namespace_id"] = request.GetNamespaceId() + details["child_workflow_id"] = request.GetWorkflowExecution().GetWorkflowId() + details["child_run_id"] = request.GetWorkflowExecution().GetRunId() + workflowresend.EmitLifecycleEvent(workflowresend.LifecycleEvent{ + ShardContext: shardContext, + NamespaceID: namespace.ID(request.GetNamespaceId()), + Execution: request.GetWorkflowExecution(), + ResendPhase: wideevents.ParentChildPhaseChildResend, + Message: "Child workflow resend checkpoint", + Outcome: outcome, + Err: err, + Details: details, + }) } diff --git a/service/history/api/workflowresend/wide_events.go b/service/history/api/workflowresend/wide_events.go new file mode 100644 index 00000000000..d7ac9b74a0d --- /dev/null +++ b/service/history/api/workflowresend/wide_events.go @@ -0,0 +1,81 @@ +package workflowresend + +import ( + "maps" + + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/util" + "go.temporal.io/server/common/wideevents" + historyi "go.temporal.io/server/service/history/interfaces" +) + +type LifecycleEvent struct { + ShardContext historyi.ShardContext + NamespaceID namespace.ID + Execution *commonpb.WorkflowExecution + ResendPhase string + Message string + Outcome string + Err error + Details map[string]any +} + +// EmitLifecycleEvent emits a parent or child workflow resend checkpoint. +func EmitLifecycleEvent(event LifecycleEvent) { + eventDetails := make(map[string]any, len(event.Details)+4) + maps.Copy(eventDetails, event.Details) + eventDetails["event_type"] = wideevents.ParentChildLifecycleEventType + eventDetails["phase"] = event.ResendPhase + eventDetails["outcome"] = event.Outcome + eventDetails["local_cluster"] = event.ShardContext.GetClusterMetadata().GetCurrentClusterName() + + namespaceName := "" + if entry, namespaceErr := event.ShardContext.GetNamespaceRegistry().GetNamespaceByID(event.NamespaceID); namespaceErr == nil { + namespaceName = entry.Name().String() + } + sourceCluster, ok := eventDetails["source_cluster"].(string) + if !ok { + sourceCluster = "" + } + delete(eventDetails, "source_cluster") + if event.Err != nil { + eventDetails["error"] = event.Err.Error() + eventDetails["error_type"] = util.ErrorType(event.Err) + } + payload := wideevents.ReplicationLifecyclePayload{ + TaskType: wideevents.ReplTaskSyncWorkflowState, + Shard: event.ShardContext.GetShardID(), + Namespace: namespaceName, + NamespaceID: event.NamespaceID.String(), + WorkflowID: event.Execution.GetWorkflowId(), + RunID: event.Execution.GetRunId(), + SourceCluster: sourceCluster, + } + switch event.Outcome { + case wideevents.ParentChildOutcomeScheduled, + wideevents.ParentChildOutcomeStarted, + wideevents.ParentChildOutcomeDeduplicated: + eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState + eventDetails["message"] = event.Message + payload.Phase = wideevents.ReplicationExecuting + payload.Details = eventDetails + wideevents.Emit(event.ShardContext.GetEventLogger(), payload) + case wideevents.ParentChildOutcomeSucceeded, wideevents.ParentChildOutcomeSourceNotFound: + eventDetails["operation"] = wideevents.ReplOperationStandbyVerificationSyncState + eventDetails["message"] = event.Message + payload.Phase = wideevents.ReplicationApplied + payload.Outcome = wideevents.ParentChildOutcomeVerified + payload.Details = eventDetails + wideevents.Emit(event.ShardContext.GetEventLogger(), payload) + default: + wideevents.EmitReplicationError( + event.ShardContext.GetEventLogger(), + payload, + wideevents.ReplOperationStandbyVerificationSyncState, + event.Message, + event.Err, + eventDetails, + ) + } +} diff --git a/service/history/history_engine2_test.go b/service/history/history_engine2_test.go index eeef5f471c6..9a7a140f93f 100644 --- a/service/history/history_engine2_test.go +++ b/service/history/history_engine2_test.go @@ -2684,6 +2684,8 @@ func (s *engine2Suite) TestRecordChildExecutionCompleted_MissingChildStartedEven func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildAsync() { s.config.EnableChildWorkflowResend = func() bool { return true } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ NamespaceId: tests.NamespaceID.String(), @@ -2768,6 +2770,66 @@ func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildAsync() { case <-time.After(10 * time.Second): s.Fail("background child resend was not re-verified") } + await.RequireTrue(s.T(), func() bool { + return len(parentChildOutcomes(capture)) >= 3 + }, 10*time.Second, 10*time.Millisecond) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeSucceeded), + }, parentChildOutcomes(capture)) + records := parentChildRecords(capture) + details := wideEventDetails(records[1]) + attributes := wideEventAttributes(records[1]) + s.Equal(wideevents.ParentChildPhaseChildResend, details["phase"]) + s.Equal(request.GetNamespaceId(), details["child_namespace_id"]) + s.Equal(request.GetWorkflowExecution().GetWorkflowId(), details["child_workflow_id"]) + s.Equal(request.GetWorkflowExecution().GetRunId(), details["child_run_id"]) + s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) + s.Equal("sync_workflow_state", details["stage"]) + s.Equal(cluster.TestCurrentClusterName, attributes["source_cluster"].AsString()) + s.Equal(string(wideevents.ReplicationApplied), wideEventAttributes(records[2])["phase"].AsString()) + s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(records[2])["outcome"].AsString()) +} + +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildSourceNotFound() { + s.config.EnableChildWorkflowResend = func() bool { return true } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + mockClusterMetadata := cluster.NewMockMetadata(s.controller) + mockClusterMetadata.EXPECT().GetClusterID().Return(tests.Version).AnyTimes() + mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestAlternativeClusterName).AnyTimes() + mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo) + mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, tests.Version).Return(cluster.TestCurrentClusterName).AnyTimes() + s.mockShard.SetClusterMetadata(mockClusterMetadata) + s.mockShard.Resource.RemoteAdminClient.EXPECT().SyncWorkflowState(gomock.Any(), gomock.Any()). + Return(nil, serviceerror.NewNotFound("child missing on source")) + + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + await.RequireTrue(s.T(), func() bool { + return len(parentChildOutcomes(capture)) >= 3 + }, 10*time.Second, 10*time.Millisecond) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeSourceNotFound), + }, parentChildOutcomes(capture)) + record := parentChildRecords(capture)[2] + s.Equal(string(wideevents.ReplicationApplied), wideEventAttributes(record)["phase"].AsString()) + s.Equal(wideevents.ParentChildOutcomeVerified, wideEventAttributes(record)["outcome"].AsString()) + s.Equal(util.ErrorType(serviceerror.NewNotFound("")), wideEventDetails(record)["error_type"]) } func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_NilSchedulerResendsSynchronously() { @@ -2997,9 +3059,11 @@ func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_SkipsApplyWhenActive func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildDeduped() { s.config.EnableChildWorkflowResend = func() bool { return true } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + eventCapture := s.parentChildEventCapture metricsHandler := metricstest.NewCaptureHandler() - capture := metricsHandler.StartCapture() - defer metricsHandler.StopCapture(capture) + metricsCapture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(metricsCapture) s.mockShard.SetMetricsHandler(metricsHandler) request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ @@ -3048,6 +3112,11 @@ func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildDeduped() err = s.historyEngine.VerifyFirstWorkflowTaskScheduled(ctx, request) s.ErrorAs(err, ¬Found) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeDeduplicated), + }, parentChildOutcomes(eventCapture)) close(release) select { @@ -3055,15 +3124,49 @@ func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildDeduped() case <-time.After(10 * time.Second): s.Fail("background child resend did not finish") } + await.RequireTrue(s.T(), func() bool { + return len(parentChildOutcomes(eventCapture)) >= 4 + }, 10*time.Second, 10*time.Millisecond) + s.Require().Equal([]string{ + string(wideevents.ParentChildOutcomeScheduled), + string(wideevents.ParentChildOutcomeStarted), + string(wideevents.ParentChildOutcomeDeduplicated), + string(wideevents.ParentChildOutcomeFailed), + }, parentChildOutcomes(eventCapture)) s.resendScheduler.InitiateShutdown() s.resendScheduler.WaitShutdown() - metricSnapshot := capture.Snapshot() + metricSnapshot := metricsCapture.Snapshot() s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendAttempts.Name()], 1) s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendSkipped.Name()], 1) s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendFailures.Name()], 1) s.Require().Len(metricSnapshot[metrics.ChildWorkflowResendLatency.Name()], 1) } +func (s *engine2Suite) TestVerifyFirstWorkflowTaskScheduled_ResendChildLimited() { + s.config.EnableChildWorkflowResend = func() bool { return true } + s.config.WorkflowResendHostMaxInFlight = func() int { return 0 } + s.config.EmitReplicationLifecycleEvents = dynamicconfig.GetBoolPropertyFn(true) + capture := s.parentChildEventCapture + + request := &historyservice.VerifyFirstWorkflowTaskScheduledRequest{ + NamespaceId: tests.NamespaceID.String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + ResendChild: true, + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(nil, &serviceerror.NotFound{}) + + err := s.historyEngine.VerifyFirstWorkflowTaskScheduled(metrics.AddMetricsContext(s.T().Context()), request) + s.Require().ErrorAs(err, new(*serviceerror.NotFound)) + s.Require().Equal([]string{string(wideevents.ParentChildOutcomeLimited)}, parentChildOutcomes(capture)) + details := wideEventDetails(parentChildRecords(capture)[0]) + s.Equal(wideevents.ParentChildPhaseChildResend, details["phase"]) + s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) + s.NotContains(details, "max_in_flight") +} + func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_WorkflowNotExist() { request := &historyservice.VerifyChildExecutionCompletionRecordedRequest{ @@ -3400,7 +3503,7 @@ func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_ResendParentLi s.Require().Equal([]string{string(wideevents.ParentChildOutcomeLimited)}, parentChildOutcomes(capture)) details := wideEventDetails(parentChildRecords(capture)[0]) s.Equal(util.ErrorType(&serviceerror.NotFound{}), details["initial_error_type"]) - s.InDelta(0, details["max_in_flight"], 0) + s.NotContains(details, "max_in_flight") } func (s *engine2Suite) TestVerifyChildExecutionCompletionRecorded_SkipsResendForRemovedNamespace() { From 5aa7a471d84de3149a18d868d2247038a71b6475 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 27 Aug 2026 15:07:29 -0700 Subject: [PATCH 085/139] Make Nexus log messages aggregatable (#11765) ## What changed Kept dynamic data out of Nexus log messages and moved it to tags. Added a review guideline requiring static logger messages and structured tags for all dynamic content. ## Why Ensures that Nexus logs are aggregatable. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .github/copilot-instructions.md | 1 + common/log/tag/tags.go | 10 ++++++++ service/frontend/nexus_endpoint_client.go | 25 +++++++++++++------ .../matching/task_queue_partition_manager.go | 8 ++++-- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1a9b5239556..f80f9ab2b41 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -62,6 +62,7 @@ Apply these patterns when reviewing PRs or suggesting code changes. - Follow existing patterns: "We have been passing through the frontend request in other libraries. Let's keep the same pattern here" - Use existing utilities before creating new ones +- Use static logger messages and record all dynamic content in structured tags - Follow CLI documentation conventions (capitalize proper nouns) - Match existing metric tag formats (CONSTANT_CASE for enum values) - Use the same error message style (no punctuation for single sentences) diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index 96747429378..56577d87c5b 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -66,6 +66,16 @@ func RequestID(requestID string) ZapTag { return NewStringTag("request-id", requestID) } +// NextPageToken returns tag for NextPageToken +func NextPageToken(nextPageToken []byte) ZapTag { + return NewBinaryTag("next-page-token", nextPageToken) +} + +// PageSize returns tag for PageSize +func PageSize(pageSize int) ZapTag { + return NewInt("page-size", pageSize) +} + // ========== Workflow tags defined here: ( wf is short for workflow) ========== // WorkflowAction returns tag for WorkflowAction diff --git a/service/frontend/nexus_endpoint_client.go b/service/frontend/nexus_endpoint_client.go index 268f4d25f58..7d08265e29d 100644 --- a/service/frontend/nexus_endpoint_client.go +++ b/service/frontend/nexus_endpoint_client.go @@ -195,9 +195,9 @@ func (c *NexusEndpointClient) List( return c.listAndFilterByName(ctx, request) } - pageSize := request.GetPageSize() + pageSize := int(request.GetPageSize()) if pageSize == 0 { - pageSize = int32(c.config.listDefaultPageSize()) + pageSize = c.config.listDefaultPageSize() } else if err := c.validatePageSize(pageSize); err != nil { return nil, err } @@ -205,10 +205,15 @@ func (c *NexusEndpointClient) List( resp, err := c.persistence.ListNexusEndpoints(ctx, &p.ListNexusEndpointsRequest{ LastKnownTableVersion: 0, NextPageToken: request.NextPageToken, - PageSize: int(pageSize), + PageSize: pageSize, }) if err != nil { - c.logger.Error(fmt.Sprintf("error listing Nexus endpoints from persistence. NextPageToken: %v PageSize: %d", request.NextPageToken, pageSize), tag.Error(err)) + c.logger.Error( + "error listing Nexus endpoints from persistence", + tag.Error(err), + tag.NextPageToken(request.NextPageToken), + tag.PageSize(pageSize), + ) return nil, serviceerror.NewInternal("error listing Nexus endpoints") } @@ -285,7 +290,13 @@ func (c *NexusEndpointClient) listAndFilterByName( PageSize: pageSize, }) if err != nil { - c.logger.Error(fmt.Sprintf("error listing Nexus endpoints from persistence with Name filter. CurrentPageToken: %v PageSize: %d Name: %v", currentPageToken, pageSize, request.Name), tag.Error(err)) + c.logger.Error( + "error listing Nexus endpoints from persistence with Name filter", + tag.Error(err), + tag.NextPageToken(currentPageToken), + tag.PageSize(pageSize), + tag.Endpoint(request.Name), + ) return nil, serviceerror.NewInternal("error listing Nexus endpoints") } @@ -398,14 +409,14 @@ func validateGetRequest(request *operatorservice.GetNexusEndpointRequest) error return issues.GetError() } -func (c *NexusEndpointClient) validatePageSize(pageSize int32) error { +func (c *NexusEndpointClient) validatePageSize(pageSize int) error { // pageSize == 0 is treated as unset and will be changed to the default and does not go through this validation if pageSize < 0 { return serviceerror.NewInvalidArgument("page_size is negative") } maxPageSize := c.config.listMaxPageSize() - if pageSize > int32(maxPageSize) { + if pageSize > maxPageSize { return serviceerror.NewInvalidArgumentf("page_size exceeds limit of %d", maxPageSize) } diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index fb84e134629..49e166dbbde 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -3,7 +3,6 @@ package matching import ( "context" "errors" - "fmt" "maps" "math" "math/bits" @@ -1104,7 +1103,12 @@ func (pm *taskQueuePartitionManagerImpl) DispatchNexusTask( opTimeout, err := time.ParseDuration(opTimeoutHeader) if err != nil { // Operation-Timeout header is not required so don't fail request on parsing errors. - pm.logger.Warn(fmt.Sprintf("unable to parse %v header: %v", nexus.HeaderOperationTimeout, opTimeoutHeader), tag.Error(err), tag.WorkflowNamespaceID(request.NamespaceId)) + pm.logger.Warn( + "unable to parse operation-timeout header", + tag.Error(err), + tag.NewStringTag(nexus.HeaderOperationTimeout, opTimeoutHeader), + tag.WorkflowNamespaceID(request.NamespaceId), + ) } else { opDeadline = time.Now().Add(opTimeout) } From b6dd0ef35b52c81fa7ffd2e191ad5b3763dc9a4a Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Thu, 27 Aug 2026 15:41:10 -0700 Subject: [PATCH 086/139] Capture panic in ServiceErrorInterceptor (#11813) ## What changed? Capture panic in `ServiceErrorInterceptor` ## Why? `ServiceErrorInterceptor` is a top-level interceptor for frontend, history and matching. Capturing panics to make sure unhandled panics won't crash the service. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../interceptor/mask_internal_error_test.go | 46 +++---- .../interceptor/service_error_interceptor.go | 21 +++- .../service_error_interceptor_test.go | 117 +++++++++++++++++- service/frontend/fx.go | 4 + service/frontend/fx_test.go | 24 ++-- service/history/fx.go | 4 + service/matching/fx.go | 4 + 7 files changed, 187 insertions(+), 33 deletions(-) diff --git a/common/rpc/interceptor/mask_internal_error_test.go b/common/rpc/interceptor/mask_internal_error_test.go index 6f0bfd47677..6a7c9d00736 100644 --- a/common/rpc/interceptor/mask_internal_error_test.go +++ b/common/rpc/interceptor/mask_internal_error_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common" @@ -17,18 +17,20 @@ import ( ) func TestMaskUnknownOrInternalErrors(t *testing.T) { - statusOk := status.New(codes.OK, "OK") testMaskUnknownOrInternalErrors(t, statusOk, false) - statusUnknown := status.New(codes.Unknown, "Unknown") + statusCanceled := status.New(codes.Canceled, "Canceled message") + testMaskUnknownOrInternalErrors(t, statusCanceled, false) + + statusUnknown := status.New(codes.Unknown, "Unknown message") testMaskUnknownOrInternalErrors(t, statusUnknown, true) - statusInternal := status.New(codes.Internal, "Internal") + statusInternal := status.New(codes.Internal, "Internal message") testMaskUnknownOrInternalErrors(t, statusInternal, true) } -func testMaskUnknownOrInternalErrors(t *testing.T, st *status.Status, expectRelpace bool) { +func testMaskUnknownOrInternalErrors(t *testing.T, st *status.Status, expectReplace bool) { controller := gomock.NewController(t) mockRegistry := namespace.NewMockRegistry(controller) mockLogger := log.NewMockLogger(controller) @@ -36,27 +38,27 @@ func testMaskUnknownOrInternalErrors(t *testing.T, st *status.Status, expectRelp errorMaskInterceptor := NewMaskInternalErrorDetailsInterceptor( dynamicconfig.FrontendMaskInternalErrorDetails.Get(dc), mockRegistry, mockLogger) - err := serviceerror.FromStatus(st) - if expectRelpace { + err := st.Err() + if expectReplace { mockLogger.EXPECT().Error(gomock.Any(), gomock.Any()).Times(1) } - errorMessage := errorMaskInterceptor.maskUnknownOrInternalErrors(nil, "test", err) - if expectRelpace { + gotError := errorMaskInterceptor.maskUnknownOrInternalErrors(nil, "test", err) + if expectReplace { errorHash := common.ErrorHash(err) - expectedMessage := fmt.Sprintf("rpc error: code = %s desc = %s (%s)", st.Message(), errorFrontendMasked, errorHash) - - assert.Equal(t, expectedMessage, errorMessage.Error()) + expectedMessage := fmt.Sprintf( + "rpc error: code = %s desc = %s (%s)", + st.Code(), + errorFrontendMasked, + errorHash, + ) + + require.Equal(t, expectedMessage, gotError.Error()) } else { - if err == nil { - assert.Equal(t, errorMessage, nil) - } else { - assert.Equal(t, errorMessage.Error(), st.Message()) - } + require.Equal(t, err, gotError) } } func TestMaskInternalErrorDetailsInterceptor(t *testing.T) { - controller := gomock.NewController(t) mockRegistry := namespace.NewMockRegistry(controller) dc := dynamicconfig.NewNoopCollection() @@ -68,18 +70,18 @@ func TestMaskInternalErrorDetailsInterceptor(t *testing.T) { test_namespace := "test-namespace" req := &workflowservice.StartWorkflowExecutionRequest{Namespace: test_namespace} mockRegistry.EXPECT().GetNamespace(namespace.Name(test_namespace)).Return(&namespace.Namespace{}, nil).AnyTimes() - assert.True(t, errorMask.shouldMaskErrors(req)) + require.True(t, errorMask.shouldMaskErrors(req)) namespace_not_found := "namespace-not-found" req = &workflowservice.StartWorkflowExecutionRequest{Namespace: namespace_not_found} mockRegistry.EXPECT().GetNamespace(namespace.Name(namespace_not_found)).Return(nil, serviceerror.NewNamespaceNotFound("missing-namespace")) - assert.False(t, errorMask.shouldMaskErrors(req)) + require.False(t, errorMask.shouldMaskErrors(req)) empty_namespace := "" req = &workflowservice.StartWorkflowExecutionRequest{Namespace: empty_namespace} mockRegistry.EXPECT().GetNamespace(namespace.Name(empty_namespace)).Return(nil, serviceerror.NewNamespaceNotFound("missing-namespace")) - assert.False(t, errorMask.shouldMaskErrors(req)) + require.False(t, errorMask.shouldMaskErrors(req)) var ei any - assert.False(t, errorMask.shouldMaskErrors(ei)) + require.False(t, errorMask.shouldMaskErrors(ei)) } diff --git a/common/rpc/interceptor/service_error_interceptor.go b/common/rpc/interceptor/service_error_interceptor.go index 8a6f014d73e..67abfc827fd 100644 --- a/common/rpc/interceptor/service_error_interceptor.go +++ b/common/rpc/interceptor/service_error_interceptor.go @@ -6,6 +6,8 @@ import ( "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/util" "google.golang.org/grpc" @@ -16,13 +18,21 @@ const truncatedSuffix = "... " type ServiceErrorInterceptor struct { maxMessageLength dynamicconfig.IntPropertyFn + + metricsHandler metrics.Handler + logger log.Logger } func NewServiceErrorInterceptor( maxMessageLength dynamicconfig.IntPropertyFn, + metricsHandler metrics.Handler, + logger log.Logger, ) *ServiceErrorInterceptor { return &ServiceErrorInterceptor{ maxMessageLength: maxMessageLength, + + metricsHandler: metricsHandler, + logger: logger, } } @@ -32,7 +42,7 @@ func (i *ServiceErrorInterceptor) Intercept( _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - resp, err := handler(ctx, req) + resp, err := i.capturePanicHandler(ctx, req, handler) var deserializationError *serialization.DeserializationError var serializationError *serialization.SerializationError @@ -52,3 +62,12 @@ func (i *ServiceErrorInterceptor) Intercept( return resp, st.Err() } + +func (i *ServiceErrorInterceptor) capturePanicHandler( + ctx context.Context, + req any, + handler grpc.UnaryHandler, +) (_ any, retError error) { + defer metrics.CapturePanic(i.logger, i.metricsHandler, &retError) + return handler(ctx, req) +} diff --git a/common/rpc/interceptor/service_error_interceptor_test.go b/common/rpc/interceptor/service_error_interceptor_test.go index 4b86d04d237..1ac62527b40 100644 --- a/common/rpc/interceptor/service_error_interceptor_test.go +++ b/common/rpc/interceptor/service_error_interceptor_test.go @@ -2,6 +2,8 @@ package interceptor import ( "context" + "errors" + "fmt" "strings" "testing" @@ -9,7 +11,11 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/serialization" + "go.uber.org/mock/gomock" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -31,7 +37,12 @@ func (e *ErrorWithoutStatus) Error() string { // Error returns string message. func TestServiceErrorInterceptorUnknown(t *testing.T) { - interceptor := NewServiceErrorInterceptor(dynamicconfig.GetIntPropertyFn(testMaxMessageLength)) + ctrl := gomock.NewController(t) + interceptor := NewServiceErrorInterceptor( + dynamicconfig.GetIntPropertyFn(testMaxMessageLength), + metrics.NewMockHandler(ctrl), + log.NewTestLogger(), + ) _, err := interceptor.Intercept(t.Context(), nil, nil, func(ctx context.Context, req any) (any, error) { @@ -54,7 +65,12 @@ func TestServiceErrorInterceptorUnknown(t *testing.T) { } func TestServiceErrorInterceptorSer(t *testing.T) { - interceptor := NewServiceErrorInterceptor(dynamicconfig.GetIntPropertyFn(testMaxMessageLength)) + ctrl := gomock.NewController(t) + interceptor := NewServiceErrorInterceptor( + dynamicconfig.GetIntPropertyFn(testMaxMessageLength), + metrics.NewMockHandler(ctrl), + log.NewTestLogger(), + ) serErrors := []error{ serialization.NewDeserializationError(enumspb.ENCODING_TYPE_PROTO3, nil), serialization.NewSerializationError(enumspb.ENCODING_TYPE_PROTO3, nil), @@ -69,7 +85,12 @@ func TestServiceErrorInterceptorSer(t *testing.T) { } func TestServiceErrorInterceptorTruncation(t *testing.T) { - interceptor := NewServiceErrorInterceptor(dynamicconfig.GetIntPropertyFn(testMaxMessageLength)) + ctrl := gomock.NewController(t) + interceptor := NewServiceErrorInterceptor( + dynamicconfig.GetIntPropertyFn(testMaxMessageLength), + metrics.NewMockHandler(ctrl), + log.NewTestLogger(), + ) t.Run("nil error is not affected", func(t *testing.T) { _, err := interceptor.Intercept(t.Context(), nil, nil, @@ -146,3 +167,93 @@ func TestServiceErrorInterceptorTruncation(t *testing.T) { } }) } + +func TestServiceErrorInterceptorPanic(t *testing.T) { + testCases := []struct { + name string + panicObj any + errMessage string + }{ + { + name: "panic with error is converted to internal error", + panicObj: errors.New("panic error message"), + errMessage: "panic error message", + }, + { + name: "panic with non-error value is converted to internal error", + panicObj: "something went wrong", + errMessage: "panic: something went wrong", + }, + { + name: "panic with service error is still converted to internal error", + panicObj: serviceerror.NewNotFound("not found message"), + errMessage: "not found message", + }, + { + name: "captured panic message is truncated", + panicObj: errors.New(strings.Repeat("a", testMaxMessageLength+100)), + errMessage: strings.Repeat("a", testMaxMessageLength-len(truncatedSuffix)) + truncatedSuffix, + }, + { + name: "no panic does not log", + panicObj: nil, + errMessage: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + metricsHandlerMock := metrics.NewMockHandler(ctrl) + loggerMock := log.NewMockLogger(ctrl) + interceptor := NewServiceErrorInterceptor( + dynamicconfig.GetIntPropertyFn(testMaxMessageLength), + metricsHandlerMock, + loggerMock, + ) + + var loggedTags []tag.Tag + if tc.panicObj != nil { + counterMock := metrics.NewMockCounterIface(ctrl) + counterMock.EXPECT().Record(int64(1)) + metricsHandlerMock.EXPECT().Counter(metrics.ServicePanic.Name()).Return(counterMock) + loggerMock.EXPECT().Error("Panic is captured", gomock.Any(), gomock.Any()). + Do(func(_ string, tags ...tag.Tag) { + loggedTags = tags + }). + Times(1) + } + + resp, err := interceptor.Intercept(t.Context(), nil, nil, + func(_ context.Context, _ any) (any, error) { + if tc.panicObj != nil { + panic(tc.panicObj) + } + return "ok", nil + }) + + if tc.panicObj != nil { + expectedMessage := fmt.Sprintf( + "rpc error: code = Internal desc = %s", + tc.errMessage, + ) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.Internal, status.Code(err)) + require.Equal(t, expectedMessage, err.Error()) + + // Logs contains the stack trace + tagsByKey := make(map[string]tag.Tag, len(loggedTags)) + for _, tg := range loggedTags { + tagsByKey[tg.Key()] = tg + } + require.Contains(t, tagsByKey, "sys-stack-trace") + require.Contains(t, tagsByKey["sys-stack-trace"].Value(), "service_error_interceptor_test.go") + require.Contains(t, tagsByKey, "error") + } else { + require.Equal(t, "ok", resp) + require.NoError(t, err) + } + }) + } +} diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 15b12f1575a..651551de210 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -362,9 +362,13 @@ func ConfigProvider( func ServiceErrorInterceptorProvider( dc *dynamicconfig.Collection, + metricsHandler metrics.Handler, + logger log.Logger, ) *interceptor.ServiceErrorInterceptor { return interceptor.NewServiceErrorInterceptor( dynamicconfig.MaxServiceErrorMessageLength.Get(dc), + metricsHandler, + logger, ) } diff --git a/service/frontend/fx_test.go b/service/frontend/fx_test.go index 0100ceb792d..39a7ea121d3 100644 --- a/service/frontend/fx_test.go +++ b/service/frontend/fx_test.go @@ -33,6 +33,8 @@ type rateLimitInterceptorTestCase struct { name string // t is the test object t *testing.T + // ctrl is the mock controller + ctrl *gomock.Controller // globalRPSLimit is the global RPS limit for all frontend hosts globalRPSLimit int // perInstanceRPSLimit is the RPS limit for each frontend host @@ -183,7 +185,7 @@ func TestRateLimitInterceptorProvider(t *testing.T) { tc.perInstanceRPSLimit = highPerInstanceRPSLimit tc.operatorRPSRatio = operatorRPSRatio tc.expectRateLimit = false - serviceResolver := membership.NewMockServiceResolver(gomock.NewController(tc.t)) + serviceResolver := membership.NewMockServiceResolver(tc.ctrl) serviceResolver.EXPECT().AvailableMemberCount().Return(0).AnyTimes() tc.serviceResolver = serviceResolver }, @@ -196,11 +198,11 @@ func TestRateLimitInterceptorProvider(t *testing.T) { tc.numRequests = 10 tc.t = t + tc.ctrl = gomock.NewController(t) { // Create a mock service resolver which returns the number of frontend hosts. // This may be overridden by the test case. - ctrl := gomock.NewController(t) - serviceResolver := membership.NewMockServiceResolver(ctrl) + serviceResolver := membership.NewMockServiceResolver(tc.ctrl) serviceResolver.EXPECT().AvailableMemberCount().Return(numHosts).AnyTimes() tc.serviceResolver = serviceResolver } @@ -208,6 +210,8 @@ func TestRateLimitInterceptorProvider(t *testing.T) { serviceErrorInterceptor := interceptor.NewServiceErrorInterceptor( dynamicconfig.GetIntPropertyFn(4000), + metrics.NewMockHandler(tc.ctrl), + log.NewTestLogger(), ) // Create a rate limit interceptor which uses the per-instance and global RPS limits from the test case. @@ -565,17 +569,20 @@ func TestNamespaceRateLimitInterceptorProvider(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Parallel() + ctrl := gomock.NewController(t) namespaceName := "test-namespace" - mockRegistry := namespace.NewMockRegistry(gomock.NewController(t)) + mockRegistry := namespace.NewMockRegistry(ctrl) mockRegistry.EXPECT().GetNamespace(namespace.Name(namespaceName)).Return(&namespace.Namespace{}, nil).AnyTimes() - serviceResolver := membership.NewMockServiceResolver(gomock.NewController(t)) + serviceResolver := membership.NewMockServiceResolver(ctrl) serviceResolver.EXPECT().AvailableMemberCount().Return(tc.frontendServiceCount).AnyTimes() config := getTestConfig(tc) serviceErrorInterceptor := interceptor.NewServiceErrorInterceptor( dynamicconfig.GetIntPropertyFn(4000), + metrics.NewMockHandler(ctrl), + log.NewTestLogger(), ) // Create a rate limit interceptor. @@ -752,11 +759,12 @@ func TestNamespaceRateLimitMetrics(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { t.Parallel() + ctrl := gomock.NewController(t) testNS := "test_namespace" - mockRegistry := namespace.NewMockRegistry(gomock.NewController(t)) + mockRegistry := namespace.NewMockRegistry(ctrl) mockRegistry.EXPECT().GetNamespace(namespace.Name(testNS)).Return(&namespace.Namespace{}, nil).AnyTimes() - serviceResolver := membership.NewMockServiceResolver(gomock.NewController(t)) + serviceResolver := membership.NewMockServiceResolver(ctrl) serviceResolver.EXPECT().AvailableMemberCount().Return(tc.frontendServiceCount).AnyTimes() metricsHandler := metricstest.NewCaptureHandler() capture := metricsHandler.StartCapture() @@ -779,6 +787,8 @@ func TestNamespaceRateLimitMetrics(t *testing.T) { serviceErrorInterceptor := interceptor.NewServiceErrorInterceptor( dynamicconfig.GetIntPropertyFn(4000), + metrics.NewMockHandler(ctrl), + log.NewTestLogger(), ) // Create a rate limit interceptor which uses the per-instance and global RPS limits from the test case. diff --git a/service/history/fx.go b/service/history/fx.go index 54d0432c51a..dcf233185fe 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -249,9 +249,13 @@ func ConfigProvider( func ServiceErrorInterceptorProvider( dc *dynamicconfig.Collection, + metricsHandler metrics.Handler, + logger log.Logger, ) *interceptor.ServiceErrorInterceptor { return interceptor.NewServiceErrorInterceptor( dynamicconfig.MaxServiceErrorMessageLength.Get(dc), + metricsHandler, + logger, ) } diff --git a/service/matching/fx.go b/service/matching/fx.go index 62efeebfd60..c3d1a791cf4 100644 --- a/service/matching/fx.go +++ b/service/matching/fx.go @@ -71,9 +71,13 @@ func ConfigProvider( func ServiceErrorInterceptorProvider( dc *dynamicconfig.Collection, + metricsHandler metrics.Handler, + logger log.Logger, ) *interceptor.ServiceErrorInterceptor { return interceptor.NewServiceErrorInterceptor( dynamicconfig.MaxServiceErrorMessageLength.Get(dc), + metricsHandler, + logger, ) } From 93a81a9724d4498f06c1ad8b0369051dac5694bf Mon Sep 17 00:00:00 2001 From: Alex Stanfield <13949480+chaptersix@users.noreply.github.com> Date: Thu, 27 Aug 2026 18:45:53 -0500 Subject: [PATCH 087/139] Re-enable BUFFER_ONE backfill test; drop unsupported memo-only test (#11818) ## Summary - Removed the skip on `testBackfillWithBufferOneOverlap` - verified passing on both V1 and CHASM: the deferred backfill start under `BUFFER_ONE` now correctly re-enables once the running workflow completes. - Removed `testUpdateScheduleMemoOnly` and its registration - `UpdateSchedule` has no partial-update path for the `schedule` field; every update (including memo/search-attribute-only ones) requires resending the full schedule, so the test's premise no longer applies. Other previously-skipped scheduler tests (`TestFailedStart`, `TestPauseUnpauseBetweenNominalAndJittered`) were checked and still reproduce their underlying issues, so they remain skipped. ## Test plan - [x] `go test -tags=test_dep ./tests/ -run 'TestScheduleCHASM/Backfill|TestScheduleV1/Backfill' -count=1 -v` - [x] `go build -tags=test_dep ./tests/...` --- tests/schedule_test.go | 87 +----------------------------------------- 1 file changed, 1 insertion(+), 86 deletions(-) diff --git a/tests/schedule_test.go b/tests/schedule_test.go index 2e4db2068d4..06f68e20b25 100644 --- a/tests/schedule_test.go +++ b/tests/schedule_test.go @@ -417,7 +417,6 @@ func TestScheduleCHASM(t *testing.T) { t.Run("TestSkipsWorkflowSentinelWhenDisabled", func(t *testing.T) { t.Parallel(); testSkipsWorkflowSentinelWhenDisabled(t, newContext) }) t.Run("TestLargeScheduleID", func(t *testing.T) { t.Parallel(); testLargeScheduleID(t, newContext) }) t.Run("TestUpdateScheduleMemo", func(t *testing.T) { t.Parallel(); testUpdateScheduleMemo(t, newContext) }) - t.Run("TestUpdateScheduleMemoOnly", func(t *testing.T) { t.Parallel(); testUpdateScheduleMemoOnly(t, newContext) }) t.Run("TestStateSizeBytesReported", func(t *testing.T) { t.Parallel(); testStateSizeBytesReported(t, newContext) }) t.Run("TestBufferOverrunDropsActions", func(t *testing.T) { t.Parallel(); testBufferOverrunDropsActions(t, newContext) }) t.Run("TestDescribeCatchupWindowAfterCreateAndUpdate", func(t *testing.T) { @@ -4131,80 +4130,6 @@ func testUpdateScheduleMemoRejected(t *testing.T, newContext contextFactory) { require.Contains(t, err.Error(), "memo updates are not supported on workflow-backed schedules") } -func testUpdateScheduleMemoOnly(t *testing.T, newContext contextFactory) { - // UpdateScheduleRequest uses replace semantics for the schedule field, so omitting it - // causes the schedule to be unset. Memo-only updates require the server to skip replacing - // the schedule when the field is nil, similar to how memo and search_attributes are handled. - t.Skip("memo-only updates not yet supported: omitting the schedule field unsets the schedule") - - s := newScheduleEnv(t, scheduleCommonOpts(t)...) - - sid := "sched-test-update-memo-only" - wid := "sched-test-update-memo-only-wf" - wt := "sched-test-update-memo-only-wt" - - s.SdkWorker().RegisterWorkflowWithOptions( - func(ctx workflow.Context) error { return nil }, - workflow.RegisterOptions{Name: wt}, - ) - - schedule := &schedulepb.Schedule{ - Spec: &schedulepb.ScheduleSpec{ - Interval: []*schedulepb.IntervalSpec{ - {Interval: durationpb.New(1 * time.Hour)}, - }, - }, - Action: &schedulepb.ScheduleAction{ - Action: &schedulepb.ScheduleAction_StartWorkflow{ - StartWorkflow: &workflowpb.NewWorkflowExecutionInfo{ - WorkflowId: wid, - WorkflowType: &commonpb.WorkflowType{Name: wt}, - TaskQueue: &taskqueuepb.TaskQueue{Name: s.WorkerTaskQueue(), Kind: enumspb.TASK_QUEUE_KIND_NORMAL}, - }, - }, - }, - } - - // Create schedule with initial memo. - memo1 := payload.EncodeString("val1") - ctx := newContext(s.Context()) - _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ - Namespace: s.Namespace().String(), - ScheduleId: sid, - Schedule: schedule, - Identity: "test", - RequestId: uuid.NewString(), - Memo: &commonpb.Memo{ - Fields: map[string]*commonpb.Payload{"key1": memo1}, - }, - }) - require.NoError(t, err) - - // Update only memo, without setting the schedule field. - memo2 := payload.EncodeString("val2") - _, err = s.FrontendClient().UpdateSchedule(newContext(s.Context()), &workflowservice.UpdateScheduleRequest{ - Namespace: s.Namespace().String(), - ScheduleId: sid, - Identity: "test", - RequestId: uuid.NewString(), - Memo: &commonpb.Memo{ - Fields: map[string]*commonpb.Payload{"key1": memo2}, - }, - }) - require.NoError(t, err) - - // Verify memo was updated and schedule is still intact. - describeResp, err := s.FrontendClient().DescribeSchedule(newContext(s.Context()), &workflowservice.DescribeScheduleRequest{ - Namespace: s.Namespace().String(), - ScheduleId: sid, - }) - require.NoError(t, err) - require.Equal(t, memo2.Data, describeResp.Memo.Fields["key1"].Data, "memo should be updated") - require.NotNil(t, describeResp.Schedule.Spec, "schedule spec should not be nil") - require.NotEmpty(t, describeResp.Schedule.Spec.Interval, "schedule spec intervals should be preserved") - require.NotNil(t, describeResp.Schedule.Action, "schedule action should be preserved") -} - func testCHASMUnpauseResumesProcessing(t *testing.T, newContext contextFactory) { s := newScheduleEnv(t, scheduleCommonOpts(t)...) @@ -4637,18 +4562,8 @@ func testBackfillReprocessesCompletedAction( // testBackfillWithBufferOneOverlap pins the expected behavior of BUFFER_ONE // over a multi-tick backfill: the first start runs immediately, exactly one // follow-up is buffered (Attempt=-1 deferred), the rest are dropped, and the -// deferred one runs once the first completes. Currently SKIPPED: fails on -// both V1 and CHASM because the deferred start never gets re-enabled after -// the running workflow completes. The first start fires, the rest never run. -// Likely a real bug in the BUFFER_ONE + backfill (Manual=true) interaction - -// recordCompletedAction's re-enable loop on Attempt==-1 may not be running -// against backfill-buffered starts. Worth a separate investigation. +// deferred one runs once the first completes. func testBackfillWithBufferOneOverlap(t *testing.T, newContext contextFactory) { - // TODO(temporalio/temporal): track removing this skip once the BUFFER_ONE - // backfill deferred re-enable path is fixed. Verify by running: - // go test ./tests/ -run 'TestScheduleCHASM/Backfill/BufferOneOverlap' -v - t.Skip("BUFFER_ONE backfill deferred re-enable is broken on both V1 and CHASM; see test doc") - s := newScheduleEnv(t, scheduleCommonOpts(t)...) sid := testcore.RandomizeStr("sched-backfill-buffer-one") From b0e1e4dbdf924573ff16e643d87a5f56393f75dd Mon Sep 17 00:00:00 2001 From: Gokhan Tekkaya <86028633+tekkaya@users.noreply.github.com> Date: Thu, 27 Aug 2026 16:49:10 -0700 Subject: [PATCH 088/139] Fix Nexus start-operation retry classification for wrapped service errors (#11820) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? `handleStartOperationError` (`components/nexusoperations/executors.go`), `callErrorToFailure`, and `newInvocationResult` (`chasm/lib/nexusoperation/task_handler_helpers.go`) all unwrap `callErr` into a typed `serviceerror.ServiceError` via `errors.As`/`errors.AsType`, then pass the *original wrapped* `callErr` — not the unwrapped `serviceErr` — to `common.IsRetryableRPCError`. Fix: pass the already-unwrapped `serviceErr` to `IsRetryableRPCError` at all three call sites — in both the workflow-based (`components/nexusoperations`) and CHASM standalone (`chasm/lib/nexusoperation`) Nexus operation state machines. These were the only call sites of `IsRetryableRPCError` outside its own definition/tests. ## Why? `IsRetryableRPCError` only recognizes a service error via a direct (non-unwrapping) type assertion or a gRPC status; neither sees through wrapping. So whenever the transport wraps the service error (e.g. `net/http.Client.Do` wraps `RoundTripper` errors in `*url.Error`), a transient error like `Unavailable` is always misclassified as non-retryable, permanently failing the Nexus operation instead of retrying it. Observed in practice: a Nexus `cancel` operation dispatched during a brief window where the internal service resolver had zero available frontend members failed with `Unavailable: no frontend host to route request to`. That error was wrapped by the HTTP round-tripper, misclassified as non-retryable, and permanently failed the operation — with ~19 of its 20-minute `scheduleToCloseTimeout` still unused, while sibling operations dispatched moments later succeeded normally. A single retry would have resolved it. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks Low risk: the change only affects the retryability decision for the `serviceerror` branch of Nexus start-operation error handling, and only for errors that were previously misclassified (wrapped service errors). Correctly-classified cases (direct/unwrapped service errors, gRPC status errors) are unaffected since `serviceErr` and `callErr` resolve to the same classification for those. --- .../nexusoperation/task_handler_helpers.go | 4 +- .../task_handler_helpers_test.go | 37 +++++++++++++++++++ common/util.go | 3 ++ components/nexusoperations/executors.go | 2 +- components/nexusoperations/executors_test.go | 31 +++++++++++++++- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 chasm/lib/nexusoperation/task_handler_helpers_test.go diff --git a/chasm/lib/nexusoperation/task_handler_helpers.go b/chasm/lib/nexusoperation/task_handler_helpers.go index 06a3d040193..66c783483df 100644 --- a/chasm/lib/nexusoperation/task_handler_helpers.go +++ b/chasm/lib/nexusoperation/task_handler_helpers.go @@ -141,7 +141,7 @@ func cancelCallOutcomeTag(callCtx context.Context, callErr error) string { // Always returns a non-nil failure. func callErrorToFailure(callErr error) (*failurepb.Failure, bool, error) { if serviceErr, ok := errors.AsType[serviceerror.ServiceError](callErr); ok { - retryable := common.IsRetryableRPCError(callErr) + retryable := common.IsRetryableRPCError(serviceErr) failure := &failurepb.Failure{ Message: fmt.Sprintf("%s: %s", strings.Replace(fmt.Sprintf("%T", serviceErr), "*serviceerror.", "", 1), serviceErr.Error()), FailureInfo: &failurepb.Failure_ServerFailureInfo{ @@ -233,7 +233,7 @@ func newInvocationResult( } if serviceErr, ok := errors.AsType[serviceerror.ServiceError](callErr); ok { - retryable := common.IsRetryableRPCError(callErr) + retryable := common.IsRetryableRPCError(serviceErr) failure := &failurepb.Failure{ Message: fmt.Sprintf("%s: %s", strings.Replace(fmt.Sprintf("%T", serviceErr), "*serviceerror.", "", 1), serviceErr.Error()), FailureInfo: &failurepb.Failure_ServerFailureInfo{ diff --git a/chasm/lib/nexusoperation/task_handler_helpers_test.go b/chasm/lib/nexusoperation/task_handler_helpers_test.go new file mode 100644 index 00000000000..e7a6788d4fa --- /dev/null +++ b/chasm/lib/nexusoperation/task_handler_helpers_test.go @@ -0,0 +1,37 @@ +package nexusoperation + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" +) + +// wrappedUnavailable returns a transient serviceerror reachable only via Unwrap, +// as an HTTP client wraps transport errors. +func wrappedUnavailable() error { + return &url.Error{ + Op: "Post", + URL: "https://internal", + Err: serviceerror.NewUnavailable("no frontend host to route request to"), + } +} + +func TestCallErrorToFailure_RetriesTransientServiceErrorEvenWhenWrapped(t *testing.T) { + t.Parallel() + + failure, retryable, err := callErrorToFailure(wrappedUnavailable()) + require.NoError(t, err) + require.True(t, retryable, "wrapped Unavailable must be classified as retryable") + require.False(t, failure.GetServerFailureInfo().GetNonRetryable()) +} + +func TestNewInvocationResult_RetriesTransientServiceErrorEvenWhenWrapped(t *testing.T) { + t.Parallel() + + result, err := newInvocationResult(nil, wrappedUnavailable()) + require.NoError(t, err) + require.IsType(t, invocationResultRetry{}, result, + "wrapped Unavailable must produce a retry result, not a terminal failure") +} diff --git a/common/util.go b/common/util.go index 750dabac745..87e02bbc63a 100644 --- a/common/util.go +++ b/common/util.go @@ -763,6 +763,9 @@ func getFieldNameFromStruct(structPtr any, fieldPtr any) (string, error) { } // IsRetryableRPCError checks if the error is a retryable gRPC error. +// +// This does not unwrap err: If err may be wrapped (e.g. by an HTTP client, which wraps +// RoundTripper errors in *url.Error), unwrap it yourself and pass the unwrapped error here. func IsRetryableRPCError(err error) bool { var st *status.Status stGetter, ok := err.(interface{ Status() *status.Status }) diff --git a/components/nexusoperations/executors.go b/components/nexusoperations/executors.go index 2713c657476..b7eb7bb2727 100644 --- a/components/nexusoperations/executors.go +++ b/components/nexusoperations/executors.go @@ -535,7 +535,7 @@ func (e taskExecutor) handleStartOperationError(env hsm.Environment, node *hsm.N switch { case errors.As(callErr, &serviceErr): - if !common.IsRetryableRPCError(callErr) { + if !common.IsRetryableRPCError(serviceErr) { return handleNonRetryableStartOperationError(node, operation, callErr) } // Fall through all uncaught errors to retryable diff --git a/components/nexusoperations/executors_test.go b/components/nexusoperations/executors_test.go index b5faee1b0a6..41d5bc55e32 100644 --- a/components/nexusoperations/executors_test.go +++ b/components/nexusoperations/executors_test.go @@ -5,6 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "net/url" "testing" "time" @@ -99,6 +101,9 @@ func TestProcessInvocationTask(t *testing.T) { startToCloseTimeout time.Duration schedToStartTimeout time.Duration destinationDown bool + // httpCallerErr, when set, fails the outbound HTTP call with this error instead of + // performing a real request. + httpCallerErr error }{ { name: "async start", @@ -379,6 +384,22 @@ func TestProcessInvocationTask(t *testing.T) { require.Empty(t, events) }, }, + { + name: "transient service error wrapped by HTTP caller", + requestTimeout: time.Hour, + httpCallerErr: &url.Error{ + Op: "Post", + URL: "http://unavailable", + Err: serviceerror.NewUnavailable("no frontend host to route request to"), + }, + expectedMetricOutcome: "service-error:Unavailable", + checkOutcome: func(t *testing.T, op nexusoperations.Operation, events []*historypb.HistoryEvent) { + require.Equal(t, enumsspb.NEXUS_OPERATION_STATE_BACKING_OFF, op.State()) + require.False(t, op.LastAttemptFailure.GetServerFailureInfo().GetNonRetryable()) + require.Contains(t, op.LastAttemptFailure.Message, "Unavailable") + require.Empty(t, events) + }, + }, { name: "invocation timeout by request timeout", requestTimeout: 2 * time.Millisecond, @@ -655,11 +676,17 @@ func TestProcessInvocationTask(t *testing.T) { Logger: log.NewNoopLogger(), EndpointRegistry: endpointReg, ClientProvider: func(ctx context.Context, namespaceID string, entry *persistencespb.NexusEndpointEntry, service string) (*nexusrpc.HTTPClient, error) { - return nexusrpc.NewHTTPClient(nexusrpc.HTTPClientOptions{ + options := nexusrpc.HTTPClientOptions{ BaseURL: "http://" + listenAddr, Service: service, Serializer: commonnexus.PayloadSerializer, - }) + } + if tc.httpCallerErr != nil { + options.HTTPCaller = func(*http.Request) (*http.Response, error) { + return nil, tc.httpCallerErr + } + } + return nexusrpc.NewHTTPClient(options) }, })) From 987fb31ca81b39a596ac55bf7855b487393f6be0 Mon Sep 17 00:00:00 2001 From: "temporal-cicd[bot]" <138905749+temporal-cicd[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 10:54:48 +0000 Subject: [PATCH 089/139] Update test shard salt (#11845) Automatically generated by the optimize-test-sharding workflow. Co-authored-by: Temporal Data --- tests/testcore/shard_salt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testcore/shard_salt.txt b/tests/testcore/shard_salt.txt index fd110f7d3ff..3fb5c2b5f84 100644 --- a/tests/testcore/shard_salt.txt +++ b/tests/testcore/shard_salt.txt @@ -1 +1 @@ --salt-591 +-salt-2661 From 5ed21eb39b8b46031666c59afc51ea3f87ad8fd0 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Fri, 28 Aug 2026 11:19:35 -0700 Subject: [PATCH 090/139] Preserve pending heartbeat timeout task during replication (#11811) ## Summary - preserve the local heartbeat timer-task status when replicated heartbeat progress only moves the deadline later - continue invalidating the status for a new activity attempt, options stamp, or cross-cluster version - prevent passive task refresh from creating a duplicate heartbeat timeout task while an earlier valid wake-up is already pending ## Why The active path coalesces heartbeat timeout tasks: recording a heartbeat keeps the earlier pending wake-up, which re-evaluates the latest heartbeat deadline when it fires. State-based replication previously cleared that timer status when the heartbeat deadline moved, causing passive task refresh to create an extra timeout task. ## Testing - `go test ./service/history/workflow -run "^TestMutableStateSuite$" -testify.m "^TestNextActivityTimerTaskMask_" -count=1` - verified with replication multiple batches both enabled and disabled --- .../history/workflow/mutable_state_impl.go | 35 +++++++------------ .../workflow/mutable_state_impl_test.go | 15 ++++++++ 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 201299bea8a..069c3e37a24 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -9320,28 +9320,14 @@ func (ms *MutableStateImpl) ShouldResetActivityTimerTaskMask(current, incoming * return false } -// getActivityTimerTaskStatus returns the activity timer task status bits to carry over -// when applying `incoming` on top of `current` during state-based replication. A bit is -// kept only when the timer task already created for it is still valid, so that a -// subsequent task refresh does not regenerate a timeout task that is already pending. +// getActivityTimerTaskStatus carries over bits for timeout tasks that remain valid +// after applying replicated activity state. Timer validity is normally determined by +// whether its deadline changed. // -// Validity is decided purely by the deadline. An ActivityTimeoutTask is a wake-up at a -// point in time: processSingleActivityTimeoutTask re-derives the whole timer sequence -// from current mutable state and fires whatever has expired, without consulting the -// task's attempt or stamp ("we don't need to check activity Stamps"). So a pending task -// whose deadline is unchanged is still correct no matter what else moved, and a task -// whose deadline moved is useless no matter what stayed put. -// -// Attempt and stamp are therefore deliberately not consulted here: they are only ever -// proxies for "some deadline probably moved", and comparing the deadlines answers that -// question directly. A new attempt, for instance, clears start-to-close and heartbeat -// because clearing the started state removes those timers outright, and moves -// schedule-to-start because its anchor advances — while schedule-to-close survives, -// being anchored to the untouched FirstScheduledTime. -// -// The exception is a cross-cluster version change, which is about task provenance -// rather than deadlines: tasks generated under the previous owning cluster are dropped -// as stale at execution, so everything has to be recreated. +// Heartbeat progress only moves its deadline later, so the existing earlier wake-up +// remains valid and will schedule the next task when it fires. Preserve that bit only +// within the same activity attempt and options stamp. Reset all bits across clusters +// because tasks generated by the previous owning cluster are stale. func (ms *MutableStateImpl) getActivityTimerTaskStatus(current, incoming *persistencespb.ActivityInfo) int32 { if current == nil { return TimerTaskStatusNone @@ -9349,8 +9335,11 @@ func (ms *MutableStateImpl) getActivityTimerTaskStatus(current, incoming *persis if !ms.clusterMetadata.IsVersionFromSameCluster(current.Version, incoming.Version) { return TimerTaskStatusNone } - return current.TimerTaskStatus &^ - getActivityTimerDeadlines(current).changedMask(getActivityTimerDeadlines(incoming)) + changedTimerMask := getActivityTimerDeadlines(current).changedMask(getActivityTimerDeadlines(incoming)) + if current.Attempt == incoming.Attempt && current.Stamp == incoming.Stamp { + changedTimerMask &^= TimerTaskStatusCreatedHeartbeat + } + return current.TimerTaskStatus &^ changedTimerMask } func (ms *MutableStateImpl) applyUpdatesToSubStateMachines( diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 0143a8d7083..26ee33d1763 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -6313,6 +6313,21 @@ func (s *mutableStateSuite) TestNextActivityTimerTaskMask_ClearsOnlyMovedDeadlin ) } +func (s *mutableStateSuite) TestNextActivityTimerTaskMask_HeartbeatProgressKeepsPendingWakeup() { + version := int64(99) + current := startedActivityInfoForMask(version, 1, 1) + incoming := startedActivityInfoForMask(version, 1, 1) + incoming.LastHeartbeatUpdateTime = timestamppb.New( + incoming.StartedTime.AsTime().Add(30 * time.Second), + ) + + s.mockShard.Resource.ClusterMetadata.EXPECT().IsVersionFromSameCluster(version, version).Return(true) + + // Recording heartbeat progress moves its timeout deadline, but the active path + // deliberately relies on the already-pending earlier task as the next wake-up. + s.Equal(current.TimerTaskStatus, s.mutableState.getActivityTimerTaskStatus(current, incoming)) +} + func (s *mutableStateSuite) TestNextActivityTimerTaskMask_UnrelatedOptionChanged_KeepsMask() { version := int64(99) current := startedActivityInfoForMask(version, 1, 1) From 054203952f13c174c294538e0a9b1e90d4c4bb19 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Fri, 28 Aug 2026 14:01:19 -0700 Subject: [PATCH 091/139] Handle zombie and orphan workflows on replication create path (#11052) ## What changed? On the passive replication-apply path, when `dispatchForNewWorkflow` finds no current execution record (`currentRunID == ""`), it now preserves the incoming execution state and persists the run via `CreateWorkflowModeBypassCurrent` in either of these cases: - the incoming execution is already `ZOMBIE`; or - the incoming execution is `COMPLETED` or `CORRUPTED` and already records a successor through `NewExecutionRunId` or `SuccessorRunId`. The new `nDCTransactionPolicyCreateBypassCurrent` policy makes this behavior explicit: it does not call `SuppressBy`, does not transition the workflow state, and does not create or update the current execution record. `nDCTransactionPolicyCreateAsZombie` remains reserved for paths that actually suppress an execution. A genuinely brand-new workflow, or a completed/corrupted run with no successor (the latest run), still becomes current as before.
Case 1: incoming snapshot is already Zombie ```text SOURCE / RETIRED CELL | REPLICATION TARGET ------------------------------------|-------------------------------------- | R1 [Zombie, non-current] | R1 absent | cur -> none | FORCE-REPLICATE R1 SNAPSHOT | state = Zombie ==R1==> | apply R1 (create path) | currentRunID == "" | | | v | BEFORE: CreateAsCurrent (BrandNew) | -> rejected by persistence: | "Invalid workflow create mode 0, | state: Zombie" | | AFTER: CreateBypassCurrent | -> R1 remains Zombie | -> cur -> none ``` The snapshot is already Zombie before it reaches the transaction manager. Promoting it to current would violate the Zombie/current invariant; converting it is also unnecessary. The correct operation is to preserve it as a non-current execution.
Case 2: closed orphan still points to a deleted successor ```text SOURCE | REPLICATION TARGET ------------------------------------|-------------------------------------- | [1] R1 continue-as-new -> R2 | R1 [completed, ->R2] | R1 absent R2 running | R2 absent cur -> R2 | cur -> none | [2] DELETE R2 (current run) | R1 [completed, orphan, ->R2] | R2 [deleted] | cur -> none | cur -> none | [3] FORCE-REPLICATE R1 | R1 [completed, ->R2] ==R1==> | apply R1 (create path) | run absent, cur -> none | | | v | BEFORE: CreateAsCurrent (BrandNew) | -> cur -> R1 | [deleted lineage resurrected] | | AFTER: CreateBypassCurrent | -> R1 remains Completed | -> cur -> none | [no resurrection] ``` Because R1 records R2 as its successor, R1 cannot be the lineage head even though the target no longer has a current record. Bypass-current preserves R1's history and state without promoting it.
## Why? Force replication can reach the create path with a missing current record in more than one form: 1. A migration/replication snapshot may already be `ZOMBIE`. `CreateWorkflowModeBrandNew` rejects that state because a Zombie must never own the current execution record. 2. A reset, continue-as-new, retry, or cron transition followed by deletion of the successor can leave a completed/corrupted orphan that still records its successor. Promoting that older run with `CreateWorkflowModeBrandNew` resurrects a workflow whose lineage has already moved on. In both cases, the incoming run is known to be non-current. Persisting it without changing its state and without writing a current record preserves the replicated data while maintaining the current-execution invariants. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) Added unit coverage verifies that: - a Zombie without successor metadata is persisted with `CreateWorkflowModeBypassCurrent` and remains Zombie; - Completed executions with `NewExecutionRunId` or `SuccessorRunId` remain Completed; - a Corrupted execution with a successor remains Corrupted; and - the preserve-state path does not call suppression or update the current execution record. Validated with: ```text go test ./service/history/ndc -run TestTransactionMgrForNewWorkflowSuite -count=1 ``` --------- Co-authored-by: Claude Opus 4.8 --- service/history/ndc/transaction_manager.go | 2 + .../ndc/transaction_manager_new_workflow.go | 57 +++- .../transaction_manager_new_workflow_test.go | 294 +++++++++++++++++- 3 files changed, 332 insertions(+), 21 deletions(-) diff --git a/service/history/ndc/transaction_manager.go b/service/history/ndc/transaction_manager.go index 20f434128f8..ad35b689238 100644 --- a/service/history/ndc/transaction_manager.go +++ b/service/history/ndc/transaction_manager.go @@ -48,6 +48,7 @@ import ( // create path (there will be only current branch) // 1. create as current -> nDCTransactionPolicyCreateAsCurrent // 2. create as zombie -> nDCTransactionPolicyCreateAsZombie +// 3. create without changing state or current -> nDCTransactionPolicyCreateBypassCurrent // // create path (there will be only current branch) + suppress current // 1. create as current & suppress current -> nDCTransactionPolicySuppressCurrentAndCreateAsCurrent @@ -74,6 +75,7 @@ type nDCTransactionPolicy int const ( nDCTransactionPolicyCreateAsCurrent nDCTransactionPolicy = iota nDCTransactionPolicyCreateAsZombie + nDCTransactionPolicyCreateBypassCurrent nDCTransactionPolicySuppressCurrentAndCreateAsCurrent nDCTransactionPolicyUpdateAsCurrent diff --git a/service/history/ndc/transaction_manager_new_workflow.go b/service/history/ndc/transaction_manager_new_workflow.go index e24b48c69c5..56731f293d9 100644 --- a/service/history/ndc/transaction_manager_new_workflow.go +++ b/service/history/ndc/transaction_manager_new_workflow.go @@ -6,6 +6,7 @@ import ( "context" "go.temporal.io/api/serviceerror" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" @@ -57,8 +58,9 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) dispatchForNewWorkflow( // NOTE: this function does NOT mutate current workflow or target workflow, // workflow mutation is done in methods within executeTransaction function - targetExecutionInfo := targetWorkflow.GetMutableState().GetExecutionInfo() - targetExecutionState := targetWorkflow.GetMutableState().GetExecutionState() + targetMutableState := targetWorkflow.GetMutableState() + targetExecutionInfo := targetMutableState.GetExecutionInfo() + targetExecutionState := targetMutableState.GetExecutionState() namespaceID := namespace.ID(targetExecutionInfo.NamespaceId) workflowID := targetExecutionInfo.WorkflowId targetRunID := targetExecutionState.RunId @@ -79,7 +81,24 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) dispatchForNewWorkflow( } if currentRunID == "" { - // current record does not exists + // Preserve non-current snapshots without promoting them to current. A zombie must never own the + // current execution record. A completed or corrupted run with a known successor must also remain + // non-current; otherwise a deleted successor could cause the older run to be resurrected. + targetState := targetExecutionState.GetState() + targetHasSuccessor := targetExecutionInfo.GetNewExecutionRunId() != "" || + targetExecutionInfo.GetSuccessorRunId() != "" + switch { + case targetState == enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + targetState == enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED && targetHasSuccessor, + targetState == enumsspb.WORKFLOW_EXECUTION_STATE_CORRUPTED && targetHasSuccessor: + return r.executeTransaction( + ctx, + nDCTransactionPolicyCreateBypassCurrent, + nil, // no current workflow: the current record was deleted + targetWorkflow, + ) + } + // current record does not exist, create as brand new return r.executeTransaction( ctx, nDCTransactionPolicyCreateAsCurrent, @@ -195,9 +214,7 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) createAsZombie( targetWorkflow Workflow, ) error { - targetWorkflowPolicy, err := targetWorkflow.SuppressBy( - currentWorkflow, - ) + targetWorkflowPolicy, err := targetWorkflow.SuppressBy(currentWorkflow) if err != nil { return err } @@ -208,7 +225,25 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) createAsZombie( // release lock on current workflow, since current cluster maybe the active cluster // and events maybe reapplied to current workflow currentWorkflow.GetReleaseFn()(nil) - currentWorkflow = nil + + return r.persistBypassCurrent(ctx, targetWorkflow, targetWorkflowPolicy) +} + +// createBypassCurrent persists the incoming execution state as-is without creating or updating the +// current execution record. Unlike createAsZombie, it deliberately performs no suppression and no +// workflow state transition. +func (r *nDCTransactionMgrForNewWorkflowImpl) createBypassCurrent( + ctx context.Context, + targetWorkflow Workflow, +) error { + return r.persistBypassCurrent(ctx, targetWorkflow, historyi.TransactionPolicyPassive) +} + +func (r *nDCTransactionMgrForNewWorkflowImpl) persistBypassCurrent( + ctx context.Context, + targetWorkflow Workflow, + targetWorkflowPolicy historyi.TransactionPolicy, +) error { ms := targetWorkflow.GetMutableState() @@ -247,7 +282,7 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) createAsZombie( } } - // target workflow is in zombie state, no need to update current record. + // The target workflow is non-current, so do not create or update the current record. createMode := persistence.CreateWorkflowModeBypassCurrent prevRunID := "" prevLastWriteVersion := int64(0) @@ -331,6 +366,12 @@ func (r *nDCTransactionMgrForNewWorkflowImpl) executeTransaction( targetWorkflow, ) + case nDCTransactionPolicyCreateBypassCurrent: + return r.createBypassCurrent( + ctx, + targetWorkflow, + ) + case nDCTransactionPolicySuppressCurrentAndCreateAsCurrent: return r.suppressCurrentAndCreateAsCurrent( ctx, diff --git a/service/history/ndc/transaction_manager_new_workflow_test.go b/service/history/ndc/transaction_manager_new_workflow_test.go index dd5c883b456..dbb7558e5fc 100644 --- a/service/history/ndc/transaction_manager_new_workflow_test.go +++ b/service/history/ndc/transaction_manager_new_workflow_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" historypb "go.temporal.io/api/history/v1" + enumsspb "go.temporal.io/server/api/enums/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common" @@ -78,24 +79,276 @@ func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_Dup() { s.ErrorIs(err, consts.ErrDuplicate) } -func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_BrandNew() { +func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_NoCurrentRecord_StateAndSuccessorCombinations() { + successorCases := []struct { + name string + newExecutionRunID string + successorRunID string + }{ + {name: "without successor"}, + {name: "with new execution run ID", newExecutionRunID: "successor"}, + {name: "with successor run ID", successorRunID: "successor"}, + } + + stateCases := []struct { + name string + state enumsspb.WorkflowExecutionState + expectedModes [3]persistence.CreateWorkflowMode + }{ + { + name: "created", + state: enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + expectedModes: [3]persistence.CreateWorkflowMode{ + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBrandNew, + }, + }, + { + name: "running", + state: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + expectedModes: [3]persistence.CreateWorkflowMode{ + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBrandNew, + }, + }, + { + name: "completed", + state: enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + expectedModes: [3]persistence.CreateWorkflowMode{ + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBypassCurrent, + persistence.CreateWorkflowModeBypassCurrent, + }, + }, + { + name: "zombie", + state: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + expectedModes: [3]persistence.CreateWorkflowMode{ + persistence.CreateWorkflowModeBypassCurrent, + persistence.CreateWorkflowModeBypassCurrent, + persistence.CreateWorkflowModeBypassCurrent, + }, + }, + { + name: "corrupted", + state: enumsspb.WORKFLOW_EXECUTION_STATE_CORRUPTED, + expectedModes: [3]persistence.CreateWorkflowMode{ + persistence.CreateWorkflowModeBrandNew, + persistence.CreateWorkflowModeBypassCurrent, + persistence.CreateWorkflowModeBypassCurrent, + }, + }, + } + + for _, stateCase := range stateCases { + for successorIndex, successorCase := range successorCases { + expectedMode := stateCase.expectedModes[successorIndex] + s.Run(stateCase.name+" "+successorCase.name, func() { + ctx := context.Background() + namespaceID := namespace.ID("some random namespace ID") + workflowID := "some random workflow ID" + runID := "some random run ID" + releaseCalled := false + + targetWorkflow := NewMockWorkflow(s.controller) + weContext := historyi.NewMockWorkflowContext(s.controller) + mutableState := historyi.NewMockMutableState(s.controller) + var releaseFn historyi.ReleaseWorkflowContextFunc = func(error) { releaseCalled = true } + targetWorkflow.EXPECT().GetContext().Return(weContext).AnyTimes() + targetWorkflow.EXPECT().GetMutableState().Return(mutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(releaseFn).AnyTimes() + + executionInfo := &persistencespb.WorkflowExecutionInfo{ + NamespaceId: namespaceID.String(), + WorkflowId: workflowID, + NewExecutionRunId: successorCase.newExecutionRunID, + SuccessorRunId: successorCase.successorRunID, + } + executionState := &persistencespb.WorkflowExecutionState{ + RunId: runID, + State: stateCase.state, + } + workflowSnapshot := &persistence.WorkflowSnapshot{ + ExecutionState: executionState, + } + workflowEventsSeq := []*persistence.WorkflowEvents{} + + mutableState.EXPECT().GetExecutionInfo().Return(executionInfo).AnyTimes() + mutableState.EXPECT().GetExecutionState().Return(executionState).AnyTimes() + if expectedMode == persistence.CreateWorkflowModeBypassCurrent { + mutableState.EXPECT().GetReapplyCandidateEvents().Return(nil) + } + mutableState.EXPECT().CloseTransactionAsSnapshot(ctx, historyi.TransactionPolicyPassive).Return( + workflowSnapshot, workflowEventsSeq, nil, + ) + + s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID( + ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID, + ).Return("", nil) + + weContext.EXPECT().CreateWorkflowExecution( + gomock.Any(), + s.mockShard, + gomock.Any(), + "", + int64(0), + mutableState, + workflowSnapshot, + workflowEventsSeq, + historyi.TransactionPolicyPassive, + ).DoAndReturn(func( + _ context.Context, + _ historyi.ShardContext, + createMode persistence.CreateWorkflowMode, + _ string, + _ int64, + _ historyi.MutableState, + workflowSnapshot *persistence.WorkflowSnapshot, + _ []*persistence.WorkflowEvents, + _ historyi.TransactionPolicy, + ) error { + s.Equal(expectedMode, createMode) + s.Equal(stateCase.state, workflowSnapshot.ExecutionState.State) + return persistence.ValidateCreateWorkflowModeState(createMode, *workflowSnapshot) + }) + + err := s.createMgr.dispatchForNewWorkflow(ctx, chasm.WorkflowArchetypeID, targetWorkflow) + s.Require().NoError(err) + s.Equal(stateCase.state, executionState.State) + s.True(releaseCalled) + }) + } + } +} + +func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_NoCurrentRecord_CompletedWithNewExecutionRunID_CreatesBypassCurrentPreservingState() { + s.testDispatchForNewWorkflowNoCurrentRecordPreservesState(&persistencespb.WorkflowExecutionInfo{ + NewExecutionRunId: "successor run ID", + }, enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED) +} + +func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_NoCurrentRecord_CompletedWithSuccessorRunID_CreatesBypassCurrentPreservingState() { + s.testDispatchForNewWorkflowNoCurrentRecordPreservesState(&persistencespb.WorkflowExecutionInfo{ + SuccessorRunId: "successor run ID", + }, enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED) +} + +func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_NoCurrentRecord_CorruptedWithSuccessorRunID_CreatesBypassCurrentPreservingState() { + s.testDispatchForNewWorkflowNoCurrentRecordPreservesState(&persistencespb.WorkflowExecutionInfo{ + SuccessorRunId: "successor run ID", + }, enumsspb.WORKFLOW_EXECUTION_STATE_CORRUPTED) +} + +func (s *transactionMgrForNewWorkflowSuite) testDispatchForNewWorkflowNoCurrentRecordPreservesState( + executionInfo *persistencespb.WorkflowExecutionInfo, + executionStateValue enumsspb.WorkflowExecutionState, +) { ctx := context.Background() namespaceID := namespace.ID("some random namespace ID") workflowID := "some random workflow ID" runID := "some random run ID" + executionInfo.NamespaceId = namespaceID.String() + executionInfo.WorkflowId = workflowID releaseCalled := false - newWorkflow := NewMockWorkflow(s.controller) + targetWorkflow := NewMockWorkflow(s.controller) weContext := historyi.NewMockWorkflowContext(s.controller) mutableState := historyi.NewMockMutableState(s.controller) var releaseFn historyi.ReleaseWorkflowContextFunc = func(error) { releaseCalled = true } - newWorkflow.EXPECT().GetContext().Return(weContext).AnyTimes() - newWorkflow.EXPECT().GetMutableState().Return(mutableState).AnyTimes() - newWorkflow.EXPECT().GetReleaseFn().Return(releaseFn).AnyTimes() + targetWorkflow.EXPECT().GetContext().Return(weContext).AnyTimes() + targetWorkflow.EXPECT().GetMutableState().Return(mutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(releaseFn).AnyTimes() - workflowSnapshot := &persistence.WorkflowSnapshot{} + executionState := &persistencespb.WorkflowExecutionState{ + RunId: runID, + State: executionStateValue, + } + workflowSnapshot := &persistence.WorkflowSnapshot{ + ExecutionState: executionState, + } + // Non-empty event sequence so the reapply branch of createBypassCurrent is exercised (rather than + // short-circuited by empty lists). On the passive apply path reapply is forwarded to the active + // cluster; here the mock stands in for a successful reapply. + workflowEventsSeq := []*persistence.WorkflowEvents{{ + Events: []*historypb.HistoryEvent{{ + EventId: common.FirstEventID + rand.Int63(), + }}, + }} + mutableState.EXPECT().GetExecutionInfo().Return(executionInfo).AnyTimes() + mutableState.EXPECT().GetExecutionState().Return(executionState).AnyTimes() + mutableState.EXPECT().GetReapplyCandidateEvents().Return(nil) + mutableState.EXPECT().CloseTransactionAsSnapshot(context.Background(), historyi.TransactionPolicyPassive).Return( + workflowSnapshot, workflowEventsSeq, nil, + ) + + s.mockTransactionMgr.EXPECT().GetCurrentWorkflowRunID( + ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID, + ).Return("", nil) + + // A non-current run with a successor and no current record must not resurrect as current: even + // with events to reapply, it is persisted via bypass-current without touching the (absent) + // current record. SuppressBy is never called since there is no current workflow to suppress against. + weContext.EXPECT().ReapplyEvents(gomock.Any(), s.mockShard, workflowEventsSeq).Return(nil) + weContext.EXPECT().CreateWorkflowExecution( + gomock.Any(), + s.mockShard, + persistence.CreateWorkflowModeBypassCurrent, + "", + int64(0), + mutableState, + workflowSnapshot, + workflowEventsSeq, + historyi.TransactionPolicyPassive, + ).DoAndReturn(func( + _ context.Context, + _ historyi.ShardContext, + createMode persistence.CreateWorkflowMode, + _ string, + _ int64, + _ historyi.MutableState, + workflowSnapshot *persistence.WorkflowSnapshot, + _ []*persistence.WorkflowEvents, + _ historyi.TransactionPolicy, + ) error { + s.Equal(persistence.CreateWorkflowModeBypassCurrent, createMode) + s.Equal(executionStateValue, workflowSnapshot.ExecutionState.State) + return persistence.ValidateCreateWorkflowModeState(createMode, *workflowSnapshot) + }) + + err := s.createMgr.dispatchForNewWorkflow(ctx, chasm.WorkflowArchetypeID, targetWorkflow) + s.NoError(err) + s.Equal(executionStateValue, executionState.State) + s.True(releaseCalled) +} + +func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_NoCurrentRecord_ZombieWithoutSuccessor_CreatesBypassCurrentPreservingState() { + ctx := context.Background() + + namespaceID := namespace.ID("some random namespace ID") + workflowID := "some random workflow ID" + runID := "some random run ID" + + releaseCalled := false + + targetWorkflow := NewMockWorkflow(s.controller) + weContext := historyi.NewMockWorkflowContext(s.controller) + mutableState := historyi.NewMockMutableState(s.controller) + var releaseFn historyi.ReleaseWorkflowContextFunc = func(error) { releaseCalled = true } + targetWorkflow.EXPECT().GetContext().Return(weContext).AnyTimes() + targetWorkflow.EXPECT().GetMutableState().Return(mutableState).AnyTimes() + targetWorkflow.EXPECT().GetReleaseFn().Return(releaseFn).AnyTimes() + + executionState := &persistencespb.WorkflowExecutionState{ + RunId: runID, + State: enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, + } + workflowSnapshot := &persistence.WorkflowSnapshot{ + ExecutionState: executionState, + } workflowEventsSeq := []*persistence.WorkflowEvents{{ Events: []*historypb.HistoryEvent{{ EventId: common.FirstEventID + rand.Int63(), @@ -105,9 +358,8 @@ func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_BrandNew( NamespaceId: namespaceID.String(), WorkflowId: workflowID, }).AnyTimes() - mutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ - RunId: runID, - }).AnyTimes() + mutableState.EXPECT().GetExecutionState().Return(executionState).AnyTimes() + mutableState.EXPECT().GetReapplyCandidateEvents().Return(nil) mutableState.EXPECT().CloseTransactionAsSnapshot(context.Background(), historyi.TransactionPolicyPassive).Return( workflowSnapshot, workflowEventsSeq, nil, ) @@ -116,20 +368,36 @@ func (s *transactionMgrForNewWorkflowSuite) TestDispatchForNewWorkflow_BrandNew( ctx, namespaceID, workflowID, chasm.WorkflowArchetypeID, ).Return("", nil) + weContext.EXPECT().ReapplyEvents(gomock.Any(), s.mockShard, workflowEventsSeq).Return(nil) weContext.EXPECT().CreateWorkflowExecution( gomock.Any(), s.mockShard, - persistence.CreateWorkflowModeBrandNew, + gomock.Any(), "", int64(0), mutableState, workflowSnapshot, workflowEventsSeq, gomock.Any(), - ).Return(nil) + ).DoAndReturn(func( + _ context.Context, + _ historyi.ShardContext, + createMode persistence.CreateWorkflowMode, + _ string, + _ int64, + _ historyi.MutableState, + workflowSnapshot *persistence.WorkflowSnapshot, + _ []*persistence.WorkflowEvents, + _ historyi.TransactionPolicy, + ) error { + s.Equal(persistence.CreateWorkflowModeBypassCurrent, createMode) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, workflowSnapshot.ExecutionState.State) + return persistence.ValidateCreateWorkflowModeState(createMode, *workflowSnapshot) + }) - err := s.createMgr.dispatchForNewWorkflow(ctx, chasm.WorkflowArchetypeID, newWorkflow) - s.NoError(err) + err := s.createMgr.dispatchForNewWorkflow(ctx, chasm.WorkflowArchetypeID, targetWorkflow) + s.Require().NoError(err) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE, executionState.State) s.True(releaseCalled) } From c044bf16b1cc47a4db80669a987484dba6145331 Mon Sep 17 00:00:00 2001 From: Kannan Date: Fri, 28 Aug 2026 14:39:45 -0700 Subject: [PATCH 092/139] Documentation for worker command task queue kind (#11648) Add a note that the worker command control queue uses `TASK_QUEUE_KIND_WORKER_COMMANDS`. --------- Co-authored-by: Claude Opus 4.6 --- docs/architecture/worker-commands.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/architecture/worker-commands.md b/docs/architecture/worker-commands.md index 443d11b21b7..266beb3284b 100644 --- a/docs/architecture/worker-commands.md +++ b/docs/architecture/worker-commands.md @@ -1,6 +1,8 @@ # Worker Commands -Worker commands are server-initiated instructions sent to workers via Nexus. Each worker process has a dedicated control task queue that it polls via Nexus RPC. A single control queue serves one or more workers running within the same process. This enables the server to push actions to workers without relying on heartbeats or long-poll cycles. +Worker commands are server-initiated commands sent to workers via Nexus. Each worker process has a dedicated task queue that it polls via Nexus RPC. This dedicated queue serves one or more workers running within the same process. This enables the server to push commands to workers without relying on activity heartbeats. For example, if user cancels an activity, server will send a cancel command to the worker that was running the activity. + +The dedicated queue (aka control queue in the code) uses a task queue kind: `TASK_QUEUE_KIND_WORKER_COMMANDS`. This distinguishes it from normal and sticky task queues in matching. The SDK sets this kind when polling for worker commands, and the dispatcher sets it when dispatching commands to matching. The server treats it as an internal task queue kind, meaning it is not directly accessible to users via standard task queue APIs. To route a command, the server needs to know the target worker's control queue. For commands that target activities, this information is stored directly in the mutable state (`ActivityInfo`) when the activity was dispatched to the worker. From 1b7d64e6b923602670bb22d1625bbf86d012784f Mon Sep 17 00:00:00 2001 From: Alex Stanfield <13949480+chaptersix@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:41:08 -0500 Subject: [PATCH 093/139] chore: update GitHub Actions for Node 24 (#11816) ## What changed? Updated applicable GitHub Actions references to versions that support the Node.js 24 runtime: - Upgraded Codecov from v5.5.5 to v6.0.1, removing its transitive Node.js 20 github-script dependency. - Upgraded actions/stale from v9.1.0 to v10.0.0. - Replaced the remaining actions/checkout@v4 references with the repository's pinned checkout v6 reference. ## Why? GitHub Actions reported that the Codecov dependency was forcing actions/github-script to run on deprecated Node.js 20. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing checks - [ ] added new unit test(s) - [ ] added new functional test(s) Validation: actionlint and git diff --check passed. ## Potential risks Codecov v6 and actions/stale v10 require Node.js 24-capable GitHub Actions runners. --- .github/actions/post-test-reporting/action.yml | 4 ++-- .github/workflows/claude-review-teams.yml | 4 ++-- .github/workflows/stale.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/post-test-reporting/action.yml b/.github/actions/post-test-reporting/action.yml index e5a1990bd67..27e78da502f 100644 --- a/.github/actions/post-test-reporting/action.yml +++ b/.github/actions/post-test-reporting/action.yml @@ -64,7 +64,7 @@ runs: - name: Upload code coverage to Codecov if: ${{ !failure() && !cancelled() }} - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ env.CODECOV_TOKEN }} directory: ./.testoutput @@ -72,7 +72,7 @@ runs: - name: Upload test results to Codecov if: ${{ !cancelled() }} - uses: codecov/codecov-action@0fb7174895f61a3b6b78fc075e0cd60383518dac # v5.5.5 + uses: codecov/codecov-action@e79a6962e0d4c0c17b229090214935d2e33f8354 # v6.0.1 with: token: ${{ env.CODECOV_TOKEN }} directory: ./.testoutput diff --git a/.github/workflows/claude-review-teams.yml b/.github/workflows/claude-review-teams.yml index 073e581c313..2b066898d7d 100644 --- a/.github/workflows/claude-review-teams.yml +++ b/.github/workflows/claude-review-teams.yml @@ -226,8 +226,8 @@ jobs: pull-requests: write issues: write steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: ref: ${{ needs.check-review-eligibility.outputs.head_sha }} path: pr-head diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2c6b0e22e37..8f251e763d3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: label stale pull requests - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 + uses: actions/stale@3a9db7e6a41a89f618792c92c0e97cc736e1b13f # v10.0.0 with: days-before-close: -1 # ie disabled days-before-stale: 120 From 935fdf5232955b880d7579597e97575541566c3d Mon Sep 17 00:00:00 2001 From: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Date: Mon, 31 Aug 2026 08:41:28 -0700 Subject: [PATCH 094/139] Fix activity operator command metric label collision (#11806) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? - Changed activity_{pause,unpause,reset,update_options} to emit once per affected activity when the mutation is applied. - Updated both WFA and SAA to use the standard per-activity label schema: namespace, operation, activity type, workflow type, task queue, versioning behavior, and service name. - Added shared construction for the standard per-activity metric scope. - Added functional coverage comparing one WFA request affecting multiple activities with equivalent SAA per-ID requests. - Added unit coverage verifying metrics capture the activity’s updated labels. ## Why? WFA and SAA emitted the same Prometheus metric names with different label sets, causing descriptor registration failures and dropped metrics. Recording these metrics per affected activity makes their semantics and labels consistent with other activity event metrics such as cancel, terminate, success, and failure, while eliminating the Prometheus collision. ## How did you test it? - [x] built - [ ] run locally and tested manually - [X] covered by existing tests - [ ] added new unit test(s) - [X] added new functional test(s) ## Potential risks WFA metrics change from one recording per API request to one per affected activity and no longer include activity_targeting_method. This may increase metric volume for multi-activity requests and require dashboards using the previous label or request-count semantics to be updated. This is intentional to establish one consistent per-activity schema across WFA and SAA. --- chasm/lib/activity/metrics.go | 10 +- common/metrics/activity.go | 29 ++ common/metrics/metric_defs.go | 10 +- common/metrics/tags.go | 6 - service/history/api/pauseactivity/api.go | 25 +- service/history/api/resetactivity/api.go | 24 +- service/history/api/unpauseactivity/api.go | 34 +-- .../history/api/updateactivityoptions/api.go | 63 +++-- .../api/updateactivityoptions/api_test.go | 86 +++++- service/history/workflow/metrics.go | 45 +++ tests/activity_api_pause_test.go | 19 ++ tests/activity_api_reset_test.go | 18 ++ tests/activity_metrics_parity_test.go | 260 +++++++++++++++--- 13 files changed, 487 insertions(+), 142 deletions(-) create mode 100644 common/metrics/activity.go diff --git a/chasm/lib/activity/metrics.go b/chasm/lib/activity/metrics.go index 84ae751a471..6e680062a9a 100644 --- a/chasm/lib/activity/metrics.go +++ b/chasm/lib/activity/metrics.go @@ -41,15 +41,15 @@ func (a *Activity) enrichedMetricsHandler(ctx chasm.Context, operation string) m actCtx := activityContextFromChasm(ctx) breakdownMetricsByTaskQueue := actCtx.config.BreakdownMetricsByTaskQueue taskQueueFamily := a.GetTaskQueue().GetName() - return metrics.GetPerTaskQueueFamilyScope( + return metrics.GetPerActivityScope( ctx.MetricsHandler(), namespaceName.String(), tqid.UnsafeTaskQueueFamily(namespaceName.String(), taskQueueFamily), breakdownMetricsByTaskQueue(namespaceName.String(), taskQueueFamily, enumspb.TASK_QUEUE_TYPE_ACTIVITY), - metrics.OperationTag(operation), - metrics.ActivityTypeTag(a.GetActivityType().GetName()), - metrics.VersioningBehaviorTag(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED), - metrics.WorkflowTypeTag(WorkflowTypeTag), + operation, + a.GetActivityType().GetName(), + WorkflowTypeTag, + enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED, ) } diff --git a/common/metrics/activity.go b/common/metrics/activity.go new file mode 100644 index 00000000000..ca1b377828c --- /dev/null +++ b/common/metrics/activity.go @@ -0,0 +1,29 @@ +package metrics + +import ( + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/common/tqid" +) + +// GetPerActivityScope returns the standard tags for metrics associated with an activity. +func GetPerActivityScope( + handler Handler, + namespaceName string, + taskQueueFamily *tqid.TaskQueueFamily, + taskQueueBreakdown bool, + operation string, + activityType string, + workflowType string, + versioningBehavior enumspb.VersioningBehavior, +) Handler { + return GetPerTaskQueueFamilyScope( + handler, + namespaceName, + taskQueueFamily, + taskQueueBreakdown, + OperationTag(operation), + ActivityTypeTag(activityType), + VersioningBehaviorTag(versioningBehavior), + WorkflowTypeTag(workflowType), + ) +} diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 3b845ef2358..ce57ce21e6c 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -347,15 +347,15 @@ const ( HistoryRespondActivityTaskFailedScope = "RespondActivityTaskFailed" // HistoryRespondActivityTaskCanceledScope tracks RespondActivityTaskCanceled API calls received by service HistoryRespondActivityTaskCanceledScope = "RespondActivityTaskCanceled" - // ActivityTerminatedScope tracks TerminateActivityExecution API calls received by service + // ActivityTerminatedScope identifies applied activity termination mutations. ActivityTerminatedScope = "ActivityTerminated" - // ActivityPausedScope tracks PauseActivityExecution API calls received by service + // ActivityPausedScope identifies applied activity pause mutations. ActivityPausedScope = "ActivityPaused" - // ActivityUnpausedScope tracks UnpauseActivityExecution API calls received by service + // ActivityUnpausedScope identifies applied activity unpause mutations. ActivityUnpausedScope = "ActivityUnpaused" - // ActivityResetScope tracks ResetActivityExecution API calls received by service + // ActivityResetScope identifies applied activity reset mutations. ActivityResetScope = "ActivityReset" - // ActivityUpdateOptionsScope tracks UpdateActivityExecutionOptions API calls received by service + // ActivityUpdateOptionsScope identifies applied activity option mutations. ActivityUpdateOptionsScope = "ActivityUpdateOptions" // HistoryGetWorkflowExecutionHistoryScope is the metric scope for non-long-poll frontend.GetWorkflowExecutionHistory HistoryGetWorkflowExecutionHistoryScope = "GetWorkflowExecutionHistory" diff --git a/common/metrics/tags.go b/common/metrics/tags.go index b0d60c018e2..7d764f8f6a4 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -64,7 +64,6 @@ const ( namespaceAllValue = "all" clientName = "client_name" isInternal = "is_internal" - activityTargetingMethod = "activity_targeting_method" unknownValue = "_unknown_" totalMetricSuffix = "_total" tagExcludedValue = "_tag_excluded_" @@ -227,11 +226,6 @@ func ActivityTypeTag(value string) Tag { return Tag{Key: activityType, Value: value} } -// ActivityTargetingMethodTag returns a tag indicating how the activity was targeted: "id" or "type". -func ActivityTargetingMethodTag(value string) Tag { - return Tag{Key: activityTargetingMethod, Value: value} -} - // CommandTypeTag returns a new command type tag. func CommandTypeTag(value string) Tag { if len(value) == 0 { diff --git a/service/history/api/pauseactivity/api.go b/service/history/api/pauseactivity/api.go index 5a7278a4757..2d3b2a59dce 100644 --- a/service/history/api/pauseactivity/api.go +++ b/service/history/api/pauseactivity/api.go @@ -8,7 +8,6 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -22,6 +21,7 @@ func Invoke( shardContext historyi.ShardContext, workflowConsistencyChecker api.WorkflowConsistencyChecker, ) (resp *historyservice.PauseActivityResponse, retError error) { + var activityMetrics []workflow.ActivityMetricsInfo err := api.GetAndUpdateWorkflowWithNew( ctx, @@ -34,6 +34,10 @@ func Invoke( func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) { mutableState := workflowLease.GetMutableState() frontendRequest := request.GetFrontendRequest() + if !mutableState.IsWorkflowExecutionRunning() { + return nil, consts.ErrWorkflowCompleted + } + var activityIDs []string switch a := frontendRequest.GetActivity().(type) { case *workflowservice.PauseActivityRequest_Id: @@ -63,10 +67,18 @@ func Invoke( } for _, activityId := range activityIDs { + activityInfo, activityFound := mutableState.GetActivityByActivityID(activityId) + if !activityFound { + return nil, consts.ErrActivityNotFound + } + wasPaused := activityInfo.GetPaused() err := workflow.PauseActivity(mutableState, activityId, pauseInfo) if err != nil { return nil, err } + if !wasPaused { + activityMetrics = append(activityMetrics, workflow.NewActivityMetricsInfo(mutableState, activityInfo)) + } } return &api.UpdateWorkflowAction{ Noop: false, @@ -82,15 +94,8 @@ func Invoke( return nil, err } - targetingMethod := "type" - if _, ok := request.GetFrontendRequest().GetActivity().(*workflowservice.PauseActivityRequest_Id); ok { - targetingMethod = "id" - } - if ns, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(request.NamespaceId)); err == nil { - metrics.ActivityPause.With(shardContext.GetMetricsHandler().WithTags( - metrics.NamespaceTag(ns.Name().String()), - metrics.ActivityTargetingMethodTag(targetingMethod), - )).Record(1) + for _, info := range activityMetrics { + metrics.ActivityPause.With(info.MetricsHandler(shardContext, metrics.ActivityPausedScope)).Record(1) } return &historyservice.PauseActivityResponse{}, nil diff --git a/service/history/api/resetactivity/api.go b/service/history/api/resetactivity/api.go index 260e8c35a0f..c566e7b99cd 100644 --- a/service/history/api/resetactivity/api.go +++ b/service/history/api/resetactivity/api.go @@ -7,7 +7,6 @@ import ( "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -21,6 +20,7 @@ func Invoke( workflowConsistencyChecker api.WorkflowConsistencyChecker, ) (resp *historyservice.ResetActivityResponse, retError error) { request := req.GetFrontendRequest() + var activityMetrics []workflow.ActivityMetricsInfo workflowKey := definition.NewWorkflowKey( req.NamespaceId, @@ -34,6 +34,10 @@ func Invoke( workflowKey, func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) { mutableState := workflowLease.GetMutableState() + if !mutableState.IsWorkflowExecutionRunning() { + return nil, consts.ErrWorkflowCompleted + } + var activityIDs []string switch a := request.GetActivity().(type) { case *workflowservice.ResetActivityRequest_Id: @@ -56,6 +60,10 @@ func Invoke( } for _, activityId := range activityIDs { + activityInfo, activityFound := mutableState.GetActivityByActivityID(activityId) + if !activityFound { + return nil, consts.ErrActivityNotFound + } if err := workflow.ResetActivity( ctx, shardContext, mutableState, activityId, @@ -64,6 +72,7 @@ func Invoke( ); err != nil { return nil, err } + activityMetrics = append(activityMetrics, workflow.NewActivityMetricsInfo(mutableState, activityInfo)) } return &api.UpdateWorkflowAction{ Noop: false, @@ -79,17 +88,8 @@ func Invoke( return nil, err } - targetingMethod := "type" - if _, ok := req.GetFrontendRequest().GetActivity().(*workflowservice.ResetActivityRequest_Id); ok { - targetingMethod = "id" - } else if _, ok := req.GetFrontendRequest().GetActivity().(*workflowservice.ResetActivityRequest_MatchAll); ok { - targetingMethod = "match_all" - } - if ns, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(req.NamespaceId)); err == nil { - metrics.ActivityReset.With(shardContext.GetMetricsHandler().WithTags( - metrics.NamespaceTag(ns.Name().String()), - metrics.ActivityTargetingMethodTag(targetingMethod), - )).Record(1) + for _, info := range activityMetrics { + metrics.ActivityReset.With(info.MetricsHandler(shardContext, metrics.ActivityResetScope)).Record(1) } return &historyservice.ResetActivityResponse{}, nil diff --git a/service/history/api/unpauseactivity/api.go b/service/history/api/unpauseactivity/api.go index 83f2f7a6c71..6e762551e62 100644 --- a/service/history/api/unpauseactivity/api.go +++ b/service/history/api/unpauseactivity/api.go @@ -8,7 +8,6 @@ import ( "go.temporal.io/server/common/definition" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -22,6 +21,7 @@ func Invoke( workflowConsistencyChecker api.WorkflowConsistencyChecker, ) (resp *historyservice.UnpauseActivityResponse, retError error) { var response *historyservice.UnpauseActivityResponse + var activityMetrics []workflow.ActivityMetricsInfo err := api.GetAndUpdateWorkflowWithNew( ctx, @@ -34,7 +34,7 @@ func Invoke( func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) { mutableState := workflowLease.GetMutableState() var err error - response, err = processUnpauseActivityRequest(shardContext, mutableState, request) + response, activityMetrics, err = processUnpauseActivityRequest(shardContext, mutableState, request) if err != nil { return nil, err } @@ -52,20 +52,11 @@ func Invoke( return nil, err } - frontendReq := request.GetFrontendRequest() - targetingMethod := "type" - if _, ok := frontendReq.GetActivity().(*workflowservice.UnpauseActivityRequest_Id); ok { - targetingMethod = "id" - } else if _, ok := frontendReq.GetActivity().(*workflowservice.UnpauseActivityRequest_UnpauseAll); ok { - targetingMethod = "unpause_all" - } - if ns, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(request.NamespaceId)); err == nil { - metrics.ActivityUnpause.With(shardContext.GetMetricsHandler().WithTags( - metrics.NamespaceTag(ns.Name().String()), - metrics.ActivityTargetingMethodTag(targetingMethod), - )).Record(1) + for _, info := range activityMetrics { + metrics.ActivityUnpause.With(info.MetricsHandler(shardContext, metrics.ActivityUnpausedScope)).Record(1) } + frontendReq := request.GetFrontendRequest() shardContext.GetLogger().Info("unpauseactivity: activity unpaused", tag.WorkflowNamespaceID(request.GetNamespaceId()), tag.WorkflowID(frontendReq.GetExecution().GetWorkflowId()), @@ -82,10 +73,10 @@ func processUnpauseActivityRequest( shardContext historyi.ShardContext, mutableState historyi.MutableState, request *historyservice.UnpauseActivityRequest, -) (*historyservice.UnpauseActivityResponse, error) { +) (*historyservice.UnpauseActivityResponse, []workflow.ActivityMetricsInfo, error) { if !mutableState.IsWorkflowExecutionRunning() { - return nil, consts.ErrWorkflowCompleted + return nil, nil, consts.ErrWorkflowCompleted } frontendRequest := request.GetFrontendRequest() var activityIDs []string @@ -106,15 +97,16 @@ func processUnpauseActivityRequest( } if len(activityIDs) == 0 { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } + var activityMetrics []workflow.ActivityMetricsInfo for _, activityId := range activityIDs { ai, activityFound := mutableState.GetActivityByActivityID(activityId) if !activityFound { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } if !ai.Paused { @@ -127,10 +119,10 @@ func processUnpauseActivityRequest( frontendRequest.GetResetAttempts(), frontendRequest.GetResetHeartbeat(), frontendRequest.GetJitter().AsDuration()); err != nil { - return nil, err + return nil, nil, err } - + activityMetrics = append(activityMetrics, workflow.NewActivityMetricsInfo(mutableState, ai)) } - return &historyservice.UnpauseActivityResponse{}, nil + return &historyservice.UnpauseActivityResponse{}, activityMetrics, nil } diff --git a/service/history/api/updateactivityoptions/api.go b/service/history/api/updateactivityoptions/api.go index 74930fdbf61..e361971eae8 100644 --- a/service/history/api/updateactivityoptions/api.go +++ b/service/history/api/updateactivityoptions/api.go @@ -51,6 +51,7 @@ func Invoke( ) var response *historyservice.UpdateActivityOptionsResponse + var activityMetrics []workflow.ActivityMetricsInfo err := api.GetAndUpdateWorkflowWithNew( ctx, @@ -62,16 +63,25 @@ func Invoke( ), func(workflowLease api.WorkflowLease) (*api.UpdateWorkflowAction, error) { mutableState := workflowLease.GetMutableState() + var updatedActivities []*persistencespb.ActivityInfo var err error if updateRequest.RestoreOriginal { - response, err = restoreOriginalOptions(ctx, mutableState, updateRequest) + response, updatedActivities, err = restoreOriginalOptions(ctx, mutableState, updateRequest) } else { - response, err = processActivityOptionsRequest(validator, mutableState, updateRequest, request.GetNamespaceId()) + response, updatedActivities, err = processActivityOptionsRequest( + validator, + mutableState, + updateRequest, + request.GetNamespaceId(), + ) } if err != nil { return nil, err } + for _, activityInfo := range updatedActivities { + activityMetrics = append(activityMetrics, workflow.NewActivityMetricsInfo(mutableState, activityInfo)) + } return &api.UpdateWorkflowAction{ Noop: false, CreateWorkflowTask: false, @@ -86,17 +96,8 @@ func Invoke( return nil, err } - targetingMethod := "type" - if _, ok := updateRequest.GetActivity().(*workflowservice.UpdateActivityOptionsRequest_Id); ok { - targetingMethod = "id" - } else if _, ok := updateRequest.GetActivity().(*workflowservice.UpdateActivityOptionsRequest_MatchAll); ok { - targetingMethod = "match_all" - } - if ns, err := shardContext.GetNamespaceRegistry().GetNamespaceByID(namespace.ID(request.NamespaceId)); err == nil { - metrics.ActivityUpdateOptions.With(shardContext.GetMetricsHandler().WithTags( - metrics.NamespaceTag(ns.Name().String()), - metrics.ActivityTargetingMethodTag(targetingMethod), - )).Record(1) + for _, info := range activityMetrics { + metrics.ActivityUpdateOptions.With(info.MetricsHandler(shardContext, metrics.ActivityUpdateOptionsScope)).Record(1) } logger := shardContext.GetLogger() @@ -128,47 +129,49 @@ func processActivityOptionsRequest( mutableState historyi.MutableState, updateRequest *workflowservice.UpdateActivityOptionsRequest, namespaceID string, -) (*historyservice.UpdateActivityOptionsResponse, error) { +) (*historyservice.UpdateActivityOptionsResponse, []*persistencespb.ActivityInfo, error) { if !mutableState.IsWorkflowExecutionRunning() { - return nil, consts.ErrWorkflowCompleted + return nil, nil, consts.ErrWorkflowCompleted } mergeFrom := updateRequest.GetActivityOptions() if mergeFrom == nil { - return nil, serviceerror.NewInvalidArgument("ActivityOptions are not provided") + return nil, nil, serviceerror.NewInvalidArgument("ActivityOptions are not provided") } activityIDs := getActivityIDs(updateRequest, mutableState) if len(activityIDs) == 0 { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } mask := updateRequest.GetUpdateMask() if mask == nil { - return nil, serviceerror.NewInvalidArgument("UpdateMask is not provided") + return nil, nil, serviceerror.NewInvalidArgument("UpdateMask is not provided") } updateFields := util.ParseFieldMask(mask) var adjustedOptions *activitypb.ActivityOptions + updatedActivities := make([]*persistencespb.ActivityInfo, 0, len(activityIDs)) var err error for _, activityId := range activityIDs { ai, activityFound := mutableState.GetActivityByActivityID(activityId) if !activityFound { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } if adjustedOptions, err = processActivityOptionsUpdate(validator, mutableState, namespaceID, ai, mergeFrom, updateFields); err != nil { - return nil, err + return nil, nil, err } + updatedActivities = append(updatedActivities, ai) } // fill the response response := &historyservice.UpdateActivityOptionsResponse{ ActivityOptions: adjustedOptions, } - return response, nil + return response, updatedActivities, nil } func processActivityOptionsUpdate( @@ -323,33 +326,34 @@ func restoreOriginalOptions( ctx context.Context, ms historyi.MutableState, updateRequest *workflowservice.UpdateActivityOptionsRequest, -) (*historyservice.UpdateActivityOptionsResponse, error) { +) (*historyservice.UpdateActivityOptionsResponse, []*persistencespb.ActivityInfo, error) { activityIDs := getActivityIDs(updateRequest, ms) if len(activityIDs) == 0 { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } var updatedOptions *activitypb.ActivityOptions + updatedActivities := make([]*persistencespb.ActivityInfo, 0, len(activityIDs)) for _, activityId := range activityIDs { ai, activityFound := ms.GetActivityByActivityID(activityId) if !activityFound { - return nil, consts.ErrActivityNotFound + return nil, nil, consts.ErrActivityNotFound } event, err := ms.GetActivityScheduledEvent(ctx, ai.ScheduledEventId) if err != nil { - return nil, err + return nil, nil, err } attrs, ok := event.Attributes.(*historypb.HistoryEvent_ActivityTaskScheduledEventAttributes) if !ok { - return nil, serviceerror.NewInvalidArgument("ActivityTaskScheduledEvent is invalid") + return nil, nil, serviceerror.NewInvalidArgument("ActivityTaskScheduledEvent is invalid") } if attrs == nil || attrs.ActivityTaskScheduledEventAttributes == nil { - return nil, serviceerror.NewInvalidArgument("ActivityTaskScheduledEvent is incomplete") + return nil, nil, serviceerror.NewInvalidArgument("ActivityTaskScheduledEvent is incomplete") } originalOptions := attrs.ActivityTaskScheduledEventAttributes @@ -367,12 +371,13 @@ func restoreOriginalOptions( } if updatedOptions, err = updateActivityOptions(ms, ai, activityOptions); err != nil { - return nil, err + return nil, nil, err } + updatedActivities = append(updatedActivities, ai) } return &historyservice.UpdateActivityOptionsResponse{ ActivityOptions: updatedOptions, - }, nil + }, updatedActivities, nil } diff --git a/service/history/api/updateactivityoptions/api_test.go b/service/history/api/updateactivityoptions/api_test.go index 5e207809786..8f48fd1f6a4 100644 --- a/service/history/api/updateactivityoptions/api_test.go +++ b/service/history/api/updateactivityoptions/api_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/suite" activitypb "go.temporal.io/api/activity/v1" commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" @@ -20,7 +21,10 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/activityoptions" "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives" @@ -193,7 +197,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptionsWfNotRunning() { s.mockMutableState.EXPECT().IsWorkflowExecutionRunning().Return(false) - _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) + _, _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.Error(err) s.ErrorAs(err, &consts.ErrWorkflowCompleted) } @@ -215,7 +219,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptionsWfNoActivity() { s.mockMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) s.mockMutableState.EXPECT().GetActivityByActivityID(gomock.Any()).Return(nil, false) - _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) + _, _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.Error(err) s.ErrorAs(err, &consts.ErrActivityNotFound) } @@ -278,13 +282,75 @@ func (s *activityOptionsSuite) Test_updateActivityOptionsAcceptance() { }, } - response, err := processActivityOptionsRequest( + response, _, err := processActivityOptionsRequest( s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.NoError(err) s.NotNil(response) } +func (s *activityOptionsSuite) TestInvokeCapturesMetricsFromUpdatedActivity() { + s.mockShard.GetConfig().BreakdownMetricsByTaskQueue = dynamicconfig.GetBoolPropertyFnFilteredByTaskQueue(true) + metricsHandler := metricstest.NewCaptureHandler() + metricCapture := metricsHandler.StartCapture() + s.T().Cleanup(func() { + metricsHandler.StopCapture(metricCapture) + }) + s.mockShard.SetMetricsHandler(metricsHandler) + + activityInfo := &persistencespb.ActivityInfo{ + ActivityId: "activity_id", + ActivityType: &commonpb.ActivityType{Name: "activity_type"}, + TaskQueue: "old-task-queue", + ScheduledEventId: 1, + StartedEventId: 2, + StartToCloseTimeout: durationpb.New(time.Second), + } + workflowContext := historyi.NewMockWorkflowContext(s.controller) + workflowContext.EXPECT().LoadMutableState(gomock.Any(), s.mockShard).Return(s.mockMutableState, nil) + workflowContext.EXPECT().UpdateWorkflowExecutionAsActive(gomock.Any(), s.mockShard).Return(nil) + s.workflowCache.EXPECT().GetOrCreateChasmExecution( + gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), + ).Return(workflowContext, wcache.NoopReleaseFn, nil) + + s.mockMutableState.EXPECT().GetExecutionState().Return(&persistencespb.WorkflowExecutionState{ + RunId: tests.RunID, + }).AnyTimes() + s.mockMutableState.EXPECT().GetNamespaceEntry().Return(tests.LocalNamespaceEntry) + s.mockMutableState.EXPECT().GetWorkflowType().Return(&commonpb.WorkflowType{Name: "workflow_type"}) + s.mockMutableState.EXPECT().GetEffectiveVersioningBehavior().Return(enumspb.VERSIONING_BEHAVIOR_UNSPECIFIED) + s.mockMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) + s.mockMutableState.EXPECT().GetActivityByActivityID("activity_id").Return(activityInfo, true).Times(1) + s.mockMutableState.EXPECT().UpdateActivity(activityInfo.ScheduledEventId, gomock.Any()).DoAndReturn( + func(_ int64, updater historyi.ActivityUpdater) error { + return updater(activityInfo, s.mockMutableState) + }, + ) + + response, err := Invoke(context.Background(), &historyservice.UpdateActivityOptionsRequest{ + NamespaceId: tests.NamespaceID.String(), + UpdateRequest: &workflowservice.UpdateActivityOptionsRequest{ + Execution: &commonpb.WorkflowExecution{ + WorkflowId: tests.WorkflowID, + RunId: tests.RunID, + }, + Activity: &workflowservice.UpdateActivityOptionsRequest_Id{Id: activityInfo.ActivityId}, + ActivityOptions: &activitypb.ActivityOptions{ + TaskQueue: &taskqueuepb.TaskQueue{Name: "new-task-queue"}, + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"task_queue.name"}}, + }, + }, s.mockShard, s.workflowConsistencyChecker) + + s.Require().NoError(err) + s.Require().NotNil(response) + s.Require().Equal("new-task-queue", activityInfo.TaskQueue) + recordings := metricCapture.Snapshot()[metrics.ActivityUpdateOptions.Name()] + s.Require().Len(recordings, 1) + s.Require().Equal("new-task-queue", recordings[0].Tags["taskqueue"]) + s.Require().Equal(metrics.ActivityUpdateOptionsScope, recordings[0].Tags["operation"]) +} + func (s *activityOptionsSuite) Test_updateActivityOptionsRejectsInvalidMergedRetryPolicy() { activityInfo := &persistencespb.ActivityInfo{ TaskQueue: "task_queue_name", @@ -317,7 +383,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptionsRejectsInvalidMergedRet s.mockMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) s.mockMutableState.EXPECT().GetActivityByActivityID("activity_id").Return(activityInfo, true) - _, err := processActivityOptionsRequest( + _, _, err := processActivityOptionsRequest( s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.ErrorContains(err, "MaximumInterval cannot be less than InitialInterval") } @@ -371,7 +437,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_RestoreDefaultFail() { request.UpdateRequest.Activity = &workflowservice.UpdateActivityOptionsRequest_Type{Type: "activity_type"} activityInfos := map[int64]*persistencespb.ActivityInfo{} s.mockMutableState.EXPECT().GetPendingActivityInfos().Return(activityInfos) - _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) + _, _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) s.Error(err) // not pending activity with such id @@ -379,7 +445,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_RestoreDefaultFail() { request.UpdateRequest.UpdateMask = nil request.UpdateRequest.Activity = &workflowservice.UpdateActivityOptionsRequest_Id{Id: "activity_id"} s.mockMutableState.EXPECT().GetActivityByActivityID(gomock.Any()).Return(nil, false) - _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) + _, _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) s.Error(err) ai := &persistencespb.ActivityInfo{ @@ -395,7 +461,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_RestoreDefaultFail() { err = errors.New("some error") s.mockMutableState.EXPECT().GetActivityScheduledEvent(gomock.Any(), gomock.Any()).Return(nil, err) s.mockMutableState.EXPECT().GetActivityByActivityID(gomock.Any()).Return(ai, true) - _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) + _, _, err = restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) s.Error(err) } @@ -448,7 +514,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_RestoreDefaultSuccess( s.mockMutableState.EXPECT().GetActivityScheduledEvent(gomock.Any(), gomock.Any()).Return(he, nil) s.mockMutableState.EXPECT().GetActivityByActivityID(gomock.Any()).Return(ai, true) s.mockMutableState.EXPECT().UpdateActivity(gomock.Any(), gomock.Any()).Return(nil) - response, err := restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) + response, _, err := restoreOriginalOptions(ctx, s.mockMutableState, request.GetUpdateRequest()) s.NotNil(response) s.NoError(err) } @@ -480,7 +546,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_PerNSTQ_Blocked() { s.mockMutableState.EXPECT().IsWorkflowExecutionRunning().Return(true) s.mockMutableState.EXPECT().GetActivityByActivityID("activity_id").Return(activityInfo, true) - _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) + _, _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.Error(err) s.Contains(err.Error(), "internal per-namespace task queue") } @@ -515,7 +581,7 @@ func (s *activityOptionsSuite) Test_updateActivityOptions_PerNSTQ_Allowed() { s.mockMutableState.EXPECT().RegenerateActivityRetryTask(gomock.Any(), gomock.Any()).Return(nil) s.mockMutableState.EXPECT().UpdateActivity(gomock.Any(), gomock.Any()).Return(nil) - resp, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) + resp, _, err := processActivityOptionsRequest(s.validator, s.mockMutableState, request.GetUpdateRequest(), request.GetNamespaceId()) s.NoError(err) s.NotNil(resp) } diff --git a/service/history/workflow/metrics.go b/service/history/workflow/metrics.go index 6b44ad18882..b06224c26ac 100644 --- a/service/history/workflow/metrics.go +++ b/service/history/workflow/metrics.go @@ -7,6 +7,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" deploymentspb "go.temporal.io/server/api/deployment/v1" enumsspb "go.temporal.io/server/api/enums/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -324,3 +325,47 @@ func RecordActivityCompletionMetrics( // Do nothing } } + +// ActivityMetricsInfo captures activity metric tags until the mutation commits. +type ActivityMetricsInfo struct { + namespaceName string + taskQueue string + activityType string + workflowType string + versioningBehavior enumspb.VersioningBehavior +} + +// NewActivityMetricsInfo captures activity metric tags from mutable state. +func NewActivityMetricsInfo( + mutableState historyi.MutableState, + activityInfo *persistencespb.ActivityInfo, +) ActivityMetricsInfo { + return ActivityMetricsInfo{ + namespaceName: mutableState.GetNamespaceEntry().Name().String(), + taskQueue: activityInfo.GetTaskQueue(), + activityType: activityInfo.GetActivityType().GetName(), + workflowType: mutableState.GetWorkflowType().GetName(), + versioningBehavior: mutableState.GetEffectiveVersioningBehavior(), + } +} + +// MetricsHandler returns a metrics handler with the captured activity tags. +func (i ActivityMetricsInfo) MetricsHandler( + shardContext historyi.ShardContext, + operation string, +) metrics.Handler { + return metrics.GetPerActivityScope( + shardContext.GetMetricsHandler(), + i.namespaceName, + tqid.UnsafeTaskQueueFamily(i.namespaceName, i.taskQueue), + shardContext.GetConfig().BreakdownMetricsByTaskQueue( + i.namespaceName, + i.taskQueue, + enumspb.TASK_QUEUE_TYPE_ACTIVITY, + ), + operation, + i.activityType, + i.workflowType, + i.versioningBehavior, + ) +} diff --git a/tests/activity_api_pause_test.go b/tests/activity_api_pause_test.go index a0c4580e136..a681138da59 100644 --- a/tests/activity_api_pause_test.go +++ b/tests/activity_api_pause_test.go @@ -111,6 +111,25 @@ func TestActivityApiPauseClientTestSuite(t *testing.T) { t.Run(api.name, func(t *testing.T) { t.Parallel() + t.Run("TestActivityPauseApi_AfterWorkflowCompleted", func(t *testing.T) { + s := testcore.NewEnv(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + workflowFn := func(workflow.Context) error { return nil } + s.SdkWorker().RegisterWorkflow(workflowFn) + + workflowRun, err := s.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: testcore.RandomizeStr("wf_id-" + t.Name()), + TaskQueue: s.WorkerTaskQueue(), + }, workflowFn) + require.NoError(t, err) + require.NoError(t, workflowRun.Get(ctx, nil)) + + err = api.pause(ctx, s, workflowRun.GetID(), "activity-id", "test-identity", "test-reason", "test-request-id") + require.ErrorContains(t, err, "workflow execution already completed") + }) + t.Run("TestActivityPauseApi_WhileRunning", func(t *testing.T) { s := testcore.NewEnv(t) diff --git a/tests/activity_api_reset_test.go b/tests/activity_api_reset_test.go index bf558143000..b71d519367e 100644 --- a/tests/activity_api_reset_test.go +++ b/tests/activity_api_reset_test.go @@ -104,6 +104,24 @@ func (s *ActivityApiResetClientTestSuite) makeWorkflowFunc(activityFunction Acti } } +func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_AfterWorkflowCompleted() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + workflowFn := func(workflow.Context) error { return nil } + s.SdkWorker().RegisterWorkflow(workflowFn) + + workflowRun, err := s.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: testcore.RandomizeStr("wf_id-" + s.T().Name()), + TaskQueue: s.TaskQueue(), + }, workflowFn) + s.Require().NoError(err) + s.Require().NoError(workflowRun.Get(ctx, nil)) + + err = s.resetFn(ctx, workflowRun.GetID(), "activity-id", false, false) + s.Require().ErrorContains(err, "workflow execution already completed") +} + func (s *ActivityApiResetClientTestSuite) TestActivityResetApi_AfterRetry() { // activity reset is called after multiple attempts, ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/tests/activity_metrics_parity_test.go b/tests/activity_metrics_parity_test.go index f695ffc5ff8..f66907fee31 100644 --- a/tests/activity_metrics_parity_test.go +++ b/tests/activity_metrics_parity_test.go @@ -3,19 +3,30 @@ package tests // SAA vs WFA metrics parity tests. import ( - "sort" + "fmt" + "slices" "strings" "testing" "time" "github.com/stretchr/testify/require" + activitypb "go.temporal.io/api/activity/v1" + commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/workflowservice/v1" + sdkclient "go.temporal.io/sdk/client" + sdkworker "go.temporal.io/sdk/worker" + "go.temporal.io/sdk/workflow" "go.temporal.io/server/chasm/lib/activity/model" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/testing/await" + "go.temporal.io/server/common/testing/testcontext" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/fieldmaskpb" ) // TestScheduleToStartMetric asserts that SAA and WFA record task_schedule_to_start_latency @@ -111,16 +122,16 @@ func (s *activityParityTestSuite) TestMetrics() { compared bool counter bool baseHandler bool - legacyWFAHandler bool workerDeploymentTags bool recordingTagKeys []string } type scenario struct { - name string - trace []model.Event - cfg activityConfig - saaOnly bool - anchor string + name string + trace []model.Event + cfg activityConfig + saaOnly bool + anchor string + requiredMetric string } type recordings map[string][]*metricstest.CapturedRecording @@ -137,11 +148,6 @@ func (s *activityParityTestSuite) TestMetrics() { "worker_build_id", "worker_deployment_name", } - legacyWFATagKeys := []string{ - "activity_targeting_method", - "namespace", - "service_name", - } catalog := []activityMetric{ {name: metrics.ActivitySuccess.Name(), compared: true, counter: true, workerDeploymentTags: true}, {name: metrics.ActivityFail.Name(), compared: true, counter: true, workerDeploymentTags: true}, @@ -152,10 +158,10 @@ func (s *activityParityTestSuite) TestMetrics() { {name: metrics.ActivityTaskTimeout.Name(), compared: true, counter: true, workerDeploymentTags: true, recordingTagKeys: []string{"timeout_type"}}, {name: metrics.ActivityStartToCloseLatency.Name(), compared: true, workerDeploymentTags: true}, {name: metrics.ActivityScheduleToCloseLatency.Name(), compared: true, workerDeploymentTags: true}, - {name: metrics.ActivityPause.Name(), compared: true, counter: true, legacyWFAHandler: true}, - {name: metrics.ActivityUnpause.Name(), compared: true, counter: true, legacyWFAHandler: true}, - {name: metrics.ActivityReset.Name(), compared: true, counter: true, legacyWFAHandler: true}, - {name: metrics.ActivityUpdateOptions.Name(), compared: true, counter: true, legacyWFAHandler: true}, + {name: metrics.ActivityPause.Name(), compared: true, counter: true}, + {name: metrics.ActivityUnpause.Name(), compared: true, counter: true}, + {name: metrics.ActivityReset.Name(), compared: true, counter: true}, + {name: metrics.ActivityUpdateOptions.Name(), compared: true, counter: true}, {name: metrics.ActivityHeartbeatCount.Name(), compared: true, counter: true, baseHandler: true}, {name: metrics.ActivityPayloadSize.Name(), compared: true, counter: true, baseHandler: true}, } @@ -168,10 +174,10 @@ func (s *activityParityTestSuite) TestMetrics() { {name: "RetryableTaskFailure", trace: []model.Event{model.Poll, model.FailRetryably}, cfg: activityConfig{MaxAttempts: 2}}, {name: "RetryableTaskFailureWithHeartbeatDetails", trace: []model.Event{model.Poll, {Type: model.RespondFailedType, Failure: &model.Failure{Retryable: true}, HasHeartbeatDetails: true}}, cfg: activityConfig{MaxAttempts: 2}}, {name: "Heartbeat", trace: []model.Event{model.Poll, model.Heartbeat}, cfg: activityConfig{MaxAttempts: 1}}, - {name: "Pause", trace: []model.Event{model.Poll, model.Pause}, cfg: activityConfig{MaxAttempts: 1}}, - {name: "Unpause", trace: []model.Event{model.Poll, model.Pause, model.Unpause}, cfg: activityConfig{MaxAttempts: 1}}, - {name: "Reset", trace: []model.Event{model.Poll, model.Reset}, cfg: activityConfig{MaxAttempts: 1}}, - {name: "UpdateOptions", trace: []model.Event{model.Poll, model.UpdateOptions}, cfg: activityConfig{MaxAttempts: 1}}, + {name: "Pause", trace: []model.Event{model.Poll, model.Pause}, cfg: activityConfig{MaxAttempts: 1}, requiredMetric: metrics.ActivityPause.Name()}, + {name: "Unpause", trace: []model.Event{model.Poll, model.Pause, model.Unpause}, cfg: activityConfig{MaxAttempts: 1}, requiredMetric: metrics.ActivityUnpause.Name()}, + {name: "Reset", trace: []model.Event{model.Poll, model.Reset}, cfg: activityConfig{MaxAttempts: 1}, requiredMetric: metrics.ActivityReset.Name()}, + {name: "UpdateOptions", trace: []model.Event{model.Poll, model.UpdateOptions}, cfg: activityConfig{MaxAttempts: 1}, requiredMetric: metrics.ActivityUpdateOptions.Name()}, {name: "Terminate", trace: []model.Event{model.Poll, model.Terminate}, cfg: activityConfig{MaxAttempts: 1}, saaOnly: true}, } expectedTimeoutType := func(trace []model.Event) string { @@ -224,7 +230,7 @@ func (s *activityParityTestSuite) TestMetrics() { for key := range seen { keys = append(keys, key) } - sort.Strings(keys) + slices.Sort(keys) return keys } metricSeries := func( @@ -264,6 +270,9 @@ func (s *activityParityTestSuite) TestMetrics() { s.Run(sc.name, func(s *activityParityTestSuite) { t := s.T() saa, saaNS := captureMetrics(t, sc, true) + if sc.requiredMetric != "" { + require.NotEmpty(t, saa[sc.requiredMetric], "SAA must emit %s", sc.requiredMetric) + } checkTags := func(implementation string, emitted recordings, namespace string) { for name, recs := range emitted { for _, rec := range recs { @@ -289,6 +298,9 @@ func (s *activityParityTestSuite) TestMetrics() { } wfa, wfaNS := captureMetrics(t, sc, false) + if sc.requiredMetric != "" { + require.NotEmpty(t, wfa[sc.requiredMetric], "WFA must emit %s", sc.requiredMetric) + } checkTags("WFA", wfa, wfaNS) for _, metric := range catalog { if !metric.compared { @@ -298,31 +310,19 @@ func (s *activityParityTestSuite) TestMetrics() { t := s.T() wfaTagKeys := seriesTagKeys(wfa[metric.name]) saaTagKeys := seriesTagKeys(saa[metric.name]) - comparisonTagKeys := wfaTagKeys - if metric.legacyWFAHandler && len(wfa[metric.name]) > 0 { - require.Equal(t, legacyWFATagKeys, wfaTagKeys, - "WFA currently emits this metric from its legacy activity handler") - for _, rec := range wfa[metric.name] { - require.Equal(t, "id", rec.Tags["activity_targeting_method"]) - } - require.Equal(t, perActivityTagKeys, saaTagKeys, - "SAA must use the standard per-activity tag keys") - comparisonTagKeys = []string{"namespace", "service_name"} - } else { - if !metric.baseHandler && len(wfa[metric.name]) > 0 { - expectedTagKeys := append([]string{}, perActivityTagKeys...) - if metric.workerDeploymentTags { - expectedTagKeys = append(expectedTagKeys, workerDeploymentTagKeys...) - } - expectedTagKeys = append(expectedTagKeys, metric.recordingTagKeys...) - sort.Strings(expectedTagKeys) - require.Equal(t, expectedTagKeys, wfaTagKeys, - "WFA must use the standard per-activity tag keys") + if !metric.baseHandler && len(wfa[metric.name]) > 0 { + expectedTagKeys := append([]string{}, perActivityTagKeys...) + if metric.workerDeploymentTags { + expectedTagKeys = append(expectedTagKeys, workerDeploymentTagKeys...) } - require.Equal(t, wfaTagKeys, saaTagKeys, "WFA and SAA tag keys must match") + expectedTagKeys = append(expectedTagKeys, metric.recordingTagKeys...) + slices.Sort(expectedTagKeys) + require.Equal(t, expectedTagKeys, wfaTagKeys, + "WFA must use the standard per-activity tag keys") } - wfaSeries := metricSeries(t, "WFA", wfa[metric.name], comparisonTagKeys, metric.counter) - saaSeries := metricSeries(t, "SAA", saa[metric.name], comparisonTagKeys, metric.counter) + require.Equal(t, wfaTagKeys, saaTagKeys, "WFA and SAA tag keys must match") + wfaSeries := metricSeries(t, "WFA", wfa[metric.name], wfaTagKeys, metric.counter) + saaSeries := metricSeries(t, "SAA", saa[metric.name], wfaTagKeys, metric.counter) if metric.counter { require.Equal(t, wfaSeries, saaSeries, "WFA and SAA counter values must match") } else { @@ -333,3 +333,175 @@ func (s *activityParityTestSuite) TestMetrics() { }) } } + +// TestOperatorMetricsPerActivityParity verifies that one WFA operator +// request targeting multiple activities in a workflow emits the same per-activity +// metrics as equivalent SAA per-ID requests. +func (s *activityParityTestSuite) TestOperatorMetricsPerActivityParity() { + type scenario struct { + name string + metric string + operation string + event model.Event + } + scenarios := []scenario{ + {name: "Pause", metric: metrics.ActivityPause.Name(), operation: metrics.ActivityPausedScope, event: model.Pause}, + {name: "Unpause", metric: metrics.ActivityUnpause.Name(), operation: metrics.ActivityUnpausedScope, event: model.Unpause}, + {name: "Reset", metric: metrics.ActivityReset.Name(), operation: metrics.ActivityResetScope, event: model.Reset}, + {name: "UpdateOptions", metric: metrics.ActivityUpdateOptions.Name(), operation: metrics.ActivityUpdateOptionsScope, event: model.UpdateOptions}, + } + + captureWFA := func(t *testing.T, sc scenario) ([]*metricstest.CapturedRecording, string) { + env := newActivityParityEnv(t) + ctx := testcontext.For(t) + workflowTaskQueue := testcore.RandomizeStr("operator-metrics-wfa-workflow") + activityTaskQueue := testcore.RandomizeStr("operator-metrics-wfa-activity") + worker := sdkworker.New(env.SdkClient(), workflowTaskQueue, sdkworker.Options{}) + worker.RegisterWorkflow(wfaOperatorMetricsWorkflow) + require.NoError(t, worker.Start()) + t.Cleanup(worker.Stop) + + run, err := env.SdkClient().ExecuteWorkflow( + ctx, + sdkclient.StartWorkflowOptions{ + ID: testcore.RandomizeStr("operator-metrics-wfa-run"), + TaskQueue: workflowTaskQueue, + }, + wfaOperatorMetricsWorkflow, + activityTaskQueue, + ) + require.NoError(t, err) + await.Require(ctx, t, func(t *await.T) { + description, err := env.SdkClient().DescribeWorkflowExecution(ctx, run.GetID(), run.GetRunID()) + t.Require().NoError(err) + t.Require().Len(description.GetPendingActivities(), 2) + }, activityDriverTimeout, activityDriverPollInterval) + + execution := &commonpb.WorkflowExecution{WorkflowId: run.GetID(), RunId: run.GetRunID()} + if sc.event.Type == model.UnpauseType { + _, err = env.FrontendClient().PauseActivity(ctx, &workflowservice.PauseActivityRequest{ + Namespace: env.Namespace().String(), + Execution: execution, + Activity: &workflowservice.PauseActivityRequest_Type{Type: wfaOperatorMetricsActivityType}, + }) + require.NoError(t, err) + } + + capture := env.StartNamespaceMetricCapture() + switch sc.event.Type { + case model.PauseType: + _, err = env.FrontendClient().PauseActivity(ctx, &workflowservice.PauseActivityRequest{ + Namespace: env.Namespace().String(), + Execution: execution, + Activity: &workflowservice.PauseActivityRequest_Type{Type: wfaOperatorMetricsActivityType}, + }) + case model.UnpauseType: + _, err = env.FrontendClient().UnpauseActivity(ctx, &workflowservice.UnpauseActivityRequest{ + Namespace: env.Namespace().String(), + Execution: execution, + Activity: &workflowservice.UnpauseActivityRequest_UnpauseAll{UnpauseAll: true}, + }) + case model.ResetType: + _, err = env.FrontendClient().ResetActivity(ctx, &workflowservice.ResetActivityRequest{ + Namespace: env.Namespace().String(), + Execution: execution, + Activity: &workflowservice.ResetActivityRequest_MatchAll{MatchAll: true}, + }) + case model.UpdateOptionsType: + _, err = env.FrontendClient().UpdateActivityOptions(ctx, &workflowservice.UpdateActivityOptionsRequest{ + Namespace: env.Namespace().String(), + Execution: execution, + Activity: &workflowservice.UpdateActivityOptionsRequest_MatchAll{MatchAll: true}, + ActivityOptions: &activitypb.ActivityOptions{ + HeartbeatTimeout: durationpb.New(time.Hour), + }, + UpdateMask: &fieldmaskpb.FieldMask{Paths: []string{"heartbeat_timeout"}}, + }) + default: + t.Fatalf("unsupported operator event %v", sc.event.Type) + } + require.NoError(t, err) + return capture.Metric(sc.metric), activityTaskQueue + } + + captureSAA := func(t *testing.T, sc scenario) []*metricstest.CapturedRecording { + env := newActivityParityEnv(t) + driver := newSAADriver(t, env, activityConfig{MaxAttempts: 1}) + activities := []*saaHandle{ + driver.start(t, activityConfig{MaxAttempts: 1}), + driver.start(t, activityConfig{MaxAttempts: 1}), + } + if sc.event.Type == model.UnpauseType { + for _, activity := range activities { + activity.driveEvent(t, model.Pause) + } + } + + capture := env.StartNamespaceMetricCapture() + for _, activity := range activities { + activity.driveEvent(t, sc.event) + } + return capture.Metric(sc.metric) + } + + normalizedSeries := func(t *testing.T, recordings []*metricstest.CapturedRecording) map[string]int64 { + series := make(map[string]int64) + for _, recording := range recordings { + tags := make([]string, 0, len(recording.Tags)) + for key, value := range recording.Tags { + switch key { + case "activityType", "namespace", "taskqueue", "workflowType": + value = "" + default: + } + tags = append(tags, key+"="+value) + } + slices.Sort(tags) + value, ok := recording.Value.(int64) + require.True(t, ok) + series[strings.Join(tags, "\x00")] += value + } + return series + } + + for _, sc := range scenarios { + s.Run(sc.name, func(s *activityParityTestSuite) { + t := s.T() + wfa, activityTaskQueue := captureWFA(t, sc) + saa := captureSAA(t, sc) + require.Len(t, wfa, 2, "one WFA request affecting two activities must emit two recordings") + require.Len(t, saa, 2, "two SAA requests affecting one activity each must emit two recordings") + for _, recording := range wfa { + require.Equal(t, activityTaskQueue, recording.Tags["taskqueue"], + "WFA must tag the activity task queue") + } + for _, recordings := range [][]*metricstest.CapturedRecording{wfa, saa} { + for _, recording := range recordings { + require.Equal(t, sc.operation, recording.Tags["operation"]) + require.Equal(t, int64(1), recording.Value) + } + } + require.Equal(t, normalizedSeries(t, wfa), normalizedSeries(t, saa)) + }) + } +} + +const wfaOperatorMetricsActivityType = "operatorMetricsActivity" + +func wfaOperatorMetricsWorkflow(ctx workflow.Context, activityTaskQueue string) error { + futures := make([]workflow.Future, 0, 2) + for i := range 2 { + activityCtx := workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + ActivityID: fmt.Sprintf("activity-%d", i), + TaskQueue: activityTaskQueue, + StartToCloseTimeout: activityLongDuration, + }) + futures = append(futures, workflow.ExecuteActivity(activityCtx, wfaOperatorMetricsActivityType)) + } + for _, future := range futures { + if err := future.Get(ctx, nil); err != nil { + return err + } + } + return nil +} From 2526165713d5f638c749d6fb0d4a9c2f625240d2 Mon Sep 17 00:00:00 2001 From: Alex Stanfield <13949480+chaptersix@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:42:13 -0500 Subject: [PATCH 095/139] Validate schedule inputs and preserve V2 IDs (#11629) ## What changed? - Validate ScheduleOverlapPolicy values in CreateSchedule, UpdateSchedule, and PatchSchedule, including initial patch and backfill/trigger overrides. - Reject malformed protobuf timestamps in schedule specs, backfill bounds, immediate-trigger scheduled times, and ListScheduleMatchingTimes ranges. - Reject negative RemainingActions at the API boundary. - Document the namespace-scoped disabledScheduleValidations entries for every new validation. - Preserve V2 schedule IDs that begin with the legacy V1 workflow-ID prefix when listing schedules. - Add table-driven frontend and public functional coverage, including nil and epoch timestamp behavior and malformed matching-time input. ## Why? Malformed schedule input could otherwise reach the scheduler and produce inconsistent errors later. Separately, V2 schedules named foo and temporal-sys-scheduler:foo could both be returned as foo by ListSchedules. ## Verification - [x] git diff --check - [x] Focused frontend schedule-validation tests. - [x] Focused scheduler workflow test. - [x] V1 and CHASM ListScheduleMatchingTimes functional coverage. - [x] make lint-code-fast ## Potential risks The new API validation is a behavior change for malformed inputs. Each check can be disabled per namespace through frontend.disabledScheduleValidations during rollout. ScheduleId query rewriting has a separate source-ambiguity follow-up and is intentionally not changed here. --- common/dynamicconfig/constants.go | 2 +- service/frontend/service.go | 3 + service/frontend/workflow_handler.go | 141 +++++++- .../frontend/workflow_handler_second_test.go | 46 +++ service/frontend/workflow_handler_test.go | 325 +++++++++++++++++ tests/schedule_test.go | 332 +++++++++++++++++- 6 files changed, 836 insertions(+), 13 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 72548ca6a1d..1e6594b7e42 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -679,7 +679,7 @@ ScheduleInvariantsScannerParams comments for details.`, FrontendDisabledScheduleValidations = NewNamespaceTypedSetting( "frontend.disabledScheduleValidations", []string(nil), - `FrontendDisabledScheduleValidations is a list of schedule validation names that should log and continue instead of rejecting the request for a specific namespace. Valid values: versioning-override, scheduler-duration.`, + `FrontendDisabledScheduleValidations is a list of schedule validation names that should log and continue instead of rejecting the request for a specific namespace. Valid values: versioning-override, scheduler-duration, scheduler-overlap-policy, scheduler-timestamp, scheduler-remaining-actions.`, ) FrontendHTTPAllowedHosts = NewGlobalTypedSettingWithConverter( "frontend.httpAllowedHosts", diff --git a/service/frontend/service.go b/service/frontend/service.go index f48d86c33bc..922fde36945 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -31,6 +31,9 @@ import ( const ( scheduleValidationVersioningOverride = "versioning-override" scheduleValidationScheduleDuration = "scheduler-duration" + scheduleValidationOverlapPolicy = "scheduler-overlap-policy" + scheduleValidationRemainingActions = "scheduler-remaining-actions" + scheduleValidationTimestamp = "scheduler-timestamp" ) // Config represents configuration for frontend service diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index 2a1230a3c04..d410c60dbae 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -3904,6 +3904,12 @@ func (wh *WorkflowHandler) CreateSchedule( if request.Schedule == nil { request.Schedule = &schedulepb.Schedule{} } + if err := wh.validateScheduleOverlapPolicies(request.Schedule, request.InitialPatch, namespaceName.String()); err != nil { + return nil, err + } + if err := wh.validateSchedulePatchTimestamps(request.InitialPatch, namespaceName.String()); err != nil { + return nil, err + } err := wh.canonicalizeScheduleSpec(request.Schedule, namespaceName.String()) if err != nil { return nil, err @@ -4719,6 +4725,9 @@ func (wh *WorkflowHandler) UpdateSchedule( if request.Schedule == nil { request.Schedule = &schedulepb.Schedule{} } + if err := wh.validateScheduleOverlapPolicies(request.Schedule, nil, namespaceName.String()); err != nil { + return nil, err + } err := wh.canonicalizeScheduleSpec(request.Schedule, namespaceName.String()) if err != nil { return nil, err @@ -4865,6 +4874,12 @@ func (wh *WorkflowHandler) PatchSchedule( len(request.Patch.Unpause) > common.ScheduleNotesSizeLimit { return nil, errNotesTooLong } + if err := wh.validateScheduleOverlapPolicies(nil, request.Patch, request.Namespace); err != nil { + return nil, err + } + if err := wh.validateSchedulePatchTimestamps(request.Patch, request.Namespace); err != nil { + return nil, err + } if trigger := request.Patch.GetTriggerImmediately(); trigger != nil && trigger.ScheduledTime == nil { trigger.ScheduledTime = timestamppb.Now() @@ -4972,6 +4987,9 @@ func (wh *WorkflowHandler) ListScheduleMatchingTimes(ctx context.Context, reques if !wh.config.EnableSchedules(request.Namespace) { return nil, errSchedulesNotAllowed } + if err := wh.validateScheduleMatchingTimesTimestamps(request); err != nil { + return nil, err + } // Prefer CHASM scheduler if enabled. if wh.chasmSchedulerEnabled(ctx, request.Namespace) { @@ -5281,15 +5299,18 @@ func (wh *WorkflowHandler) listSchedulesChasm( // versions can be returned. listInfo := ex.ChasmMemo // V2 customMemo := ex.Memo - if listInfo.GetSpec() == nil { + isV1Schedule := listInfo.GetSpec() == nil + if isV1Schedule { listInfo = wh.decodeScheduleListInfo(customMemo) // V1 wh.cleanScheduleMemo(customMemo) } else { scheduler.CleanSpec(listInfo.Spec) // done as part of decodeScheduleListInfo for V1 } - workflowID := ex.BusinessID - scheduleID := strings.TrimPrefix(workflowID, scheduler.WorkflowIDPrefix) // needed for V1 schedules, not CHASM + scheduleID := ex.BusinessID + if isV1Schedule { + scheduleID = strings.TrimPrefix(scheduleID, scheduler.WorkflowIDPrefix) + } schedules[i] = &schedulepb.ScheduleListEntry{ ScheduleId: scheduleID, @@ -7004,10 +7025,119 @@ func validateScheduleIntervalDurations(spec *schedulepb.ScheduleSpec) error { return nil } +func validateScheduleTimestamps(spec *schedulepb.ScheduleSpec) error { + if err := validateTimestamp(spec.GetStartTime(), "start time"); err != nil { + return err + } + return validateTimestamp(spec.GetEndTime(), "end time") +} + +func validateTimestamp(value *timestamppb.Timestamp, field string) error { + if value != nil { + if err := value.CheckValid(); err != nil { + return fmt.Errorf("%s is not a valid timestamp: %w", field, err) + } + } + return nil +} + +func validateScheduleRemainingActions(schedule *schedulepb.Schedule) error { + if schedule.GetState().GetRemainingActions() < 0 { + return errors.New("remaining actions cannot be negative") + } + return nil +} + +func validateScheduleOverlapPolicy(policy enumspb.ScheduleOverlapPolicy, field string) error { + if _, ok := enumspb.ScheduleOverlapPolicy_name[int32(policy)]; !ok { + return fmt.Errorf("%s has unsupported overlap policy %v", field, policy) + } + return nil +} + +func (wh *WorkflowHandler) validateScheduleOverlapPolicies( + schedule *schedulepb.Schedule, + patch *schedulepb.SchedulePatch, + namespaceName string, +) error { + if schedule != nil { + if err := validateScheduleOverlapPolicy(schedule.GetPolicies().GetOverlapPolicy(), "schedule policies"); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationOverlapPolicy, namespaceName) + } + } + if patch == nil { + return nil + } + if trigger := patch.GetTriggerImmediately(); trigger != nil { + if err := validateScheduleOverlapPolicy(trigger.GetOverlapPolicy(), "trigger immediately request"); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationOverlapPolicy, namespaceName) + } + } + for i, backfill := range patch.GetBackfillRequest() { + if err := validateScheduleOverlapPolicy(backfill.GetOverlapPolicy(), fmt.Sprintf("backfill request %d", i)); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationOverlapPolicy, namespaceName) + } + } + return nil +} + +func (wh *WorkflowHandler) validateSchedulePatchTimestamps( + patch *schedulepb.SchedulePatch, + namespaceName string, +) error { + if patch == nil { + return nil + } + if trigger := patch.GetTriggerImmediately(); trigger != nil { + if err := validateTimestamp(trigger.GetScheduledTime(), "trigger immediately request scheduled time"); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationTimestamp, namespaceName) + } + } + for i, backfill := range patch.GetBackfillRequest() { + if err := validateTimestamp(backfill.GetStartTime(), fmt.Sprintf("backfill request %d start time", i)); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationTimestamp, namespaceName) + } + if err := validateTimestamp(backfill.GetEndTime(), fmt.Sprintf("backfill request %d end time", i)); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationTimestamp, namespaceName) + } + } + return nil +} + +func (wh *WorkflowHandler) validateScheduleMatchingTimesTimestamps( + request *workflowservice.ListScheduleMatchingTimesRequest, +) error { + if err := validateTimestamp(request.GetStartTime(), "start time"); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationTimestamp, request.GetNamespace()) + } + if err := validateTimestamp(request.GetEndTime(), "end time"); err != nil { + return wh.handleScheduleValidationError(err, scheduleValidationTimestamp, request.GetNamespace()) + } + return nil +} + +func (wh *WorkflowHandler) handleScheduleValidationError(err error, validation, namespaceName string) error { + if !wh.config.IsScheduleValidationDisabled(validation, namespaceName) { + return serviceerror.NewInvalidArgumentf("Invalid schedule: %v", err) + } + wh.throttledLogger.Warn( + "Ignoring disabled schedule validation", + tag.WorkflowNamespace(namespaceName), + tag.NewStringTag("validation", validation), + tag.Error(err), + ) + return nil +} + func (wh *WorkflowHandler) canonicalizeScheduleSpec(schedule *schedulepb.Schedule, namespaceName string) error { if schedule.Spec == nil { schedule.Spec = &schedulepb.ScheduleSpec{} } + if err := validateScheduleRemainingActions(schedule); err != nil { + if err := wh.handleScheduleValidationError(err, scheduleValidationRemainingActions, namespaceName); err != nil { + return err + } + } if err := validateScheduleIntervalDurations(schedule.Spec); err != nil { if !wh.config.IsScheduleValidationDisabled(scheduleValidationScheduleDuration, namespaceName) { return serviceerror.NewInvalidArgumentf("Invalid schedule spec: %v", err) @@ -7019,6 +7149,11 @@ func (wh *WorkflowHandler) canonicalizeScheduleSpec(schedule *schedulepb.Schedul tag.Error(err), ) } + if err := validateScheduleTimestamps(schedule.Spec); err != nil { + if err := wh.handleScheduleValidationError(err, scheduleValidationTimestamp, namespaceName); err != nil { + return err + } + } compiledSpec, err := wh.scheduleSpecBuilder.NewCompiledSpec(schedule.Spec) if err != nil { return serviceerror.NewInvalidArgumentf("Invalid schedule spec: %v", err) diff --git a/service/frontend/workflow_handler_second_test.go b/service/frontend/workflow_handler_second_test.go index ae85f18a36d..be88a473c1a 100644 --- a/service/frontend/workflow_handler_second_test.go +++ b/service/frontend/workflow_handler_second_test.go @@ -9,12 +9,17 @@ import ( enumspb "go.temporal.io/api/enums/v1" schedulepb "go.temporal.io/api/schedule/v1" workflowpb "go.temporal.io/api/workflow/v1" + "go.temporal.io/api/workflowservice/v1" + chasmspb "go.temporal.io/server/api/chasm/v1" schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/api/visibilityservice/v1" + "go.temporal.io/server/chasm" dc "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/service/worker/scheduler" "go.uber.org/mock/gomock" "google.golang.org/grpc/metadata" ) @@ -79,6 +84,47 @@ func TestCHASMSchedulerRoutingAndCreationGates(t *testing.T) { } } +func TestListSchedulesChasmPreservesV2ScheduleIDs(t *testing.T) { + controller := gomock.NewController(t) + chasmVisibilityManager := chasm.NewMockVisibilityManager(controller) + + v2ListInfoPayload, err := payload.Encode(&schedulepb.ScheduleListInfo{Spec: &schedulepb.ScheduleSpec{}}) + require.NoError(t, err) + v1ChasmMemoPayload, err := payload.Encode(&schedulepb.ScheduleListInfo{}) + require.NoError(t, err) + v1ListInfoBytes, err := (&schedulepb.ScheduleListInfo{Spec: &schedulepb.ScheduleSpec{}}).Marshal() + require.NoError(t, err) + + chasmVisibilityManager.EXPECT().ListExecutions(gomock.Any(), gomock.Any(), gomock.Any()).Return( + &visibilityservice.ListChasmExecutionsResponse{Executions: []*chasmspb.VisibilityExecutionInfo{ + {BusinessId: "foo", ChasmMemo: v2ListInfoPayload}, + {BusinessId: scheduler.WorkflowIDPrefix + "foo", ChasmMemo: v2ListInfoPayload}, + { + BusinessId: scheduler.WorkflowIDPrefix + "legacy", + ChasmMemo: v1ChasmMemoPayload, + Memo: &commonpb.Memo{Fields: map[string]*commonpb.Payload{ + scheduler.MemoFieldInfo: payload.EncodeBytes(v1ListInfoBytes), + }}, + }, + }}, nil, + ) + + wh := &WorkflowHandler{} + response, err := wh.listSchedulesChasm( + chasm.NewVisibilityManagerContext(context.Background(), chasmVisibilityManager), + &workflowservice.ListSchedulesRequest{MaximumPageSize: 10}, + "test-namespace", + "test-namespace-id", + "", + ) + require.NoError(t, err) + require.Equal(t, []string{"foo", scheduler.WorkflowIDPrefix + "foo", "legacy"}, []string{ + response.Schedules[0].ScheduleId, + response.Schedules[1].ScheduleId, + response.Schedules[2].ScheduleId, + }) +} + func TestDescribeScheduleAnnotatesScheduledWorkflowWithTypes(t *testing.T) { makeWorkflowHandler := func( visibilityManager *manager.MockVisibilityManager, diff --git a/service/frontend/workflow_handler_test.go b/service/frontend/workflow_handler_test.go index 386e18fadea..c6aee79dc7a 100644 --- a/service/frontend/workflow_handler_test.go +++ b/service/frontend/workflow_handler_test.go @@ -5779,6 +5779,168 @@ func TestCanonicalizeScheduleSpec_DurationValidationKillSwitch(t *testing.T) { } } +func TestValidateScheduleTimestamps(t *testing.T) { + for _, tc := range []struct { + name string + spec *schedulepb.ScheduleSpec + errString string + }{ + { + name: "start time", + spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }, + errString: "start time is not a valid timestamp", + }, + { + name: "end time", + spec: &schedulepb.ScheduleSpec{ + EndTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }, + errString: "end time is not a valid timestamp", + }, + } { + t.Run(tc.name, func(t *testing.T) { + require.ErrorContains(t, validateScheduleTimestamps(tc.spec), tc.errString) + }) + } +} + +func TestValidateScheduleMatchingTimesTimestamps(t *testing.T) { + for _, tc := range []struct { + name string + request *workflowservice.ListScheduleMatchingTimesRequest + errString string + }{ + { + name: "invalid start time", + request: &workflowservice.ListScheduleMatchingTimesRequest{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + EndTime: timestamppb.Now(), + }, + errString: "start time is not a valid timestamp", + }, + { + name: "invalid end time", + request: &workflowservice.ListScheduleMatchingTimesRequest{ + StartTime: timestamppb.Now(), + EndTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }, + errString: "end time is not a valid timestamp", + }, + { + name: "missing range", + request: &workflowservice.ListScheduleMatchingTimesRequest{}, + }, + { + name: "epoch range", + request: &workflowservice.ListScheduleMatchingTimesRequest{ + StartTime: ×tamppb.Timestamp{}, + EndTime: ×tamppb.Timestamp{}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + wh := newScheduleSpecHandler(nil) + err := wh.validateScheduleMatchingTimesTimestamps(tc.request) + if tc.errString == "" { + require.NoError(t, err) + return + } + + var invalidArgument *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, tc.errString) + }) + } +} + +func TestScheduleValidationKillSwitches(t *testing.T) { + testCases := []struct { + name string + validation string + errString string + schedule func() *schedulepb.Schedule + validate func(*WorkflowHandler, *schedulepb.Schedule) error + assertOK func(*testing.T, *schedulepb.Schedule) + }{ + { + name: "timestamp", + validation: scheduleValidationTimestamp, + errString: "start time is not a valid timestamp", + schedule: func() *schedulepb.Schedule { + return &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + EndTime: ×tamppb.Timestamp{Seconds: 3600, Nanos: 1_000_000_000}, + }} + }, + validate: func(wh *WorkflowHandler, schedule *schedulepb.Schedule) error { + return wh.canonicalizeScheduleSpec(schedule, "test-namespace") + }, + assertOK: func(t *testing.T, schedule *schedulepb.Schedule) { + require.Equal(t, time.Unix(1, 0).UTC(), schedule.GetSpec().GetStartTime().AsTime()) + require.Equal(t, time.Unix(3601, 0).UTC(), schedule.GetSpec().GetEndTime().AsTime()) + }, + }, + { + name: "remaining actions", + validation: scheduleValidationRemainingActions, + errString: "remaining actions cannot be negative", + schedule: func() *schedulepb.Schedule { + return &schedulepb.Schedule{State: &schedulepb.ScheduleState{RemainingActions: -1}} + }, + validate: func(wh *WorkflowHandler, schedule *schedulepb.Schedule) error { + return wh.canonicalizeScheduleSpec(schedule, "test-namespace") + }, + }, + { + name: "overlap policy", + validation: scheduleValidationOverlapPolicy, + errString: "unsupported overlap policy", + schedule: func() *schedulepb.Schedule { + return &schedulepb.Schedule{Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: enumspb.ScheduleOverlapPolicy(99), + }} + }, + validate: func(wh *WorkflowHandler, schedule *schedulepb.Schedule) error { + return wh.validateScheduleOverlapPolicies(schedule, nil, "test-namespace") + }, + }, + } + + for _, validation := range testCases { + for _, toggle := range []struct { + name string + disabled []string + wantError bool + }{ + {name: "enforced by default", wantError: true}, + {name: "unrelated validation disabled", disabled: []string{scheduleValidationScheduleDuration}, wantError: true}, + {name: "validation disabled", disabled: []string{validation.validation}}, + } { + t.Run(validation.name+"/"+toggle.name, func(t *testing.T) { + wh := newScheduleSpecHandler(func(c *Config) { + if toggle.disabled != nil { + c.DisabledScheduleValidations = dc.GetTypedPropertyFnFilteredByNamespace(toggle.disabled) + } + }) + schedule := validation.schedule() + err := validation.validate(wh, schedule) + if toggle.wantError { + var invalidArgument *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, validation.errString) + return + } + require.NoError(t, err) + if validation.assertOK != nil { + validation.assertOK(t, schedule) + } + }) + } + } +} + // Regression test for SCH-057: CreateSchedule and UpdateSchedule must reject malformed // interval duration protobufs with InvalidArgument before either the V1 or the CHASM // backend is invoked. @@ -5833,6 +5995,169 @@ func (s *WorkflowHandlerSuite) TestCreateUpdateSchedule_RejectsMalformedInterval }) } +func (s *WorkflowHandlerSuite) TestScheduleValidation() { + config := s.newConfig() + config.EnableSchedules = dc.GetBoolPropertyFnFilteredByNamespace(true) + wh := s.getWorkflowHandler(config) + ctx := context.Background() + create := func(schedule *schedulepb.Schedule, initialPatch *schedulepb.SchedulePatch) error { + _, err := wh.CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + RequestId: uuid.NewString(), + Schedule: schedule, + InitialPatch: initialPatch, + }) + return err + } + update := func(schedule *schedulepb.Schedule) error { + _, err := wh.UpdateSchedule(ctx, &workflowservice.UpdateScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + RequestId: uuid.NewString(), + Schedule: schedule, + }) + return err + } + assertInvalidArgument := func(err error, want string) { + s.T().Helper() + var invalidArgument *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArgument) + s.ErrorContains(err, want) + } + + invalidPolicy := enumspb.ScheduleOverlapPolicy(99) + malformedTimestamp := func() *schedulepb.Schedule { + return &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + EndTime: ×tamppb.Timestamp{Seconds: 3600, Nanos: 1_000_000_000}, + }} + } + negativeRemainingActions := func() *schedulepb.Schedule { + return &schedulepb.Schedule{State: &schedulepb.ScheduleState{RemainingActions: -1}} + } + invalidSchedulePolicy := func() *schedulepb.Schedule { + return &schedulepb.Schedule{Policies: &schedulepb.SchedulePolicies{OverlapPolicy: invalidPolicy}} + } + invalidBackfillPolicy := func() *schedulepb.SchedulePatch { + return &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + OverlapPolicy: invalidPolicy, + }}} + } + invalidBackfillTimestamp := func() *schedulepb.SchedulePatch { + return &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}} + } + invalidTriggerTimestamp := func() *schedulepb.SchedulePatch { + return &schedulepb.SchedulePatch{TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{ + ScheduledTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }} + } + + // No backend call is expected: every case is rejected before either schedule backend is selected. + for _, tc := range []struct { + name string + errString string + invoke func() error + }{ + { + name: "CreateSchedule timestamp", + errString: "start time is not a valid timestamp", + invoke: func() error { return create(malformedTimestamp(), nil) }, + }, + { + name: "UpdateSchedule timestamp", + errString: "start time is not a valid timestamp", + invoke: func() error { return update(malformedTimestamp()) }, + }, + { + name: "CreateSchedule remaining actions", + errString: "remaining actions cannot be negative", + invoke: func() error { return create(negativeRemainingActions(), nil) }, + }, + { + name: "UpdateSchedule remaining actions", + errString: "remaining actions cannot be negative", + invoke: func() error { return update(negativeRemainingActions()) }, + }, + { + name: "CreateSchedule schedule policy", + errString: "unsupported overlap policy", + invoke: func() error { return create(invalidSchedulePolicy(), nil) }, + }, + { + name: "CreateSchedule initial patch", + errString: "unsupported overlap policy", + invoke: func() error { return create(&schedulepb.Schedule{}, invalidBackfillPolicy()) }, + }, + { + name: "CreateSchedule initial patch timestamp", + errString: "backfill request 0 start time is not a valid timestamp", + invoke: func() error { return create(&schedulepb.Schedule{}, invalidBackfillTimestamp()) }, + }, + { + name: "UpdateSchedule policy", + errString: "unsupported overlap policy", + invoke: func() error { return update(invalidSchedulePolicy()) }, + }, + { + name: "PatchSchedule trigger", + errString: "unsupported overlap policy", + invoke: func() error { + _, err := wh.PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + Patch: &schedulepb.SchedulePatch{TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{ + OverlapPolicy: invalidPolicy, + }}, + }) + return err + }, + }, + { + name: "PatchSchedule trigger timestamp", + errString: "trigger immediately request scheduled time is not a valid timestamp", + invoke: func() error { + _, err := wh.PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + Patch: invalidTriggerTimestamp(), + }) + return err + }, + }, + { + name: "PatchSchedule backfill", + errString: "unsupported overlap policy", + invoke: func() error { + _, err := wh.PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + Patch: invalidBackfillPolicy(), + }) + return err + }, + }, + { + name: "PatchSchedule backfill timestamp", + errString: "backfill request 0 start time is not a valid timestamp", + invoke: func() error { + _, err := wh.PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.testNamespace.String(), + ScheduleId: "test-schedule", + Patch: invalidBackfillTimestamp(), + }) + return err + }, + }, + } { + s.Run(tc.name, func() { + assertInvalidArgument(tc.invoke(), tc.errString) + }) + } +} + func (s *WorkflowHandlerSuite) TestUpdateSchedule_ValidationAndErrors() { config := s.newConfig() config.EnableSchedules = dc.GetBoolPropertyFnFilteredByNamespace(true) diff --git a/tests/schedule_test.go b/tests/schedule_test.go index 06f68e20b25..6abb995b528 100644 --- a/tests/schedule_test.go +++ b/tests/schedule_test.go @@ -2016,6 +2016,7 @@ func testListScheduleMatchingTimes(t *testing.T, newContext contextFactory) { schedule := &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{}, Interval: []*schedulepb.IntervalSpec{ {Interval: durationpb.New(1 * time.Hour)}, }, @@ -2047,15 +2048,49 @@ func testListScheduleMatchingTimes(t *testing.T, newContext contextFactory) { startTime := timestamppb.New(now) endTime := timestamppb.New(now.Add(5 * time.Hour)) - resp, err := s.FrontendClient().ListScheduleMatchingTimes(ctx, &workflowservice.ListScheduleMatchingTimesRequest{ - Namespace: s.Namespace().String(), - ScheduleId: sid, - StartTime: startTime, - EndTime: endTime, - }) - s.NoError(err) - // With 1-hour interval over 5 hours, we expect 5 matching times. - s.Len(resp.GetStartTime(), 5) + for _, tc := range []struct { + name string + startTime *timestamppb.Timestamp + endTime *timestamppb.Timestamp + expectedTimes int + errorMessage string + }{ + { + name: "valid range", + startTime: startTime, + endTime: endTime, + expectedTimes: 5, + }, + { + name: "epoch range", + startTime: ×tamppb.Timestamp{}, + endTime: ×tamppb.Timestamp{}, + }, + { + name: "invalid start time", + startTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + endTime: endTime, + errorMessage: "start time is not a valid timestamp", + }, + } { + t.Run(tc.name, func(t *testing.T) { + resp, err := s.FrontendClient().ListScheduleMatchingTimes(ctx, &workflowservice.ListScheduleMatchingTimesRequest{ + Namespace: s.Namespace().String(), + ScheduleId: sid, + StartTime: tc.startTime, + EndTime: tc.endTime, + }) + if tc.errorMessage != "" { + var invalidArgument *serviceerror.InvalidArgument + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, tc.errorMessage) + return + } + + require.NoError(t, err) + require.Len(t, resp.GetStartTime(), tc.expectedTimes) + }) + } } func testLimitMemoSpecSize(t *testing.T, newContext contextFactory) { @@ -5280,6 +5315,285 @@ func TestScheduleNextActionTimeVisibility(t *testing.T) { v2Sid, query) } +func TestListSchedulesPreservesV2ScheduleIDWithV1Prefix(t *testing.T) { + s := newScheduleEnv(t, scheduleCommonOpts(t)...) + ctx := chasmContextFactory(testcontext.For(t)) + + suffix := testcore.RandomizeStr("sched-list-v2-prefix") + plainID := "foo-" + suffix + prefixedID := scheduler.WorkflowIDPrefix + plainID + for _, scheduleID := range []string{plainID, prefixedID} { + createSchedule(ctx, t, s, scheduleID, &schedulepb.Schedule{ + Spec: intervalSpec(noOpInterval), + State: &schedulepb.ScheduleState{Paused: true}, + Action: startWorkflowAction(s, "wf-"+scheduleID, "wt-"+scheduleID), + }) + } + + require.Eventually(t, func() bool { + response, err := s.FrontendClient().ListSchedules(ctx, &workflowservice.ListSchedulesRequest{ + Namespace: s.Namespace().String(), + MaximumPageSize: 10, + }) + if err != nil { + return false + } + ids := make(map[string]struct{}, len(response.Schedules)) + for _, schedule := range response.Schedules { + ids[schedule.GetScheduleId()] = struct{}{} + } + _, foundPlainID := ids[plainID] + _, foundPrefixedID := ids[prefixedID] + return foundPlainID && foundPrefixedID + }, awaitTimeout, pollInterval, + "ListSchedules must preserve both V2 IDs %q and %q", plainID, prefixedID) +} + +func TestScheduleRejectsInvalidRequests(t *testing.T) { + s := newScheduleEnv(t, scheduleCommonOpts(t)...) + ctx := chasmContextFactory(testcontext.For(t)) + invalidPolicy := enumspb.ScheduleOverlapPolicy(99) + + validSchedule := func(scheduleID string) *schedulepb.Schedule { + return &schedulepb.Schedule{ + Spec: intervalSpec(noOpInterval), + State: &schedulepb.ScheduleState{Paused: true}, + Action: startWorkflowAction(s, "wf-"+scheduleID, "wt-"+scheduleID), + } + } + + testCases := []struct { + name string + requiresExisting bool + errorMessage string + request func(scheduleID string) error + }{ + { + name: "CreateSchedule schedule policy", + errorMessage: "unsupported overlap policy", + request: func(scheduleID string) error { + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: invalidPolicy, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "CreateSchedule initial patch", + errorMessage: "unsupported overlap policy", + request: func(scheduleID string) error { + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{}, + InitialPatch: &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + OverlapPolicy: invalidPolicy, + }}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "CreateSchedule initial patch timestamp", + errorMessage: "backfill request 0 start time is not a valid timestamp", + request: func(scheduleID string) error { + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{}, + InitialPatch: &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "CreateSchedule timestamp", + errorMessage: "start time is not a valid timestamp", + request: func(scheduleID string) error { + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "CreateSchedule remaining actions", + errorMessage: "remaining actions cannot be negative", + request: func(scheduleID string) error { + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{State: &schedulepb.ScheduleState{RemainingActions: -1}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "UpdateSchedule schedule policy", + requiresExisting: true, + errorMessage: "unsupported overlap policy", + request: func(scheduleID string) error { + _, err := s.FrontendClient().UpdateSchedule(ctx, &workflowservice.UpdateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: invalidPolicy, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "UpdateSchedule timestamp", + requiresExisting: true, + errorMessage: "start time is not a valid timestamp", + request: func(scheduleID string) error { + _, err := s.FrontendClient().UpdateSchedule(ctx, &workflowservice.UpdateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{Spec: &schedulepb.ScheduleSpec{ + StartTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "UpdateSchedule remaining actions", + requiresExisting: true, + errorMessage: "remaining actions cannot be negative", + request: func(scheduleID string) error { + _, err := s.FrontendClient().UpdateSchedule(ctx, &workflowservice.UpdateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Schedule: &schedulepb.Schedule{State: &schedulepb.ScheduleState{RemainingActions: -1}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "PatchSchedule trigger", + requiresExisting: true, + errorMessage: "unsupported overlap policy", + request: func(scheduleID string) error { + _, err := s.FrontendClient().PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Patch: &schedulepb.SchedulePatch{TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{ + OverlapPolicy: invalidPolicy, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "PatchSchedule backfill", + requiresExisting: true, + errorMessage: "unsupported overlap policy", + request: func(scheduleID string) error { + _, err := s.FrontendClient().PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Patch: &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + OverlapPolicy: invalidPolicy, + }}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "PatchSchedule trigger timestamp", + requiresExisting: true, + errorMessage: "trigger immediately request scheduled time is not a valid timestamp", + request: func(scheduleID string) error { + _, err := s.FrontendClient().PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Patch: &schedulepb.SchedulePatch{TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{ + ScheduledTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + { + name: "PatchSchedule backfill timestamp", + requiresExisting: true, + errorMessage: "backfill request 0 end time is not a valid timestamp", + request: func(scheduleID string) error { + _, err := s.FrontendClient().PatchSchedule(ctx, &workflowservice.PatchScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + Patch: &schedulepb.SchedulePatch{BackfillRequest: []*schedulepb.BackfillRequest{{ + EndTime: ×tamppb.Timestamp{Nanos: 1_000_000_000}, + }}}, + RequestId: uuid.NewString(), + Identity: "test", + }) + return err + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + scheduleID := testcore.RandomizeStr("sched-invalid-request") + if tc.requiresExisting { + createSchedule(ctx, t, s, scheduleID, validSchedule(scheduleID)) + } + + var invalidArgument *serviceerror.InvalidArgument + err := tc.request(scheduleID) + require.ErrorAs(t, err, &invalidArgument) + require.ErrorContains(t, err, tc.errorMessage) + + describeResponse, err := s.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: scheduleID, + }) + if tc.requiresExisting { + require.NoError(t, err) + require.NotNil(t, describeResponse) + return + } + var notFound *serviceerror.NotFound + require.ErrorAs(t, err, ¬Found) + }) + } +} + // TestMirroredIncludeExcludeSpec sets identical interval and exclusion // specifications that match every 1s, effectively cancelling each other out. func TestMirroredIncludeExcludeSpec(t *testing.T) { From a3ebb1052dfa5f7089b8259ec9ca7a2854ff9e6e Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 31 Aug 2026 09:23:45 -0700 Subject: [PATCH 096/139] Programmable grpc fault injection (#9076) ## What changed? Add gRPC fault injection for functional tests. ## Why? Today this only addresses a single use case, but going forward it offers the ability to write tests for error cases that were difficult to test before (at least not without brittle and slow `time.Sleep`s). ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [x] added new functional test(s) --- common/rpc/grpcfaults/generator.go | 242 +++++++++++++++ common/rpc/grpcfaults/generator_test.go | 280 ++++++++++++++++++ common/rpc/grpcfaults/interceptor.go | 45 +++ common/rpc/grpcfaults/interceptor_test.go | 91 ++++++ common/testing/grpcfaultstest/hooks_noop.go | 18 ++ .../testing/grpcfaultstest/hooks_testdep.go | 98 ++++++ .../grpcfaultstest/hooks_testdep_test.go | 131 ++++++++ common/testing/testhooks/hooks.go | 35 ++- docs/development/testing.md | 25 ++ service/frontend/fx.go | 7 + service/fx.go | 9 + tests/testcore/fault_injection.go | 51 ++++ tests/testcore/onebox.go | 8 + tests/testcore/test_env.go | 41 +++ tests/update_workflow_test.go | 103 +++---- 15 files changed, 1104 insertions(+), 80 deletions(-) create mode 100644 common/rpc/grpcfaults/generator.go create mode 100644 common/rpc/grpcfaults/generator_test.go create mode 100644 common/rpc/grpcfaults/interceptor.go create mode 100644 common/rpc/grpcfaults/interceptor_test.go create mode 100644 common/testing/grpcfaultstest/hooks_noop.go create mode 100644 common/testing/grpcfaultstest/hooks_testdep.go create mode 100644 common/testing/grpcfaultstest/hooks_testdep_test.go create mode 100644 tests/testcore/fault_injection.go diff --git a/common/rpc/grpcfaults/generator.go b/common/rpc/grpcfaults/generator.go new file mode 100644 index 00000000000..ed24a5be0b7 --- /dev/null +++ b/common/rpc/grpcfaults/generator.go @@ -0,0 +1,242 @@ +package grpcfaults + +import ( + "context" + "sync" + + "go.temporal.io/server/common/namespace" +) + +type ( + namespaceNameGetter interface { + GetNamespace() string + } + + namespaceIDGetter interface { + GetNamespaceId() string + } +) + +type faultStage int + +const ( + faultStageRequest faultStage = iota + faultStageResponse +) + +// Outcome contains the response and error returned when a gRPC fault matches. +type Outcome struct { + Response any + Error error +} + +// RequestCallback is a callback function for pre-handler gRPC fault injection. +type RequestCallback func(ctx context.Context, fullMethod string, req any) *Outcome + +// ResponseCallback is a callback function for post-handler gRPC fault injection. +type ResponseCallback func(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome + +// Hooks connects a CallbackGenerator to externally managed fault callbacks. +type Hooks interface { + InstallRequestCallback(Scope, RequestCallback) func() + InstallResponseCallback(Scope, ResponseCallback) func() +} + +type callback func(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome + +// callbackEntry represents a registered gRPC callback with its ID. +type callbackEntry struct { + id uint64 + callback callback +} + +// Scope identifies a namespace by ID, name, or both. An empty scope applies globally. +type Scope struct { + NamespaceID namespace.ID + NamespaceName namespace.Name +} + +type callbackScope struct { + Scope + stage faultStage +} + +type callbackBucket struct { + callbacks []callbackEntry + unregister func() +} + +// CallbackGenerator handles fault injection for gRPC requests and responses. +type CallbackGenerator struct { + mu sync.RWMutex + callbacks map[callbackScope]*callbackBucket + nextID uint64 + hooks Hooks +} + +// NewCallbackGenerator creates a new CallbackGenerator instance. +func NewCallbackGenerator() *CallbackGenerator { + return NewCallbackGeneratorWithHooks(nil) +} + +// NewCallbackGeneratorWithHooks creates a CallbackGenerator connected to external fault hooks. +func NewCallbackGeneratorWithHooks(hooks Hooks) *CallbackGenerator { + return &CallbackGenerator{ + callbacks: make(map[callbackScope]*callbackBucket), + hooks: hooks, + } +} + +// RegisterRequestCallback registers a pre-handler gRPC fault injection callback and returns a +// cleanup function that removes the callback when called. +func (r *CallbackGenerator) RegisterRequestCallback(scope Scope, cb RequestCallback) func() { + return r.registerCallback(scope, faultStageRequest, func(ctx context.Context, fullMethod string, req, _ any, _ error) *Outcome { + return cb(ctx, fullMethod, req) + }) +} + +// RegisterResponseCallback registers a post-handler gRPC fault injection callback and returns a +// cleanup function that removes the callback when called. +func (r *CallbackGenerator) RegisterResponseCallback(scope Scope, cb ResponseCallback) func() { + return r.registerCallback(scope, faultStageResponse, func(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome { + return cb(ctx, fullMethod, req, resp, err) + }) +} + +func (r *CallbackGenerator) registerCallback(faultScope Scope, stage faultStage, cb callback) func() { + if r == nil { + return func() {} + } + + var callbackScopes []callbackScope + if faultScope.NamespaceID != "" { + callbackScopes = append(callbackScopes, callbackScope{Scope: Scope{NamespaceID: faultScope.NamespaceID}, stage: stage}) + } + if faultScope.NamespaceName != "" { + callbackScopes = append(callbackScopes, callbackScope{Scope: Scope{NamespaceName: faultScope.NamespaceName}, stage: stage}) + } + if len(callbackScopes) == 0 { + callbackScopes = append(callbackScopes, callbackScope{stage: stage}) + } + + r.mu.Lock() + r.nextID++ + entry := callbackEntry{id: r.nextID, callback: cb} + for _, callbackScope := range callbackScopes { + bucket := r.callbacks[callbackScope] + if bucket == nil { + bucket = &callbackBucket{} + if r.hooks != nil { + bucket.unregister = r.installHooks(callbackScope) + } + r.callbacks[callbackScope] = bucket + } + bucket.callbacks = append(bucket.callbacks, entry) + } + r.mu.Unlock() + + return func() { + r.unregisterCallback(callbackScopes, entry.id) + } +} + +func (r *CallbackGenerator) unregisterCallback(callbackScopes []callbackScope, id uint64) { + r.mu.Lock() + defer r.mu.Unlock() + for _, callbackScope := range callbackScopes { + bucket := r.callbacks[callbackScope] + if bucket == nil { + continue + } + for i, existing := range bucket.callbacks { + if existing.id == id { + bucket.callbacks = append(bucket.callbacks[:i], bucket.callbacks[i+1:]...) + break + } + } + if len(bucket.callbacks) == 0 { + if bucket.unregister != nil { + bucket.unregister() + } + delete(r.callbacks, callbackScope) + } + } +} + +func (r *CallbackGenerator) installHooks(scope callbackScope) func() { + if scope.stage == faultStageRequest { + return r.hooks.InstallRequestCallback(scope.Scope, func(ctx context.Context, fullMethod string, req any) *Outcome { + return r.generate(ctx, fullMethod, scope, req, nil, nil) + }) + } + return r.hooks.InstallResponseCallback(scope.Scope, func(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome { + return r.generate(ctx, fullMethod, scope, req, resp, err) + }) +} + +// GenerateRequest checks registered gRPC callbacks before the handler runs. +func (r *CallbackGenerator) GenerateRequest(ctx context.Context, fullMethod string, req any) *Outcome { + return r.generateFaultForStage(ctx, fullMethod, faultStageRequest, req, nil, nil) +} + +// GenerateResponse checks registered gRPC callbacks after the handler runs. +func (r *CallbackGenerator) GenerateResponse(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome { + return r.generateFaultForStage(ctx, fullMethod, faultStageResponse, req, resp, err) +} + +func (r *CallbackGenerator) generateFaultForStage(ctx context.Context, fullMethod string, stage faultStage, req, resp any, err error) *Outcome { + if namespaceID, ok := namespaceIDFromRequest(req); ok { + if outcome := r.generate(ctx, fullMethod, callbackScope{ + Scope: Scope{NamespaceID: namespaceID}, + stage: stage, + }, req, resp, err); outcome != nil { + return outcome + } + } else if namespaceName, ok := namespaceNameFromRequest(req); ok { + if outcome := r.generate(ctx, fullMethod, callbackScope{ + Scope: Scope{NamespaceName: namespaceName}, + stage: stage, + }, req, resp, err); outcome != nil { + return outcome + } + } + return r.generate(ctx, fullMethod, callbackScope{stage: stage}, req, resp, err) +} + +func (r *CallbackGenerator) generate(ctx context.Context, fullMethod string, scope callbackScope, req, resp any, err error) *Outcome { + if r == nil { + return nil + } + r.mu.RLock() + bucket := r.callbacks[scope] + if bucket == nil || len(bucket.callbacks) == 0 { + r.mu.RUnlock() + return nil + } + callbacks := make([]callbackEntry, len(bucket.callbacks)) + copy(callbacks, bucket.callbacks) + r.mu.RUnlock() + + for _, entry := range callbacks { + if outcome := entry.callback(ctx, fullMethod, req, resp, err); outcome != nil { + return outcome + } + } + return nil +} + +func namespaceIDFromRequest(req any) (namespace.ID, bool) { + request, ok := req.(namespaceIDGetter) + if !ok || request.GetNamespaceId() == "" { + return "", false + } + return namespace.ID(request.GetNamespaceId()), true +} + +func namespaceNameFromRequest(req any) (namespace.Name, bool) { + request, ok := req.(namespaceNameGetter) + if !ok || request.GetNamespace() == "" { + return "", false + } + return namespace.Name(request.GetNamespace()), true +} diff --git a/common/rpc/grpcfaults/generator_test.go b/common/rpc/grpcfaults/generator_test.go new file mode 100644 index 00000000000..e1b097b0d06 --- /dev/null +++ b/common/rpc/grpcfaults/generator_test.go @@ -0,0 +1,280 @@ +//go:build test_dep + +package grpcfaults_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/rpc/grpcfaults" + "go.temporal.io/server/common/testing/await" +) + +func TestCallbackGenerator_Generate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fullMethod := "/test.Service/Method" + handlerResponse := &struct{}{} + handlerErr := errors.New("handler") + injectedErr := errors.New("injected") + bothNamespaces := grpcfaults.Scope{NamespaceID: "namespace-id", NamespaceName: "namespace-name"} + + type callback struct { + name string + scope grpcfaults.Scope + miss bool + response any + err error + } + + for _, stage := range []struct { + name string + response bool + }{ + { + name: "request", + }, + { + name: "response", + response: true, + }, + } { + stage := stage + for _, test := range []struct { + name string + request any + callbacks []callback + expectedMiss bool + expectedResp any + expectedErr error + expectedCalls []string + }{ + { + name: "no callbacks", + request: "request", + expectedMiss: true, + }, + { + name: "namespace before global", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + callbacks: []callback{ + {name: "namespace", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, response: "namespace response"}, + {name: "global", response: "global response"}, + }, + expectedResp: "namespace response", + expectedCalls: []string{"namespace"}, + }, + { + name: "global after namespace miss", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + callbacks: []callback{ + {name: "namespace", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, miss: true}, + {name: "global", response: "response"}, + }, + expectedResp: "response", + expectedCalls: []string{"namespace", "global"}, + }, + { + name: "namespace mismatch", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "other-namespace-id"}, + callbacks: []callback{ + {name: "namespace", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, err: injectedErr}, + }, + expectedMiss: true, + }, + { + name: "matched error", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + callbacks: []callback{ + {name: "namespace", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, err: injectedErr}, + }, + expectedErr: injectedErr, + expectedCalls: []string{"namespace"}, + }, + { + name: "first match wins", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + callbacks: []callback{ + {name: "first", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, miss: true}, // no match! + {name: "second", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, response: "response"}, + {name: "third", scope: grpcfaults.Scope{NamespaceID: "namespace-id"}, response: "other response"}, // never reached! + }, + expectedResp: "response", + expectedCalls: []string{"first", "second"}, + }, + { + name: "scope with namespace ID and name/namespace ID", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + callbacks: []callback{ + {name: "namespace", scope: bothNamespaces, response: "response"}, + }, + expectedResp: "response", + expectedCalls: []string{"namespace"}, + }, + { + name: "scope with namespace ID and name/namespace name", + request: &workflowservice.UpdateNamespaceRequest{Namespace: "namespace-name"}, + callbacks: []callback{ + {name: "namespace", scope: bothNamespaces, response: "response"}, + }, + expectedResp: "response", + expectedCalls: []string{"namespace"}, + }, + } { + test := test + t.Run(stage.name+"/"+test.name, func(t *testing.T) { + t.Parallel() + + generator := grpcfaults.NewCallbackGenerator() + var calls []string + invoke := func(callback callback, actualCtx context.Context, actualMethod string, actualRequest, actualResponse any, actualErr error) *grpcfaults.Outcome { + calls = append(calls, callback.name) + require.Equal(t, ctx, actualCtx) + require.Equal(t, fullMethod, actualMethod) + require.Equal(t, test.request, actualRequest) + if stage.response { + require.Equal(t, handlerResponse, actualResponse) + require.ErrorIs(t, actualErr, handlerErr) + } + if callback.miss { + return nil + } + return &grpcfaults.Outcome{Response: callback.response, Error: callback.err} + } + + for _, callback := range test.callbacks { + callback := callback + if stage.response { + generator.RegisterResponseCallback(callback.scope, func(actualCtx context.Context, actualMethod string, actualRequest, actualResponse any, actualErr error) *grpcfaults.Outcome { + return invoke(callback, actualCtx, actualMethod, actualRequest, actualResponse, actualErr) + }) + } else { + generator.RegisterRequestCallback(callback.scope, func(actualCtx context.Context, actualMethod string, actualRequest any) *grpcfaults.Outcome { + return invoke(callback, actualCtx, actualMethod, actualRequest, nil, nil) + }) + } + } + + var outcome *grpcfaults.Outcome + if stage.response { + outcome = generator.GenerateResponse(ctx, fullMethod, test.request, handlerResponse, handlerErr) + } else { + outcome = generator.GenerateRequest(ctx, fullMethod, test.request) + } + + if test.expectedMiss { + require.Nil(t, outcome) + require.Equal(t, test.expectedCalls, calls) + return + } + require.NotNil(t, outcome) + if test.expectedErr == nil { + require.NoError(t, outcome.Error) + } else { + require.ErrorIs(t, outcome.Error, test.expectedErr) + } + require.Equal(t, test.expectedResp, outcome.Response) + require.Equal(t, test.expectedCalls, calls) + }) + } + } +} + +func TestCallbackGenerator_Unregister(t *testing.T) { + t.Parallel() + + t.Run("idempotent", func(t *testing.T) { + t.Parallel() + + generator := grpcfaults.NewCallbackGenerator() + unregister := generator.RegisterRequestCallback(grpcfaults.Scope{NamespaceID: "namespace-id"}, func(context.Context, string, any) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Response: "response"} + }) + + unregister() + unregister() + outcome := generator.GenerateRequest( + context.Background(), + "/test.Service/Method", + &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + ) + + require.Nil(t, outcome) + }) + + t.Run("removes namespace ID and name registrations", func(t *testing.T) { + t.Parallel() + + generator := grpcfaults.NewCallbackGenerator() + unregister := generator.RegisterRequestCallback(grpcfaults.Scope{NamespaceID: "namespace-id", NamespaceName: "namespace-name"}, func(context.Context, string, any) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Response: "response"} + }) + unregister() + + for _, test := range []struct { + name string + request any + }{ + { + name: "namespace ID", + request: &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + }, + { + name: "namespace name", + request: &workflowservice.UpdateNamespaceRequest{Namespace: "namespace-name"}, + }, + } { + t.Run(test.name, func(t *testing.T) { + outcome := generator.GenerateRequest(context.Background(), "/test.Service/Method", test.request) + + require.Nil(t, outcome) + }) + } + }) + + t.Run("during generate", func(t *testing.T) { + t.Parallel() + + generator := grpcfaults.NewCallbackGenerator() + callbackStarted := make(chan struct{}) + continueCallback := make(chan struct{}) + unregister := generator.RegisterRequestCallback(grpcfaults.Scope{NamespaceID: "namespace-id"}, func(context.Context, string, any) *grpcfaults.Outcome { + await.Snd(t, callbackStarted, struct{}{}) + await.Rcv(t, continueCallback) + return &grpcfaults.Outcome{Response: "response"} + }) + type result struct { + outcome *grpcfaults.Outcome + } + resultCh := make(chan result, 1) + go func() { + outcome := generator.GenerateRequest( + context.Background(), + "/test.Service/Method", + &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + ) + await.Snd(t, resultCh, result{outcome: outcome}) + }() + + await.Rcv(t, callbackStarted) + unregister() + await.Snd(t, continueCallback, struct{}{}) + generateResult := await.Rcv(t, resultCh) + + require.NotNil(t, generateResult.outcome) + require.Equal(t, "response", generateResult.outcome.Response) + require.NoError(t, generateResult.outcome.Error) + + outcome := generator.GenerateRequest( + context.Background(), + "/test.Service/Method", + &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + ) + require.Nil(t, outcome) + }) +} diff --git a/common/rpc/grpcfaults/interceptor.go b/common/rpc/grpcfaults/interceptor.go new file mode 100644 index 00000000000..1d7828f65d7 --- /dev/null +++ b/common/rpc/grpcfaults/interceptor.go @@ -0,0 +1,45 @@ +package grpcfaults + +import ( + "context" + + "google.golang.org/grpc" +) + +// Generator checks for gRPC faults before and after a handler runs. +type Generator interface { + GenerateRequest(ctx context.Context, fullMethod string, req any) *Outcome + GenerateResponse(ctx context.Context, fullMethod string, req, resp any, err error) *Outcome +} + +// UnaryServerInterceptor returns a unary server interceptor that checks for +// dynamically registered fault injection callbacks before and after the handler. +// +// This allows tests and configured generators to register callbacks that can +// inspect requests/responses and inject faults on demand. +// +// Behavior: +// - If no request generator is registered, the handler proceeds normally. +// - Request callbacks are checked before the handler. If matched, the handler is skipped. +// - Response callbacks are checked after the handler with actual resp/err. If matched, returned values are used. +// - If no callbacks match, the handler's response/error is returned. +func UnaryServerInterceptor(generator Generator) grpc.UnaryServerInterceptor { + if generator == nil { + return nil + } + return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + // Check before handler (can short-circuit) + if outcome := generator.GenerateRequest(ctx, info.FullMethod, req); outcome != nil { + return outcome.Response, outcome.Error + } + + // Call handler + resp, err := handler(ctx, req) + + // Check after handler (can modify response/error) + if outcome := generator.GenerateResponse(ctx, info.FullMethod, req, resp, err); outcome != nil { + return outcome.Response, outcome.Error + } + return resp, err + } +} diff --git a/common/rpc/grpcfaults/interceptor_test.go b/common/rpc/grpcfaults/interceptor_test.go new file mode 100644 index 00000000000..7c803944aa6 --- /dev/null +++ b/common/rpc/grpcfaults/interceptor_test.go @@ -0,0 +1,91 @@ +package grpcfaults_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/rpc/grpcfaults" + "google.golang.org/grpc" +) + +func TestUnaryServerInterceptor_NilGenerator(t *testing.T) { + t.Parallel() + + require.Nil(t, grpcfaults.UnaryServerInterceptor(nil)) +} + +func TestUnaryServerInterceptor_ConfiguredBeforeHandler(t *testing.T) { + t.Parallel() + + injectedErr := errors.New("injected") + generator := grpcfaults.NewCallbackGenerator() + generator.RegisterRequestCallback(grpcfaults.Scope{}, func(context.Context, string, any) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Error: injectedErr} + }) + interceptor := grpcfaults.UnaryServerInterceptor(generator) + handlerCalled := false + + response, err := interceptor( + context.Background(), + "request", + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + handlerCalled = true + return "response", nil + }, + ) + + require.ErrorIs(t, err, injectedErr) + require.Nil(t, response) + require.False(t, handlerCalled) +} + +func TestUnaryServerInterceptor_ConfiguredAfterHandler(t *testing.T) { + t.Parallel() + + generator := grpcfaults.NewCallbackGenerator() + generator.RegisterResponseCallback(grpcfaults.Scope{}, func(context.Context, string, any, any, error) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Response: "replacement"} + }) + interceptor := grpcfaults.UnaryServerInterceptor(generator) + + response, err := interceptor( + context.Background(), + "request", + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + return "response", nil + }, + ) + + require.NoError(t, err) + require.Equal(t, "replacement", response) +} + +func TestUnaryServerInterceptor_ConfiguredHandlerError(t *testing.T) { + t.Parallel() + + handlerErr := errors.New("handler") + injectedErr := errors.New("injected") + generator := grpcfaults.NewCallbackGenerator() + generator.RegisterResponseCallback(grpcfaults.Scope{}, func(_ context.Context, _ string, _, response any, err error) *grpcfaults.Outcome { + require.Nil(t, response) + require.ErrorIs(t, err, handlerErr) + return &grpcfaults.Outcome{Error: injectedErr} + }) + interceptor := grpcfaults.UnaryServerInterceptor(generator) + + response, err := interceptor( + context.Background(), + "request", + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + return nil, handlerErr + }, + ) + + require.ErrorIs(t, err, injectedErr) + require.Nil(t, response) +} diff --git a/common/testing/grpcfaultstest/hooks_noop.go b/common/testing/grpcfaultstest/hooks_noop.go new file mode 100644 index 00000000000..a85dc153950 --- /dev/null +++ b/common/testing/grpcfaultstest/hooks_noop.go @@ -0,0 +1,18 @@ +//go:build !test_dep + +package grpcfaultstest + +import ( + "go.temporal.io/server/common/rpc/grpcfaults" + commontesthooks "go.temporal.io/server/common/testing/testhooks" +) + +// NewCallbackGenerator creates a CallbackGenerator without test hooks. +func NewCallbackGenerator(commontesthooks.TestHooks) *grpcfaults.CallbackGenerator { + return grpcfaults.NewCallbackGenerator() +} + +// NewGenerator returns nil when test hooks are disabled. +func NewGenerator(commontesthooks.TestHooks) grpcfaults.Generator { + return nil +} diff --git a/common/testing/grpcfaultstest/hooks_testdep.go b/common/testing/grpcfaultstest/hooks_testdep.go new file mode 100644 index 00000000000..840f1ca7a73 --- /dev/null +++ b/common/testing/grpcfaultstest/hooks_testdep.go @@ -0,0 +1,98 @@ +//go:build test_dep + +package grpcfaultstest + +import ( + "context" + + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/grpcfaults" + rpcinterceptor "go.temporal.io/server/common/rpc/interceptor" + commontesthooks "go.temporal.io/server/common/testing/testhooks" +) + +type adapter struct { + testHooks commontesthooks.TestHooks +} + +// NewCallbackGenerator creates a CallbackGenerator connected to namespace-scoped test hooks. +func NewCallbackGenerator(testHooks commontesthooks.TestHooks) *grpcfaults.CallbackGenerator { + return grpcfaults.NewCallbackGeneratorWithHooks(adapter{testHooks: testHooks}) +} + +func (a adapter) InstallRequestCallback(scope grpcfaults.Scope, callback grpcfaults.RequestCallback) func() { + switch { + case scope.NamespaceID != "": + return commontesthooks.Set(a.testHooks, commontesthooks.GRPCRequestFaultGeneratorByNamespaceID, callback, scope.NamespaceID) + case scope.NamespaceName != "": + return commontesthooks.Set(a.testHooks, commontesthooks.GRPCRequestFaultGeneratorByNamespaceName, callback, scope.NamespaceName) + default: + return func() {} + } +} + +func (a adapter) InstallResponseCallback(scope grpcfaults.Scope, callback grpcfaults.ResponseCallback) func() { + switch { + case scope.NamespaceID != "": + return commontesthooks.Set(a.testHooks, commontesthooks.GRPCResponseFaultGeneratorByNamespaceID, callback, scope.NamespaceID) + case scope.NamespaceName != "": + return commontesthooks.Set(a.testHooks, commontesthooks.GRPCResponseFaultGeneratorByNamespaceName, callback, scope.NamespaceName) + default: + return func() {} + } +} + +// NewGenerator creates a Generator backed by namespace-scoped test hooks. +func NewGenerator(testHooks commontesthooks.TestHooks) grpcfaults.Generator { + return adapter{testHooks: testHooks} +} + +func (a adapter) GenerateRequest(ctx context.Context, fullMethod string, req any) *grpcfaults.Outcome { + if generate, ok := getRequestFaultGenerator(a.testHooks, req); ok { + return generate(ctx, fullMethod, req) + } + return nil +} + +func (a adapter) GenerateResponse(ctx context.Context, fullMethod string, req, resp any, err error) *grpcfaults.Outcome { + if generate, ok := getResponseFaultGenerator(a.testHooks, req); ok { + return generate(ctx, fullMethod, req, resp, err) + } + return nil +} + +func getRequestFaultGenerator(testHooks commontesthooks.TestHooks, req any) (grpcfaults.RequestCallback, bool) { + if namespaceID, ok := namespaceIDFromRequest(req); ok { + return commontesthooks.Get(testHooks, commontesthooks.GRPCRequestFaultGeneratorByNamespaceID, namespaceID) + } + if namespaceName, ok := namespaceNameFromRequest(req); ok { + return commontesthooks.Get(testHooks, commontesthooks.GRPCRequestFaultGeneratorByNamespaceName, namespaceName) + } + return nil, false +} + +func getResponseFaultGenerator(testHooks commontesthooks.TestHooks, req any) (grpcfaults.ResponseCallback, bool) { + if namespaceID, ok := namespaceIDFromRequest(req); ok { + return commontesthooks.Get(testHooks, commontesthooks.GRPCResponseFaultGeneratorByNamespaceID, namespaceID) + } + if namespaceName, ok := namespaceNameFromRequest(req); ok { + return commontesthooks.Get(testHooks, commontesthooks.GRPCResponseFaultGeneratorByNamespaceName, namespaceName) + } + return nil, false +} + +func namespaceIDFromRequest(req any) (namespace.ID, bool) { + request, ok := req.(rpcinterceptor.NamespaceIDGetter) + if !ok || request.GetNamespaceId() == "" { + return "", false + } + return namespace.ID(request.GetNamespaceId()), true +} + +func namespaceNameFromRequest(req any) (namespace.Name, bool) { + request, ok := req.(rpcinterceptor.NamespaceNameGetter) + if !ok || request.GetNamespace() == "" { + return "", false + } + return namespace.Name(request.GetNamespace()), true +} diff --git a/common/testing/grpcfaultstest/hooks_testdep_test.go b/common/testing/grpcfaultstest/hooks_testdep_test.go new file mode 100644 index 00000000000..a4c71ef338e --- /dev/null +++ b/common/testing/grpcfaultstest/hooks_testdep_test.go @@ -0,0 +1,131 @@ +//go:build test_dep + +package grpcfaultstest_test + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/grpcfaults" + "go.temporal.io/server/common/testing/grpcfaultstest" + "go.temporal.io/server/common/testing/testhooks" + "google.golang.org/grpc" +) + +func TestUnaryServerInterceptor_NoGenerator(t *testing.T) { + t.Parallel() + + interceptor := grpcfaults.UnaryServerInterceptor(grpcfaultstest.NewGenerator(testhooks.NewTestHooks())) + response, err := interceptor( + context.Background(), + "request", + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + return "response", nil + }, + ) + + require.NoError(t, err) + require.Equal(t, "response", response) +} + +func TestUnaryServerInterceptor_BeforeHandler(t *testing.T) { + t.Parallel() + + testHooks := testhooks.NewTestHooks() + injectedErr := errors.New("injected") + generator := grpcfaultstest.NewCallbackGenerator(testHooks) + generator.RegisterRequestCallback(grpcfaults.Scope{NamespaceID: "namespace-id"}, func(context.Context, string, any) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Error: injectedErr} + }) + interceptor := grpcfaults.UnaryServerInterceptor(grpcfaultstest.NewGenerator(testHooks)) + handlerCalled := false + + response, err := interceptor( + context.Background(), + &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + handlerCalled = true + return "response", nil + }, + ) + + require.ErrorIs(t, err, injectedErr) + require.Nil(t, response) + require.False(t, handlerCalled) +} + +func TestUnaryServerInterceptor_AfterHandler(t *testing.T) { + t.Parallel() + + testHooks := testhooks.NewTestHooks() + generator := grpcfaultstest.NewCallbackGenerator(testHooks) + generator.RegisterResponseCallback(grpcfaults.Scope{NamespaceName: "namespace-name"}, func(context.Context, string, any, any, error) *grpcfaults.Outcome { + return &grpcfaults.Outcome{Response: "replacement"} + }) + interceptor := grpcfaults.UnaryServerInterceptor(grpcfaultstest.NewGenerator(testHooks)) + + response, err := interceptor( + context.Background(), + &workflowservice.UpdateNamespaceRequest{Namespace: "namespace-name"}, + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + return "response", nil + }, + ) + + require.NoError(t, err) + require.Equal(t, "replacement", response) +} + +func TestUnaryServerInterceptor_HandlerError(t *testing.T) { + t.Parallel() + + testHooks := testhooks.NewTestHooks() + handlerErr := errors.New("handler") + injectedErr := errors.New("injected") + generator := grpcfaultstest.NewCallbackGenerator(testHooks) + generator.RegisterResponseCallback(grpcfaults.Scope{NamespaceID: "namespace-id"}, func(_ context.Context, _ string, _, response any, err error) *grpcfaults.Outcome { + require.Nil(t, response) + require.ErrorIs(t, err, handlerErr) + return &grpcfaults.Outcome{Error: injectedErr} + }) + interceptor := grpcfaults.UnaryServerInterceptor(grpcfaultstest.NewGenerator(testHooks)) + handlerCalled := false + + response, err := interceptor( + context.Background(), + &matchingservice.AddWorkflowTaskRequest{NamespaceId: "namespace-id"}, + &grpc.UnaryServerInfo{FullMethod: "/test.Service/Method"}, + func(context.Context, any) (any, error) { + handlerCalled = true + return nil, handlerErr + }, + ) + + require.ErrorIs(t, err, injectedErr) + require.Nil(t, response) + require.True(t, handlerCalled) +} + +func TestNewCallbackGenerator_UnregisterRemovesHook(t *testing.T) { + t.Parallel() + + testHooks := testhooks.NewTestHooks() + generator := grpcfaultstest.NewCallbackGenerator(testHooks) + unregister := generator.RegisterRequestCallback(grpcfaults.Scope{NamespaceID: "namespace-id"}, func(context.Context, string, any) *grpcfaults.Outcome { + return nil + }) + _, ok := testhooks.Get(testHooks, testhooks.GRPCRequestFaultGeneratorByNamespaceID, namespace.ID("namespace-id")) + require.True(t, ok) + + unregister() + _, ok = testhooks.Get(testHooks, testhooks.GRPCRequestFaultGeneratorByNamespaceID, namespace.ID("namespace-id")) + require.False(t, ok) +} diff --git a/common/testing/testhooks/hooks.go b/common/testing/testhooks/hooks.go index bfeea7580b8..474f2eb1ecb 100644 --- a/common/testing/testhooks/hooks.go +++ b/common/testing/testhooks/hooks.go @@ -9,27 +9,32 @@ import ( replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/grpcfaults" historytasks "go.temporal.io/server/service/history/tasks" ) // Test hook keys with their return type and scope. // Try to avoid global scope as it requires a dedicated test cluster. var ( - MatchingDisableSyncMatch = newKey[bool, namespace.ID]() - MatchingLBForceReadPartition = newKey[int, namespace.ID]() - MatchingLBForceWritePartition = newKey[int, namespace.ID]() - UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() - UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() - TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() - MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() - MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() - MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() - HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() - HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() - HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() - HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() - HistoryDLQTaskDeleteInterceptor = newKey[func(context.Context, *historyservice.DeleteDLQTasksRequest, func(context.Context, *historyservice.DeleteDLQTasksRequest) (*historyservice.DeleteDLQTasksResponse, error)) (*historyservice.DeleteDLQTasksResponse, error), global]() - NamespaceReplicationTaskInterceptor = newKey[func(context.Context, *replicationspb.NamespaceTaskAttributes, func() error) error, namespace.Name]() + MatchingDisableSyncMatch = newKey[bool, namespace.ID]() + MatchingLBForceReadPartition = newKey[int, namespace.ID]() + MatchingLBForceWritePartition = newKey[int, namespace.ID]() + UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() + UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() + TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() + MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() + MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() + MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() + HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() + HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() + HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() + HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() + HistoryDLQTaskDeleteInterceptor = newKey[func(context.Context, *historyservice.DeleteDLQTasksRequest, func(context.Context, *historyservice.DeleteDLQTasksRequest) (*historyservice.DeleteDLQTasksResponse, error)) (*historyservice.DeleteDLQTasksResponse, error), global]() + NamespaceReplicationTaskInterceptor = newKey[func(context.Context, *replicationspb.NamespaceTaskAttributes, func() error) error, namespace.Name]() + GRPCRequestFaultGeneratorByNamespaceID = newKey[grpcfaults.RequestCallback, namespace.ID]() + GRPCRequestFaultGeneratorByNamespaceName = newKey[grpcfaults.RequestCallback, namespace.Name]() + GRPCResponseFaultGeneratorByNamespaceID = newKey[grpcfaults.ResponseCallback, namespace.ID]() + GRPCResponseFaultGeneratorByNamespaceName = newKey[grpcfaults.ResponseCallback, namespace.Name]() ) // keyID is a unique identifier for a key, used as a map key. diff --git a/docs/development/testing.md b/docs/development/testing.md index b8857dc6655..e5c3d733290 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -199,6 +199,31 @@ You'll find a fully initialized task poller in any functional test suite, look f _NOTE: The previous `testcore.TaskPoller` has been deprecated and should not be used in new code._ +### gRPC fault injection + +The `testcore` package injects faults into gRPC calls by intercepting requests and responses. +The fault function determines which RPCs trigger a fault and returns the error to inject. +It returns a cleanup function that can be used to remove the fault again. + +**Example:** + +```go +env.InjectRequestFault( + func(req any) error { + if _, ok := req.(*matchingservice.AddWorkflowTaskRequest); ok { + return serviceerror.NewNotFound("injected fault") + } + return nil + }) +``` + +`InjectRequestFault` runs _before_ the handler, so an injected error prevents the operation from +executing. `InjectResponseFault` runs _after_ the handler, so an injected error can model an +operation that executed but whose response was lost. The test fails if the fault never triggers. +Only unary RPCs are intercepted; streaming RPCs are unaffected. + +The `env` methods scope faults to the test's namespace. + ### testhooks package The `testhooks` package injects test-specific behavior into production code paths that are otherwise diff --git a/service/frontend/fx.go b/service/frontend/fx.go index 651551de210..2ba531cf795 100644 --- a/service/frontend/fx.go +++ b/service/frontend/fx.go @@ -43,10 +43,12 @@ import ( "go.temporal.io/server/common/resource" "go.temporal.io/server/common/rpc" "go.temporal.io/server/common/rpc/encryption" + "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/sdk" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/telemetry" + "go.temporal.io/server/common/testing/grpcfaultstest" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service" "go.temporal.io/server/service/frontend/configs" @@ -259,6 +261,7 @@ func GrpcServerOptionsProvider( customInterceptors []grpc.UnaryServerInterceptor, customStreamInterceptors []grpc.StreamServerInterceptor, metricsHandler metrics.Handler, + testHooks testhooks.TestHooks, ) GrpcServerOptions { kep := keepalive.EnforcementPolicy{ MinTime: serviceConfig.KeepAliveMinTime(), @@ -318,6 +321,10 @@ func GrpcServerOptionsProvider( // TODO: Deprecate WithChainedFrontendGrpcInterceptors and provide a inner custom interceptor unaryInterceptors = append(unaryInterceptors, customInterceptors...) } + faultGenerator := grpcfaultstest.NewGenerator(testHooks) + if faultInterceptor := grpcfaults.UnaryServerInterceptor(faultGenerator); faultInterceptor != nil { + unaryInterceptors = append(unaryInterceptors, faultInterceptor) + } // retry interceptor should be the most inner interceptor unaryInterceptors = append(unaryInterceptors, retryableInterceptor.Intercept) diff --git a/service/fx.go b/service/fx.go index 5368c261ffb..f41031b62a4 100644 --- a/service/fx.go +++ b/service/fx.go @@ -13,8 +13,11 @@ import ( "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/quotas/calculator" "go.temporal.io/server/common/rpc" + "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/rpc/interceptor" "go.temporal.io/server/common/telemetry" + "go.temporal.io/server/common/testing/grpcfaultstest" + "go.temporal.io/server/common/testing/testhooks" "go.uber.org/fx" "google.golang.org/grpc" ) @@ -50,6 +53,7 @@ type ( ContextMetadataInterceptor *interceptor.ContextMetadataInterceptor `optional:"true"` AdditionalInterceptors []grpc.UnaryServerInterceptor `optional:"true"` AdditionalStreamInterceptors []grpc.StreamServerInterceptor `optional:"true"` + TestHooks testhooks.TestHooks } ) @@ -176,5 +180,10 @@ func getUnaryInterceptors(params GrpcServerOptionsParams) []grpc.UnaryServerInte interceptors = append(interceptors, params.ContextMetadataInterceptor.Intercept) } + faultGenerator := grpcfaultstest.NewGenerator(params.TestHooks) + if faultInterceptor := grpcfaults.UnaryServerInterceptor(faultGenerator); faultInterceptor != nil { + interceptors = append(interceptors, faultInterceptor) + } + return append(interceptors, params.RetryableInterceptor.Intercept) } diff --git a/tests/testcore/fault_injection.go b/tests/testcore/fault_injection.go new file mode 100644 index 00000000000..1222b1f558e --- /dev/null +++ b/tests/testcore/fault_injection.go @@ -0,0 +1,51 @@ +package testcore + +import ( + "sync" + "sync/atomic" + "testing" +) + +// RequestFault determines whether a fault should be injected before a gRPC handler runs. +// Return the error to inject, or nil to not inject a fault. +type RequestFault func(req any) error + +// ResponseFault determines whether a fault should be injected after a gRPC handler runs. +// Return the error to inject, or nil to preserve the handler response and error. +type ResponseFault func(req, resp any, handlerErr error) error + +type faultTracker struct { + t testing.TB + fired atomic.Bool + + mu sync.Mutex + loggingEnabled bool +} + +func newFaultTracker(t testing.TB) *faultTracker { + t.Helper() + return &faultTracker{t: t, loggingEnabled: true} +} + +func (f *faultTracker) markFired(req any) { + f.fired.Store(true) + f.mu.Lock() + defer f.mu.Unlock() + if f.loggingEnabled { + f.t.Logf("Fault injection fired: %T", req) + } +} + +func (f *faultTracker) attach(unregister func()) func() { + f.t.Cleanup(func() { + f.mu.Lock() + f.loggingEnabled = false + f.mu.Unlock() + unregister() + if !f.fired.Load() { + f.t.Error("fault injection was registered but never fired - the fault was never injected") + } + }) + + return unregister +} diff --git a/tests/testcore/onebox.go b/tests/testcore/onebox.go index 908241e475e..f4fe0649bc3 100644 --- a/tests/testcore/onebox.go +++ b/tests/testcore/onebox.go @@ -32,6 +32,8 @@ import ( "go.temporal.io/server/common/resolver" "go.temporal.io/server/common/rpc/auth" "go.temporal.io/server/common/rpc/encryption" + "go.temporal.io/server/common/rpc/grpcfaults" + "go.temporal.io/server/common/testing/grpcfaultstest" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/temporal" @@ -64,6 +66,7 @@ type ( replicationStreamRecorder *ReplicationStreamRecorder historyTaskRecorder *HistoryTaskRecorder + grpcFaultGenerator *grpcfaults.CallbackGenerator callbackLock sync.RWMutex onGetClaims func(*authorization.AuthInfo) (*authorization.Claims, error) @@ -129,6 +132,7 @@ func newTemporal(t *testing.T, params *temporalParams) *temporalImpl { workerConfig: params.WorkerConfig, replicationStreamRecorder: NewReplicationStreamRecorder(), } + impl.grpcFaultGenerator = grpcfaultstest.NewCallbackGenerator(impl.testHooks) // Base options are independent of which services this test cluster starts. // [Start] adds the per-service config and static host map. @@ -351,6 +355,10 @@ func (c *temporalImpl) GetHistoryTaskRecorder() *HistoryTaskRecorder { return c.historyTaskRecorder } +func (c *temporalImpl) GetGRPCFaultGenerator() *grpcfaults.CallbackGenerator { + return c.grpcFaultGenerator +} + func (c *temporalImpl) TLSConfigProvider() *encryption.FixedTLSConfigProvider { return c.tlsConfigProvider } diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index 61f1371a35d..079a81ba8b3 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/testing/taskpoller" "go.temporal.io/server/common/testing/testcontext" "go.temporal.io/server/common/testing/testhooks" @@ -458,6 +459,46 @@ func (e *TestEnv) Tv() *testvars.TestVars { return e.tv } +// InjectRequestFault registers a pre-handler gRPC fault injection scoped to this test's namespace. +// Requests match either the namespace ID or name filter, depending on which +// namespace field they expose. Requests without either field are ignored. +// Returns a cleanup function that disables the fault. +func (e *TestEnv) InjectRequestFault(fault RequestFault) func() { + scope := grpcfaults.Scope{ + NamespaceID: e.nsID, + NamespaceName: e.nsName, + } + tracker := newFaultTracker(e.t) + unregister := e.GetTestCluster().Host().GetGRPCFaultGenerator().RegisterRequestCallback(scope, func(_ context.Context, _ string, req any) *grpcfaults.Outcome { + if injectedErr := fault(req); injectedErr != nil { + tracker.markFired(req) + return &grpcfaults.Outcome{Error: injectedErr} + } + return nil + }) + return tracker.attach(unregister) +} + +// InjectResponseFault registers a post-handler gRPC fault injection scoped to this test's namespace. +// Requests match either the namespace ID or name filter, depending on which +// namespace field they expose. Requests without either field are ignored. +// Returns a cleanup function that disables the fault. +func (e *TestEnv) InjectResponseFault(fault ResponseFault) func() { + scope := grpcfaults.Scope{ + NamespaceID: e.nsID, + NamespaceName: e.nsName, + } + tracker := newFaultTracker(e.t) + unregister := e.GetTestCluster().Host().GetGRPCFaultGenerator().RegisterResponseCallback(scope, func(_ context.Context, _ string, req, resp any, err error) *grpcfaults.Outcome { + if injectedErr := fault(req, resp, err); injectedErr != nil { + tracker.markFired(req) + return &grpcfaults.Outcome{Error: injectedErr} + } + return nil + }) + return tracker.attach(unregister) +} + // Context returns the test-level timeout context with RPC version headers already included. // This context will be canceled when the test timeout occurs. Use this directly for all RPC // operations - no need to wrap with NewContext or add headers manually. diff --git a/tests/update_workflow_test.go b/tests/update_workflow_test.go index 5c0d0efb094..b9ef4d7ecc0 100644 --- a/tests/update_workflow_test.go +++ b/tests/update_workflow_test.go @@ -18,11 +18,13 @@ import ( updatepb "go.temporal.io/api/update/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/api/adminservice/v1" + "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" + serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protoutils" "go.temporal.io/server/common/testing/taskpoller" @@ -1380,15 +1382,35 @@ func (s *WorkflowUpdateSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S env := testcore.NewEnv(s.T()) mustStartWorkflow(env, env.Tv()) - wtHandlerCalls := 0 - wtHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*commandpb.Command, error) { - wtHandlerCalls++ - switch wtHandlerCalls { - case 1: - // Completes first WT with empty command list. - return nil, nil - case 2: - // Worker gets full history because update was issued after sticky worker is gone. + // Drain existing WT from regular task queue, respond with sticky attributes to enable sticky task queue. + _, err := env.TaskPoller().PollAndHandleWorkflowTask(env.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + StickyAttributes: env.Tv().StickyExecutionAttributes(10 * time.Second), + }, nil + }) + s.NoError(err) + + // Inject StickyWorkerUnavailable for AddWorkflowTask on the sticky queue. + // This causes history to fall back to the normal queue. + env.InjectRequestFault( + func(req any) error { + r, ok := req.(*matchingservice.AddWorkflowTaskRequest) + if ok && r.GetTaskQueue().GetKind() == enumspb.TASK_QUEUE_KIND_STICKY { + return serviceerrors.NewStickyWorkerUnavailable() + } + return nil + }) + + // Send an update. History tries sticky queue first, gets StickyWorkerUnavailable, + // and falls back to the normal queue. + updateResultCh := sendUpdateNoError(env, env.Tv()) + + // Process update in workflow task from non-sticky task queue. + res, err := env.TaskPoller().PollAndHandleWorkflowTask(env.Tv(), + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + // Full history from event 1 confirms the task came from the normal queue + // (sticky queue would send partial history starting after the last completed WT). s.EqualHistory(` 1 WorkflowExecutionStarted 2 WorkflowTaskScheduled @@ -1397,20 +1419,7 @@ func (s *WorkflowUpdateSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 5 WorkflowTaskScheduled // Speculative WT. 6 WorkflowTaskStarted `, task.History) - return env.UpdateAcceptCompleteCommands(env.Tv()), nil - default: - s.Failf("wtHandler called too many times", "wtHandler shouldn't be called %d times", wtHandlerCalls) - return nil, nil - } - } - msgHandlerCalls := 0 - msgHandler := func(task *workflowservice.PollWorkflowTaskQueueResponse) ([]*protocolpb.Message, error) { - msgHandlerCalls++ - switch msgHandlerCalls { - case 1: - return nil, nil - case 2: updRequestMsg := task.Messages[0] updRequest := protoutils.UnmarshalAny[*updatepb.Request](s.T(), updRequestMsg.GetBody()) @@ -1418,52 +1427,16 @@ func (s *WorkflowUpdateSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S s.Equal(env.Tv().HandlerName(), updRequest.GetInput().GetName()) s.EqualValues(5, updRequestMsg.GetEventId()) - return env.UpdateAcceptCompleteMessages(env.Tv(), updRequestMsg), nil - default: - s.Failf("msgHandler called too many times", "msgHandler shouldn't be called %d times", msgHandlerCalls) - return nil, nil - } - } - - //nolint:staticcheck // SA1019 TaskPoller replacement needed - poller := &testcore.TaskPoller{ - Client: env.FrontendClient(), - Namespace: env.Namespace().String(), - TaskQueue: env.Tv().TaskQueue(), - StickyTaskQueue: env.Tv().StickyTaskQueue(), - StickyScheduleToStartTimeout: 3 * time.Second, - Identity: env.Tv().WorkerIdentity(), - WorkflowTaskHandler: wtHandler, - MessageHandler: msgHandler, - Logger: env.Logger, - T: s.T(), - } - - // Drain existing WT from regular task queue, but respond with sticky enabled response to enable stick task queue. - _, err := poller.PollAndProcessWorkflowTask(testcore.WithRespondSticky, testcore.WithoutRetries) - s.NoError(err) - - env.Logger.Info("Sleep 10+ seconds to make sure stickyPollerUnavailableWindow time has passed.") - time.Sleep(10*time.Second + 100*time.Millisecond) //nolint:forbidigo - env.Logger.Info("Sleep 10+ seconds is done.") - - // Now send an update. It should try sticky task queue first, but got "StickyWorkerUnavailable" error - // and resend it to normal. - // This can be observed in wtHandler: if history is partial => sticky task queue is used. - updateResultCh := sendUpdateNoError(env, env.Tv()) - - // Process update in workflow task from non-sticky task queue. - res, err := poller.PollAndProcessWorkflowTask(testcore.WithoutRetries) + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: env.UpdateAcceptCompleteCommands(env.Tv()), + Messages: env.UpdateAcceptCompleteMessages(env.Tv(), updRequestMsg), + }, nil + }) s.NoError(err) s.NotNil(res) updateResult := <-updateResultCh s.Equal("success-result-of-"+env.Tv().UpdateID(), testcore.DecodeString(s.T(), updateResult.GetOutcome().GetSuccess())) - s.EqualValues(0, res.NewTask.ResetHistoryEventId) - - s.Equal(2, wtHandlerCalls) - s.Equal(2, msgHandlerCalls) - - events := env.GetHistory(env.Namespace().String(), env.Tv().WorkflowExecution()) + s.EqualValues(0, res.ResetHistoryEventId) s.EqualHistoryEvents(` 1 WorkflowExecutionStarted @@ -1475,7 +1448,7 @@ func (s *WorkflowUpdateSuite) TestStickySpeculativeWorkflowTask_AcceptComplete_S 7 WorkflowTaskCompleted 8 WorkflowExecutionUpdateAccepted {"AcceptedRequestSequencingEventId": 5} // WTScheduled event which delivered update to the worker. 9 WorkflowExecutionUpdateCompleted {"AcceptedEventId": 8} - `, events) + `, env.GetHistory(env.Namespace().String(), env.Tv().WorkflowExecution())) } func (s *WorkflowUpdateSuite) TestFirstNormalScheduledWorkflowTask_Reject() { From 47db1efe8be9161629affc2a25b2bf847304ad73 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 31 Aug 2026 09:24:27 -0700 Subject: [PATCH 097/139] Clean up persistence test assertions (#11788) Address hundreds of linter issues. --- .../cluster_metadata_manager.go | 137 +++--- .../history_v2_persistence.go | 423 +++++++++++------- .../metadata_persistence_v2.go | 199 ++++---- .../persistence-tests/queue_persistence.go | 80 ++-- 4 files changed, 455 insertions(+), 384 deletions(-) diff --git a/common/persistence/persistence-tests/cluster_metadata_manager.go b/common/persistence/persistence-tests/cluster_metadata_manager.go index 6dfef45a4ff..013f89a4382 100644 --- a/common/persistence/persistence-tests/cluster_metadata_manager.go +++ b/common/persistence/persistence-tests/cluster_metadata_manager.go @@ -12,6 +12,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/debug" p "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/testing/await" ) type ( @@ -54,7 +55,7 @@ func (s *ClusterMetadataManagerSuite) TearDownSuite() { // TestClusterMembershipEmptyInitially verifies the GetClusterMembers() works with an initial empty table func (s *ClusterMetadataManagerSuite) TestClusterMembershipEmptyInitially() { resp, err := s.ClusterMetadataManager.GetClusterMembers(s.ctx, &p.GetClusterMembersRequest{LastHeartbeatWithin: time.Minute * 10}) - s.Nil(err) + s.NoError(err) s.NotNil(resp) s.Empty(resp.ActiveMembers) } @@ -73,11 +74,11 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertCanReadAny() { } err = s.ClusterMetadataManager.UpsertClusterMembership(s.ctx, req) - s.Nil(err) + s.NoError(err) resp, err := s.ClusterMetadataManager.GetClusterMembers(s.ctx, &p.GetClusterMembersRequest{}) - s.Nil(err) + s.NoError(err) s.NotNil(resp) s.NotEmpty(resp.ActiveMembers) @@ -134,17 +135,17 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertCanPageRead() { } func (s *ClusterMetadataManagerSuite) validateUpsert(req *p.UpsertClusterMembershipRequest, resp *p.GetClusterMembersResponse, err error) { - s.Nil(err) + s.NoError(err) s.NotNil(resp) s.NotEmpty(resp.ActiveMembers) - s.Equal(len(resp.ActiveMembers), 1) + s.Len(resp.ActiveMembers, 1) // Have to round to 1 second due to SQL implementations. Cassandra truncates at 1ms. - s.Equal(resp.ActiveMembers[0].SessionStart.Round(time.Second), req.SessionStart.Round(time.Second)) - s.Equal(resp.ActiveMembers[0].RPCAddress.String(), req.RPCAddress.String()) - s.Equal(resp.ActiveMembers[0].RPCPort, req.RPCPort) + s.Equal(req.SessionStart.Round(time.Second), resp.ActiveMembers[0].SessionStart.Round(time.Second)) + s.Equal(req.RPCAddress.String(), resp.ActiveMembers[0].RPCAddress.String()) + s.Equal(req.RPCPort, resp.ActiveMembers[0].RPCPort) s.True(resp.ActiveMembers[0].RecordExpiry.After(time.Now().UTC())) - s.Equal(resp.ActiveMembers[0].HostID, req.HostID) - s.Equal(resp.ActiveMembers[0].Role, req.Role) + s.Equal(req.HostID, resp.ActiveMembers[0].HostID) + s.Equal(req.Role, resp.ActiveMembers[0].Role) } // TestClusterMembershipReadFiltersCorrectly verifies that we can UpsertClusterMembership and read our result using filters @@ -162,7 +163,7 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipReadFiltersCorrectly( } err = s.ClusterMetadataManager.UpsertClusterMembership(s.ctx, req) - s.Nil(err) + s.NoError(err) resp, err := s.ClusterMetadataManager.GetClusterMembers( s.ctx, @@ -171,22 +172,22 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipReadFiltersCorrectly( s.validateUpsert(req, resp, err) - time.Sleep(time.Second * 1) - resp, err = s.ClusterMetadataManager.GetClusterMembers( - s.ctx, - &p.GetClusterMembersRequest{LastHeartbeatWithin: time.Millisecond, HostIDEquals: req.HostID}, - ) - - s.Nil(err) - s.NotNil(resp) - s.Empty(resp.ActiveMembers) + await.Require(s.ctx, s.T(), func(t *await.T) { + resp, err := s.ClusterMetadataManager.GetClusterMembers( + s.ctx, + &p.GetClusterMembersRequest{LastHeartbeatWithin: time.Millisecond, HostIDEquals: req.HostID}, + ) + require.NoError(t, err) + require.NotNil(t, resp) + require.Empty(t, resp.ActiveMembers) + }, time.Second, 100*time.Millisecond) resp, err = s.ClusterMetadataManager.GetClusterMembers( s.ctx, &p.GetClusterMembersRequest{RoleEquals: p.Matching}, ) - s.Nil(err) + s.NoError(err) s.NotNil(resp) s.Empty(resp.ActiveMembers) @@ -195,7 +196,7 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipReadFiltersCorrectly( &p.GetClusterMembersRequest{SessionStartedAfter: time.Now().UTC()}, ) - s.Nil(err) + s.NoError(err) s.NotNil(resp) s.Empty(resp.ActiveMembers) @@ -205,7 +206,7 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipReadFiltersCorrectly( ) s.validateUpsert(req, resp, err) - s.waitForPrune(5 * time.Second) + s.waitForPrune(8 * time.Second) } // TestClusterMembershipUpsertExpiresCorrectly verifies RecordExpiry functions properly for ClusterMembership records @@ -235,14 +236,14 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertExpiresCorrectl s.NoError(err) s.NotNil(resp) s.NotEmpty(resp.ActiveMembers) - s.Equal(len(resp.ActiveMembers), 1) + s.Len(resp.ActiveMembers, 1) // Have to round to 1 second due to SQL implementations. Cassandra truncates at 1ms. - s.Equal(resp.ActiveMembers[0].SessionStart.Round(time.Second), req.SessionStart.Round(time.Second)) - s.Equal(resp.ActiveMembers[0].RPCAddress.String(), req.RPCAddress.String()) - s.Equal(resp.ActiveMembers[0].RPCPort, req.RPCPort) + s.Equal(req.SessionStart.Round(time.Second), resp.ActiveMembers[0].SessionStart.Round(time.Second)) + s.Equal(req.RPCAddress.String(), resp.ActiveMembers[0].RPCAddress.String()) + s.Equal(req.RPCPort, resp.ActiveMembers[0].RPCPort) s.True(resp.ActiveMembers[0].RecordExpiry.After(time.Now().UTC())) - s.Equal(resp.ActiveMembers[0].HostID, req.HostID) - s.Equal(resp.ActiveMembers[0].Role, req.Role) + s.Equal(req.HostID, resp.ActiveMembers[0].HostID) + s.Equal(req.Role, resp.ActiveMembers[0].Role) s.waitForPrune(5 * time.Second) } @@ -250,21 +251,18 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertExpiresCorrectl // waitForPrune waits up for the persistence backend to prune all records. Some persistence backends // may not remove TTL'd entries at the exact instant they should expire, so we allow some timing flexibility here. func (s *ClusterMetadataManagerSuite) waitForPrune(waitFor time.Duration) { - s.Eventually(func() bool { + await.Require(s.ctx, s.T(), func(t *await.T) { err := s.ClusterMetadataManager.PruneClusterMembership(s.ctx, &p.PruneClusterMembershipRequest{MaxRecordsPruned: 100}) - s.Nil(err) + require.NoError(t, err) resp, err := s.ClusterMetadataManager.GetClusterMembers( s.ctx, &p.GetClusterMembersRequest{LastHeartbeatWithin: time.Minute * 10}, ) - s.NoError(err) - s.NotNil(resp) - return len(resp.ActiveMembers) == 0 - - }, - waitFor, - 500*time.Millisecond) + require.NoError(t, err) + require.NotNil(t, resp) + require.Empty(t, resp.ActiveMembers) + }, waitFor, 500*time.Millisecond) } // TestClusterMembershipUpsertInvalidExpiry verifies we cannot specify a non-positive RecordExpiry duration @@ -281,8 +279,7 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertInvalidExpiry() } err = s.ClusterMetadataManager.UpsertClusterMembership(s.ctx, req) - s.NotNil(err) - s.IsType(err, p.ErrInvalidMembershipExpiry) + s.ErrorIs(err, p.ErrInvalidMembershipExpiry) } // TestInitImmutableMetadataReadWrite runs through the various cases of ClusterMetadata behavior @@ -296,9 +293,9 @@ func (s *ClusterMetadataManagerSuite) TestClusterMembershipUpsertInvalidExpiry() func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { clusterNameToPersist := "testing" historyShardsToPersist := int32(43) - clusterIdToPersist := "12345" + clusterIDToPersist := "12345" clusterAddress := "cluster-address" - clusterHttpAddress := "cluster-http-address" + clusterHTTPAddress := "cluster-http-address" failoverVersionIncrement := int64(10) initialFailoverVersion := int64(1) @@ -308,8 +305,7 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { getResp, err := s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) // Validate they match our initializations - s.NotNil(err) - s.IsType(&serviceerror.NotFound{}, err) + s.ErrorAs(err, new(*serviceerror.NotFound)) s.Nil(getResp) // Case 2 - Init, no data persisted yet @@ -320,16 +316,16 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { ClusterMetadata: &persistencespb.ClusterMetadata{ ClusterName: clusterNameToPersist, HistoryShardCount: historyShardsToPersist, - ClusterId: clusterIdToPersist, + ClusterId: clusterIDToPersist, ClusterAddress: clusterAddress, - HttpAddress: clusterHttpAddress, + HttpAddress: clusterHTTPAddress, FailoverVersionIncrement: failoverVersionIncrement, InitialFailoverVersion: initialFailoverVersion, IsGlobalNamespaceEnabled: true, IsConnectionEnabled: true, }}) - s.Nil(err) + s.NoError(err) s.True(initialResp) // request should be applied as this is first initialize // Case 3 - Get, data persisted @@ -337,13 +333,13 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { getResp, err = s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) // Validate they match our initializations - s.Nil(err) - s.True(getResp != nil) + s.NoError(err) + s.NotNil(getResp) s.Equal(clusterNameToPersist, getResp.ClusterName) s.Equal(historyShardsToPersist, getResp.HistoryShardCount) - s.Equal(clusterIdToPersist, getResp.ClusterId) + s.Equal(clusterIDToPersist, getResp.ClusterId) s.Equal(clusterAddress, getResp.ClusterAddress) - s.Equal(clusterHttpAddress, getResp.HttpAddress) + s.Equal(clusterHTTPAddress, getResp.HttpAddress) s.Equal(failoverVersionIncrement, getResp.FailoverVersionIncrement) s.Equal(initialFailoverVersion, getResp.InitialFailoverVersion) s.True(getResp.IsGlobalNamespaceEnabled) @@ -357,20 +353,20 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { HistoryShardCount: int32(77), }}) - s.Nil(err) + s.NoError(err) s.False(secondResp) // Should not have applied, and should match values from first request // Refetch persisted getResp, err = s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) // Validate they match our initial values - s.Nil(err) + s.NoError(err) s.NotNil(getResp) s.Equal(clusterNameToPersist, getResp.ClusterName) s.Equal(historyShardsToPersist, getResp.HistoryShardCount) - s.Equal(clusterIdToPersist, getResp.ClusterId) + s.Equal(clusterIDToPersist, getResp.ClusterId) s.Equal(clusterAddress, getResp.ClusterAddress) - s.Equal(clusterHttpAddress, getResp.HttpAddress) + s.Equal(clusterHTTPAddress, getResp.HttpAddress) s.Equal(failoverVersionIncrement, getResp.FailoverVersionIncrement) s.Equal(initialFailoverVersion, getResp.InitialFailoverVersion) s.True(getResp.IsGlobalNamespaceEnabled) @@ -386,21 +382,20 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { ClusterMetadata: getResp.ClusterMetadata, Version: getResp.Version, }) - s.Nil(err) + s.NoError(err) s.True(thirdResp) getResp, err = s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) - s.Nil(err) + s.NoError(err) s.NotNil(getResp) s.Equal("1.0", getResp.VersionInfo.Current.Version) // Case 6 - Delete Cluster Metadata err = s.ClusterMetadataManager.DeleteClusterMetadata(s.ctx, &p.DeleteClusterMetadataRequest{ClusterName: clusterNameToPersist}) - s.Nil(err) + s.NoError(err) getResp, err = s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) // Validate they match our initializations - s.NotNil(err) - s.IsType(&serviceerror.NotFound{}, err) + s.ErrorAs(err, new(*serviceerror.NotFound)) s.Nil(getResp) // Case 7 - Update current cluster metadata @@ -411,15 +406,15 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { ClusterMetadata: &persistencespb.ClusterMetadata{ ClusterName: clusterNameToPersist, HistoryShardCount: historyShardsToPersist, - ClusterId: clusterIdToPersist, + ClusterId: clusterIDToPersist, ClusterAddress: clusterAddress, - HttpAddress: clusterHttpAddress, + HttpAddress: clusterHTTPAddress, FailoverVersionIncrement: failoverVersionIncrement, InitialFailoverVersion: initialFailoverVersion, IsGlobalNamespaceEnabled: true, IsConnectionEnabled: true, }}) - s.Nil(err) + s.NoError(err) s.True(initialResp) // Case 8 - Get, data persisted @@ -427,13 +422,13 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { getResp, err = s.ClusterMetadataManager.GetClusterMetadata(s.ctx, &p.GetClusterMetadataRequest{ClusterName: clusterNameToPersist}) // Validate they match our initializations - s.Nil(err) - s.True(getResp != nil) + s.NoError(err) + s.NotNil(getResp) s.Equal(clusterNameToPersist, getResp.ClusterName) s.Equal(historyShardsToPersist, getResp.HistoryShardCount) - s.Equal(clusterIdToPersist, getResp.ClusterId) + s.Equal(clusterIDToPersist, getResp.ClusterId) s.Equal(clusterAddress, getResp.ClusterAddress) - s.Equal(clusterHttpAddress, getResp.HttpAddress) + s.Equal(clusterHTTPAddress, getResp.HttpAddress) s.Equal(failoverVersionIncrement, getResp.FailoverVersionIncrement) s.Equal(initialFailoverVersion, getResp.InitialFailoverVersion) s.True(getResp.IsGlobalNamespaceEnabled) @@ -465,9 +460,9 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { ClusterMetadata: &persistencespb.ClusterMetadata{ ClusterName: clusterNameToPersist + "2", HistoryShardCount: historyShardsToPersist, - ClusterId: clusterIdToPersist, + ClusterId: clusterIDToPersist, ClusterAddress: clusterAddress, - HttpAddress: clusterHttpAddress, + HttpAddress: clusterHTTPAddress, FailoverVersionIncrement: failoverVersionIncrement, InitialFailoverVersion: initialFailoverVersion, IsGlobalNamespaceEnabled: true, @@ -477,8 +472,8 @@ func (s *ClusterMetadataManagerSuite) TestInitImmutableMetadataReadWrite() { resp, err := s.ClusterMetadataManager.ListClusterMetadata(s.ctx, &p.ListClusterMetadataRequest{PageSize: 1}) s.NoError(err) - s.Equal(1, len(resp.ClusterMetadata)) + s.Len(resp.ClusterMetadata, 1) resp, err = s.ClusterMetadataManager.ListClusterMetadata(s.ctx, &p.ListClusterMetadataRequest{PageSize: 1, NextPageToken: resp.NextPageToken}) s.NoError(err) - s.Equal(1, len(resp.ClusterMetadata)) + s.Len(resp.ClusterMetadata, 1) } diff --git a/common/persistence/persistence-tests/history_v2_persistence.go b/common/persistence/persistence-tests/history_v2_persistence.go index 84d7bf86e50..348c2d6d2b5 100644 --- a/common/persistence/persistence-tests/history_v2_persistence.go +++ b/common/persistence/persistence-tests/history_v2_persistence.go @@ -2,6 +2,8 @@ package persistencetests import ( "context" + "errors" + "fmt" "math/rand" "sync" "sync/atomic" @@ -16,6 +18,7 @@ import ( "go.temporal.io/server/common/debug" p "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/testing/protorequire" + "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -99,8 +102,8 @@ func (s *HistoryV2PersistenceSuite) TestScanAllTrees() { resp, err := s.ExecutionManager.GetAllHistoryTreeBranches(s.ctx, &p.GetAllHistoryTreeBranchesRequest{ PageSize: 1, }) - s.Nil(err) - s.Equal(0, len(resp.Branches), "some trees were leaked in other tests") + s.NoError(err) + s.Empty(resp.Branches, "some trees were leaked in other tests") trees := map[string]bool{} totalTrees := 1002 @@ -109,11 +112,11 @@ func (s *HistoryV2PersistenceSuite) TestScanAllTrees() { for range totalTrees { treeID := uuid.NewString() bi, err := s.newHistoryBranch(treeID) - s.Nil(err) + s.NoError(err) events := s.genRandomEvents([]int64{1, 2, 3}, 1) err = s.appendNewBranchAndFirstNode(bi, events, 1, "branchInfo") - s.Nil(err) + s.NoError(err) trees[string(treeID)] = true } @@ -123,14 +126,14 @@ func (s *HistoryV2PersistenceSuite) TestScanAllTrees() { PageSize: pgSize, NextPageToken: pgToken, }) - s.Nil(err) + s.NoError(err) for _, br := range resp.Branches { - uuidTreeId := br.BranchInfo.TreeId - if trees[uuidTreeId] { - delete(trees, uuidTreeId) + uuidTreeID := br.BranchInfo.TreeId + if trees[uuidTreeID] { + delete(trees, uuidTreeID) - s.True(br.ForkTime.AsTime().UnixNano() > 0) - s.True(len(br.BranchInfo.BranchId) > 0) + s.Positive(br.ForkTime.AsTime().UnixNano()) + s.NotEmpty(br.BranchInfo.BranchId) s.Equal("branchInfo", br.Info) } else { s.Fail("treeID not found", br.BranchInfo.TreeId) @@ -143,47 +146,47 @@ func (s *HistoryV2PersistenceSuite) TestScanAllTrees() { pgToken = resp.NextPageToken } - s.Equal(0, len(trees)) + s.Empty(trees) } // TestReadBranchByPagination test func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { treeID := uuid.NewString() bi, err := s.newHistoryBranch(treeID) - s.Nil(err) + s.NoError(err) historyW := &historypb.History{} events := s.genRandomEvents([]int64{1, 2, 3}, 0) err = s.appendNewBranchAndFirstNode(bi, events, 1, "branchInfo") - s.Nil(err) + s.NoError(err) historyW.Events = events events = s.genRandomEvents([]int64{4}, 0) err = s.appendNewNode(bi, events, 2) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{5, 6, 7, 8}, 4) err = s.appendNewNode(bi, events, 6) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) // stale event batch events = s.genRandomEvents([]int64{6, 7, 8}, 1) err = s.appendNewNode(bi, events, 3) - s.Nil(err) + s.NoError(err) // stale event batch events = s.genRandomEvents([]int64{6, 7, 8}, 2) err = s.appendNewNode(bi, events, 4) - s.Nil(err) + s.NoError(err) // stale event batch events = s.genRandomEvents([]int64{6, 7, 8}, 3) err = s.appendNewNode(bi, events, 5) - s.Nil(err) + s.NoError(err) events = s.genRandomEvents([]int64{9}, 4) err = s.appendNewNode(bi, events, 7) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) // Start to read from middle, should not return error, but the first batch should be ignored by application layer @@ -197,53 +200,53 @@ func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { } // first page resp, err := s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(4, len(resp.HistoryEvents)) + s.NoError(err) + s.Len(resp.HistoryEvents, 4) s.Equal(int64(6), resp.HistoryEvents[0].GetEventId()) events = s.genRandomEvents([]int64{10}, 4) err = s.appendNewNode(bi, events, 8) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{11}, 4) err = s.appendNewNode(bi, events, 9) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{12}, 4) err = s.appendNewNode(bi, events, 10) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{13, 14, 15}, 4) err = s.appendNewNode(bi, events, 11) - s.Nil(err) + s.NoError(err) // we don't append this batch because we will fork from 13 // historyW.Events = append(historyW.Events, events...) // fork from here bi2, err := s.fork(bi, 13) - s.Nil(err) + s.NoError(err) events = s.genRandomEvents([]int64{13}, 4) err = s.appendNewNode(bi2, events, 12) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{14}, 4) err = s.appendNewNode(bi2, events, 13) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{15, 16, 17}, 4) err = s.appendNewNode(bi2, events, 14) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{18, 19, 20}, 4) err = s.appendNewNode(bi2, events, 15) - s.Nil(err) + s.NoError(err) historyW.Events = append(historyW.Events, events...) // read branch to verify @@ -260,9 +263,9 @@ func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { // first page resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) + s.NoError(err) - s.Equal(8, len(resp.HistoryEvents)) + s.Len(resp.HistoryEvents, 8) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken @@ -270,14 +273,14 @@ func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { // doe to difference in Cassandra / MySQL pagination // the stale event batch may get returned resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) + s.NoError(err) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken if len(resp.HistoryEvents) == 0 { // second page resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(3, len(resp.HistoryEvents)) + s.NoError(err) + s.Len(resp.HistoryEvents, 3) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken } else if len(resp.HistoryEvents) == 3 { @@ -288,15 +291,15 @@ func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { // 3rd page, since we fork from nodeID=13, we can only see one batch of 12 here resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(1, len(resp.HistoryEvents)) + s.NoError(err) + s.Len(resp.HistoryEvents, 1) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken // 4th page, 13~17 resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(5, len(resp.HistoryEvents)) + s.NoError(err) + s.Len(resp.HistoryEvents, 5) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken @@ -307,165 +310,221 @@ func (s *HistoryV2PersistenceSuite) TestReadBranchByPagination() { // to get history again, no error and history events should be returned. req.PageSize = 1 resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(3, len(resp.HistoryEvents)) + s.NoError(err) + s.Len(resp.HistoryEvents, 3) historyR.Events = append(historyR.Events, resp.HistoryEvents...) req.NextPageToken = resp.NextPageToken if len(resp.NextPageToken) != 0 { resp, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.Nil(err) - s.Equal(0, len(resp.HistoryEvents)) + s.NoError(err) + s.Empty(resp.HistoryEvents) } s.ProtoEqual(historyW, historyR) - s.Equal(0, len(resp.NextPageToken)) + s.Empty(resp.NextPageToken) // MinEventID is in the middle of the last batch and this is the first request (NextPageToken // is empty), the call should return an error. req.MinEventID = 19 req.NextPageToken = nil _, err = s.ExecutionManager.ReadHistoryBranch(s.ctx, req) - s.IsType(&serviceerror.NotFound{}, err) + s.ErrorAs(err, new(*serviceerror.NotFound)) err = s.deleteHistoryBranch(bi2) - s.Nil(err) + s.NoError(err) err = s.deleteHistoryBranch(bi) - s.Nil(err) + s.NoError(err) branches := s.descTree(treeID) - s.Equal(0, len(branches)) + s.Empty(branches) } // TestConcurrentlyCreateAndAppendBranches test func (s *HistoryV2PersistenceSuite) TestConcurrentlyCreateAndAppendBranches() { treeID := uuid.NewString() - wg := sync.WaitGroup{} concurrency := 1 m := &sync.Map{} + writtenHistories := make([]*historypb.History, concurrency) + readHistories := make([]*historypb.History, concurrency) // test create new branch along with appending new nodes - for i := range concurrency { - wg.Add(1) - go func(idx int) { - defer wg.Done() + var group errgroup.Group + for idx := range concurrency { + group.Go(func() error { bi, err := s.newHistoryBranch(treeID) - s.Nil(err) + if err != nil { + return fmt.Errorf("create history branch: %w", err) + } historyW := &historypb.History{} m.Store(idx, bi) events := s.genRandomEvents([]int64{1, 2, 3}, 1) err = s.appendNewBranchAndFirstNode(bi, events, 1, "branchInfo") - s.Nil(err) + if err != nil { + return fmt.Errorf("append first history node: %w", err) + } historyW.Events = events events = s.genRandomEvents([]int64{4}, 1) err = s.appendNewNode(bi, events, 2) - s.Nil(err) + if err != nil { + return fmt.Errorf("append second history node: %w", err) + } historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{5, 6, 7, 8}, 1) err = s.appendNewNode(bi, events, 3) - s.Nil(err) + if err != nil { + return fmt.Errorf("append third history node: %w", err) + } historyW.Events = append(historyW.Events, events...) events = s.genRandomEvents([]int64{9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, 1) err = s.appendNewNode(bi, events, 4000) - s.Nil(err) + if err != nil { + return fmt.Errorf("append fourth history node: %w", err) + } historyW.Events = append(historyW.Events, events...) // read branch to verify historyR := &historypb.History{} - events = s.read(bi, 1, 21) - s.Equal(20, len(events)) + events, err = s.readWithError(bi, 1, 21) + if err != nil { + return fmt.Errorf("read history branch: %w", err) + } + if len(events) != 20 { + return fmt.Errorf("read history branch: got %d events, want 20", len(events)) + } historyR.Events = events - - s.ProtoEqual(historyW, historyR) - }(i) + writtenHistories[idx] = historyW + readHistories[idx] = historyR + return nil + }) } - wg.Wait() + s.NoError(group.Wait()) + for idx := range concurrency { + s.ProtoEqual(writtenHistories[idx], readHistories[idx]) + } branches := s.descTree(treeID) - s.Equal(concurrency, len(branches)) + s.Len(branches, concurrency) - wg = sync.WaitGroup{} + group = errgroup.Group{} // test appending nodes(override and new nodes) on each branch concurrently - for i := range concurrency { - wg.Add(1) - go func(idx int) { - defer wg.Done() - - branch := s.getBranchByKey(m, idx) + for idx := range concurrency { + group.Go(func() error { + branch, ok := s.getBranchByKey(m, idx) + if !ok { + return fmt.Errorf("history branch %d not found", idx) + } // override with smaller txn_id events := s.genRandomEvents([]int64{5}, 1) err := s.appendNewNode(branch, events, 0) - s.Nil(err) + if err != nil { + return fmt.Errorf("append history node with smaller transaction ID: %w", err) + } // it shouldn't change anything - events = s.read(branch, 1, 25) - s.Equal(20, len(events)) + events, err = s.readWithError(branch, 1, 25) + if err != nil { + return fmt.Errorf("read unchanged history: %w", err) + } + if len(events) != 20 { + return fmt.Errorf("read unchanged history: got %d events, want 20", len(events)) + } // override with greatest txn_id events = s.genRandomEvents([]int64{5}, 1) err = s.appendNewNode(branch, events, 3000) - s.Nil(err) + if err != nil { + return fmt.Errorf("append history node with greater transaction ID: %w", err) + } // read to verify override success, at this point history is corrupted, missing 6/7/8, so we should only see 5 events - events = s.read(branch, 1, 6) - s.Equal(5, len(events)) + events, err = s.readWithError(branch, 1, 6) + if err != nil { + return fmt.Errorf("read first overridden history: %w", err) + } + if len(events) != 5 { + return fmt.Errorf("read first overridden history: got %d events, want 5", len(events)) + } _, err = s.readWithError(branch, 1, 25) - _, ok := err.(*serviceerror.DataLoss) - s.Equal(true, ok) + if _, ok := errors.AsType[*serviceerror.DataLoss](err); !ok { + return fmt.Errorf("read first corrupted history: got %T, want *serviceerror.DataLoss", err) + } // override with even larger txn_id and same version events = s.genRandomEvents([]int64{5, 6}, 1) err = s.appendNewNode(branch, events, 3001) - s.Nil(err) + if err != nil { + return fmt.Errorf("append history node with even larger transaction ID: %w", err) + } // read to verify override success, at this point history is corrupted, missing 7/8, so we should only see 6 events - events = s.read(branch, 1, 7) - s.Equal(6, len(events)) + events, err = s.readWithError(branch, 1, 7) + if err != nil { + return fmt.Errorf("read second overridden history: %w", err) + } + if len(events) != 6 { + return fmt.Errorf("read second overridden history: got %d events, want 6", len(events)) + } _, err = s.readWithError(branch, 1, 25) - _, ok = err.(*serviceerror.DataLoss) - s.Equal(true, ok) + if _, ok := errors.AsType[*serviceerror.DataLoss](err); !ok { + return fmt.Errorf("read second corrupted history: got %T, want *serviceerror.DataLoss", err) + } // override more with larger txn_id, this would fix the corrupted hole so that we cna get 20 events again events = s.genRandomEvents([]int64{7, 8}, 1) err = s.appendNewNode(branch, events, 3002) - s.Nil(err) + if err != nil { + return fmt.Errorf("append history node to repair corrupted history: %w", err) + } // read to verify override - events = s.read(branch, 1, 25) - s.Equal(20, len(events)) + events, err = s.readWithError(branch, 1, 25) + if err != nil { + return fmt.Errorf("read repaired history: %w", err) + } + if len(events) != 20 { + return fmt.Errorf("read repaired history: got %d events, want 20", len(events)) + } events = s.genRandomEvents([]int64{9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}, 1) err = s.appendNewNode(branch, events, 4001) - s.Nil(err) - events = s.read(branch, 1, 25) - s.Equal(23, len(events)) - }(i) + if err != nil { + return fmt.Errorf("append final history node: %w", err) + } + events, err = s.readWithError(branch, 1, 25) + if err != nil { + return fmt.Errorf("read final history: %w", err) + } + if len(events) != 23 { + return fmt.Errorf("read final history: got %d events, want 23", len(events)) + } + return nil + }) } - wg.Wait() + s.NoError(group.Wait()) // Finally lets clean up all branches m.Range(func(k, v any) bool { br := v.([]byte) // delete old branches along with create new branches err := s.deleteHistoryBranch(br) - s.Nil(err) + s.NoError(err) return true }) branches = s.descTree(treeID) - s.Equal(0, len(branches)) + s.Empty(branches) } // TestConcurrentlyForkAndAppendBranches test func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { treeID := uuid.NewString() - wg := sync.WaitGroup{} concurrency := 10 masterBr, err := s.newHistoryBranch(treeID) - s.Nil(err) + s.NoError(err) branches := s.descTree(treeID) - s.Equal(0, len(branches)) + s.Empty(branches) // append first batch to master branch eids := []int64{} @@ -474,14 +533,13 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { } events := s.genRandomEvents(eids, 1) err = s.appendNewBranchAndFirstNode(masterBr, events[0:1], 1, "masterbr") - s.Nil(err) + s.NoError(err) readEvents := s.read(masterBr, 1, int64(concurrency)+2) - s.Nil(err) - s.Equal(1, len(readEvents)) + s.Len(readEvents, 1) branches = s.descTree(treeID) - s.Equal(1, len(branches)) + s.Len(branches, 1) mbrID := branches[0].BranchId txn := int64(1) @@ -496,31 +554,31 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { } err = s.appendOneByOne(masterBr, events[1:], reserveTxn(len(events[1:]))) - s.Nil(err) + s.NoError(err) events = s.read(masterBr, 1, int64(concurrency)+2) - s.Nil(err) - s.Equal((concurrency)+1, len(events)) + s.Len(events, (concurrency)+1) level1ID := new(sync.Map) level1Br := new(sync.Map) // test forking from master branch and append nodes - for i := range concurrency { - wg.Add(1) - go func(idx int) { - defer wg.Done() - + var group errgroup.Group + for idx := range concurrency { + group.Go(func() error { forkNodeID := rand.Int63n(int64(concurrency)) + 2 level1ID.Store(idx, forkNodeID) bi, err := s.fork(masterBr, forkNodeID) - s.Nil(err) + if err != nil { + return fmt.Errorf("fork first-level history branch: %w", err) + } level1Br.Store(idx, bi) // cannot append to ancestors events := s.genRandomEvents([]int64{forkNodeID - 1}, 1) err = s.appendNewNode(bi, events, reserveTxn(1)) - _, ok := err.(*p.InvalidPersistenceRequestError) - s.Equal(true, ok) + if _, ok := errors.AsType[*p.InvalidPersistenceRequestError](err); !ok { + return fmt.Errorf("append to ancestor: got %T, want *persistence.InvalidPersistenceRequestError", err) + } // append second batch to first level eids := make([]int64, 0) @@ -530,47 +588,63 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { events = s.genRandomEvents(eids, 1) err = s.appendNewNode(bi, events[0:1], reserveTxn(1)) - s.Nil(err) + if err != nil { + return fmt.Errorf("append first node to first-level branch: %w", err) + } err = s.appendOneByOne(bi, events[1:], reserveTxn(len(events[1:]))) - s.Nil(err) + if err != nil { + return fmt.Errorf("append remaining nodes to first-level branch: %w", err) + } - events = s.read(bi, 1, int64(concurrency)*2+2) - s.Nil(err) - s.Equal((concurrency)*2+1, len(events)) + events, err = s.readWithError(bi, 1, int64(concurrency)*2+2) + if err != nil { + return fmt.Errorf("read first-level history branch: %w", err) + } + if len(events) != concurrency*2+1 { + return fmt.Errorf("read first-level history branch: got %d events, want %d", len(events), concurrency*2+1) + } if idx == 0 { err = s.deleteHistoryBranch(bi) - s.Nil(err) + if err != nil { + return fmt.Errorf("delete first-level history branch: %w", err) + } } - }(i) + return nil + }) } - wg.Wait() + s.NoError(group.Wait()) branches = s.descTree(treeID) - s.Equal(concurrency, len(branches)) + s.Len(branches, concurrency) forkOnLevel1 := int32(0) level2Br := new(sync.Map) - wg = sync.WaitGroup{} + group = errgroup.Group{} // test forking for second level of branch - for i := 1; i < concurrency; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() - + for idx := 1; idx < concurrency; idx++ { + group.Go(func() error { // Event we fork from level1 branch, it is possible that the new branch will fork from master branch forkNodeID := rand.Int63n(int64(concurrency)*2) + 2 - forkBr := s.getBranchByKey(level1Br, idx) - lastForkNodeID := s.getIDByKey(level1ID, idx) + forkBr, ok := s.getBranchByKey(level1Br, idx) + if !ok { + return fmt.Errorf("first-level history branch %d not found", idx) + } + lastForkNodeID, ok := s.getIDByKey(level1ID, idx) + if !ok { + return fmt.Errorf("first-level fork node ID %d not found", idx) + } if forkNodeID > lastForkNodeID { atomic.AddInt32(&forkOnLevel1, int32(1)) } bi, err := s.fork(forkBr, forkNodeID) - s.Nil(err) + if err != nil { + return fmt.Errorf("fork second-level history branch: %w", err) + } level2Br.Store(idx, bi) // append second batch to second level @@ -580,36 +654,55 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { } events := s.genRandomEvents(eids, 1) err = s.appendNewNode(bi, events[0:1], reserveTxn(1)) - s.Nil(err) + if err != nil { + return fmt.Errorf("append first node to second-level branch: %w", err) + } err = s.appendOneByOne(bi, events[1:], reserveTxn(len(events[1:]))) - s.Nil(err) - events = s.read(bi, 1, int64(concurrency)*3+2) - s.Nil(err) - s.Equal((concurrency)*3+1, len(events)) + if err != nil { + return fmt.Errorf("append remaining nodes to second-level branch: %w", err) + } + events, err = s.readWithError(bi, 1, int64(concurrency)*3+2) + if err != nil { + return fmt.Errorf("read second-level history branch: %w", err) + } + if len(events) != concurrency*3+1 { + return fmt.Errorf("read second-level history branch: got %d events, want %d", len(events), concurrency*3+1) + } // try override last event events = s.genRandomEvents([]int64{int64(concurrency)*3 + 1}, 1) err = s.appendNewNode(bi, events, reserveTxn(1)) - s.Nil(err) - events = s.read(bi, 1, int64(concurrency)*3+2) - s.Nil(err) - s.Equal((concurrency)*3+1, len(events)) + if err != nil { + return fmt.Errorf("override final second-level history event: %w", err) + } + events, err = s.readWithError(bi, 1, int64(concurrency)*3+2) + if err != nil { + return fmt.Errorf("read overridden second-level history branch: %w", err) + } + if len(events) != concurrency*3+1 { + return fmt.Errorf("read overridden second-level history branch: got %d events, want %d", len(events), concurrency*3+1) + } // test fork and newBranch concurrently bi, err = s.newHistoryBranch(treeID) - s.Nil(err) + if err != nil { + return fmt.Errorf("create concurrent history branch: %w", err) + } level2Br.Store(concurrency+idx, bi) events = s.genRandomEvents([]int64{1}, 1) err = s.appendNewBranchAndFirstNode(bi, events, reserveTxn(1), "newbr") - s.Nil(err) + if err != nil { + return fmt.Errorf("append first node to concurrent history branch: %w", err) + } - }(i) + return nil + }) } - wg.Wait() + s.NoError(group.Wait()) branches = s.descTree(treeID) - s.Equal(concurrency*3-2, len(branches)) + s.Len(branches, concurrency*3-2) actualForkOnLevel1 := int32(0) masterCnt := 0 for _, b := range branches { @@ -618,7 +711,7 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { } else if len(b.Ancestors) == 0 { masterCnt++ } else { - s.Equal(1, len(b.Ancestors)) + s.Len(b.Ancestors, 1) s.Equal(mbrID, b.Ancestors[0].GetBranchId()) } } @@ -630,7 +723,7 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { br := v.([]byte) // delete old branches along with create new branches err := s.deleteHistoryBranch(br) - s.Nil(err) + s.NoError(err) return true }) @@ -638,30 +731,34 @@ func (s *HistoryV2PersistenceSuite) TestConcurrentlyForkAndAppendBranches() { br := v.([]byte) // delete old branches along with create new branches err := s.deleteHistoryBranch(br) - s.Nil(err) + s.NoError(err) return true }) err = s.deleteHistoryBranch(masterBr) - s.Nil(err) + s.NoError(err) branches = s.descTree(treeID) - s.Equal(0, len(branches)) + s.Empty(branches) } -func (s *HistoryV2PersistenceSuite) getBranchByKey(m *sync.Map, k int) []byte { +func (s *HistoryV2PersistenceSuite) getBranchByKey(m *sync.Map, k int) ([]byte, bool) { v, ok := m.Load(k) - s.Equal(true, ok) - br := v.([]byte) - return br + if !ok { + return nil, false + } + br, ok := v.([]byte) + return br, ok } -func (s *HistoryV2PersistenceSuite) getIDByKey(m *sync.Map, k int) int64 { +func (s *HistoryV2PersistenceSuite) getIDByKey(m *sync.Map, k int) (int64, bool) { v, ok := m.Load(k) - s.Equal(true, ok) - id := v.(int64) - return id + if !ok { + return 0, false + } + id, ok := v.(int64) + return id, ok } func (s *HistoryV2PersistenceSuite) genRandomEvents(eventIDs []int64, version int64) []*historypb.HistoryEvent { @@ -734,7 +831,7 @@ func (s *HistoryV2PersistenceSuite) descTree(treeID string) []*persistencespb.Hi // persistence helper func (s *HistoryV2PersistenceSuite) read(branch []byte, minID, maxID int64) []*historypb.HistoryEvent { res, err := s.readWithError(branch, minID, maxID) - s.Nil(err) + s.NoError(err) return res } @@ -756,8 +853,8 @@ func (s *HistoryV2PersistenceSuite) readWithError(branch []byte, minID, maxID in if err != nil { return nil, err } - if len(resp.HistoryEvents) > 0 { - s.True(resp.Size > 0) + if len(resp.HistoryEvents) > 0 && resp.Size <= 0 { + return nil, fmt.Errorf("history response size must be positive: %d", resp.Size) } res = append(res, resp.HistoryEvents...) token = resp.NextPageToken @@ -809,7 +906,9 @@ func (s *HistoryV2PersistenceSuite) append(branch []byte, events []*historypb.Hi if err != nil { return err } - s.True(resp.Size > 0) + if resp.Size <= 0 { + return fmt.Errorf("append response size must be positive: %d", resp.Size) + } return err } diff --git a/common/persistence/persistence-tests/metadata_persistence_v2.go b/common/persistence/persistence-tests/metadata_persistence_v2.go index 5363298b23e..ceef7870b43 100644 --- a/common/persistence/persistence-tests/metadata_persistence_v2.go +++ b/common/persistence/persistence-tests/metadata_persistence_v2.go @@ -2,7 +2,6 @@ package persistencetests import ( "context" - "errors" "fmt" "strconv" "strings" @@ -22,6 +21,7 @@ import ( "go.temporal.io/server/common/persistence/cassandra" "go.temporal.io/server/common/persistence/sql" "go.temporal.io/server/common/primitives/timestamp" + "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/protorequire" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -150,31 +150,31 @@ func (m *MetadataPersistenceSuiteV2) TestCreateWithPartialNamespaceSameNameSameI ) m.NoError(err0) m.NotNil(resp0) - m.EqualValues(id, resp0.ID) + m.Equal(id, resp0.ID) // for namespace which do not have replication config set, will default to // use current cluster as active, with current cluster as all clusters resp1, err1 := m.GetNamespace(id, "") m.NoError(err1) m.NotNil(resp1) - m.EqualValues(id, resp1.Namespace.Info.Id) + m.Equal(id, resp1.Namespace.Info.Id) m.Equal(name, resp1.Namespace.Info.Name) m.Equal(state, resp1.Namespace.Info.State) m.Equal(description, resp1.Namespace.Info.Description) m.Equal(owner, resp1.Namespace.Info.Owner) m.Equal(data, resp1.Namespace.Info.Data) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp1.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp1.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp1.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp1.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(badBinaries, resp1.Namespace.Config.BadBinaries) m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(1, len(resp1.Namespace.ReplicationConfig.Clusters)) + m.Len(resp1.Namespace.ReplicationConfig.Clusters, 1) m.Equal(isGlobalNamespace, resp1.IsGlobalNamespace) m.Equal(configVersion, resp1.Namespace.ConfigVersion) m.Equal(failoverVersion, resp1.Namespace.FailoverVersion) - m.True(resp1.Namespace.ReplicationConfig.Clusters[0] == cluster.TestCurrentClusterName) + m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.Clusters[0]) m.Equal(p.InitialFailoverNotificationVersion, resp1.Namespace.FailoverNotificationVersion) m.truncatePartialNamespace() } @@ -229,31 +229,31 @@ func (m *MetadataPersistenceSuiteV2) TestCreateWithPartialNamespaceSameNameDiffe ) m.NoError(err0) m.NotNil(resp0) - m.EqualValues(id, resp0.ID) + m.Equal(id, resp0.ID) // for namespace which do not have replication config set, will default to // use current cluster as active, with current cluster as all clusters resp1, err1 := m.GetNamespace(id, "") m.NoError(err1) m.NotNil(resp1) - m.EqualValues(id, resp1.Namespace.Info.Id) + m.Equal(id, resp1.Namespace.Info.Id) m.Equal(name, resp1.Namespace.Info.Name) m.Equal(state, resp1.Namespace.Info.State) m.Equal(description, resp1.Namespace.Info.Description) m.Equal(owner, resp1.Namespace.Info.Owner) m.Equal(data, resp1.Namespace.Info.Data) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp1.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp1.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp1.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp1.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(badBinaries, resp1.Namespace.Config.BadBinaries) m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(1, len(resp1.Namespace.ReplicationConfig.Clusters)) + m.Len(resp1.Namespace.ReplicationConfig.Clusters, 1) m.Equal(isGlobalNamespace, resp1.IsGlobalNamespace) m.Equal(configVersion, resp1.Namespace.ConfigVersion) m.Equal(failoverVersion, resp1.Namespace.FailoverVersion) - m.True(resp1.Namespace.ReplicationConfig.Clusters[0] == cluster.TestCurrentClusterName) + m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.Clusters[0]) m.Equal(p.InitialFailoverNotificationVersion, resp1.Namespace.FailoverNotificationVersion) m.truncatePartialNamespace() } @@ -305,8 +305,7 @@ func (m *MetadataPersistenceSuiteV2) TestCreateWithPartialNamespaceDifferentName configVersion, failoverVersion, ) - m.Error(err0) - m.IsType(&serviceerror.NamespaceAlreadyExists{}, err0) + m.ErrorAs(err0, new(*serviceerror.NamespaceAlreadyExists)) m.Nil(resp0) m.truncatePartialNamespace() } @@ -353,31 +352,31 @@ func (m *MetadataPersistenceSuiteV2) TestCreateNamespace() { ) m.NoError(err0) m.NotNil(resp0) - m.EqualValues(id, resp0.ID) + m.Equal(id, resp0.ID) // for namespace which do not have replication config set, will default to // use current cluster as active, with current cluster as all clusters resp1, err1 := m.GetNamespace(id, "") m.NoError(err1) m.NotNil(resp1) - m.EqualValues(id, resp1.Namespace.Info.Id) + m.Equal(id, resp1.Namespace.Info.Id) m.Equal(name, resp1.Namespace.Info.Name) m.Equal(state, resp1.Namespace.Info.State) m.Equal(description, resp1.Namespace.Info.Description) m.Equal(owner, resp1.Namespace.Info.Owner) m.Equal(data, resp1.Namespace.Info.Data) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp1.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp1.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp1.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp1.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp1.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(badBinaries, resp1.Namespace.Config.BadBinaries) m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(1, len(resp1.Namespace.ReplicationConfig.Clusters)) + m.Len(resp1.Namespace.ReplicationConfig.Clusters, 1) m.Equal(isGlobalNamespace, resp1.IsGlobalNamespace) m.Equal(configVersion, resp1.Namespace.ConfigVersion) m.Equal(failoverVersion, resp1.Namespace.FailoverVersion) - m.True(resp1.Namespace.ReplicationConfig.Clusters[0] == cluster.TestCurrentClusterName) + m.Equal(cluster.TestCurrentClusterName, resp1.Namespace.ReplicationConfig.Clusters[0]) m.Equal(p.InitialFailoverNotificationVersion, resp1.Namespace.FailoverNotificationVersion) resp2, err2 := m.CreateNamespace( @@ -401,8 +400,7 @@ func (m *MetadataPersistenceSuiteV2) TestCreateNamespace() { configVersion, failoverVersion, ) - m.Error(err2) - m.IsType(&serviceerror.NamespaceAlreadyExists{}, err2) + m.ErrorAs(err2, new(*serviceerror.NamespaceAlreadyExists)) m.Nil(resp2) } @@ -429,8 +427,7 @@ func (m *MetadataPersistenceSuiteV2) TestGetNamespace() { resp0, err0 := m.GetNamespace("", "does-not-exist") m.Nil(resp0) - m.Error(err0) - m.IsType(&serviceerror.NamespaceNotFound{}, err0) + m.ErrorAs(err0, new(*serviceerror.NamespaceNotFound)) testBinaries := &namespacepb.BadBinaries{ Binaries: map[string]*namespacepb.BadBinaryInfo{ "abc": { @@ -468,25 +465,25 @@ func (m *MetadataPersistenceSuiteV2) TestGetNamespace() { ) m.NoError(err1) m.NotNil(resp1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) resp2, err2 := m.GetNamespace(id, "") m.NoError(err2) m.NotNil(resp2) - m.EqualValues(id, resp2.Namespace.Info.Id) + m.Equal(id, resp2.Namespace.Info.Id) m.Equal(name, resp2.Namespace.Info.Name) m.Equal(state, resp2.Namespace.Info.State) m.Equal(description, resp2.Namespace.Info.Description) m.Equal(owner, resp2.Namespace.Info.Owner) m.Equal(data, resp2.Namespace.Info.Data) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp2.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp2.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp2.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp2.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp2.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp2.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(testBinaries, resp2.Namespace.Config.BadBinaries) m.Equal(clusterActive, resp2.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(clusters), len(resp2.Namespace.ReplicationConfig.Clusters)) + m.Len(resp2.Namespace.ReplicationConfig.Clusters, len(clusters)) for index := range clusters { m.Equal(clusters[index], resp2.Namespace.ReplicationConfig.Clusters[index]) } @@ -498,19 +495,19 @@ func (m *MetadataPersistenceSuiteV2) TestGetNamespace() { resp3, err3 := m.GetNamespace("", name) m.NoError(err3) m.NotNil(resp3) - m.EqualValues(id, resp3.Namespace.Info.Id) + m.Equal(id, resp3.Namespace.Info.Id) m.Equal(name, resp3.Namespace.Info.Name) m.Equal(state, resp3.Namespace.Info.State) m.Equal(description, resp3.Namespace.Info.Description) m.Equal(owner, resp3.Namespace.Info.Owner) m.Equal(data, resp3.Namespace.Info.Data) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp3.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp3.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp3.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp3.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp3.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp3.Namespace.Config.VisibilityArchivalUri) m.Equal(clusterActive, resp3.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(clusters), len(resp3.Namespace.ReplicationConfig.Clusters)) + m.Len(resp3.Namespace.ReplicationConfig.Clusters, len(clusters)) for index := range clusters { m.Equal(clusters[index], resp3.Namespace.ReplicationConfig.Clusters[index]) } @@ -520,13 +517,12 @@ func (m *MetadataPersistenceSuiteV2) TestGetNamespace() { m.Equal(p.InitialFailoverNotificationVersion, resp3.Namespace.FailoverNotificationVersion) resp4, err4 := m.GetNamespace(id, name) - m.Error(err4) - m.IsType(&serviceerror.InvalidArgument{}, err4) + m.ErrorAs(err4, new(*serviceerror.InvalidArgument)) m.Nil(resp4) resp5, err5 := m.GetNamespace("", "") m.Nil(resp5) - m.IsType(&serviceerror.InvalidArgument{}, err5) + m.ErrorAs(err5, new(*serviceerror.InvalidArgument)) } // TestConcurrentCreateNamespace test @@ -564,8 +560,8 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentCreateNamespace() { var wg sync.WaitGroup for i := 1; i <= concurrency; i++ { newValue := fmt.Sprintf("v-%v", i) - wg.Add(1) - go func(data map[string]string) { + data := map[string]string{"k0": newValue} + wg.Go(func() { _, err1 := m.CreateNamespace( &persistencespb.NamespaceInfo{ Id: id, @@ -594,8 +590,7 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentCreateNamespace() { if err1 == nil { atomic.AddInt32(&successCount, 1) } - wg.Done() - }(map[string]string{"k0": newValue}) + }) } wg.Wait() m.Equal(int32(1), successCount) @@ -607,14 +602,14 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentCreateNamespace() { m.Equal(state, resp.Namespace.Info.State) m.Equal(description, resp.Namespace.Info.Description) m.Equal(owner, resp.Namespace.Info.Owner) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(testBinaries, resp.Namespace.Config.BadBinaries) m.Equal(clusterActive, resp.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(clusters), len(resp.Namespace.ReplicationConfig.Clusters)) + m.Len(resp.Namespace.ReplicationConfig.Clusters, len(clusters)) for index := range clusters { m.Equal(clusters[index], resp.Namespace.ReplicationConfig.Clusters[index]) } @@ -624,10 +619,10 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentCreateNamespace() { // check namespace data ss := strings.Split(resp.Namespace.Info.Data["k0"], "-") - m.Equal(2, len(ss)) + m.Len(ss, 2) vi, err := strconv.Atoi(ss[1]) m.NoError(err) - m.Equal(true, vi > 0 && vi <= concurrency) + m.True(vi > 0 && vi <= concurrency) } // TestConcurrentUpdateNamespace test @@ -678,7 +673,7 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentUpdateNamespace() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) resp2, err2 := m.GetNamespace(id, "") m.NoError(err2) @@ -701,8 +696,8 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentUpdateNamespace() { var wg sync.WaitGroup for i := 1; i <= concurrency; i++ { newValue := fmt.Sprintf("v-%v", i) - wg.Add(1) - go func(updatedData map[string]string) { + updatedData := map[string]string{"k0": newValue} + wg.Go(func() { err3 := m.UpdateNamespace( &persistencespb.NamespaceInfo{ Id: resp2.Namespace.Info.Id, @@ -734,8 +729,7 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentUpdateNamespace() { if err3 == nil { atomic.AddInt32(&successCount, 1) } - wg.Done() - }(map[string]string{"k0": newValue}) + }) } wg.Wait() m.Equal(int32(1), successCount) @@ -743,21 +737,21 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentUpdateNamespace() { resp3, err3 := m.GetNamespace("", name) m.NoError(err3) m.NotNil(resp3) - m.EqualValues(id, resp3.Namespace.Info.Id) + m.Equal(id, resp3.Namespace.Info.Id) m.Equal(name, resp3.Namespace.Info.Name) m.Equal(state, resp3.Namespace.Info.State) m.Equal(isGlobalNamespace, resp3.IsGlobalNamespace) m.Equal(description, resp3.Namespace.Info.Description) m.Equal(owner, resp3.Namespace.Info.Owner) - m.EqualValues(time.Duration(retention)*time.Hour*24, resp3.Namespace.Config.Retention.AsDuration()) + m.Equal(time.Duration(retention)*time.Hour*24, resp3.Namespace.Config.Retention.AsDuration()) m.Equal(historyArchivalState, resp3.Namespace.Config.HistoryArchivalState) m.Equal(historyArchivalURI, resp3.Namespace.Config.HistoryArchivalUri) m.Equal(visibilityArchivalState, resp3.Namespace.Config.VisibilityArchivalState) m.Equal(visibilityArchivalURI, resp3.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(testBinaries, resp3.Namespace.Config.BadBinaries) m.Equal(clusterActive, resp3.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(clusters), len(resp3.Namespace.ReplicationConfig.Clusters)) + m.Len(resp3.Namespace.ReplicationConfig.Clusters, len(clusters)) for index := range clusters { m.Equal(clusters[index], resp3.Namespace.ReplicationConfig.Clusters[index]) } @@ -767,10 +761,10 @@ func (m *MetadataPersistenceSuiteV2) TestConcurrentUpdateNamespace() { // check namespace data ss := strings.Split(resp3.Namespace.Info.Data["k0"], "-") - m.Equal(2, len(ss)) + m.Len(ss, 2) vi, err := strconv.Atoi(ss[1]) m.NoError(err) - m.Equal(true, vi > 0 && vi <= concurrency) + m.True(vi > 0 && vi <= concurrency) } // TestUpdateNamespace test @@ -820,7 +814,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) resp2, err2 := m.GetNamespace(id, "") m.NoError(err2) @@ -889,7 +883,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { resp4, err4 := m.GetNamespace("", name) m.NoError(err4) m.NotNil(resp4) - m.EqualValues(id, resp4.Namespace.Info.Id) + m.Equal(id, resp4.Namespace.Info.Id) m.Equal(name, resp4.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp4.IsGlobalNamespace) m.Equal(updatedState, resp4.Namespace.Info.State) @@ -903,7 +897,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { m.Equal(updatedVisibilityArchivalURI, resp4.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(testBinaries, resp4.Namespace.Config.BadBinaries) m.Equal(updateClusterActive, resp4.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(updateClusters), len(resp4.Namespace.ReplicationConfig.Clusters)) + m.Len(resp4.Namespace.ReplicationConfig.Clusters, len(updateClusters)) for index := range clusters { m.Equal(updateClusters[index], resp4.Namespace.ReplicationConfig.Clusters[index]) } @@ -916,7 +910,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { resp5, err5 := m.GetNamespace(id, "") m.NoError(err5) m.NotNil(resp5) - m.EqualValues(id, resp5.Namespace.Info.Id) + m.Equal(id, resp5.Namespace.Info.Id) m.Equal(name, resp5.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp5.IsGlobalNamespace) m.Equal(updatedState, resp5.Namespace.Info.State) @@ -929,7 +923,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { m.Equal(updatedVisibilityArchivalState, resp5.Namespace.Config.VisibilityArchivalState) m.Equal(updatedVisibilityArchivalURI, resp5.Namespace.Config.VisibilityArchivalUri) m.Equal(updateClusterActive, resp5.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(updateClusters), len(resp5.Namespace.ReplicationConfig.Clusters)) + m.Len(resp5.Namespace.ReplicationConfig.Clusters, len(updateClusters)) for index := range clusters { m.Equal(updateClusters[index], resp5.Namespace.ReplicationConfig.Clusters[index]) } @@ -973,7 +967,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { resp6, err6 := m.GetNamespace(id, "") m.NoError(err6) m.NotNil(resp6) - m.EqualValues(id, resp6.Namespace.Info.Id) + m.Equal(id, resp6.Namespace.Info.Id) m.Equal(name, resp6.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp6.IsGlobalNamespace) m.Equal(updatedState, resp6.Namespace.Info.State) @@ -987,7 +981,7 @@ func (m *MetadataPersistenceSuiteV2) TestUpdateNamespace() { m.Equal(updatedVisibilityArchivalURI, resp6.Namespace.Config.VisibilityArchivalUri) m.ProtoEqual(testBinaries, resp6.Namespace.Config.BadBinaries) m.Equal(updateClusterActive, resp6.Namespace.ReplicationConfig.ActiveClusterName) - m.Equal(len(updateClusters), len(resp6.Namespace.ReplicationConfig.Clusters)) + m.Len(resp6.Namespace.ReplicationConfig.Clusters, len(updateClusters)) for index := range clusters { m.Equal(updateClusters[index], resp4.Namespace.ReplicationConfig.Clusters[index]) } @@ -1045,7 +1039,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespace() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) _, err2 := m.GetNamespace(id, "") m.NoError(err2) @@ -1059,14 +1053,14 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespace() { resp4, err4 := m.GetNamespace("", newName) m.NoError(err4) m.NotNil(resp4) - m.EqualValues(id, resp4.Namespace.Info.Id) + m.Equal(id, resp4.Namespace.Info.Id) m.Equal(newName, resp4.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp4.IsGlobalNamespace) resp5, err5 := m.GetNamespace(id, "") m.NoError(err5) m.NotNil(resp5) - m.EqualValues(id, resp5.Namespace.Info.Id) + m.Equal(id, resp5.Namespace.Info.Id) m.Equal(newName, resp5.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp5.IsGlobalNamespace) @@ -1079,7 +1073,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespace() { resp6, err6 := m.GetNamespace(id, "") m.NoError(err6) m.NotNil(resp6) - m.EqualValues(id, resp6.Namespace.Info.Id) + m.Equal(id, resp6.Namespace.Info.Id) m.Equal(newNewName, resp6.Namespace.Info.Name) m.Equal(isGlobalNamespace, resp6.IsGlobalNamespace) } @@ -1130,7 +1124,7 @@ func (m *MetadataPersistenceSuiteV2) TestDeleteNamespace() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) resp2, err2 := m.GetNamespace("", name) m.NoError(err2) @@ -1140,22 +1134,14 @@ func (m *MetadataPersistenceSuiteV2) TestDeleteNamespace() { m.NoError(err3) // May need to loop here to avoid potential inconsistent read-after-write in cassandra - m.Eventually( - func() bool { - resp, err := m.GetNamespace("", name) - if errors.As(err, new(*serviceerror.NamespaceNotFound)) { - m.Nil(resp) - return true - } - return false - }, - 10*time.Second, - 100*time.Millisecond, - ) + await.Require(m.ctx, m.T(), func(t *await.T) { + resp, err := m.GetNamespace("", name) + require.ErrorAs(t, err, new(*serviceerror.NamespaceNotFound)) + require.Nil(t, resp) + }, 10*time.Second, 100*time.Millisecond) resp5, err5 := m.GetNamespace(id, "") - m.Error(err5) - m.IsType(&serviceerror.NamespaceNotFound{}, err5) + m.ErrorAs(err5, new(*serviceerror.NamespaceNotFound)) m.Nil(resp5) id = uuid.NewString() @@ -1184,19 +1170,17 @@ func (m *MetadataPersistenceSuiteV2) TestDeleteNamespace() { failoverVersion, ) m.NoError(err6) - m.EqualValues(id, resp6.ID) + m.Equal(id, resp6.ID) err7 := m.DeleteNamespace(id, "") m.NoError(err7) resp8, err8 := m.GetNamespace("", name) - m.Error(err8) - m.IsType(&serviceerror.NamespaceNotFound{}, err8) + m.ErrorAs(err8, new(*serviceerror.NamespaceNotFound)) m.Nil(resp8) resp9, err9 := m.GetNamespace(id, "") - m.Error(err9) - m.IsType(&serviceerror.NamespaceNotFound{}, err9) + m.ErrorAs(err9, new(*serviceerror.NamespaceNotFound)) m.Nil(resp9) } @@ -1322,14 +1306,14 @@ func (m *MetadataPersistenceSuiteV2) TestListNamespaces() { } // There should be 2 non-empty pages. - m.Equal(pageCount, 2) - m.Equal(len(inputNamespaces), len(outputNamespaces)) + m.Equal(2, pageCount) + m.Len(outputNamespaces, len(inputNamespaces)) for _, namespace := range inputNamespaces { m.DeepEqual(namespace, outputNamespaces[namespace.Namespace.Info.Id]) } } -func (m *MetadataPersistenceSuiteV2) TestListNamespaces_DeletedNamespace() { +func (m *MetadataPersistenceSuiteV2) TestListNamespacesDeletedNamespace() { inputNamespaces := []*p.GetNamespaceResponse{ { Namespace: &persistencespb.NamespaceDetail{ @@ -1409,7 +1393,7 @@ func (m *MetadataPersistenceSuiteV2) TestListNamespaces_DeletedNamespace() { m.Equal(1, pageCount) m.Len(listNamespacesPageSize2, 2) for _, namespace := range listNamespacesPageSize2 { - m.NotEqual(namespace.Namespace.Info.State, enumspb.NAMESPACE_STATE_DELETED) + m.NotEqual(enumspb.NAMESPACE_STATE_DELETED, namespace.Namespace.Info.State) } pageCount = 0 @@ -1432,7 +1416,7 @@ func (m *MetadataPersistenceSuiteV2) TestListNamespaces_DeletedNamespace() { m.Equal(2, pageCount) m.Len(listNamespacesPageSize1, 2) for _, namespace := range listNamespacesPageSize1 { - m.NotEqual(namespace.Namespace.Info.State, enumspb.NAMESPACE_STATE_DELETED) + m.NotEqual(enumspb.NAMESPACE_STATE_DELETED, namespace.Namespace.Info.State) } } @@ -1550,7 +1534,7 @@ func (m *MetadataPersistenceSuiteV2) TestCASFailureUpdateNamespace() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) resp2, err2 := m.GetNamespace(id, "") m.NoError(err2) @@ -1646,7 +1630,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceWithNameConflict() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id1, resp1.ID) + m.Equal(id1, resp1.ID) // Create second namespace resp2, err2 := m.CreateNamespace( @@ -1674,7 +1658,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceWithNameConflict() { failoverVersion, ) m.NoError(err2) - m.EqualValues(id2, resp2.ID) + m.Equal(id2, resp2.ID) // Try to rename namespace1 to the same name as namespace2 (should fail) err3 := m.MetadataManager.RenameNamespace(m.ctx, &p.RenameNamespaceRequest{ @@ -2028,18 +2012,11 @@ func (m *MetadataPersistenceSuiteV2) TestDeleteNamespaceIdempotency() { m.NoError(err3) // May need to loop here to avoid potential inconsistent read-after-write in cassandra - m.Eventually( - func() bool { - resp, err := m.GetNamespace(id, "") - if errors.As(err, new(*serviceerror.NamespaceNotFound)) { - m.Nil(resp) - return true - } - return false - }, - 10*time.Second, - 100*time.Millisecond, - ) + await.Require(m.ctx, m.T(), func(t *await.T) { + resp, err := m.GetNamespace(id, "") + require.ErrorAs(t, err, new(*serviceerror.NamespaceNotFound)) + require.Nil(t, resp) + }, 10*time.Second, 100*time.Millisecond) // Delete again - This should NOT error (deleting a non-existent namespace is a no-op) err5 := m.DeleteNamespace(id, "") @@ -2170,7 +2147,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceCassandra() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) // Verify namespace exists with original name resp2, err2 := m.GetNamespace(id, "") @@ -2188,7 +2165,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceCassandra() { resp4, err4 := m.GetNamespace("", newName) m.NoError(err4) m.NotNil(resp4) - m.EqualValues(id, resp4.Namespace.Info.Id) + m.Equal(id, resp4.Namespace.Info.Id) m.Equal(newName, resp4.Namespace.Info.Name) m.Equal(description, resp4.Namespace.Info.Description) m.Equal(owner, resp4.Namespace.Info.Owner) @@ -2200,14 +2177,10 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceCassandra() { m.Equal(newName, resp5.Namespace.Info.Name) // Verify old name no longer exists (may need eventual consistency check for Cassandra) - m.Eventually( - func() bool { - _, err := m.GetNamespace("", name) - return errors.As(err, new(*serviceerror.NamespaceNotFound)) - }, - 10*time.Second, - 100*time.Millisecond, - ) + await.Require(m.ctx, m.T(), func(t *await.T) { + _, err := m.GetNamespace("", name) + require.ErrorAs(t, err, new(*serviceerror.NamespaceNotFound)) + }, 10*time.Second, 100*time.Millisecond) // Fetch metadata version before renaming metadataBeforeRename, err9 := m.MetadataManager.GetMetadata(m.ctx) @@ -2294,7 +2267,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceSQL() { failoverVersion, ) m.NoError(err1) - m.EqualValues(id, resp1.ID) + m.Equal(id, resp1.ID) // Verify namespace exists with original name resp2, err2 := m.GetNamespace(id, "") @@ -2312,7 +2285,7 @@ func (m *MetadataPersistenceSuiteV2) TestRenameNamespaceSQL() { resp4, err4 := m.GetNamespace("", newName) m.NoError(err4) m.NotNil(resp4) - m.EqualValues(id, resp4.Namespace.Info.Id) + m.Equal(id, resp4.Namespace.Info.Id) m.Equal(newName, resp4.Namespace.Info.Name) m.Equal(description, resp4.Namespace.Info.Description) m.Equal(owner, resp4.Namespace.Info.Owner) diff --git a/common/persistence/persistence-tests/queue_persistence.go b/common/persistence/persistence-tests/queue_persistence.go index 0960538bfb6..b0e7fe9b3ad 100644 --- a/common/persistence/persistence-tests/queue_persistence.go +++ b/common/persistence/persistence-tests/queue_persistence.go @@ -3,7 +3,6 @@ package persistencetests import ( "context" "fmt" - "sync" "time" "github.com/stretchr/testify/require" @@ -11,6 +10,7 @@ import ( replicationspb "go.temporal.io/server/api/replication/v1" "go.temporal.io/server/common/debug" "go.temporal.io/server/common/persistence" + "golang.org/x/sync/errgroup" ) type ( @@ -68,24 +68,26 @@ func (s *QueuePersistenceSuite) TestNamespaceReplicationQueue() { close(messageChan) }() - wg := sync.WaitGroup{} - wg.Add(concurrentSenders) + var group errgroup.Group - for i := range concurrentSenders { - go func(senderNum int) { - defer wg.Done() + for senderNum := range concurrentSenders { + group.Go(func() error { + var publishErr error for message := range messageChan { err := s.Publish(s.ctx, message) - id := message.Attributes.(*replicationspb.ReplicationTask_NamespaceTaskAttributes).NamespaceTaskAttributes.Id - s.Nil(err, "Enqueue message failed when sender %d tried to send %s", senderNum, id) + if err != nil && publishErr == nil { + id := message.Attributes.(*replicationspb.ReplicationTask_NamespaceTaskAttributes).NamespaceTaskAttributes.Id + publishErr = fmt.Errorf("enqueue message failed when sender %d tried to send %s: %w", senderNum, id, err) + } } - }(i) + return publishErr + }) } - wg.Wait() + s.NoError(group.Wait()) result, lastRetrievedMessageID, err := s.GetReplicationMessages(s.ctx, persistence.EmptyQueueMessageID, numMessages) - s.Nil(err, "GetReplicationMessages failed.") + s.NoError(err, "GetReplicationMessages failed.") s.Len(result, numMessages) s.Equal(int64(numMessages-1), lastRetrievedMessageID) for i, task := range result { @@ -97,32 +99,32 @@ func (s *QueuePersistenceSuite) TestNamespaceReplicationQueue() { func (s *QueuePersistenceSuite) TestQueueMetadataOperations() { clusterAckLevels, err := s.GetAckLevels(s.ctx) s.Require().NoError(err) - s.Assert().Len(clusterAckLevels, 0) + s.Empty(clusterAckLevels) err = s.UpdateAckLevel(s.ctx, 10, "test1") s.Require().NoError(err) clusterAckLevels, err = s.GetAckLevels(s.ctx) s.Require().NoError(err) - s.Assert().Len(clusterAckLevels, 1) - s.Assert().Equal(int64(10), clusterAckLevels["test1"]) + s.Len(clusterAckLevels, 1) + s.Equal(int64(10), clusterAckLevels["test1"]) err = s.UpdateAckLevel(s.ctx, 20, "test1") s.Require().NoError(err) clusterAckLevels, err = s.GetAckLevels(s.ctx) s.Require().NoError(err) - s.Assert().Len(clusterAckLevels, 1) - s.Assert().Equal(int64(20), clusterAckLevels["test1"]) + s.Len(clusterAckLevels, 1) + s.Equal(int64(20), clusterAckLevels["test1"]) err = s.UpdateAckLevel(s.ctx, 25, "test2") s.Require().NoError(err) clusterAckLevels, err = s.GetAckLevels(s.ctx) s.Require().NoError(err) - s.Assert().Len(clusterAckLevels, 2) - s.Assert().Equal(int64(20), clusterAckLevels["test1"]) - s.Assert().Equal(int64(25), clusterAckLevels["test2"]) + s.Len(clusterAckLevels, 2) + s.Equal(int64(20), clusterAckLevels["test1"]) + s.Equal(int64(25), clusterAckLevels["test2"]) } // TestNamespaceReplicationDLQ tests namespace DLQ operations @@ -148,47 +150,49 @@ func (s *QueuePersistenceSuite) TestNamespaceReplicationDLQ() { close(messageChan) }() - wg := sync.WaitGroup{} - wg.Add(concurrentSenders) + var group errgroup.Group - for i := range concurrentSenders { - go func(senderNum int) { - defer wg.Done() + for senderNum := range concurrentSenders { + group.Go(func() error { + var publishErr error for message := range messageChan { err := s.PublishToNamespaceDLQ(s.ctx, message) - id := message.Attributes.(*replicationspb.ReplicationTask_NamespaceTaskAttributes).NamespaceTaskAttributes.Id - s.Nil(err, "Enqueue message failed when sender %d tried to send %s", senderNum, id) + if err != nil && publishErr == nil { + id := message.Attributes.(*replicationspb.ReplicationTask_NamespaceTaskAttributes).NamespaceTaskAttributes.Id + publishErr = fmt.Errorf("enqueue message failed when sender %d tried to send %s: %w", senderNum, id, err) + } } - }(i) + return publishErr + }) } - wg.Wait() + s.NoError(group.Wait()) result1, token, err := s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, maxMessageID, numMessages/2, nil) - s.Nil(err, "GetReplicationMessages failed.") + s.NoError(err, "GetReplicationMessages failed.") s.NotNil(token) result2, token, err := s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, maxMessageID, numMessages, token) - s.Nil(err, "GetReplicationMessages failed.") - s.Equal(len(token), 0) + s.NoError(err, "GetReplicationMessages failed.") + s.Empty(token) s.Equal(len(result1)+len(result2), numMessages) - _, _, err = s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, 1<<63-1, numMessages, nil) + _, nextToken, err := s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, 1<<63-1, numMessages, nil) s.NoError(err, "GetReplicationMessages failed.") - s.Equal(len(token), 0) + s.NotEmpty(nextToken) lastMessageID := result2[len(result2)-1].SourceTaskId err = s.DeleteMessageFromNamespaceDLQ(s.ctx, lastMessageID) s.NoError(err) result3, token, err := s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, maxMessageID, numMessages, token) - s.Nil(err, "GetReplicationMessages failed.") - s.Equal(len(token), 0) + s.NoError(err, "GetReplicationMessages failed.") + s.Empty(token) s.Equal(len(result3), numMessages-1) err = s.RangeDeleteMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, lastMessageID) s.NoError(err) result4, token, err := s.GetMessagesFromNamespaceDLQ(s.ctx, persistence.EmptyQueueMessageID, maxMessageID, numMessages, token) - s.Nil(err, "GetReplicationMessages failed.") - s.Equal(len(token), 0) - s.Equal(len(result4), 0) + s.NoError(err, "GetReplicationMessages failed.") + s.Empty(token) + s.Empty(result4) } // TestNamespaceDLQMetadataOperations tests queue metadata operations From b8c7ef4966183a04079d1a0aba16643f7c3fccf8 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 31 Aug 2026 09:24:59 -0700 Subject: [PATCH 098/139] Verify Nexus log correlation across hops (#11689) ## What changed A functional test asserting that the request ID the Nexus handler observes is the same one the caller-side failure log carries, along with the rest of the invocation tag set. ## Why Ensure we can correlate a caller-side failure with its handler-side counterpart. --------- Co-authored-by: Claude Opus 5 (1M context) --- tests/nexus_workflow_test.go | 35 ++++++++++++++++++++++++++++-- tests/testcore/test_env.go | 17 +++++++++++++++ tests/testcore/test_env_test.go | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index 72a016951b9..eb156da738e 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -48,6 +48,7 @@ import ( "go.temporal.io/server/common/testing/historyrequire" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testlogger" "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/frontend/configs" "go.temporal.io/server/tests/testcore" @@ -2631,9 +2632,11 @@ func (s *NexusWorkflowTestSuite) TestNexusCallbackAfterCallerComplete(chasmEnabl func (s *NexusWorkflowTestSuite) TestNexusOperationSyncNexusFailure(chasmEnabled bool) { env := s.newTestEnv(chasmEnabled) taskQueue := testcore.RandomizeStr(s.T().Name()) + handlerRequestIDs := make(chan string, 1) h := nexustest.Handler{ OnStartOperation: func(ctx context.Context, service, operation string, input *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + handlerRequestIDs <- options.RequestID return nil, &nexus.HandlerError{ Type: nexus.HandlerErrorTypeBadRequest, Cause: &nexus.FailureError{ @@ -2664,7 +2667,8 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationSyncNexusFailure(chasmEnabled s.NoError(w.Start()) s.T().Cleanup(w.Stop) - capture := env.StartNamespaceMetricCapture() + logCapture := env.StartNamespaceLogCapture() + metricCapture := env.StartNamespaceMetricCapture() run, err := env.SdkClient().ExecuteWorkflow(s.Context(), client.StartWorkflowOptions{ TaskQueue: taskQueue, }, callerWF) @@ -2685,7 +2689,34 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationSyncNexusFailure(chasmEnabled s.NoError(json.Unmarshal(failure.Details, &details)) s.Equal("details", details) - outboundRequests := capture.Metric("nexus_outbound_requests") + handlerRequestID := await.Rcv(s.T(), handlerRequestIDs) + s.Require().NotEmpty(handlerRequestID) + + // The attempt tag is off by one between implementations: HSM's task carries the count of + // *completed* attempts, CHASM's is 1-based. + attempt := int32(0) + if chasmEnabled { + attempt = 1 + } + logCapture.RequireContains(s.T(), testlogger.CapturedLogPattern{ + Level: testlogger.Error, + Message: "Nexus StartOperation request failed", + Tags: map[string]any{ + "operation": "StartOperation", + "wf-namespace": env.Namespace().String(), + "nexus-endpoint-target-namespace-id": "", + "request-id": handlerRequestID, + "nexus-operation": "operation", + "endpoint": endpointName, + "wf-id": run.GetID(), + "wf-run-id": run.GetRunID(), + "attempt-start": testlogger.AnyTagValue, + "attempt": attempt, + "error": "handler error (BAD_REQUEST)", + }, + }) + + outboundRequests := metricCapture.Metric("nexus_outbound_requests") s.Len(outboundRequests, 1) // Confirming that requests which do not go through our frontend are not tagged with `failure_source` s.Subset(outboundRequests[0].Tags, map[string]string{"namespace": env.Namespace().String(), "method": "StartOperation", "failure_source": "_unknown_", "outcome": "handler-error:BAD_REQUEST"}) diff --git a/tests/testcore/test_env.go b/tests/testcore/test_env.go index 079a81ba8b3..3ebe727ba72 100644 --- a/tests/testcore/test_env.go +++ b/tests/testcore/test_env.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/testing/taskpoller" @@ -593,6 +594,22 @@ func (e *TestEnv) OverrideDynamicConfig(setting dynamicconfig.GenericSetting, va return e.cluster.host.overrideDynamicConfigForTest(e.t, setting.Key(), value) } +// StartNamespaceLogCapture starts a log capture scoped to this test environment's namespace. +func (e *TestEnv) StartNamespaceLogCapture() *testlogger.Capture { + testLogger, ok := e.Logger.(*testlogger.TestLogger) + if !ok { + e.t.Fatalf("StartNamespaceLogCapture requires a *testlogger.TestLogger logger, got %T", e.Logger) + } + capture := testLogger.StartCapture( + tag.WorkflowNamespace(e.Namespace().String()), + tag.WorkflowNamespaceID(e.NamespaceID().String()), + ) + e.t.Cleanup(func() { + testLogger.StopCapture(capture) + }) + return capture +} + // StartGlobalMetricCapture starts a cluster-global metrics capture for this test and automatically stops it during cleanup. // Metric capture is cluster-global, so it is only safe on dedicated clusters. // Misuse detection is best-effort and only applies to queried metrics that produced recordings. diff --git a/tests/testcore/test_env_test.go b/tests/testcore/test_env_test.go index f8ad7234dd2..65e9a757d02 100644 --- a/tests/testcore/test_env_test.go +++ b/tests/testcore/test_env_test.go @@ -4,7 +4,10 @@ import ( "sync" "testing" + "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/testing/parallelsuite" + "go.temporal.io/server/common/testing/testlogger" ) type TestEnvSuite struct { @@ -46,3 +49,38 @@ func (s *TestEnvSuite) TestDedicatedClusterGuard_ConcurrentRecord() { wg.Wait() s.NoError(guard.validate()) } + +func (s *TestEnvSuite) TestStartNamespaceLogCapture() { + testLogger := testlogger.NewTestLogger(s.T(), testlogger.FailOnExpectedErrorOnly) + env := &TestEnv{ + FunctionalTestBase: &FunctionalTestBase{externalNamespace: namespace.Name("external")}, + Logger: testLogger, + nsName: namespace.Name("primary"), + nsID: namespace.ID("primary-id"), + t: s.T(), + } + + capture := env.StartNamespaceLogCapture() + + testLogger.Info("primary name", tag.WorkflowNamespace("primary")) + testLogger.Info("primary ID", tag.WorkflowNamespaceID("primary-id")) + testLogger.Info("external name", tag.WorkflowNamespace("external")) + testLogger.Info("unrelated name", tag.WorkflowNamespace("unrelated")) + testLogger.Info("unrelated ID", tag.WorkflowNamespaceID("unrelated-id")) + + testLogger.Info("target only", tag.NexusEndpointTargetNamespaceID("primary-id")) + testLogger.Info("unscoped") + + s.ElementsMatch([]testlogger.CapturedLog{ + { + Level: testlogger.Info, + Message: "primary name", + Tags: []tag.Tag{tag.WorkflowNamespace("primary")}, + }, + { + Level: testlogger.Info, + Message: "primary ID", + Tags: []tag.Tag{tag.WorkflowNamespaceID("primary-id")}, + }, + }, capture.Snapshot()) +} From 44131b26ad32aa5b8cfa4ac86298021483c4735d Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 31 Aug 2026 09:53:20 -0700 Subject: [PATCH 099/139] Clean up version rule test assertions (#11787) Address hundreds of linter issues. --- service/matching/version_rule_test.go | 414 ++++++++++++-------------- 1 file changed, 183 insertions(+), 231 deletions(-) diff --git a/service/matching/version_rule_test.go b/service/matching/version_rule_test.go index ad9444bf249..f012b0dbe19 100644 --- a/service/matching/version_rule_test.go +++ b/service/matching/version_rule_test.go @@ -6,13 +6,13 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" commonclock "go.temporal.io/server/common/clock" hlc "go.temporal.io/server/common/clock/hybrid_logical_clock" - "go.temporal.io/server/common/testing/protoassert" + "go.temporal.io/server/common/testing/protorequire" "google.golang.org/protobuf/proto" ) @@ -43,11 +43,11 @@ func mkNewDeleteAssignmentReq(ruleIdx int32, force bool) *workflowservice.Update } } -func mkAssignmentRulePersistence(rule *taskqueuepb.BuildIdAssignmentRule, createTs, deleteTs *hlc.Clock) *persistencespb.AssignmentRule { +func mkAssignmentRulePersistence(rule *taskqueuepb.BuildIdAssignmentRule, createTS, deleteTS *hlc.Clock) *persistencespb.AssignmentRule { return &persistencespb.AssignmentRule{ Rule: rule, - CreateTimestamp: createTs, - DeleteTimestamp: deleteTs, + CreateTimestamp: createTS, + DeleteTimestamp: deleteTS, } } @@ -84,18 +84,18 @@ func mkNewDeleteRedirectReq(source string) *workflowservice.UpdateWorkerVersioni } } -func mkNewCommitBuildIdReq(target string, force bool) *workflowservice.UpdateWorkerVersioningRulesRequest_CommitBuildId { +func mkNewCommitBuildIDReq(target string, force bool) *workflowservice.UpdateWorkerVersioningRulesRequest_CommitBuildId { return &workflowservice.UpdateWorkerVersioningRulesRequest_CommitBuildId{ TargetBuildId: target, Force: force, } } -func mkRedirectRulePersistence(rule *taskqueuepb.CompatibleBuildIdRedirectRule, createTs, deleteTs *hlc.Clock) *persistencespb.RedirectRule { +func mkRedirectRulePersistence(rule *taskqueuepb.CompatibleBuildIdRedirectRule, createTS, deleteTS *hlc.Clock) *persistencespb.RedirectRule { return &persistencespb.RedirectRule{ Rule: rule, - CreateTimestamp: createTs, - DeleteTimestamp: deleteTs, + CreateTimestamp: createTS, + DeleteTimestamp: deleteTS, } } @@ -189,44 +189,44 @@ func TestInsertAssignmentRuleBasic(t *testing.T) { maxRules := 10 clock := hlc.Zero(1) initialData := mkInitialData(0, clock) - assert.False(t, containsFullyRamped(initialData.GetAssignmentRules())) + require.False(t, containsFullyRamped(initialData.GetAssignmentRules())) expected := &persistencespb.VersioningData{AssignmentRules: []*persistencespb.AssignmentRule{}} // insert at index 0 rule1 := mkAssignmentRuleWithoutRamp("1") data, err := insertAssignmentRule(rule1, initialData, clock, 0, maxRules) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 0, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("1", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) rule2 := mkAssignmentRuleWithoutRamp("2") data, err = insertAssignmentRule(rule2, data, clock, 0, maxRules) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 0, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("2", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) rule3 := mkAssignmentRuleWithoutRamp("3") data, err = insertAssignmentRule(rule3, data, clock, 0, maxRules) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 0, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("3", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // insert into the middle rule4 := mkAssignmentRuleWithoutRamp("4") data, err = insertAssignmentRule(rule4, data, clock, 2, maxRules) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 2, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("4", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // insert with a too-big index, it should be at the back rule5 := mkAssignmentRuleWithoutRamp("5") data, err = insertAssignmentRule(rule5, data, clock, 100, maxRules) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules = append(expected.AssignmentRules, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("5", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // initial data should be unmodified - protoassert.ProtoEqual(t, mkInitialData(0, clock), initialData) + protorequire.ProtoEqual(t, mkInitialData(0, clock), initialData) } func TestInsertAssignmentRuleMaxRules(t *testing.T) { @@ -239,13 +239,12 @@ func TestInsertAssignmentRuleMaxRules(t *testing.T) { // insert 3x --> success for range 3 { data, err = insertAssignmentRule(mkAssignmentRuleWithoutRamp("1"), data, clock, 0, maxRules) - assert.NoError(t, err) + require.NoError(t, err) } // insert fourth --> error _, err = insertAssignmentRule(mkAssignmentRuleWithoutRamp("1"), data, clock, 0, maxRules) - assert.Error(t, err) - assert.Equal(t, errExceedsMaxAssignmentRules(4, maxRules), err) + require.Equal(t, errExceedsMaxAssignmentRules(4, maxRules), err) } // Test requirement that target id isn't in a version set @@ -256,20 +255,18 @@ func TestInsertAssignmentRuleInVersionSet(t *testing.T) { // target 0 --> failure _, err := insertAssignmentRule(mkAssignmentRuleWithoutRamp("0"), data, clock, 0, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errTargetIsVersionSetMember, err) + require.Equal(t, errTargetIsVersionSetMember, err) } func TestInsertAssignmentRulePartiallyRampedRuleIsRedirectSource(t *testing.T) { t.Parallel() clock := hlc.Zero(1) data, err := insertRedirectRule(mkRedirectRule("0", "1"), mkInitialData(0, clock), clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // insert 1 --> failure _, err = insertAssignmentRule(mkAssignmentRuleWithRamp("0", 10), data, clock, 0, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errPartiallyRampedAssignmentRuleIsRedirectRuleSource, err) + require.Equal(t, errPartiallyRampedAssignmentRuleIsRedirectRuleSource, err) } func TestInsertAssignmentRuleInvalidNegativeIndex(t *testing.T) { @@ -279,8 +276,7 @@ func TestInsertAssignmentRuleInvalidNegativeIndex(t *testing.T) { // insert @ -1 --> failure _, err := insertAssignmentRule(mkAssignmentRuleWithoutRamp("0"), data, clock, -1, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errInvalidNegativeIndex, err) + require.Equal(t, errInvalidNegativeIndex, err) } func TestInsertAssignmentRuleInvalidRampPercentage(t *testing.T) { @@ -290,13 +286,11 @@ func TestInsertAssignmentRuleInvalidRampPercentage(t *testing.T) { // insert with ramp percent < 0 --> failure _, err := insertAssignmentRule(mkAssignmentRuleWithRamp("0", -1), data, clock, 0, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errInvalidRampPercentage, err) + require.Equal(t, errInvalidRampPercentage, err) // insert with ramp percent > 100 --> failure _, err = insertAssignmentRule(mkAssignmentRuleWithRamp("0", 101), data, clock, 0, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errInvalidRampPercentage, err) + require.Equal(t, errInvalidRampPercentage, err) } func TestReplaceAssignmentRuleBasic(t *testing.T) { @@ -325,37 +319,37 @@ func TestReplaceAssignmentRuleBasic(t *testing.T) { rule2 := mkAssignmentRuleWithoutRamp("2") clock = hlc.Next(clock, timesource) data, err = replaceAssignmentRule(rule2, data, clock, 2, false) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules[2].DeleteTimestamp = clock expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 2, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("2", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // [1, 1, 2] --> [0, 1, 2] // [1A, 1A, 2A, 1D] --> [0A, 1D, 1A, 2A, 1D] rule0 := mkAssignmentRuleWithoutRamp("0") clock = hlc.Next(clock, timesource) data, err = replaceAssignmentRule(rule0, data, clock, 0, false) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules[0].DeleteTimestamp = clock expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 0, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("0", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // [0, 1, 2] --> [0, 11, 2] // [0A, 1D, 1A, 2A, 1D] --> [0A, 1D, 11A, 1D, 2A, 1D] rule11 := mkAssignmentRuleWithoutRamp("11") clock = hlc.Next(clock, timesource) data, err = replaceAssignmentRule(rule11, data, clock, 1, false) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules[2].DeleteTimestamp = clock expected.AssignmentRules = slices.Insert(expected.AssignmentRules, 2, mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("11", 100), clock, nil)) - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // out-of-bounds indices --> failure _, err = replaceAssignmentRule(rule11, data, clock, 99, false) - assert.Error(t, err) + require.Error(t, err) - _, err = replaceAssignmentRule(rule11, data, clock, 99, false) - assert.Error(t, err) + _, err = replaceAssignmentRule(rule11, data, clock, -1, false) + require.Error(t, err) } func TestReplaceAssignmentRuleInVersionSet(t *testing.T) { @@ -369,8 +363,7 @@ func TestReplaceAssignmentRuleInVersionSet(t *testing.T) { // replace 0 --> failure _, err = replaceAssignmentRule(mkAssignmentRuleWithoutRamp("0"), data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errTargetIsVersionSetMember, err) + require.Equal(t, errTargetIsVersionSetMember, err) } func TestReplaceAssignmentRulePartiallyRampedRuleIsRedirectSource(t *testing.T) { @@ -387,8 +380,7 @@ func TestReplaceAssignmentRulePartiallyRampedRuleIsRedirectSource(t *testing.T) // replace with target isSource and ramp < 100 --> failure _, err := replaceAssignmentRule(mkAssignmentRuleWithRamp("0", 10), data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errPartiallyRampedAssignmentRuleIsRedirectRuleSource, err) + require.Equal(t, errPartiallyRampedAssignmentRuleIsRedirectRuleSource, err) } func TestReplaceAssignmentRuleTestRequireFullyRamped(t *testing.T) { @@ -396,21 +388,17 @@ func TestReplaceAssignmentRuleTestRequireFullyRamped(t *testing.T) { clock := hlc.Zero(1) data := mkInitialData(0, clock) var err error - data.AssignmentRules = []*persistencespb.AssignmentRule{ - mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("1", 10), clock, nil), - } // replace fully-ramped rule with partially-ramped rule --> failure data.AssignmentRules = []*persistencespb.AssignmentRule{ mkAssignmentRulePersistence(mkAssignmentRuleWithoutRamp("1"), clock, nil), } _, err = replaceAssignmentRule(mkAssignmentRuleWithRamp("2", 20), data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errRequireFullyRampedAssignmentRule, err) + require.Equal(t, errRequireFullyRampedAssignmentRule, err) // same as above but with force --> success _, err = replaceAssignmentRule(mkAssignmentRuleWithRamp("4", 20), data, clock, 0, true) - assert.NoError(t, err) + require.NoError(t, err) } func TestReplaceAssignmentRuleIndexOutOfBounds(t *testing.T) { @@ -423,13 +411,11 @@ func TestReplaceAssignmentRuleIndexOutOfBounds(t *testing.T) { // replace @ -1 --> failure _, err := replaceAssignmentRule(mkAssignmentRuleWithoutRamp("0"), data, clock, -1, false) - assert.Error(t, err) - assert.Equal(t, errAssignmentRuleIndexOutOfBounds(-1, len(data.AssignmentRules)), err) + require.Equal(t, errAssignmentRuleIndexOutOfBounds(-1, len(data.AssignmentRules)), err) // replace @ 1 --> failure _, err = replaceAssignmentRule(mkAssignmentRuleWithoutRamp("0"), data, clock, 1, false) - assert.Error(t, err) - assert.Equal(t, errAssignmentRuleIndexOutOfBounds(1, len(data.AssignmentRules)), err) + require.Equal(t, errAssignmentRuleIndexOutOfBounds(1, len(data.AssignmentRules)), err) } func TestReplaceAssignmentRuleInvalidRampPercentage(t *testing.T) { @@ -442,13 +428,11 @@ func TestReplaceAssignmentRuleInvalidRampPercentage(t *testing.T) { // replace with ramp percent < 0 --> failure _, err := replaceAssignmentRule(mkAssignmentRuleWithRamp("0", -1), data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errInvalidRampPercentage, err) + require.Equal(t, errInvalidRampPercentage, err) // replace with ramp percent > 100 --> failure _, err = replaceAssignmentRule(mkAssignmentRuleWithRamp("0", 101), data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errInvalidRampPercentage, err) + require.Equal(t, errInvalidRampPercentage, err) } func TestDeleteAssignmentRuleBasic(t *testing.T) { @@ -477,21 +461,21 @@ func TestDeleteAssignmentRuleBasic(t *testing.T) { // in-bounds index --> success data, err = deleteAssignmentRule(data, nextClock(), 2, false) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules[2].DeleteTimestamp = clock - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) data, err = deleteAssignmentRule(data, nextClock(), 0, false) - assert.NoError(t, err) + require.NoError(t, err) expected.AssignmentRules[0].DeleteTimestamp = clock - protoassert.ProtoEqual(t, expected, data) + protorequire.ProtoEqual(t, expected, data) // out-of-bounds index --> failure _, err = deleteAssignmentRule(data, nextClock(), 99, false) - assert.Error(t, err) + require.Error(t, err) _, err = deleteAssignmentRule(data, nextClock(), -1, false) - assert.Error(t, err) + require.Error(t, err) } func TestDeleteAssignmentRuleTestRequireFullyRamped(t *testing.T) { @@ -499,21 +483,17 @@ func TestDeleteAssignmentRuleTestRequireFullyRamped(t *testing.T) { clock := hlc.Zero(1) data := mkInitialData(0, clock) var err error - data.AssignmentRules = []*persistencespb.AssignmentRule{ - mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("1", 10), clock, nil), - } // delete only fully-ramped rule --> failure data.AssignmentRules = []*persistencespb.AssignmentRule{ mkAssignmentRulePersistence(mkAssignmentRuleWithoutRamp("1"), clock, nil), } _, err = deleteAssignmentRule(data, clock, 0, false) - assert.Error(t, err) - assert.Equal(t, errRequireFullyRampedAssignmentRule, err) + require.Equal(t, errRequireFullyRampedAssignmentRule, err) // same as above but with force --> success _, err = deleteAssignmentRule(data, clock, 0, true) - assert.NoError(t, err) + require.NoError(t, err) // delete one of two fully-ramped rules --> success data.AssignmentRules = []*persistencespb.AssignmentRule{ @@ -521,7 +501,7 @@ func TestDeleteAssignmentRuleTestRequireFullyRamped(t *testing.T) { mkAssignmentRulePersistence(mkAssignmentRuleWithoutRamp("1"), clock, nil), } _, err = deleteAssignmentRule(data, clock, 0, false) - assert.NoError(t, err) + require.NoError(t, err) } func TestDeleteAssignmentRuleIndexOutOfBounds(t *testing.T) { @@ -534,13 +514,11 @@ func TestDeleteAssignmentRuleIndexOutOfBounds(t *testing.T) { // delete @ -1 --> failure _, err := deleteAssignmentRule(data, clock, -1, false) - assert.Error(t, err) - assert.Equal(t, errAssignmentRuleIndexOutOfBounds(-1, len(data.AssignmentRules)), err) + require.Equal(t, errAssignmentRuleIndexOutOfBounds(-1, len(data.AssignmentRules)), err) // delete @ 1 --> failure _, err = deleteAssignmentRule(data, clock, 1, false) - assert.Error(t, err) - assert.Equal(t, errAssignmentRuleIndexOutOfBounds(1, len(data.AssignmentRules)), err) + require.Equal(t, errAssignmentRuleIndexOutOfBounds(1, len(data.AssignmentRules)), err) } func TestAddRedirectRuleBasic(t *testing.T) { @@ -551,29 +529,23 @@ func TestAddRedirectRuleBasic(t *testing.T) { rule1 := mkRedirectRule("1", "0") data, err := insertRedirectRule(rule1, initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) expectedSet = append(expectedSet, mkRedirectRulePersistence(rule1, clock, nil)) - for _, r := range data.RedirectRules { - assert.Contains(t, expectedSet, r) - } + protorequire.ProtoElementsMatch(t, expectedSet, data.RedirectRules) rule2 := mkRedirectRule("2", "0") data, err = insertRedirectRule(rule2, data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) expectedSet = append(expectedSet, mkRedirectRulePersistence(rule2, clock, nil)) - for _, r := range data.RedirectRules { - assert.Contains(t, expectedSet, r) - } + protorequire.ProtoElementsMatch(t, expectedSet, data.RedirectRules) rule3 := mkRedirectRule("3", "0") data, err = insertRedirectRule(rule3, data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) expectedSet = append(expectedSet, mkRedirectRulePersistence(rule3, clock, nil)) - for _, r := range data.RedirectRules { - assert.Contains(t, expectedSet, r) - } + protorequire.ProtoElementsMatch(t, expectedSet, data.RedirectRules) - protoassert.ProtoEqual(t, mkInitialData(0, clock), initialData) + protorequire.ProtoEqual(t, mkInitialData(0, clock), initialData) } func TestAddRedirectRuleMaxRules(t *testing.T) { @@ -588,13 +560,12 @@ func TestAddRedirectRuleMaxRules(t *testing.T) { src := fmt.Sprintf("%d", i) dst := fmt.Sprintf("%d", i+1) data, err = insertRedirectRule(mkRedirectRule(src, dst), data, clock, maxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) } // insert fourth --> error _, err = insertRedirectRule(mkRedirectRule("10", "20"), data, clock, maxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errExceedsMaxRedirectRules(4, maxRules), err) + require.Equal(t, errExceedsMaxRedirectRules(4, maxRules), err) } func TestAddRedirectRuleInVersionSet(t *testing.T) { @@ -605,13 +576,11 @@ func TestAddRedirectRuleInVersionSet(t *testing.T) { // insert with source build id "0" --> failure _, err := insertRedirectRule(mkRedirectRule("0", "1"), initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errSourceIsVersionSetMember, err) + require.Equal(t, errSourceIsVersionSetMember, err) // insert with target build id "0" --> failure _, err = insertRedirectRule(mkRedirectRule("1", "0"), initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errTargetIsVersionSetMember, err) + require.Equal(t, errTargetIsVersionSetMember, err) } func TestAddRedirectRuleSourceIsPartiallyRampedAssignmentRuleTarget(t *testing.T) { @@ -625,8 +594,7 @@ func TestAddRedirectRuleSourceIsPartiallyRampedAssignmentRuleTarget(t *testing.T // insert redirect rule with target 1 --> failure _, err := insertRedirectRule(mkRedirectRule("1", "0"), data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errSourceIsPartiallyRampedAssignmentRuleTarget, err) + require.Equal(t, errSourceIsPartiallyRampedAssignmentRuleTarget, err) } func TestAddRedirectRuleAlreadyExists(t *testing.T) { @@ -636,12 +604,11 @@ func TestAddRedirectRuleAlreadyExists(t *testing.T) { // insert with source build id "0" data, err := insertRedirectRule(mkRedirectRule("0", "1"), initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // insert with source build id "0" --> failure _, err = insertRedirectRule(mkRedirectRule("0", "6"), data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errSourceAlreadyExists("0", "1"), err) + require.Equal(t, errSourceAlreadyExists("0", "1"), err) } func TestAddRedirectRuleCreateCycle(t *testing.T) { @@ -651,17 +618,15 @@ func TestAddRedirectRuleCreateCycle(t *testing.T) { // insert with source -> target == "0" -> "0" --> failure _, err := insertRedirectRule(mkRedirectRule("0", "0"), initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) // insert with source -> target == "0" -> "1" --> success data, err := insertRedirectRule(mkRedirectRule("0", "1"), initialData, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // insert with source build id "1" -> "0" --> failure _, err = insertRedirectRule(mkRedirectRule("1", "0"), data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) } func TestAddRedirectRuleMaxUpstreamBuildIDs(t *testing.T) { @@ -673,18 +638,17 @@ func TestAddRedirectRuleMaxUpstreamBuildIDs(t *testing.T) { // insert (4->5) // 4 ---> 5 data, err := insertRedirectRule(mkRedirectRule("4", "5"), data, clock, ignoreMaxRules, maxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // insert (5->6) // 4 ---> 5 ---> 6 data, err = insertRedirectRule(mkRedirectRule("5", "6"), data, clock, ignoreMaxRules, maxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // insert (6->7) // 4 ---> 5 ---> 6 ---> 7 _, err = insertRedirectRule(mkRedirectRule("6", "7"), data, clock, ignoreMaxRules, maxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errExceedsMaxUpstreamBuildIDs(3, maxUpstreamBuildIDs), err) + require.Equal(t, errExceedsMaxUpstreamBuildIDs(3, maxUpstreamBuildIDs), err) } func TestAddRedirectRuleUnversionedTarget(t *testing.T) { @@ -694,7 +658,7 @@ func TestAddRedirectRuleUnversionedTarget(t *testing.T) { // insert (1->"") errors _, err := insertRedirectRule(mkRedirectRule("1", ""), data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.ErrorIs(t, err, errUnversionedRedirectRuleTarget) + require.ErrorIs(t, err, errUnversionedRedirectRuleTarget) } func TestReplaceRedirectRuleBasic(t *testing.T) { @@ -713,14 +677,15 @@ func TestReplaceRedirectRuleBasic(t *testing.T) { prevRule := getActiveRedirectRuleBySrc(source, data) rule := mkRedirectRule(source, target) data, err = replaceRedirectRule(rule, data, clock, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) newActive := getActiveRedirectRuleBySrc(source, data) - protoassert.ProtoEqual(t, newActive.GetRule(), rule) + protorequire.ProtoEqual(t, rule, newActive.GetRule()) deleted := getDeletedRedirectRuleBySrc(source, data) - assert.Equal(t, deleted[0].GetRule().GetSourceBuildId(), prevRule.GetRule().GetSourceBuildId()) - assert.Equal(t, deleted[0].GetRule().GetTargetBuildId(), prevRule.GetRule().GetTargetBuildId()) + require.Len(t, deleted, 1) + require.Equal(t, prevRule.GetRule().GetSourceBuildId(), deleted[0].GetRule().GetSourceBuildId()) + require.Equal(t, prevRule.GetRule().GetTargetBuildId(), deleted[0].GetRule().GetTargetBuildId()) for _, dr := range deleted { - assert.GreaterOrEqual(t, newActive.GetCreateTimestamp().GetWallClock(), dr.GetDeleteTimestamp().GetWallClock()) + require.GreaterOrEqual(t, newActive.GetCreateTimestamp().GetWallClock(), dr.GetDeleteTimestamp().GetWallClock()) } } @@ -743,8 +708,7 @@ func TestReplaceRedirectRuleInVersionSet(t *testing.T) { // replace with target 0 --> failure _, err = replaceRedirectRule(mkRedirectRule("1", "0"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errTargetIsVersionSetMember, err) + require.Equal(t, errTargetIsVersionSetMember, err) } func TestReplaceRedirectRuleCreateCycle(t *testing.T) { @@ -759,20 +723,16 @@ func TestReplaceRedirectRuleCreateCycle(t *testing.T) { var err error _, err = replaceRedirectRule(mkRedirectRule("0", "0"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) _, err = replaceRedirectRule(mkRedirectRule("2", "0"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) _, err = replaceRedirectRule(mkRedirectRule("1", "0"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) _, err = replaceRedirectRule(mkRedirectRule("2", "1"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errIsCyclic, err) + require.Equal(t, errIsCyclic, err) } func TestReplaceRedirectRuleMaxUpstreamBuildIDs(t *testing.T) { @@ -791,13 +751,12 @@ func TestReplaceRedirectRuleMaxUpstreamBuildIDs(t *testing.T) { // replace(2, new_target=1) // 2 ---> 1, 4 ---> 5 ---> 6 data, err := replaceRedirectRule(mkRedirectRule("2", "1"), data, clock, maxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // replace(2, new_target=4) // 2 ---> 4 ---> 5 ---> 6 _, err = replaceRedirectRule(mkRedirectRule("2", "4"), data, clock, maxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errExceedsMaxUpstreamBuildIDs(3, maxUpstreamBuildIDs), err) + require.Equal(t, errExceedsMaxUpstreamBuildIDs(3, maxUpstreamBuildIDs), err) } func TestReplaceRedirectRuleUnversionedTarget(t *testing.T) { @@ -807,11 +766,11 @@ func TestReplaceRedirectRuleUnversionedTarget(t *testing.T) { // insert (1->2) so that we can replace data, err := insertRedirectRule(mkRedirectRule("1", "2"), data, clock, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // replace (1->"") errors _, err = replaceRedirectRule(mkRedirectRule("1", ""), data, clock, ignoreMaxUpstreamBuildIDs) - assert.ErrorIs(t, err, errUnversionedRedirectRuleTarget) + require.ErrorIs(t, err, errUnversionedRedirectRuleTarget) } func TestReplaceRedirectRuleNotFound(t *testing.T) { @@ -822,8 +781,7 @@ func TestReplaceRedirectRuleNotFound(t *testing.T) { // fails because no rules to replace _, err = replaceRedirectRule(mkRedirectRule("1", "100"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errSourceNotFound("1"), err) + require.Equal(t, errSourceNotFound("1"), err) data.RedirectRules = []*persistencespb.RedirectRule{ mkRedirectRulePersistence(mkRedirectRule("0", "1"), clock, nil), @@ -831,8 +789,7 @@ func TestReplaceRedirectRuleNotFound(t *testing.T) { // fails because source doesnt exist _, err = replaceRedirectRule(mkRedirectRule("1", "100"), data, clock, ignoreMaxUpstreamBuildIDs) - assert.Error(t, err) - assert.Equal(t, errSourceNotFound("1"), err) + require.Equal(t, errSourceNotFound("1"), err) } func TestDeleteRedirectRuleBasic(t *testing.T) { @@ -849,21 +806,21 @@ func TestDeleteRedirectRuleBasic(t *testing.T) { clock1 := hlc.Next(clock, timesource) data, err = deleteRedirectRule("1", data, clock1) - assert.NoError(t, err) + require.NoError(t, err) deleted := getDeletedRedirectRuleBySrc("1", data) - assert.Equal(t, 1, len(deleted)) - assert.Equal(t, clock1.GetWallClock(), deleted[0].GetDeleteTimestamp().GetWallClock()) - assert.Equal(t, "1", deleted[0].GetRule().GetSourceBuildId()) - assert.Equal(t, "2", deleted[0].GetRule().GetTargetBuildId()) + require.Len(t, deleted, 1) + require.Equal(t, clock1.GetWallClock(), deleted[0].GetDeleteTimestamp().GetWallClock()) + require.Equal(t, "1", deleted[0].GetRule().GetSourceBuildId()) + require.Equal(t, "2", deleted[0].GetRule().GetTargetBuildId()) clock2 := hlc.Next(clock1, timesource) data, err = deleteRedirectRule("2", data, clock2) - assert.NoError(t, err) + require.NoError(t, err) deleted = getDeletedRedirectRuleBySrc("2", data) - assert.Equal(t, 1, len(deleted)) - assert.Equal(t, clock2.GetWallClock(), deleted[0].GetDeleteTimestamp().GetWallClock()) - assert.Equal(t, "2", deleted[0].GetRule().GetSourceBuildId()) - assert.Equal(t, "3", deleted[0].GetRule().GetTargetBuildId()) + require.Len(t, deleted, 1) + require.Equal(t, clock2.GetWallClock(), deleted[0].GetDeleteTimestamp().GetWallClock()) + require.Equal(t, "2", deleted[0].GetRule().GetSourceBuildId()) + require.Equal(t, "3", deleted[0].GetRule().GetTargetBuildId()) } func TestDeleteRedirectRuleNotFound(t *testing.T) { @@ -873,8 +830,7 @@ func TestDeleteRedirectRuleNotFound(t *testing.T) { // fails because no rules to delete _, err := deleteRedirectRule("1", data, clock) - assert.Error(t, err) - assert.Equal(t, errSourceNotFound("1"), err) + require.Equal(t, errSourceNotFound("1"), err) // insert a rule to replace data.RedirectRules = []*persistencespb.RedirectRule{ @@ -883,8 +839,7 @@ func TestDeleteRedirectRuleNotFound(t *testing.T) { // fails because no rule with that source _, err = deleteRedirectRule("1", data, clock) - assert.Error(t, err) - assert.Equal(t, errSourceNotFound("1"), err) + require.Equal(t, errSourceNotFound("1"), err) } func TestGetWorkerVersioningRules(t *testing.T) { @@ -909,27 +864,27 @@ func TestGetWorkerVersioningRules(t *testing.T) { // Call list successfully dummyClock := hlc.Zero(99) // used to generate conflict token, but not in this test resp, err := GetTimestampedWorkerVersioningRules(data, dummyClock) - assert.NoError(t, err) + require.NoError(t, err) // check assignment rules assignmentRules := resp.GetResponse().GetAssignmentRules() - assert.Equal(t, 3, len(assignmentRules)) - protoassert.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ + require.Len(t, assignmentRules, 3) + protorequire.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ Rule: mkAssignmentRuleWithoutRamp("1"), CreateTime: hlc.ProtoTimestamp(clock1), }, assignmentRules[0]) - protoassert.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ + protorequire.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ Rule: mkAssignmentRuleWithoutRamp("10"), CreateTime: hlc.ProtoTimestamp(clock2), }, assignmentRules[1]) - protoassert.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ + protorequire.ProtoEqual(t, &taskqueuepb.TimestampedBuildIdAssignmentRule{ Rule: mkAssignmentRuleWithoutRamp("100"), CreateTime: hlc.ProtoTimestamp(clock2), }, assignmentRules[2]) // check redirect rules, no ordering guarantee redirectRules := resp.GetResponse().GetCompatibleRedirectRules() - assert.Equal(t, 3, len(redirectRules)) + require.Len(t, redirectRules, 3) contains := func(expected *taskqueuepb.TimestampedCompatibleBuildIdRedirectRule) bool { for _, r := range redirectRules { if proto.Equal(expected, r) { @@ -938,19 +893,19 @@ func TestGetWorkerVersioningRules(t *testing.T) { } return false } - assert.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ + require.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ Rule: mkRedirectRule("1", "2"), CreateTime: hlc.ProtoTimestamp(clock1), })) - assert.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ + require.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ Rule: mkRedirectRule("3", "4"), CreateTime: hlc.ProtoTimestamp(clock2), })) - assert.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ + require.True(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ Rule: mkRedirectRule("4", "5"), CreateTime: hlc.ProtoTimestamp(clock2), })) - assert.False(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ + require.False(t, contains(&taskqueuepb.TimestampedCompatibleBuildIdRedirectRule{ Rule: mkRedirectRule("4", "6"), CreateTime: hlc.ProtoTimestamp(clock1), })) @@ -968,28 +923,28 @@ func TestCleanupRedirectRuleTombstones(t *testing.T) { rule1 := mkRedirectRule("1", "10") clock1 := hlc.Next(clock, timesource) data, err := insertRedirectRule(rule1, initialData, clock1, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) rule2 := mkRedirectRule("2", "10") data, err = insertRedirectRule(rule2, data, clock1, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) rule3 := mkRedirectRule("3", "10") data, err = insertRedirectRule(rule3, data, clock1, ignoreMaxRules, ignoreMaxUpstreamBuildIDs) - assert.NoError(t, err) + require.NoError(t, err) // delete "now," ~1 hour ago clock4 := hlc.Next(clock, timesource) data, err = deleteRedirectRule("1", data, clock4) - assert.NoError(t, err) + require.NoError(t, err) // delete 35 min later, ~25 min ago timesource.Advance(35 * time.Minute) clock5 := hlc.Next(clock, timesource) data, err = deleteRedirectRule("2", data, clock5) - assert.NoError(t, err) + require.NoError(t, err) // delete 25 min later, ~now (real time.Now()) timesource.Advance(25 * time.Minute) clock6 := hlc.Next(clock, timesource) data, err = deleteRedirectRule("3", data, clock6) - assert.NoError(t, err) + require.NoError(t, err) // Remove data that was deleted > 30 min ago --> remove first rule data = CleanupRuleTombstones(data, 30*time.Minute) @@ -997,9 +952,9 @@ func TestCleanupRedirectRuleTombstones(t *testing.T) { for _, r := range data.GetRedirectRules() { sources = append(sources, r.GetRule().GetSourceBuildId()) } - assert.NotContains(t, sources, "1") - assert.Contains(t, sources, "2") - assert.Contains(t, sources, "3") + require.NotContains(t, sources, "1") + require.Contains(t, sources, "2") + require.Contains(t, sources, "3") // Remove data that was deleted > 5 min ago --> remove second rule data = CleanupRuleTombstones(data, 5*time.Minute) @@ -1007,9 +962,9 @@ func TestCleanupRedirectRuleTombstones(t *testing.T) { for _, r := range data.GetRedirectRules() { sources = append(sources, r.GetRule().GetSourceBuildId()) } - assert.NotContains(t, sources, "1") - assert.NotContains(t, sources, "2") - assert.Contains(t, sources, "3") + require.NotContains(t, sources, "1") + require.NotContains(t, sources, "2") + require.Contains(t, sources, "3") } func TestCommitBuildIDBasic(t *testing.T) { @@ -1034,9 +989,9 @@ func TestCommitBuildIDBasic(t *testing.T) { } var err error - data, err = CommitBuildID(clock2, data, mkNewCommitBuildIdReq("10", false), true, ignoreMaxRules) - assert.NoError(t, err) - protoassert.ProtoEqual(t, expected, data) + data, err = CommitBuildID(clock2, data, mkNewCommitBuildIDReq("10", false), true, ignoreMaxRules) + require.NoError(t, err) + protorequire.ProtoEqual(t, expected, data) // make sure multiple commits are idempotent except for timestamps clock3 := hlc.Next(clock2, timesource) @@ -1049,9 +1004,9 @@ func TestCommitBuildIDBasic(t *testing.T) { mkAssignmentRulePersistence(mkAssignmentRuleWithRamp("10", 100), clock3, nil), }, } - data, err = CommitBuildID(clock3, data, mkNewCommitBuildIdReq("10", false), true, ignoreMaxRules) - assert.NoError(t, err) - protoassert.ProtoEqual(t, expected, data) + data, err = CommitBuildID(clock3, data, mkNewCommitBuildIDReq("10", false), true, ignoreMaxRules) + require.NoError(t, err) + protorequire.ProtoEqual(t, expected, data) } func TestCommitBuildIDNoRecentPoller(t *testing.T) { @@ -1070,13 +1025,12 @@ func TestCommitBuildIDNoRecentPoller(t *testing.T) { var err error // without force --> fail - _, err = CommitBuildID(clock2, data, mkNewCommitBuildIdReq("10", false), false, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errNoRecentPollerOnCommitVersion("10"), err) + _, err = CommitBuildID(clock2, data, mkNewCommitBuildIDReq("10", false), false, ignoreMaxRules) + require.Equal(t, errNoRecentPollerOnCommitVersion("10"), err) // with force --> success - _, err = CommitBuildID(clock2, data, mkNewCommitBuildIdReq("10", true), false, ignoreMaxRules) - assert.NoError(t, err) + _, err = CommitBuildID(clock2, data, mkNewCommitBuildIDReq("10", true), false, ignoreMaxRules) + require.NoError(t, err) } func TestCommitBuildIDInVersionSet(t *testing.T) { @@ -1093,9 +1047,8 @@ func TestCommitBuildIDInVersionSet(t *testing.T) { var err error // with target 0 --> fail - _, err = CommitBuildID(clock2, data, mkNewCommitBuildIdReq("0", false), true, ignoreMaxRules) - assert.Error(t, err) - assert.Equal(t, errTargetIsVersionSetMember, err) + _, err = CommitBuildID(clock2, data, mkNewCommitBuildIDReq("0", false), true, ignoreMaxRules) + require.Equal(t, errTargetIsVersionSetMember, err) } func TestCommitBuildIDMaxAssignmentRules(t *testing.T) { @@ -1114,9 +1067,8 @@ func TestCommitBuildIDMaxAssignmentRules(t *testing.T) { var err error // commit a new target, no rules to be deleted --> fail - _, err = CommitBuildID(clock2, data, mkNewCommitBuildIdReq("1000", false), true, maxRules) - assert.Error(t, err) - assert.Equal(t, errExceedsMaxAssignmentRules(4, maxRules), err) + _, err = CommitBuildID(clock2, data, mkNewCommitBuildIDReq("1000", false), true, maxRules) + require.Equal(t, errExceedsMaxAssignmentRules(4, maxRules), err) } /* @@ -1165,40 +1117,40 @@ func TestFindTerminalBuildId(t *testing.T) { | 5 <------ 3 <------ 4 */ - createTs := hlc.Zero(1) + createTS := hlc.Zero(1) redirectRules := []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("1", "10"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("4", "3"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("1", "10"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("4", "3"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTS, nil), } - assert.Equal(t, "10", findTerminalBuildId("1", redirectRules)) - assert.Equal(t, "10", findTerminalBuildId("2", redirectRules)) - assert.Equal(t, "10", findTerminalBuildId("3", redirectRules)) - assert.Equal(t, "10", findTerminalBuildId("4", redirectRules)) - assert.Equal(t, "10", findTerminalBuildId("5", redirectRules)) - assert.Equal(t, "10", findTerminalBuildId("10", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("1", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("2", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("3", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("4", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("5", redirectRules)) + require.Equal(t, "10", findTerminalBuildId("10", redirectRules)) // empty rule set - assert.Equal(t, "11", findTerminalBuildId("11", []*persistencespb.RedirectRule{})) + require.Equal(t, "11", findTerminalBuildId("11", []*persistencespb.RedirectRule{})) // single rule redirectRules = []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTS, nil), } - assert.Equal(t, "2", findTerminalBuildId("1", redirectRules)) - assert.Equal(t, "2", findTerminalBuildId("2", redirectRules)) + require.Equal(t, "2", findTerminalBuildId("1", redirectRules)) + require.Equal(t, "2", findTerminalBuildId("2", redirectRules)) // cyclic rule set redirectRules = []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTS, nil), } - assert.Equal(t, "", findTerminalBuildId("1", redirectRules)) - assert.Equal(t, "", findTerminalBuildId("2", redirectRules)) + require.Empty(t, findTerminalBuildId("1", redirectRules)) + require.Empty(t, findTerminalBuildId("2", redirectRules)) } func TestGetUpstreamBuildIds_NoCycle(t *testing.T) { @@ -1214,21 +1166,21 @@ func TestGetUpstreamBuildIds_NoCycle(t *testing.T) { | 5 <------ 3 <------ 4 */ - createTs := hlc.Zero(1) + createTS := hlc.Zero(1) redirectRules := []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("1", "10"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("4", "3"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("1", "10"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("2", "1"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("4", "3"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTS, nil), } expectedUpstreamBuildIds := []string{"2", "5", "3", "4"} upstreamBuildIds := getUpstreamBuildIds("1", redirectRules) slices.Sort(expectedUpstreamBuildIds) slices.Sort(upstreamBuildIds) - assert.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) + require.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) } func TestGetUpstreamBuildIds_WithCycle(t *testing.T) { @@ -1241,19 +1193,19 @@ func TestGetUpstreamBuildIds_WithCycle(t *testing.T) { | v 5 <------ 3 ------> 4 */ - createTs := hlc.Zero(1) + createTS := hlc.Zero(1) redirectRules := []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("2", "3"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "4"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("1", "2"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("2", "3"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "4"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTS, nil), } expectedUpstreamBuildIds := []string{"5", "3", "2"} upstreamBuildIds := getUpstreamBuildIds("1", redirectRules) slices.Sort(expectedUpstreamBuildIds) slices.Sort(upstreamBuildIds) - assert.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) + require.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) /* e.g. @@ -1264,15 +1216,15 @@ func TestGetUpstreamBuildIds_WithCycle(t *testing.T) { 5 <------ 3 ------> 4 */ redirectRules = []*persistencespb.RedirectRule{ - mkRedirectRulePersistence(mkRedirectRule("2", "3"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "4"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("4", "2"), createTs, nil), - mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTs, nil), + mkRedirectRulePersistence(mkRedirectRule("2", "3"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "4"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("3", "5"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("4", "2"), createTS, nil), + mkRedirectRulePersistence(mkRedirectRule("5", "1"), createTS, nil), } expectedUpstreamBuildIds = []string{"5", "3", "2", "4"} upstreamBuildIds = getUpstreamBuildIds("1", redirectRules) slices.Sort(expectedUpstreamBuildIds) slices.Sort(upstreamBuildIds) - assert.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) + require.Equal(t, expectedUpstreamBuildIds, upstreamBuildIds) } From 2374d91a5242b9015e40ed05d5886edb3fedf4cc Mon Sep 17 00:00:00 2001 From: David Porter Date: Mon, 31 Aug 2026 14:50:30 -0600 Subject: [PATCH 100/139] feat: Upgades the SDK to v1.48.0 (#11874) ## What changed? Upgrades the Go SDK to latest release version to v1.48.0. ## Why? Pulling in the latest fixes, including some important performance fixes ## How did you test it? - [ ] built - [ ] run locally and tested manually - [X] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks None identified --- common/testing/mocksdk/client_mock.go | 28 ++++++++++++++++++++++ common/testing/mocksdk/workflowrun_mock.go | 14 +++++++++++ go.mod | 6 ++--- go.sum | 12 +++++----- tests/mixedbrain/go.mod | 2 +- tests/mixedbrain/go.sum | 4 ++-- 6 files changed, 54 insertions(+), 12 deletions(-) diff --git a/common/testing/mocksdk/client_mock.go b/common/testing/mocksdk/client_mock.go index dcb559961bf..b25824a1c6e 100644 --- a/common/testing/mocksdk/client_mock.go +++ b/common/testing/mocksdk/client_mock.go @@ -59,6 +59,20 @@ func (mr *MockClientMockRecorder) CancelWorkflow(ctx, workflowID, runID any) *go return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelWorkflow", reflect.TypeOf((*MockClient)(nil).CancelWorkflow), ctx, workflowID, runID) } +// CancelWorkflowWithOptions mocks base method. +func (m *MockClient) CancelWorkflowWithOptions(ctx context.Context, options client.CancelWorkflowOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CancelWorkflowWithOptions", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// CancelWorkflowWithOptions indicates an expected call of CancelWorkflowWithOptions. +func (mr *MockClientMockRecorder) CancelWorkflowWithOptions(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CancelWorkflowWithOptions", reflect.TypeOf((*MockClient)(nil).CancelWorkflowWithOptions), ctx, options) +} + // CheckHealth mocks base method. func (m *MockClient) CheckHealth(ctx context.Context, request *client.CheckHealthRequest) (*client.CheckHealthResponse, error) { m.ctrl.T.Helper() @@ -795,6 +809,20 @@ func (mr *MockClientMockRecorder) TerminateWorkflow(ctx, workflowID, runID, reas return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateWorkflow", reflect.TypeOf((*MockClient)(nil).TerminateWorkflow), varargs...) } +// TerminateWorkflowWithOptions mocks base method. +func (m *MockClient) TerminateWorkflowWithOptions(ctx context.Context, options client.TerminateWorkflowOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "TerminateWorkflowWithOptions", ctx, options) + ret0, _ := ret[0].(error) + return ret0 +} + +// TerminateWorkflowWithOptions indicates an expected call of TerminateWorkflowWithOptions. +func (mr *MockClientMockRecorder) TerminateWorkflowWithOptions(ctx, options any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "TerminateWorkflowWithOptions", reflect.TypeOf((*MockClient)(nil).TerminateWorkflowWithOptions), ctx, options) +} + // UpdateWithStartWorkflow mocks base method. func (m *MockClient) UpdateWithStartWorkflow(ctx context.Context, options client.UpdateWithStartWorkflowOptions) (client.WorkflowUpdateHandle, error) { m.ctrl.T.Helper() diff --git a/common/testing/mocksdk/workflowrun_mock.go b/common/testing/mocksdk/workflowrun_mock.go index f9aad181ede..29dbb4b9c9e 100644 --- a/common/testing/mocksdk/workflowrun_mock.go +++ b/common/testing/mocksdk/workflowrun_mock.go @@ -55,6 +55,20 @@ func (mr *MockWorkflowRunMockRecorder) Get(ctx, valuePtr any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockWorkflowRun)(nil).Get), ctx, valuePtr) } +// GetFirstExecutionRunID mocks base method. +func (m *MockWorkflowRun) GetFirstExecutionRunID() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetFirstExecutionRunID") + ret0, _ := ret[0].(string) + return ret0 +} + +// GetFirstExecutionRunID indicates an expected call of GetFirstExecutionRunID. +func (mr *MockWorkflowRunMockRecorder) GetFirstExecutionRunID() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFirstExecutionRunID", reflect.TypeOf((*MockWorkflowRun)(nil).GetFirstExecutionRunID)) +} + // GetID mocks base method. func (m *MockWorkflowRun) GetID() string { m.ctrl.T.Helper() diff --git a/go.mod b/go.mod index 7a9d0965bba..42dea9ec419 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( github.com/maruel/panicparse/v2 v2.5.0 github.com/mitchellh/mapstructure v1.5.0 github.com/nexus-rpc/nexus-proto-annotations v0.1.0 - github.com/nexus-rpc/sdk-go v0.6.0 + github.com/nexus-rpc/sdk-go v0.7.0 github.com/olekukonko/tablewriter v0.0.5 github.com/olivere/elastic/v7 v7.0.32 github.com/prometheus/client_golang v1.21.0 @@ -68,7 +68,7 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 go.temporal.io/api v1.63.5 go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab - go.temporal.io/sdk v1.44.0 + go.temporal.io/sdk v1.48.0 go.uber.org/fx v1.24.0 go.uber.org/goleak v1.3.0 go.uber.org/mock v0.6.0 @@ -81,7 +81,7 @@ require ( golang.org/x/text v0.40.0 golang.org/x/time v0.15.0 google.golang.org/api v0.276.0 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 gopkg.in/validator.v2 v2.0.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 95797622c25..c16758ef876 100644 --- a/go.sum +++ b/go.sum @@ -327,8 +327,8 @@ github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOF github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/nexus-rpc/nexus-proto-annotations v0.1.0 h1:2fELd+9sqUtNu6Fg//pw8YFsxOvp8vZ8hfP0nHhNI80= github.com/nexus-rpc/nexus-proto-annotations v0.1.0/go.mod h1:n3UjF1bPCW8llR8tHvbxJ+27yPWrhpo8w/Yg1IOuY0Y= -github.com/nexus-rpc/sdk-go v0.6.0 h1:QRgnP2zTbxEbiyWG/aXH8uSC5LV/Mg1fqb19jb4DBlo= -github.com/nexus-rpc/sdk-go v0.6.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= +github.com/nexus-rpc/sdk-go v0.7.0 h1:38NrfY5rLnZAiMMs2ZfCKI/CSDzdfJG+27iAgfA8bUI= +github.com/nexus-rpc/sdk-go v0.7.0/go.mod h1:FHdPfVQwRuJFZFTF0Y2GOAxCrbIBNrcPna9slkGKPYk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= @@ -483,8 +483,8 @@ go.temporal.io/api v1.63.5 h1:c11+kPYHkXXL3UiShPdbMD+xtvqGsbTibUA9ypmiCa4= go.temporal.io/api v1.63.5/go.mod h1:SrlW2JMwVlDP4nRWSNznUFqnSHd+YeMDS1BkYo63HCQ= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab h1:99wXW0317BBi49d6xgMdA0EZtvA+xbBUWV4HsTEGEcg= go.temporal.io/auto-scaled-workers v0.0.0-20260811170210-91f6fe1d10ab/go.mod h1:hhHijO9XRPIkAflLJJHix61M9FzbRPqk8fSydkcLkqw= -go.temporal.io/sdk v1.44.0 h1:suitPDukX74rW3/N1FqvEbZTZVJJsxMKhv0KMa/j7pU= -go.temporal.io/sdk v1.44.0/go.mod h1:vkApR12F9/Y8OR+hkxe7WyXQFuCX6clhzqnAk6rzDAM= +go.temporal.io/sdk v1.48.0 h1:WDctKDVuh0Z8Nf7euAyqs/EwcPg1JTIIq1Fut8Tq118= +go.temporal.io/sdk v1.48.0/go.mod h1:SHv3+fLzD0GGZAwf0xNSvu8UmO1nFgG9WBSYoowApIk= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= @@ -636,8 +636,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260420184626-e10c466a9529 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260420184626-e10c466a9529/go.mod h1:a5OGAgyRr4lqco7AG9hQM9Fwh0N2ZV4grR0eXFEsXQg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/mixedbrain/go.mod b/tests/mixedbrain/go.mod index dde79a7698e..f4cb971775b 100644 --- a/tests/mixedbrain/go.mod +++ b/tests/mixedbrain/go.mod @@ -9,7 +9,7 @@ require ( github.com/temporalio/omes v0.0.0-20260529203146-c6ee1f56c726 go.temporal.io/api v1.63.5 go.temporal.io/server v0.0.0-00010101000000-000000000000 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 ) diff --git a/tests/mixedbrain/go.sum b/tests/mixedbrain/go.sum index f235f9afc83..fd3ce9ed7e2 100644 --- a/tests/mixedbrain/go.sum +++ b/tests/mixedbrain/go.sum @@ -75,8 +75,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260420184626-e10c466a9529 h1: google.golang.org/genproto/googleapis/api v0.0.0-20260420184626-e10c466a9529/go.mod h1:a5OGAgyRr4lqco7AG9hQM9Fwh0N2ZV4grR0eXFEsXQg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529 h1:XF8+t6QQiS0o9ArVan/HW8Q7cycNPGsJf6GA2nXxYAg= google.golang.org/genproto/googleapis/rpc v0.0.0-20260420184626-e10c466a9529/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7757511e41c79025f5588da37e0c19afbfa7be57 Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Mon, 31 Aug 2026 17:08:12 -0700 Subject: [PATCH 101/139] [Visibility Elasticsearch] Return empty page token when result size is less than page size (#11829) ## What changed? Return empty page token when result size is less than page size in Visibility with Elasticsearch. This is a partial revert of https://github.com/temporalio/temporal/pull/10540. ## Why? Since we disabled partial results in search queries, always returning page token is unnecessary. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../visibility/store/elasticsearch/visibility_store.go | 2 +- .../store/elasticsearch/visibility_store_read_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store.go b/common/persistence/visibility/store/elasticsearch/visibility_store.go index 5a56408befe..f72695a162e 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store.go @@ -908,7 +908,7 @@ func (s *VisibilityStore) GetListWorkflowExecutionsResponse( lastHitSort = hit.Sort } - if len(searchResult.Hits.Hits) > 0 { // this means the response might not the last page + if len(searchResult.Hits.Hits) == pageSize { // this means the response might not the last page response.NextPageToken, err = s.serializePageToken(&visibilityPageToken{ SearchAfter: lastHitSort, }) diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go index 1ed80816334..c07882f6778 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go @@ -683,7 +683,7 @@ func (s *ESVisibilitySuite) TestGetListWorkflowExecutionsResponse() { // test page size > number of results resp, err = s.visibilityStore.GetListWorkflowExecutionsResponse(searchResult, testNamespace, 2, nil) s.NoError(err) - s.Equal(serializedToken, resp.NextPageToken) + s.Empty(resp.NextPageToken) s.Equal(1, len(resp.Executions)) // test for search after @@ -704,7 +704,7 @@ func (s *ESVisibilitySuite) TestGetListWorkflowExecutionsResponse() { // test page size > number of results resp, err = s.visibilityStore.GetListWorkflowExecutionsResponse(searchResult, testNamespace, numOfHits+1, nil) s.NoError(err) - s.Equal(serializedToken, resp.NextPageToken) + s.Empty(resp.NextPageToken) s.Equal(numOfHits, len(resp.Executions)) } From bde624efd13fbd3843654058db6d9c716166318b Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Mon, 31 Aug 2026 17:14:07 -0700 Subject: [PATCH 102/139] Tag Nexus logs by lifecycle stage (#11757) ## What changed? Added a `nexus-stage` tag to Nexus-related logs. ## Why? Help operators understand which parts of a Nexus lifecycle logs relate to; and correlate them across via RequestID and such. --- chasm/lib/callback/library.go | 3 + chasm/lib/callback/tasks.go | 3 +- chasm/lib/nexusoperation/task_handler_base.go | 2 +- common/log/tag/tags.go | 4 + common/log/tag/values.go | 8 ++ components/callbacks/chasm_invocation.go | 4 +- components/callbacks/executors.go | 2 + components/callbacks/executors_test.go | 16 +++- components/nexusoperations/executors.go | 1 + .../frontend/nexus_completion_http_handler.go | 6 +- .../frontend/nexus_operation_http_handler.go | 1 + service/history/circuitbreakerpool/fx.go | 27 +++++-- service/history/circuitbreakerpool/fx_test.go | 75 +++++++++++++++++++ 13 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 service/history/circuitbreakerpool/fx_test.go diff --git a/chasm/lib/callback/library.go b/chasm/lib/callback/library.go index 5d9b4f0003e..176303a477d 100644 --- a/chasm/lib/callback/library.go +++ b/chasm/lib/callback/library.go @@ -5,6 +5,8 @@ import ( "google.golang.org/grpc" ) +const InvocationTaskGroup = chasm.CallbackLibraryName + ".invoke" + type ( Library struct { chasm.UnimplementedLibrary @@ -48,6 +50,7 @@ func (l *Library) Tasks() []*chasm.RegistrableTask { chasm.NewRegistrableSideEffectTask( "invoke", l.InvocationTaskHandler, + chasm.WithTaskGroup(InvocationTaskGroup), ), chasm.NewRegistrablePureTask( "backoff", diff --git a/chasm/lib/callback/tasks.go b/chasm/lib/callback/tasks.go index c54e065e073..e54be304b6c 100644 --- a/chasm/lib/callback/tasks.go +++ b/chasm/lib/callback/tasks.go @@ -8,6 +8,7 @@ import ( "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" @@ -97,7 +98,7 @@ func newInvocationTaskHandler(opts invocationTaskHandlerOptions) *invocationTask config: opts.Config, namespaceRegistry: opts.NamespaceRegistry, metricsHandler: opts.MetricsHandler, - logger: opts.Logger, + logger: log.With(opts.Logger, tag.NexusStageHandlerOutbound), httpCallerProvider: opts.HTTPCallerProvider, httpTraceProvider: opts.HTTPTraceProvider, historyClient: opts.HistoryClient, diff --git a/chasm/lib/nexusoperation/task_handler_base.go b/chasm/lib/nexusoperation/task_handler_base.go index 2c8f8520541..46982846d89 100644 --- a/chasm/lib/nexusoperation/task_handler_base.go +++ b/chasm/lib/nexusoperation/task_handler_base.go @@ -57,7 +57,7 @@ func (o InvocationTaskHandlerOptions) toBase() nexusTaskHandlerBase { config: o.Config, namespaceRegistry: o.NamespaceRegistry, metricsHandler: o.MetricsHandler, - logger: o.Logger, + logger: log.With(o.Logger, tag.NexusStageCallerOutbound), clientProvider: o.ClientProvider, endpointRegistry: o.EndpointRegistry, httpTraceProvider: o.HTTPTraceProvider, diff --git a/common/log/tag/tags.go b/common/log/tag/tags.go index 56577d87c5b..bf915309d0e 100644 --- a/common/log/tag/tags.go +++ b/common/log/tag/tags.go @@ -394,6 +394,10 @@ func component(component string) ZapTag { return NewStringTag("component", component) } +func nexusStage(stage string) ZapTag { + return NewStringTag("nexus-stage", stage) +} + // Lifecycle returns tag for Lifecycle func lifecycle(lifecycle string) ZapTag { return NewStringTag("lifecycle", lifecycle) diff --git a/common/log/tag/values.go b/common/log/tag/values.go index 3a3838e4c87..f945e60097f 100644 --- a/common/log/tag/values.go +++ b/common/log/tag/values.go @@ -138,6 +138,14 @@ var ( VersionChecker = component("version-checker") ) +// Pre-defined values for Nexus stage tag. Nexus stage identifies the request lifecycle position. +var ( + NexusStageCallerOutbound = nexusStage("caller-outbound") + NexusStageCallerInbound = nexusStage("caller-inbound") + NexusStageHandlerInbound = nexusStage("handler-inbound") + NexusStageHandlerOutbound = nexusStage("handler-outbound") +) + // Pre-defined values for scope tag var ( ScopeHost = scope("host") diff --git a/components/callbacks/chasm_invocation.go b/components/callbacks/chasm_invocation.go index 568ac78a4af..bb7db429f12 100644 --- a/components/callbacks/chasm_invocation.go +++ b/components/callbacks/chasm_invocation.go @@ -64,13 +64,13 @@ func (c chasmInvocation) Invoke(ctx context.Context, ns *namespace.Namespace, e request, err := c.getHistoryRequest(ref, requestID) if err != nil { - return invocationResultFail{logInternalError(e.Logger, "failed to build history request: %v", err)} + return invocationResultFail{logInternalError(e.Logger, "failed to build history request", err)} } // RPC to History for cross-shard completion delivery. _, err = e.HistoryClient.CompleteNexusOperationChasm(ctx, request) if err != nil { - redactedErr := logInternalError(e.Logger, "failed to complete Nexus operation: %v", err) + redactedErr := logInternalError(e.Logger, "failed to complete Nexus operation", err) if isRetryableRPCResponse(err) { return invocationResultRetry{redactedErr} } diff --git a/components/callbacks/executors.go b/components/callbacks/executors.go index 8253a992423..bb3e92678d5 100644 --- a/components/callbacks/executors.go +++ b/components/callbacks/executors.go @@ -7,6 +7,7 @@ import ( "go.temporal.io/server/chasm" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" @@ -25,6 +26,7 @@ func RegisterExecutor( registry *hsm.Registry, executorOptions TaskExecutorOptions, ) error { + executorOptions.Logger = log.With(executorOptions.Logger, tag.NexusStageHandlerOutbound) exec := taskExecutor{executorOptions} if err := hsm.RegisterImmediateExecutor( registry, diff --git a/components/callbacks/executors_test.go b/components/callbacks/executors_test.go index 1a0015b58c1..47bf8dcb834 100644 --- a/components/callbacks/executors_test.go +++ b/components/callbacks/executors_test.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/testing/testlogger" "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" @@ -317,6 +318,7 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { completion nexusrpc.CompleteOperationOptions headerValue string expectsInternalError bool + expectedLogMessage string assertOutcome func(*testing.T, callbacks.Callback) }{ { @@ -405,6 +407,7 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { }(), headerValue: encodedRef, expectsInternalError: true, + expectedLogMessage: "failed to complete Nexus operation", assertOutcome: func(t *testing.T, cb callbacks.Callback) { require.Equal(t, enumsspb.CALLBACK_STATE_BACKING_OFF, cb.State()) }, @@ -426,6 +429,7 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { }(), headerValue: encodedRef, expectsInternalError: true, + expectedLogMessage: "failed to complete Nexus operation", assertOutcome: func(t *testing.T, cb callbacks.Callback) { require.Equal(t, enumsspb.CALLBACK_STATE_FAILED, cb.State()) }, @@ -443,6 +447,7 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { }(), headerValue: "invalid-base64!!!", expectsInternalError: true, + expectedLogMessage: "failed to decode CHASM callback token", assertOutcome: func(t *testing.T, cb callbacks.Callback) { require.Equal(t, enumsspb.CALLBACK_STATE_FAILED, cb.State()) }, @@ -451,6 +456,8 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + capture := logger.StartCapture() ctrl := gomock.NewController(t) namespaceRegistryMock := namespace.NewMockRegistry(ctrl) factory := namespace.NewDefaultReplicationResolverFactory() @@ -503,7 +510,7 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { NamespaceRegistry: namespaceRegistryMock, MetricsHandler: metrics.NoopMetricsHandler, HistoryClient: historyClient, - Logger: log.NewNoopLogger(), + Logger: logger, Config: &callbacks.Config{ RequestTimeout: dynamicconfig.GetDurationPropertyFnFilteredByDestination(time.Second), RetryPolicy: func() backoff.RetryPolicy { @@ -536,6 +543,13 @@ func TestProcessInvocationTaskChasm_Outcomes(t *testing.T) { if tc.expectsInternalError { require.ErrorContains(t, err, "internal error, reference-id:") + capture.RequireContains(t, testlogger.CapturedLogPattern{ + Level: testlogger.Error, + Message: tc.expectedLogMessage, + Tags: map[string]any{ + "nexus-stage": "handler-outbound", + }, + }) } else { require.NoError(t, err) } diff --git a/components/nexusoperations/executors.go b/components/nexusoperations/executors.go index b7eb7bb2727..0436d9c880e 100644 --- a/components/nexusoperations/executors.go +++ b/components/nexusoperations/executors.go @@ -75,6 +75,7 @@ func RegisterExecutor( registry *hsm.Registry, options TaskExecutorOptions, ) error { + options.Logger = log.With(options.Logger, tag.NexusStageCallerOutbound) exec := taskExecutor{options} if err := hsm.RegisterImmediateExecutor( registry, diff --git a/service/frontend/nexus_completion_http_handler.go b/service/frontend/nexus_completion_http_handler.go index c97a7bca6c3..66b323612ba 100644 --- a/service/frontend/nexus_completion_http_handler.go +++ b/service/frontend/nexus_completion_http_handler.go @@ -90,7 +90,7 @@ func newNexusCompletionHandler( return &nexusCompletionHandler{ ClusterMetadata: clusterMetadata, NamespaceRegistry: namespaceRegistry, - Logger: logger, + Logger: log.With(logger, tag.NexusStageCallerInbound), MetricsHandler: metricsHandler, Config: serviceConfig, CallbackTokenGenerator: callbackTokenGenerator, @@ -110,11 +110,11 @@ func newNexusCompletionHandler( } } -func newNexusCompletionHTTPHandler(handler *nexusCompletionHandler, logger log.Logger) *nexusCompletionHTTPHandler { +func newNexusCompletionHTTPHandler(handler *nexusCompletionHandler) *nexusCompletionHTTPHandler { return &nexusCompletionHTTPHandler{ httpHandler: nexusrpc.NewCompletionHTTPHandler(nexusrpc.CompletionHandlerOptions{ Handler: handler, - Logger: log.NewSlogLogger(logger), + Logger: log.NewSlogLogger(handler.Logger), Serializer: commonnexus.PayloadSerializer, }), } diff --git a/service/frontend/nexus_operation_http_handler.go b/service/frontend/nexus_operation_http_handler.go index 8623dd202bb..93998d2f17e 100644 --- a/service/frontend/nexus_operation_http_handler.go +++ b/service/frontend/nexus_operation_http_handler.go @@ -70,6 +70,7 @@ func NewNexusOperationHTTPHandler( httpTraceProvider commonnexus.HTTPClientTraceProvider, httpServerHandlerInstrumenter telemetry.HTTPServerHandlerInstrumenter, ) *NexusOperationHTTPHandler { + logger = log.With(logger, tag.NexusStageHandlerInbound) return &NexusOperationHTTPHandler{ base: nexusrpc.BaseHTTPHandler{ Logger: log.NewSlogLogger(logger), diff --git a/service/history/circuitbreakerpool/fx.go b/service/history/circuitbreakerpool/fx.go index 8cc393f2210..e995a57e0e9 100644 --- a/service/history/circuitbreakerpool/fx.go +++ b/service/history/circuitbreakerpool/fx.go @@ -4,10 +4,14 @@ import ( "fmt" "github.com/sony/gobreaker" + chasmcallback "go.temporal.io/server/chasm/lib/callback" + chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/circuitbreaker" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" + hsmcallbacks "go.temporal.io/server/components/callbacks" + hsmnexus "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/tasks" "go.uber.org/fx" @@ -60,15 +64,28 @@ func OutboundQueueCircuitBreakerPoolProvider( func onStateChange( key tasks.TaskGroupNamespaceIDAndDestination, nsName string, - logger log.SnTaggedLogger, + logger log.Logger, ) func(name string, from gobreaker.State, to gobreaker.State) { + logger = log.With( + logger, + tag.ComponentOutboundQueue, + tag.WorkflowNamespace(nsName), + tag.WorkflowNamespaceID(key.NamespaceID), + tag.Destination(key.Destination), + tag.NewStringTag("task-group", key.TaskGroup), + ) + + switch key.TaskGroup { + case chasmnexus.TaskGroupName, hsmnexus.TaskTypeInvocation, hsmnexus.TaskTypeCancelation: + logger = log.With(logger, tag.NexusStageCallerOutbound) + case hsmcallbacks.TaskTypeInvocation, chasmcallback.InvocationTaskGroup: + logger = log.With(logger, tag.NexusStageHandlerOutbound) + default: + } + return func(_ string, from gobreaker.State, to gobreaker.State) { logger.Warn( "outbound queue circuit breaker state change", - tag.WorkflowNamespace(nsName), - tag.WorkflowNamespaceID(key.NamespaceID), - tag.Destination(key.Destination), - tag.NewStringTag("task-group", key.TaskGroup), tag.NewStringTag("from-state", from.String()), tag.NewStringTag("to-state", to.String()), ) diff --git a/service/history/circuitbreakerpool/fx_test.go b/service/history/circuitbreakerpool/fx_test.go new file mode 100644 index 00000000000..a6d3f1858b7 --- /dev/null +++ b/service/history/circuitbreakerpool/fx_test.go @@ -0,0 +1,75 @@ +package circuitbreakerpool + +import ( + "testing" + + "github.com/sony/gobreaker" + "github.com/stretchr/testify/require" + chasmcallback "go.temporal.io/server/chasm/lib/callback" + chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" + "go.temporal.io/server/common/testing/testlogger" + hsmcallbacks "go.temporal.io/server/components/callbacks" + hsmnexus "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/tasks" +) + +func TestOnStateChangeNexusStage(t *testing.T) { + for _, testCase := range []struct { + name string + taskGroup string + stage string + }{ + { + name: "HSM caller invocation", + taskGroup: hsmnexus.TaskTypeInvocation, + stage: "caller-outbound", + }, + { + name: "HSM caller cancellation", + taskGroup: hsmnexus.TaskTypeCancelation, + stage: "caller-outbound", + }, + { + name: "CHASM caller outbound", + taskGroup: chasmnexus.TaskGroupName, + stage: "caller-outbound", + }, + { + name: "CHASM handler outbound", + taskGroup: chasmcallback.InvocationTaskGroup, + stage: "handler-outbound", + }, + { + name: "HSM handler outbound", + taskGroup: hsmcallbacks.TaskTypeInvocation, + stage: "handler-outbound", + }, + { + name: "non-Nexus task group", + taskGroup: "other.Task", + }, + } { + t.Run(testCase.name, func(t *testing.T) { + logger := testlogger.NewTestLogger(t, testlogger.FailOnExpectedErrorOnly) + capture := logger.StartCapture() + onStateChange( + tasks.TaskGroupNamespaceIDAndDestination{TaskGroup: testCase.taskGroup}, + "namespace", + logger, + )("", gobreaker.StateClosed, gobreaker.StateOpen) + + records := capture.Snapshot() + require.Len(t, records, 1) + actualTags := make(map[string]any, len(records[0].Tags)) + for _, actual := range records[0].Tags { + actualTags[actual.Key()] = actual.Value() + } + require.Equal(t, "outbound-queue-processor", actualTags["component"]) + if testCase.stage == "" { + require.NotContains(t, actualTags, "nexus-stage") + } else { + require.Equal(t, testCase.stage, actualTags["nexus-stage"]) + } + }) + } +} From 2a63a6f8500bbdb0ba782ce0b7950c28b02098ac Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 1 Sep 2026 09:35:08 -0700 Subject: [PATCH 103/139] Move legacy HSM implementations under history service (#11836) ## What changed? Moved the `components/` package to `service/history/hsm`. ## Why? 1. It's deprecated/in maintenance mode, the top-level location is not earned anymore. 2. Calling it `hsm` to make it easier to map to and tell apart from `chasm`. --- .github/.golangci.yml | 5 +++++ .github/CODEOWNERS | 4 ++-- docs/architecture/circuit-breaker.md | 2 +- docs/architecture/nexus.md | 19 ++++++++++--------- nexusworkflowref/nexusworkflowref.go | 2 +- nexusworkflowref/nexusworkflowref_test.go | 2 +- .../nexus_completion_http_handler_test.go | 2 +- service/frontend/service.go | 2 +- service/history/api/describeworkflow/api.go | 4 ++-- service/history/circuitbreakerpool/fx.go | 4 ++-- service/history/circuitbreakerpool/fx_test.go | 4 ++-- service/history/fx.go | 6 +++--- service/history/handler.go | 2 +- .../hsm}/callbacks/chasm_invocation.go | 0 .../history/hsm}/callbacks/config.go | 0 .../history/hsm}/callbacks/config_test.go | 0 service/history/hsm/callbacks/doc.go | 4 ++++ .../history/hsm}/callbacks/executors.go | 0 .../history/hsm}/callbacks/executors_test.go | 2 +- .../history/hsm}/callbacks/fx.go | 0 .../history/hsm}/callbacks/metrics.go | 0 .../hsm}/callbacks/nexus_invocation.go | 0 .../history/hsm}/callbacks/request.go | 0 .../history/hsm}/callbacks/request_test.go | 0 .../history/hsm}/callbacks/statemachine.go | 0 .../hsm}/callbacks/statemachine_test.go | 2 +- .../history/hsm}/callbacks/tasks.go | 0 service/history/hsm/dummy/doc.go | 2 ++ .../history/hsm}/dummy/executors.go | 0 .../history/hsm}/dummy/statemachine.go | 0 .../history/hsm}/dummy/tasks.go | 0 .../build_callback_url_test.go | 0 .../hsm}/nexusoperations/completion.go | 0 .../hsm}/nexusoperations/completion_test.go | 2 +- .../history/hsm}/nexusoperations/config.go | 0 service/history/hsm/nexusoperations/doc.go | 4 ++++ .../history/hsm}/nexusoperations/events.go | 0 .../hsm}/nexusoperations/events_test.go | 2 +- .../history/hsm}/nexusoperations/executors.go | 0 .../hsm}/nexusoperations/executors_test.go | 2 +- .../history/hsm}/nexusoperations/fx.go | 0 .../hsm}/nexusoperations/helpers_test.go | 2 +- .../history/hsm}/nexusoperations/metrics.go | 0 .../hsm}/nexusoperations/statemachine.go | 0 .../hsm}/nexusoperations/statemachine_test.go | 2 +- .../history/hsm}/nexusoperations/tasks.go | 0 .../hsm}/nexusoperations/workflow/commands.go | 2 +- .../nexusoperations/workflow/commands_test.go | 4 ++-- service/history/ndc/workflow_resetter_test.go | 2 +- .../history/statemachine_environment_test.go | 2 +- .../timer_queue_active_task_executor_test.go | 2 +- .../timer_queue_standby_task_executor_test.go | 2 +- .../history/workflow/mutable_state_impl.go | 2 +- .../workflow/mutable_state_impl_test.go | 4 ++-- .../workflow/mutable_state_rebuilder_test.go | 2 +- .../history/workflow/task_generator_test.go | 4 ++-- service/history/workflow/timeskipping.go | 2 +- service/history/workflow/timeskipping_test.go | 2 +- service/history/workflow/util.go | 2 +- service/matching/config.go | 2 +- service/matching/matching_engine_test.go | 2 +- tests/nexus_api_test.go | 4 ++-- tests/nexus_workflow_test.go | 2 +- tests/testcore/dynamic_config_overrides.go | 2 +- tests/testcore/functional_test_base.go | 2 +- tests/testcore/onebox.go | 2 +- ...uffered_events_replication_helpers_test.go | 2 +- tests/xdc/nexus_request_forwarding_test.go | 2 +- tests/xdc/nexus_state_replication_test.go | 4 ++-- 69 files changed, 77 insertions(+), 61 deletions(-) rename {components => service/history/hsm}/callbacks/chasm_invocation.go (100%) rename {components => service/history/hsm}/callbacks/config.go (100%) rename {components => service/history/hsm}/callbacks/config_test.go (100%) create mode 100644 service/history/hsm/callbacks/doc.go rename {components => service/history/hsm}/callbacks/executors.go (100%) rename {components => service/history/hsm}/callbacks/executors_test.go (99%) rename {components => service/history/hsm}/callbacks/fx.go (100%) rename {components => service/history/hsm}/callbacks/metrics.go (100%) rename {components => service/history/hsm}/callbacks/nexus_invocation.go (100%) rename {components => service/history/hsm}/callbacks/request.go (100%) rename {components => service/history/hsm}/callbacks/request_test.go (100%) rename {components => service/history/hsm}/callbacks/statemachine.go (100%) rename {components => service/history/hsm}/callbacks/statemachine_test.go (99%) rename {components => service/history/hsm}/callbacks/tasks.go (100%) create mode 100644 service/history/hsm/dummy/doc.go rename {components => service/history/hsm}/dummy/executors.go (100%) rename {components => service/history/hsm}/dummy/statemachine.go (100%) rename {components => service/history/hsm}/dummy/tasks.go (100%) rename {components => service/history/hsm}/nexusoperations/build_callback_url_test.go (100%) rename {components => service/history/hsm}/nexusoperations/completion.go (100%) rename {components => service/history/hsm}/nexusoperations/completion_test.go (98%) rename {components => service/history/hsm}/nexusoperations/config.go (100%) create mode 100644 service/history/hsm/nexusoperations/doc.go rename {components => service/history/hsm}/nexusoperations/events.go (100%) rename {components => service/history/hsm}/nexusoperations/events_test.go (99%) rename {components => service/history/hsm}/nexusoperations/executors.go (100%) rename {components => service/history/hsm}/nexusoperations/executors_test.go (99%) rename {components => service/history/hsm}/nexusoperations/fx.go (100%) rename {components => service/history/hsm}/nexusoperations/helpers_test.go (98%) rename {components => service/history/hsm}/nexusoperations/metrics.go (100%) rename {components => service/history/hsm}/nexusoperations/statemachine.go (100%) rename {components => service/history/hsm}/nexusoperations/statemachine_test.go (99%) rename {components => service/history/hsm}/nexusoperations/tasks.go (100%) rename {components => service/history/hsm}/nexusoperations/workflow/commands.go (99%) rename {components => service/history/hsm}/nexusoperations/workflow/commands_test.go (99%) diff --git a/.github/.golangci.yml b/.github/.golangci.yml index 6595477d84d..e82b7d31edf 100644 --- a/.github/.golangci.yml +++ b/.github/.golangci.yml @@ -171,6 +171,7 @@ linters: - ^api - ^proto - ^.git + - ^\.\./service/history/hsm/(callbacks|dummy|nexusoperations)/ rules: - path-except: _test\.go|tests/.+\.go text: "time.Sleep" @@ -234,6 +235,10 @@ linters: - path: ^tools\/.+\.go linters: - revive + # These packages remain imported to read and maintain persisted HSM state. + - text: "SA1019: go.temporal.io/server/service/history/hsm/(callbacks|nexusoperations) is deprecated" + linters: + - staticcheck issues: max-issues-per-linter: 0 max-same-issues: 0 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3e3fd350385..2129a3c5c0b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -35,9 +35,9 @@ /common/persistence/ @temporalio/oss-foundations @temporalio/oss-matching @temporalio/cgs /common/persistence/**/nexus_endpoint* @temporalio/oss-foundations @temporalio/oss-matching @temporalio/nexus -# Components +# Legacy HSM implementations -/components/ @temporalio/act @temporalio/nexus +/service/history/hsm/ @temporalio/act @temporalio/nexus # Proto Definitions diff --git a/docs/architecture/circuit-breaker.md b/docs/architecture/circuit-breaker.md index 8b37c24c377..d2a5752c30d 100644 --- a/docs/architecture/circuit-breaker.md +++ b/docs/architecture/circuit-breaker.md @@ -4,7 +4,7 @@ Most work in Temporal's history service drives workflows forward internally. But *outbound* calls to systems outside the server. The main example is [Nexus](./nexus.md): invoking a Nexus operation makes the server issue an HTTP request to a Nexus endpoint (which may be another Temporal namespace or an external service). Another example is delivering a workflow-completion -[callback](../../components/callbacks) (an HTTP request the server sends when a workflow finishes). +[callback](../../chasm/lib/callback) (an HTTP request the server sends when a workflow finishes). In these cases, the server — not a user Worker — is the client making the request. diff --git a/docs/architecture/nexus.md b/docs/architecture/nexus.md index ddbcd1a852c..057a5330b56 100644 --- a/docs/architecture/nexus.md +++ b/docs/architecture/nexus.md @@ -233,7 +233,7 @@ The circuit breaker is dynamically configurable via: ## Nexus Operations -The [`nexusoperations` component](../../components/nexusoperations) holds all of the logic for scheduling Nexus +The [`nexusoperations` component](../../service/history/hsm/nexusoperations) holds all of the logic for scheduling Nexus Operations from a workflow and maintaining the Operations's lifecycle. There are some other pieces of Nexus logic spread around other parts of the server (`common`, `frontend`, and @@ -245,7 +245,7 @@ Machine framework (docs TBD). ### `Operation` State Machine The -[Operation](https://github.com/temporalio/temporal/blob/a0fdea5319be5f1631d7e2b0f6f06c38dae3d413/components/nexusoperations/statemachine.go#L65) +[Operation](../../service/history/hsm/nexusoperations/statemachine.go) state machine manages the lifetime of an Operation the StartOperation request. The state machine transitions between these states (as defined in code): @@ -325,7 +325,7 @@ releases the mutable state lock while making the HTTP request, and reacquires th Since Nexus Operations can provide their results asynchronously, when constructing the `StartOperation` call, the executor adds a callback URL along with a set of headers to later correlate the operation completion with the calling workflow and operation state machine. The -[component.nexusoperations.callback.endpoint.template](https://github.com/temporalio/temporal/blob/7c8025aff96af7d72a91af615f1d625817842894/components/nexusoperations/config.go#L69) +[component.nexusoperations.callback.endpoint.template](../../service/history/hsm/nexusoperations/config.go) global dynamic config must be set to construct callback URLs or the executor will fail to process invocation tasks. When routing callbacks to external clusters and non-Temporal destinations, the URL is used and should be a value that is publically accessible to those external destinations. Callbacks that are routed internally within the cluster resolve @@ -342,14 +342,15 @@ The workflow closed trigger will trigger a callback when a workflow completes su cancellation, or termination, as well as carry over the callbacks to the next execution in a chain when a workflow is retried or continues-as-new. -Similarly to Nexus Operations, callbacks are implemented via a hierarchical state machine and a set of executors, which -are located in [the components directory](../../components/callbacks). +Callbacks are implemented as CHASM components and task handlers in the +[callback library](../../chasm/lib/callback). The +[legacy HSM callbacks package](../../service/history/hsm/callbacks) remains for persisted HSM state. Callbacks are continously retried using a [configurable retry policy][callback-retry-policy] until they succeed, permanently fail, or the workflow's retention period expires. -The timeout for making a single callback HTTP call is configurable via: `component.callbacks.request.timeout` -(default is 10 seconds). +The timeout for making a single callback HTTP call is configurable via: `callback.request.timeout` +(default is 10 seconds). The legacy HSM implementation uses `component.callbacks.request.timeout`. ### `Callback` State Machine @@ -442,5 +443,5 @@ sequenceDiagram Frontend -->>- Client: Response ``` -[nexus-retry-policy]: https://github.com/temporalio/temporal/blob/f0f5539c8de71f7f9dec22e4e30a5cb9ded7b945/components/nexusoperations/config.go#L77-L87 -[callback-retry-policy]: https://github.com/temporalio/temporal/blob/f0f5539c8de71f7f9dec22e4e30a5cb9ded7b945/components/callbacks/config.go#L40-L50 +[nexus-retry-policy]: ../../service/history/hsm/nexusoperations/config.go +[callback-retry-policy]: ../../chasm/lib/callback/config.go diff --git a/nexusworkflowref/nexusworkflowref.go b/nexusworkflowref/nexusworkflowref.go index a865faa43b0..3aa38d38eac 100644 --- a/nexusworkflowref/nexusworkflowref.go +++ b/nexusworkflowref/nexusworkflowref.go @@ -15,7 +15,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" ) // operationsFieldName is the first segment of a workflow Nexus operation's CHASM component path (the diff --git a/nexusworkflowref/nexusworkflowref_test.go b/nexusworkflowref/nexusworkflowref_test.go index cb586b5f95c..c677d023a9e 100644 --- a/nexusworkflowref/nexusworkflowref_test.go +++ b/nexusworkflowref/nexusworkflowref_test.go @@ -8,7 +8,7 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" ) const ( diff --git a/service/frontend/nexus_completion_http_handler_test.go b/service/frontend/nexus_completion_http_handler_test.go index 571b4286189..135f13eb4e5 100644 --- a/service/frontend/nexus_completion_http_handler_test.go +++ b/service/frontend/nexus_completion_http_handler_test.go @@ -14,8 +14,8 @@ import ( tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/common/log" "go.temporal.io/server/common/nexus/nexusrpc" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/nexusworkflowref" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.uber.org/mock/gomock" "google.golang.org/grpc" ) diff --git a/service/frontend/service.go b/service/frontend/service.go index 922fde36945..f30a7b2c03e 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -21,7 +21,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/retrypolicy" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "google.golang.org/grpc" "google.golang.org/grpc/health" healthpb "google.golang.org/grpc/health/grpc_health_v1" diff --git a/service/history/api/describeworkflow/api.go b/service/history/api/describeworkflow/api.go index 94cfaf60535..15b428d38c4 100644 --- a/service/history/api/describeworkflow/api.go +++ b/service/history/api/describeworkflow/api.go @@ -30,11 +30,11 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/visibility/manager" - "go.temporal.io/server/components/callbacks" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/circuitbreakerpool" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/service/history/workflow" diff --git a/service/history/circuitbreakerpool/fx.go b/service/history/circuitbreakerpool/fx.go index e995a57e0e9..5179693271d 100644 --- a/service/history/circuitbreakerpool/fx.go +++ b/service/history/circuitbreakerpool/fx.go @@ -10,9 +10,9 @@ import ( "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" - hsmcallbacks "go.temporal.io/server/components/callbacks" - hsmnexus "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/configs" + hsmcallbacks "go.temporal.io/server/service/history/hsm/callbacks" + hsmnexus "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/service/history/tasks" "go.uber.org/fx" ) diff --git a/service/history/circuitbreakerpool/fx_test.go b/service/history/circuitbreakerpool/fx_test.go index a6d3f1858b7..a226be941c6 100644 --- a/service/history/circuitbreakerpool/fx_test.go +++ b/service/history/circuitbreakerpool/fx_test.go @@ -8,8 +8,8 @@ import ( chasmcallback "go.temporal.io/server/chasm/lib/callback" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/testing/testlogger" - hsmcallbacks "go.temporal.io/server/components/callbacks" - hsmnexus "go.temporal.io/server/components/nexusoperations" + hsmcallbacks "go.temporal.io/server/service/history/hsm/callbacks" + hsmnexus "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/service/history/tasks" ) diff --git a/service/history/fx.go b/service/history/fx.go index dcf233185fe..ebf488380f7 100644 --- a/service/history/fx.go +++ b/service/history/fx.go @@ -37,9 +37,6 @@ import ( "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/callbacks" - hsmnexusoperations "go.temporal.io/server/components/nexusoperations" - hsmnexusworkflow "go.temporal.io/server/components/nexusoperations/workflow" "go.temporal.io/server/service" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/workflowresend" @@ -48,6 +45,9 @@ import ( "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" + hsmnexusoperations "go.temporal.io/server/service/history/hsm/nexusoperations" + hsmnexusworkflow "go.temporal.io/server/service/history/hsm/nexusoperations/workflow" "go.temporal.io/server/service/history/replication" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/workflow" diff --git a/service/history/handler.go b/service/history/handler.go index d25e52b32bc..42e88cfc4d5 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -53,7 +53,6 @@ import ( serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/testhooks" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/deletedlqtasks" "go.temporal.io/server/service/history/api/deleteexecution" @@ -64,6 +63,7 @@ import ( "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/service/history/replication" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tasks" diff --git a/components/callbacks/chasm_invocation.go b/service/history/hsm/callbacks/chasm_invocation.go similarity index 100% rename from components/callbacks/chasm_invocation.go rename to service/history/hsm/callbacks/chasm_invocation.go diff --git a/components/callbacks/config.go b/service/history/hsm/callbacks/config.go similarity index 100% rename from components/callbacks/config.go rename to service/history/hsm/callbacks/config.go diff --git a/components/callbacks/config_test.go b/service/history/hsm/callbacks/config_test.go similarity index 100% rename from components/callbacks/config_test.go rename to service/history/hsm/callbacks/config_test.go diff --git a/service/history/hsm/callbacks/doc.go b/service/history/hsm/callbacks/doc.go new file mode 100644 index 00000000000..dc17c5c7478 --- /dev/null +++ b/service/history/hsm/callbacks/doc.go @@ -0,0 +1,4 @@ +// Package callbacks implements the legacy HSM-backed callback state machine. +// +// Deprecated: New callback functionality belongs in chasm/lib/callback. +package callbacks diff --git a/components/callbacks/executors.go b/service/history/hsm/callbacks/executors.go similarity index 100% rename from components/callbacks/executors.go rename to service/history/hsm/callbacks/executors.go diff --git a/components/callbacks/executors_test.go b/service/history/hsm/callbacks/executors_test.go similarity index 99% rename from components/callbacks/executors_test.go rename to service/history/hsm/callbacks/executors_test.go index 47bf8dcb834..bbd119af354 100644 --- a/components/callbacks/executors_test.go +++ b/service/history/hsm/callbacks/executors_test.go @@ -27,8 +27,8 @@ import ( commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/testing/testlogger" - "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" "go.temporal.io/server/service/history/hsm/hsmtest" queuescommon "go.temporal.io/server/service/history/queues/common" queueserrors "go.temporal.io/server/service/history/queues/errors" diff --git a/components/callbacks/fx.go b/service/history/hsm/callbacks/fx.go similarity index 100% rename from components/callbacks/fx.go rename to service/history/hsm/callbacks/fx.go diff --git a/components/callbacks/metrics.go b/service/history/hsm/callbacks/metrics.go similarity index 100% rename from components/callbacks/metrics.go rename to service/history/hsm/callbacks/metrics.go diff --git a/components/callbacks/nexus_invocation.go b/service/history/hsm/callbacks/nexus_invocation.go similarity index 100% rename from components/callbacks/nexus_invocation.go rename to service/history/hsm/callbacks/nexus_invocation.go diff --git a/components/callbacks/request.go b/service/history/hsm/callbacks/request.go similarity index 100% rename from components/callbacks/request.go rename to service/history/hsm/callbacks/request.go diff --git a/components/callbacks/request_test.go b/service/history/hsm/callbacks/request_test.go similarity index 100% rename from components/callbacks/request_test.go rename to service/history/hsm/callbacks/request_test.go diff --git a/components/callbacks/statemachine.go b/service/history/hsm/callbacks/statemachine.go similarity index 100% rename from components/callbacks/statemachine.go rename to service/history/hsm/callbacks/statemachine.go diff --git a/components/callbacks/statemachine_test.go b/service/history/hsm/callbacks/statemachine_test.go similarity index 99% rename from components/callbacks/statemachine_test.go rename to service/history/hsm/callbacks/statemachine_test.go index df50252d2f3..eb300a580c8 100644 --- a/components/callbacks/statemachine_test.go +++ b/service/history/hsm/callbacks/statemachine_test.go @@ -10,8 +10,8 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" - "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" ) func TestValidTransitions(t *testing.T) { diff --git a/components/callbacks/tasks.go b/service/history/hsm/callbacks/tasks.go similarity index 100% rename from components/callbacks/tasks.go rename to service/history/hsm/callbacks/tasks.go diff --git a/service/history/hsm/dummy/doc.go b/service/history/hsm/dummy/doc.go new file mode 100644 index 00000000000..5d1b49dffc5 --- /dev/null +++ b/service/history/hsm/dummy/doc.go @@ -0,0 +1,2 @@ +// Package dummy provides HSM fixtures for history queue tests. +package dummy diff --git a/components/dummy/executors.go b/service/history/hsm/dummy/executors.go similarity index 100% rename from components/dummy/executors.go rename to service/history/hsm/dummy/executors.go diff --git a/components/dummy/statemachine.go b/service/history/hsm/dummy/statemachine.go similarity index 100% rename from components/dummy/statemachine.go rename to service/history/hsm/dummy/statemachine.go diff --git a/components/dummy/tasks.go b/service/history/hsm/dummy/tasks.go similarity index 100% rename from components/dummy/tasks.go rename to service/history/hsm/dummy/tasks.go diff --git a/components/nexusoperations/build_callback_url_test.go b/service/history/hsm/nexusoperations/build_callback_url_test.go similarity index 100% rename from components/nexusoperations/build_callback_url_test.go rename to service/history/hsm/nexusoperations/build_callback_url_test.go diff --git a/components/nexusoperations/completion.go b/service/history/hsm/nexusoperations/completion.go similarity index 100% rename from components/nexusoperations/completion.go rename to service/history/hsm/nexusoperations/completion.go diff --git a/components/nexusoperations/completion_test.go b/service/history/hsm/nexusoperations/completion_test.go similarity index 98% rename from components/nexusoperations/completion_test.go rename to service/history/hsm/nexusoperations/completion_test.go index c8787e7bb2d..c33c0baa29b 100644 --- a/components/nexusoperations/completion_test.go +++ b/service/history/hsm/nexusoperations/completion_test.go @@ -11,9 +11,9 @@ import ( historypb "go.temporal.io/api/history/v1" chasmnexus "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/metrics/metricstest" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" "google.golang.org/protobuf/types/known/durationpb" ) diff --git a/components/nexusoperations/config.go b/service/history/hsm/nexusoperations/config.go similarity index 100% rename from components/nexusoperations/config.go rename to service/history/hsm/nexusoperations/config.go diff --git a/service/history/hsm/nexusoperations/doc.go b/service/history/hsm/nexusoperations/doc.go new file mode 100644 index 00000000000..078ef28a601 --- /dev/null +++ b/service/history/hsm/nexusoperations/doc.go @@ -0,0 +1,4 @@ +// Package nexusoperations implements the legacy HSM-backed Nexus operation state machines. +// +// Deprecated: New Nexus operation functionality belongs in chasm/lib/nexusoperation. +package nexusoperations diff --git a/components/nexusoperations/events.go b/service/history/hsm/nexusoperations/events.go similarity index 100% rename from components/nexusoperations/events.go rename to service/history/hsm/nexusoperations/events.go diff --git a/components/nexusoperations/events_test.go b/service/history/hsm/nexusoperations/events_test.go similarity index 99% rename from components/nexusoperations/events_test.go rename to service/history/hsm/nexusoperations/events_test.go index 3a5cfc8a97d..75e0838b106 100644 --- a/components/nexusoperations/events_test.go +++ b/service/history/hsm/nexusoperations/events_test.go @@ -8,9 +8,9 @@ import ( "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/components/nexusoperations/executors.go b/service/history/hsm/nexusoperations/executors.go similarity index 100% rename from components/nexusoperations/executors.go rename to service/history/hsm/nexusoperations/executors.go diff --git a/components/nexusoperations/executors_test.go b/service/history/hsm/nexusoperations/executors_test.go similarity index 99% rename from components/nexusoperations/executors_test.go rename to service/history/hsm/nexusoperations/executors_test.go index 41d5bc55e32..bf0510e789a 100644 --- a/components/nexusoperations/executors_test.go +++ b/service/history/hsm/nexusoperations/executors_test.go @@ -39,10 +39,10 @@ import ( "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/resource" "go.temporal.io/server/common/testing/protorequire" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" queueserrors "go.temporal.io/server/service/history/queues/errors" "go.uber.org/mock/gomock" "google.golang.org/grpc" diff --git a/components/nexusoperations/fx.go b/service/history/hsm/nexusoperations/fx.go similarity index 100% rename from components/nexusoperations/fx.go rename to service/history/hsm/nexusoperations/fx.go diff --git a/components/nexusoperations/helpers_test.go b/service/history/hsm/nexusoperations/helpers_test.go similarity index 98% rename from components/nexusoperations/helpers_test.go rename to service/history/hsm/nexusoperations/helpers_test.go index 7a06ef45208..1c959a63458 100644 --- a/components/nexusoperations/helpers_test.go +++ b/service/history/hsm/nexusoperations/helpers_test.go @@ -14,9 +14,9 @@ import ( "go.temporal.io/sdk/converter" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/service/history/workflow" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/components/nexusoperations/metrics.go b/service/history/hsm/nexusoperations/metrics.go similarity index 100% rename from components/nexusoperations/metrics.go rename to service/history/hsm/nexusoperations/metrics.go diff --git a/components/nexusoperations/statemachine.go b/service/history/hsm/nexusoperations/statemachine.go similarity index 100% rename from components/nexusoperations/statemachine.go rename to service/history/hsm/nexusoperations/statemachine.go diff --git a/components/nexusoperations/statemachine_test.go b/service/history/hsm/nexusoperations/statemachine_test.go similarity index 99% rename from components/nexusoperations/statemachine_test.go rename to service/history/hsm/nexusoperations/statemachine_test.go index 4943feacaa8..2a4b0dd9e53 100644 --- a/components/nexusoperations/statemachine_test.go +++ b/service/history/hsm/nexusoperations/statemachine_test.go @@ -12,9 +12,9 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common" "go.temporal.io/server/common/backoff" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) diff --git a/components/nexusoperations/tasks.go b/service/history/hsm/nexusoperations/tasks.go similarity index 100% rename from components/nexusoperations/tasks.go rename to service/history/hsm/nexusoperations/tasks.go diff --git a/components/nexusoperations/workflow/commands.go b/service/history/hsm/nexusoperations/workflow/commands.go similarity index 99% rename from components/nexusoperations/workflow/commands.go rename to service/history/hsm/nexusoperations/workflow/commands.go index 2b9bb3f903e..b39a5f3a6bf 100644 --- a/components/nexusoperations/workflow/commands.go +++ b/service/history/hsm/nexusoperations/workflow/commands.go @@ -18,8 +18,8 @@ import ( chasmworkflow "go.temporal.io/server/chasm/lib/workflow" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/primitives/timestamp" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" "google.golang.org/protobuf/types/known/durationpb" diff --git a/components/nexusoperations/workflow/commands_test.go b/service/history/hsm/nexusoperations/workflow/commands_test.go similarity index 99% rename from components/nexusoperations/workflow/commands_test.go rename to service/history/hsm/nexusoperations/workflow/commands_test.go index 7da08528204..d7dca109390 100644 --- a/components/nexusoperations/workflow/commands_test.go +++ b/service/history/hsm/nexusoperations/workflow/commands_test.go @@ -23,9 +23,9 @@ import ( "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexustest" - "go.temporal.io/server/components/nexusoperations" - opsworkflow "go.temporal.io/server/components/nexusoperations/workflow" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/nexusoperations" + opsworkflow "go.temporal.io/server/service/history/hsm/nexusoperations/workflow" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/tests" "go.temporal.io/server/service/history/workflow" diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index 7d1f69b2350..126a95058cb 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -38,9 +38,9 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/testing/testlogger" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" diff --git a/service/history/statemachine_environment_test.go b/service/history/statemachine_environment_test.go index 935c619812e..af66194022f 100644 --- a/service/history/statemachine_environment_test.go +++ b/service/history/statemachine_environment_test.go @@ -26,10 +26,10 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/tasktoken" - "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/notification" "go.temporal.io/server/service/history/queues" diff --git a/service/history/timer_queue_active_task_executor_test.go b/service/history/timer_queue_active_task_executor_test.go index 5dc5a930cfc..88cc7e2bfbc 100644 --- a/service/history/timer_queue_active_task_executor_test.go +++ b/service/history/timer_queue_active_task_executor_test.go @@ -37,12 +37,12 @@ import ( "go.temporal.io/server/common/telemetry" pm "go.temporal.io/server/common/testing/protomock" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/dummy" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/deletemanager" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/dummy" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/notification" "go.temporal.io/server/service/history/queues" diff --git a/service/history/timer_queue_standby_task_executor_test.go b/service/history/timer_queue_standby_task_executor_test.go index 104c6209e36..844cbd4eb1d 100644 --- a/service/history/timer_queue_standby_task_executor_test.go +++ b/service/history/timer_queue_standby_task_executor_test.go @@ -37,12 +37,12 @@ import ( "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/dummy" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/deletemanager" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/dummy" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/notification" "go.temporal.io/server/service/history/queues" diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 069c3e37a24..5e0de6e88f3 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -65,12 +65,12 @@ import ( "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/util" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/historybuilder" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/service/history/workflow/update" diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 26ee33d1763..ac97ddec479 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -54,13 +54,13 @@ import ( "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/callbacks" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/historybuilder" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tasks" diff --git a/service/history/workflow/mutable_state_rebuilder_test.go b/service/history/workflow/mutable_state_rebuilder_test.go index ce7261b3936..4ced2dfd279 100644 --- a/service/history/workflow/mutable_state_rebuilder_test.go +++ b/service/history/workflow/mutable_state_rebuilder_test.go @@ -31,10 +31,10 @@ import ( "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/testing/protomock" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/historybuilder" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tasks" diff --git a/service/history/workflow/task_generator_test.go b/service/history/workflow/task_generator_test.go index a4e04472830..5af41d20264 100644 --- a/service/history/workflow/task_generator_test.go +++ b/service/history/workflow/task_generator_test.go @@ -28,11 +28,11 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testlogger" - "go.temporal.io/server/components/callbacks" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" "go.temporal.io/server/service/history/hsm/hsmtest" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tasks" diff --git a/service/history/workflow/timeskipping.go b/service/history/workflow/timeskipping.go index 65d9c311c95..12f047239b1 100644 --- a/service/history/workflow/timeskipping.go +++ b/service/history/workflow/timeskipping.go @@ -12,7 +12,7 @@ import ( "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/log/tag" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/tasks" "google.golang.org/protobuf/types/known/durationpb" diff --git a/service/history/workflow/timeskipping_test.go b/service/history/workflow/timeskipping_test.go index c16db2d08bb..27f89d1add9 100644 --- a/service/history/workflow/timeskipping_test.go +++ b/service/history/workflow/timeskipping_test.go @@ -14,7 +14,7 @@ import ( "go.temporal.io/server/chasm/lib/activity" "go.temporal.io/server/common" "go.temporal.io/server/common/clock" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/service/history/tests" diff --git a/service/history/workflow/util.go b/service/history/workflow/util.go index adfa7ad78d7..3098ada5e2a 100644 --- a/service/history/workflow/util.go +++ b/service/history/workflow/util.go @@ -16,9 +16,9 @@ import ( "go.temporal.io/server/common/effect" "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/callbacks" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" + "go.temporal.io/server/service/history/hsm/callbacks" historyi "go.temporal.io/server/service/history/interfaces" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" diff --git a/service/matching/config.go b/service/matching/config.go index 36af2818c2c..763bfb972f4 100644 --- a/service/matching/config.go +++ b/service/matching/config.go @@ -10,7 +10,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/tqid" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/service/matching/counter" ) diff --git a/service/matching/matching_engine_test.go b/service/matching/matching_engine_test.go index 62cce1c554c..93e82d09dca 100644 --- a/service/matching/matching_engine_test.go +++ b/service/matching/matching_engine_test.go @@ -72,8 +72,8 @@ import ( "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/worker_versioning" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/history/consts" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.uber.org/mock/gomock" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/durationpb" diff --git a/tests/nexus_api_test.go b/tests/nexus_api_test.go index a68d2d1a5ce..42e47dee0f4 100644 --- a/tests/nexus_api_test.go +++ b/tests/nexus_api_test.go @@ -16,6 +16,7 @@ import ( "go.temporal.io/api/workflowservice/v1" sdkclient "go.temporal.io/sdk/client" "go.temporal.io/sdk/temporal" + "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" @@ -25,7 +26,6 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/frontend/configs" "go.temporal.io/server/tests/testcore" "google.golang.org/grpc/metadata" @@ -251,7 +251,7 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Outcomes(useTemporalFailures require.True(t, set) timeout, err := time.ParseDuration(timeoutStr) - var dispatchTimeoutBuffer = nexusoperations.MinDispatchTaskTimeout.Get(dynamicconfig.NewNoopCollection())("test") + var dispatchTimeoutBuffer = nexusoperation.MinDispatchTaskTimeout.Get(dynamicconfig.NewNoopCollection())("test") expectedMaxTimeout := 2*time.Second - dispatchTimeoutBuffer require.LessOrEqual(t, timeout, expectedMaxTimeout, "timeout should be buffered") diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index eb156da738e..699395d9735 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -49,8 +49,8 @@ import ( "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/testlogger" - "go.temporal.io/server/components/nexusoperations" "go.temporal.io/server/service/frontend/configs" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" ) diff --git a/tests/testcore/dynamic_config_overrides.go b/tests/testcore/dynamic_config_overrides.go index c7e4ac8f74c..21e9bd18581 100644 --- a/tests/testcore/dynamic_config_overrides.go +++ b/tests/testcore/dynamic_config_overrides.go @@ -5,7 +5,7 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/persistence/visibility" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" ) var ( diff --git a/tests/testcore/functional_test_base.go b/tests/testcore/functional_test_base.go index d9f5b325c06..e2a84e100ae 100644 --- a/tests/testcore/functional_test_base.go +++ b/tests/testcore/functional_test_base.go @@ -45,7 +45,7 @@ import ( "go.temporal.io/server/common/testing/testlogger" "go.temporal.io/server/common/testing/testtelemetry" "go.temporal.io/server/common/testing/updateutils" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/temporal" ) diff --git a/tests/testcore/onebox.go b/tests/testcore/onebox.go index f4fe0649bc3..7de9f536ef7 100644 --- a/tests/testcore/onebox.go +++ b/tests/testcore/onebox.go @@ -35,7 +35,7 @@ import ( "go.temporal.io/server/common/rpc/grpcfaults" "go.temporal.io/server/common/testing/grpcfaultstest" "go.temporal.io/server/common/testing/testhooks" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/temporal" "go.temporal.io/server/tests/testutils" "go.uber.org/multierr" diff --git a/tests/xdc/buffered_events_replication_helpers_test.go b/tests/xdc/buffered_events_replication_helpers_test.go index 1a21145756e..b220b0dbfa5 100644 --- a/tests/xdc/buffered_events_replication_helpers_test.go +++ b/tests/xdc/buffered_events_replication_helpers_test.go @@ -39,7 +39,7 @@ import ( "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/protoutils" "go.temporal.io/server/common/testing/testhooks" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" diff --git a/tests/xdc/nexus_request_forwarding_test.go b/tests/xdc/nexus_request_forwarding_test.go index 4d3ffab5de8..e73a4ab33bf 100644 --- a/tests/xdc/nexus_request_forwarding_test.go +++ b/tests/xdc/nexus_request_forwarding_test.go @@ -33,7 +33,7 @@ import ( "go.temporal.io/server/common/nexus/nexusrpc" "go.temporal.io/server/common/nexus/nexustest" "go.temporal.io/server/common/payload" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/tests/testcore" ) diff --git a/tests/xdc/nexus_state_replication_test.go b/tests/xdc/nexus_state_replication_test.go index 2a6df4e7e35..843afd3f0e9 100644 --- a/tests/xdc/nexus_state_replication_test.go +++ b/tests/xdc/nexus_state_replication_test.go @@ -37,8 +37,8 @@ import ( "go.temporal.io/server/common/nexus/nexustest" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/testvars" - "go.temporal.io/server/components/callbacks" - "go.temporal.io/server/components/nexusoperations" + "go.temporal.io/server/service/history/hsm/callbacks" + "go.temporal.io/server/service/history/hsm/nexusoperations" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" ) From 250603e3d724763ef6dd891c883232cca720e4cd Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 1 Sep 2026 10:23:34 -0700 Subject: [PATCH 104/139] Annotate worker task spans (#10739) ## What changed? Annotated worker RPCs' OTEL spans with "worker task ID". ## Why? Today, the OTEL chain is broken once workers are involved. This new attribute is needed to allow joining these separate traces (e.g. one from poll, one from respond) back into a chain. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [x] added new functional test(s) --- common/tasktoken/worker_task_id.go | 30 +++ common/tasktoken/worker_task_id_test.go | 16 ++ common/telemetry/grpc.go | 85 ++++++++- common/telemetry/grpc_test.go | 239 +++++++++++++++++++----- common/telemetry/tags.go | 2 + service/matching/matching_engine.go | 42 ++++- tests/nexus_otel_test.go | 73 ++++++++ 7 files changed, 429 insertions(+), 58 deletions(-) create mode 100644 common/tasktoken/worker_task_id.go create mode 100644 common/tasktoken/worker_task_id_test.go diff --git a/common/tasktoken/worker_task_id.go b/common/tasktoken/worker_task_id.go new file mode 100644 index 00000000000..396416a1a56 --- /dev/null +++ b/common/tasktoken/worker_task_id.go @@ -0,0 +1,30 @@ +package tasktoken + +import ( + "strconv" + "strings" +) + +// WorkflowWorkerTaskID returns the correlation ID shared by a workflow task's producer, poll, and response traces. +func WorkflowWorkerTaskID(namespaceID string, runID string, scheduledEventID int64) string { + return workerTaskID("workflow", namespaceID, runID, strconv.FormatInt(scheduledEventID, 10)) +} + +// ActivityWorkerTaskID returns the correlation ID shared by an activity task's producer, poll, and response traces. +func ActivityWorkerTaskID(namespaceID string, runID string, scheduledEventID int64) string { + return workerTaskID("activity", namespaceID, runID, strconv.FormatInt(scheduledEventID, 10)) +} + +// QueryWorkerTaskID returns the correlation ID shared by a query task's producer, poll, and response traces. +func QueryWorkerTaskID(namespaceID string, taskID string) string { + return workerTaskID("query", namespaceID, taskID) +} + +// NexusWorkerTaskID returns the correlation ID shared by a Nexus task's producer, poll, and response traces. +func NexusWorkerTaskID(namespaceID string, taskID string) string { + return workerTaskID("nexus", namespaceID, taskID) +} + +func workerTaskID(taskType string, parts ...string) string { + return taskType + "/" + strings.Join(parts, "/") +} diff --git a/common/tasktoken/worker_task_id_test.go b/common/tasktoken/worker_task_id_test.go new file mode 100644 index 00000000000..3479d349b02 --- /dev/null +++ b/common/tasktoken/worker_task_id_test.go @@ -0,0 +1,16 @@ +package tasktoken + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWorkerTaskID(t *testing.T) { + t.Parallel() + + require.Equal(t, "workflow/namespace-id/run-id/42", WorkflowWorkerTaskID("namespace-id", "run-id", 42)) + require.Equal(t, "activity/namespace-id/run-id/42", ActivityWorkerTaskID("namespace-id", "run-id", 42)) + require.Equal(t, "query/namespace-id/query-id", QueryWorkerTaskID("namespace-id", "query-id")) + require.Equal(t, "nexus/namespace-id/nexus-id", NexusWorkerTaskID("namespace-id", "nexus-id")) +} diff --git a/common/telemetry/grpc.go b/common/telemetry/grpc.go index d31d9a46124..36de2b5ef3b 100644 --- a/common/telemetry/grpc.go +++ b/common/telemetry/grpc.go @@ -8,6 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" + "go.temporal.io/api/workflowservice/v1" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/rpc/interceptor/logtags" @@ -27,10 +28,16 @@ type ( // ClientStatsHandler gives a named type to the grpc.UnaryClientInterceptor implementation provided by otelgrpc. ClientStatsHandler stats.Handler + // taskTokenCarrier is implemented by RPC payloads carrying serialized task tokens. + taskTokenCarrier interface { + GetTaskToken() []byte + } + customServerStatsHandler struct { - isDebug bool - wrapped stats.Handler - tags *logtags.WorkflowTags + isDebug bool + wrapped stats.Handler + tags *logtags.WorkflowTags + tokenSerializer *tasktoken.Serializer } ) @@ -78,9 +85,10 @@ func newCustomServerStatsHandler( logger log.Logger, ) *customServerStatsHandler { return &customServerStatsHandler{ - wrapped: handler, - isDebug: DebugMode(), - tags: logtags.NewWorkflowTags(tasktoken.NewSerializer(), logger), + wrapped: handler, + isDebug: DebugMode(), + tags: logtags.NewWorkflowTags(tasktoken.NewSerializer(), logger), + tokenSerializer: tasktoken.NewSerializer(), } } @@ -162,6 +170,10 @@ func (c *customServerStatsHandler) annotateTags( span trace.Span, payload any, ) { + if !span.IsRecording() { + return + } + methodName, ok := ctx.Value(methodNameKey{}).(string) if !ok { methodName = "unknown" @@ -180,6 +192,67 @@ func (c *customServerStatsHandler) annotateTags( } span.SetAttributes(attribute.Key(k).String(logTag.Value().(string))) } + + if workerTaskID := c.workerTaskID(payload); workerTaskID != "" { + span.SetAttributes(attribute.String(WorkerTaskIDKey, workerTaskID)) + } +} + +// workerTaskID returns the correlation ID from a supported task-token payload, or empty if unavailable. +func (c *customServerStatsHandler) workerTaskID(payload any) string { + tokenCarrier, ok := payload.(taskTokenCarrier) + if !ok { + return "" + } + taskToken := tokenCarrier.GetTaskToken() + if len(taskToken) == 0 { + return "" + } + + switch message := payload.(type) { + case *workflowservice.PollNexusTaskQueueResponse, + *workflowservice.RespondNexusTaskCompletedRequest, + *workflowservice.RespondNexusTaskFailedRequest: + token, err := c.tokenSerializer.DeserializeNexusTaskToken(taskToken) + if err == nil { + return tasktoken.NexusWorkerTaskID(token.GetNamespaceId(), token.GetTaskId()) + } + case *workflowservice.RespondQueryTaskCompletedRequest: + token, err := c.tokenSerializer.DeserializeQueryTaskToken(taskToken) + if err == nil { + return tasktoken.QueryWorkerTaskID(token.GetNamespaceId(), token.GetTaskId()) + } + case *workflowservice.PollWorkflowTaskQueueResponse: + if message.GetQuery() != nil { + token, err := c.tokenSerializer.DeserializeQueryTaskToken(taskToken) + if err == nil { + return tasktoken.QueryWorkerTaskID(token.GetNamespaceId(), token.GetTaskId()) + } + } else { + token, err := c.tokenSerializer.Deserialize(taskToken) + if err == nil { + return tasktoken.WorkflowWorkerTaskID(token.GetNamespaceId(), token.GetRunId(), token.GetScheduledEventId()) + } + } + case *workflowservice.RespondWorkflowTaskCompletedRequest, + *workflowservice.RespondWorkflowTaskFailedRequest: + token, err := c.tokenSerializer.Deserialize(taskToken) + if err == nil { + return tasktoken.WorkflowWorkerTaskID(token.GetNamespaceId(), token.GetRunId(), token.GetScheduledEventId()) + } + case *workflowservice.PollActivityTaskQueueResponse, + *workflowservice.RecordActivityTaskHeartbeatRequest, + *workflowservice.RespondActivityTaskCompletedRequest, + *workflowservice.RespondActivityTaskFailedRequest, + *workflowservice.RespondActivityTaskCanceledRequest: + token, err := c.tokenSerializer.Deserialize(taskToken) + if err == nil { + return tasktoken.ActivityWorkerTaskID(token.GetNamespaceId(), token.GetRunId(), token.GetScheduledEventId()) + } + default: + return "" + } + return "" } func (c *customServerStatsHandler) TagConn(ctx context.Context, info *stats.ConnTagInfo) context.Context { diff --git a/common/telemetry/grpc_test.go b/common/telemetry/grpc_test.go index b85f68a093e..fe7d0d79bd8 100644 --- a/common/telemetry/grpc_test.go +++ b/common/telemetry/grpc_test.go @@ -1,9 +1,7 @@ package telemetry_test import ( - "context" "encoding/json" - "os" "testing" "github.com/stretchr/testify/require" @@ -12,55 +10,22 @@ import ( "go.opentelemetry.io/otel/sdk/trace" "go.opentelemetry.io/otel/sdk/trace/tracetest" commonpb "go.temporal.io/api/common/v1" + querypb "go.temporal.io/api/query/v1" "go.temporal.io/api/workflowservice/v1" + tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/common/api" + "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/telemetry" "google.golang.org/grpc/codes" "google.golang.org/grpc/stats" "google.golang.org/grpc/status" ) -func Test_ServerStatsHandler(t *testing.T) { - - makeRequest := func(responseErr error) map[string]attribute.KeyValue { - t.Helper() - - exporter := tracetest.NewInMemoryExporter() - tp := trace.NewTracerProvider(trace.WithSyncer(exporter)) - tmp := propagation.TraceContext{} - otelStatsHandler := telemetry.NewServerStatsHandler(tp, tmp, nil) - - ctx := otelStatsHandler.TagRPC(context.Background(), &stats.RPCTagInfo{ - FullMethodName: api.WorkflowServicePrefix, - }) - otelStatsHandler.HandleRPC(ctx, &stats.InPayload{ - Payload: &workflowservice.TerminateWorkflowExecutionRequest{ - WorkflowExecution: &commonpb.WorkflowExecution{ - WorkflowId: "WF-ID", - RunId: "RUN-ID", - }, - }, - }) - if responseErr == nil { - otelStatsHandler.HandleRPC(ctx, &stats.OutPayload{ - Payload: &workflowservice.TerminateWorkflowExecutionResponse{}, - }) - } - otelStatsHandler.HandleRPC(ctx, &stats.End{ - Error: responseErr, - }) - - exportedSpans := exporter.GetSpans() - require.Len(t, exportedSpans, 1) - attrByKey := map[string]attribute.KeyValue{} - for _, a := range exportedSpans[0].Attributes { - attrByKey[string(a.Key)] = a - } - return attrByKey - } - +func TestServerStatsHandler(t *testing.T) { t.Run("annotate span with workflow tags", func(t *testing.T) { - spanAttrsByKey := makeRequest(nil) + t.Parallel() + + spanAttrsByKey := captureTerminateWorkflowAttributes(t, nil) require.Equal(t, "WF-ID", spanAttrsByKey["temporalWorkflowID"].Value.AsString()) require.Equal(t, "RUN-ID", spanAttrsByKey["temporalRunID"].Value.AsString()) @@ -71,10 +36,9 @@ func Test_ServerStatsHandler(t *testing.T) { }) t.Run("annotate span with request/response payload in debug mode", func(t *testing.T) { - os.Setenv("TEMPORAL_OTEL_DEBUG", "true") - defer os.Unsetenv("TEMPORAL_OTEL_DEBUG") + t.Setenv("TEMPORAL_OTEL_DEBUG", "true") - spanAttrsByKey := makeRequest(nil) + spanAttrsByKey := captureTerminateWorkflowAttributes(t, nil) require.JSONEq(t, `{"workflowExecution":{"workflowId":"WF-ID","runId":"RUN-ID"}}`, @@ -83,10 +47,9 @@ func Test_ServerStatsHandler(t *testing.T) { }) t.Run("annotate span with response error payload in debug mode", func(t *testing.T) { - os.Setenv("TEMPORAL_OTEL_DEBUG", "true") - defer os.Unsetenv("TEMPORAL_OTEL_DEBUG") + t.Setenv("TEMPORAL_OTEL_DEBUG", "true") - spanAttrsByKey := makeRequest(status.Errorf(codes.Internal, "Something went wrong")) + spanAttrsByKey := captureTerminateWorkflowAttributes(t, status.Errorf(codes.Internal, "Something went wrong")) require.JSONEq(t, `{"code":13,"message":"Something went wrong"}`, @@ -94,16 +57,150 @@ func Test_ServerStatsHandler(t *testing.T) { }) t.Run("skip if noop trace provider", func(t *testing.T) { + t.Parallel() + tp := telemetry.NoopTracerProvider tmp := propagation.TraceContext{} otelStatsHandler := telemetry.NewServerStatsHandler(tp, tmp, nil) require.Nil(t, otelStatsHandler) }) + + t.Run("annotate spans with worker task ID", func(t *testing.T) { + t.Parallel() + + serializer := tasktoken.NewSerializer() + taskToken, err := serializer.Serialize(&tokenspb.Task{ + NamespaceId: "namespace-id", + RunId: "run-id", + ScheduledEventId: 42, + }) + require.NoError(t, err) + queryToken, err := serializer.SerializeQueryTaskToken(&tokenspb.QueryTask{ + NamespaceId: "namespace-id", + TaskId: "query-id", + }) + require.NoError(t, err) + nexusToken, err := serializer.SerializeNexusTaskToken(&tokenspb.NexusTask{ + NamespaceId: "namespace-id", + TaskId: "nexus-id", + }) + require.NoError(t, err) + + for _, tc := range []struct { + name string + method string + payload any + outbound bool + workerTaskID string + }{ + { + name: "WorkflowPoll", + method: "PollWorkflowTaskQueue", + payload: &workflowservice.PollWorkflowTaskQueueResponse{TaskToken: taskToken}, + outbound: true, + workerTaskID: "workflow/namespace-id/run-id/42", + }, + { + name: "WorkflowCompletion", + method: "RespondWorkflowTaskCompleted", + payload: &workflowservice.RespondWorkflowTaskCompletedRequest{TaskToken: taskToken}, + workerTaskID: "workflow/namespace-id/run-id/42", + }, + { + name: "WorkflowFailure", + method: "RespondWorkflowTaskFailed", + payload: &workflowservice.RespondWorkflowTaskFailedRequest{TaskToken: taskToken}, + workerTaskID: "workflow/namespace-id/run-id/42", + }, + { + name: "ActivityPoll", + method: "PollActivityTaskQueue", + payload: &workflowservice.PollActivityTaskQueueResponse{TaskToken: taskToken}, + outbound: true, + workerTaskID: "activity/namespace-id/run-id/42", + }, + { + name: "ActivityCompletion", + method: "RespondActivityTaskCompleted", + payload: &workflowservice.RespondActivityTaskCompletedRequest{TaskToken: taskToken}, + workerTaskID: "activity/namespace-id/run-id/42", + }, + { + name: "ActivityHeartbeat", + method: "RecordActivityTaskHeartbeat", + payload: &workflowservice.RecordActivityTaskHeartbeatRequest{TaskToken: taskToken}, + workerTaskID: "activity/namespace-id/run-id/42", + }, + { + name: "ActivityFailure", + method: "RespondActivityTaskFailed", + payload: &workflowservice.RespondActivityTaskFailedRequest{TaskToken: taskToken}, + workerTaskID: "activity/namespace-id/run-id/42", + }, + { + name: "ActivityCancellation", + method: "RespondActivityTaskCanceled", + payload: &workflowservice.RespondActivityTaskCanceledRequest{TaskToken: taskToken}, + workerTaskID: "activity/namespace-id/run-id/42", + }, + { + name: "QueryPoll", + method: "PollWorkflowTaskQueue", + payload: &workflowservice.PollWorkflowTaskQueueResponse{TaskToken: queryToken, Query: &querypb.WorkflowQuery{}}, + outbound: true, + workerTaskID: "query/namespace-id/query-id", + }, + { + name: "QueryCompletion", + method: "RespondQueryTaskCompleted", + payload: &workflowservice.RespondQueryTaskCompletedRequest{TaskToken: queryToken}, + workerTaskID: "query/namespace-id/query-id", + }, + { + name: "NexusPoll", + method: "PollNexusTaskQueue", + payload: &workflowservice.PollNexusTaskQueueResponse{TaskToken: nexusToken}, + outbound: true, + workerTaskID: "nexus/namespace-id/nexus-id", + }, + { + name: "NexusCompletion", + method: "RespondNexusTaskCompleted", + payload: &workflowservice.RespondNexusTaskCompletedRequest{TaskToken: nexusToken}, + workerTaskID: "nexus/namespace-id/nexus-id", + }, + { + name: "NexusFailure", + method: "RespondNexusTaskFailed", + payload: &workflowservice.RespondNexusTaskFailedRequest{TaskToken: nexusToken}, + workerTaskID: "nexus/namespace-id/nexus-id", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + var payloadStats stats.RPCStats + if tc.outbound { + payloadStats = &stats.OutPayload{Payload: tc.payload} + } else { + payloadStats = &stats.InPayload{Payload: tc.payload} + } + + attrs := captureServerRPCAttributes(t, tc.method, payloadStats, &stats.End{}) + workerTaskID, ok := attrs[telemetry.WorkerTaskIDKey] + require.True(t, ok) + require.Equal(t, tc.workerTaskID, workerTaskID.Value.AsString()) + }) + } + }) } -func Test_ClientStatsHandler(t *testing.T) { +func TestClientStatsHandler(t *testing.T) { + t.Parallel() t.Run("skip if noop trace provider", func(t *testing.T) { + t.Parallel() + tp := telemetry.NoopTracerProvider tmp := propagation.TraceContext{} otelStatsHandler := telemetry.NewClientStatsHandler(tp, tmp) @@ -111,6 +208,50 @@ func Test_ClientStatsHandler(t *testing.T) { }) } +func captureTerminateWorkflowAttributes(t *testing.T, responseErr error) map[string]attribute.KeyValue { + t.Helper() + + rpcStats := []stats.RPCStats{&stats.InPayload{ + Payload: &workflowservice.TerminateWorkflowExecutionRequest{ + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: "WF-ID", + RunId: "RUN-ID", + }, + }, + }} + if responseErr == nil { + rpcStats = append(rpcStats, &stats.OutPayload{ + Payload: &workflowservice.TerminateWorkflowExecutionResponse{}, + }) + } + rpcStats = append(rpcStats, &stats.End{ + Error: responseErr, + }) + return captureServerRPCAttributes(t, "TerminateWorkflowExecution", rpcStats...) +} + +func captureServerRPCAttributes(t *testing.T, method string, rpcStats ...stats.RPCStats) map[string]attribute.KeyValue { + t.Helper() + + exporter := tracetest.NewInMemoryExporter() + tp := trace.NewTracerProvider(trace.WithSyncer(exporter)) + handler := telemetry.NewServerStatsHandler(tp, propagation.TraceContext{}, nil) + ctx := handler.TagRPC(t.Context(), &stats.RPCTagInfo{ + FullMethodName: api.WorkflowServicePrefix + method, + }) + for _, rpcStat := range rpcStats { + handler.HandleRPC(ctx, rpcStat) + } + + exportedSpans := exporter.GetSpans() + require.Len(t, exportedSpans, 1) + attrByKey := map[string]attribute.KeyValue{} + for _, a := range exportedSpans[0].Attributes { + attrByKey[string(a.Key)] = a + } + return attrByKey +} + func toStr(t *testing.T, v attribute.Value) string { t.Helper() var payload map[string]json.RawMessage diff --git a/common/telemetry/tags.go b/common/telemetry/tags.go index c10d6e16083..12e39bb54fa 100644 --- a/common/telemetry/tags.go +++ b/common/telemetry/tags.go @@ -19,4 +19,6 @@ const ( NexusOperationKey = "nexus.operation" NexusRequestIDKey = "nexus.request_id" NexusServiceKey = "nexus.service" + + WorkerTaskIDKey = "worker_task.id" ) diff --git a/service/matching/matching_engine.go b/service/matching/matching_engine.go index 049da260866..9333d491712 100644 --- a/service/matching/matching_engine.go +++ b/service/matching/matching_engine.go @@ -14,6 +14,8 @@ import ( "github.com/google/uuid" "github.com/nexus-rpc/sdk-go/nexus" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" commonpb "go.temporal.io/api/common/v1" deploymentpb "go.temporal.io/api/deployment/v1" enumspb "go.temporal.io/api/enums/v1" @@ -57,6 +59,7 @@ import ( "go.temporal.io/server/common/stream_batcher" "go.temporal.io/server/common/taskqueue" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/tqid" "go.temporal.io/server/common/util" @@ -622,6 +625,17 @@ func (e *matchingEngineImpl) AddWorkflowTask( Priority: addRequest.Priority, } + if span := trace.SpanFromContext(ctx); span.IsRecording() { + // Later poll and response spans derive the same ID from the task token, + // allowing them to be correlated with this span. + workerTaskID := tasktoken.WorkflowWorkerTaskID( + taskInfo.GetNamespaceId(), + taskInfo.GetRunId(), + taskInfo.GetScheduledEventId(), + ) + span.SetAttributes(attribute.String(telemetry.WorkerTaskIDKey, workerTaskID)) + } + return pm.AddTask(ctx, addTaskParams{ taskInfo: taskInfo, forwardInfo: addRequest.ForwardInfo, @@ -662,6 +676,17 @@ func (e *matchingEngineImpl) AddActivityTask( ComponentRef: addRequest.ComponentRef, } + if span := trace.SpanFromContext(ctx); span.IsRecording() { + // Later poll and response spans derive the same ID from the task token, + // allowing them to be correlated with this span. + workerTaskID := tasktoken.ActivityWorkerTaskID( + taskInfo.GetNamespaceId(), + taskInfo.GetRunId(), + taskInfo.GetScheduledEventId(), + ) + span.SetAttributes(attribute.String(telemetry.WorkerTaskIDKey, workerTaskID)) + } + return pm.AddTask(ctx, addTaskParams{ taskInfo: taskInfo, forwardInfo: addRequest.ForwardInfo, @@ -1146,6 +1171,12 @@ func (e *matchingEngineImpl) QueryWorkflow( if resp != nil || err != nil { return resp, err } + if span := trace.SpanFromContext(ctx); span.IsRecording() { + // Later poll and response spans derive the same ID from the task token, + // allowing them to be correlated with this span. + workerTaskID := tasktoken.QueryWorkerTaskID(queryRequest.GetNamespaceId(), taskID) + span.SetAttributes(attribute.String(telemetry.WorkerTaskIDKey, workerTaskID)) + } // if we get here it means that dispatch of query task has occurred locally // must wait on result channel to get query result @@ -2659,7 +2690,6 @@ func (e *matchingEngineImpl) DispatchNexusTask(ctx context.Context, request *mat if err != nil { return nil, err } - // Buffer the deadline so we can still respond with timeout if we hit the deadline while dispatching ctx, cancel := contextutil.WithDeadlineBuffer(ctx, matching.DefaultTimeout, e.config.MinDispatchTaskTimeout(ns.Name().String())) defer cancel() @@ -2687,6 +2717,12 @@ func (e *matchingEngineImpl) DispatchNexusTask(ctx context.Context, request *mat if resp != nil { return resp, nil } + if span := trace.SpanFromContext(ctx); span.IsRecording() { + // Later poll and response spans derive the same ID from the task token, + // allowing them to be correlated with this span. + workerTaskID := tasktoken.NexusWorkerTaskID(request.GetNamespaceId(), taskID) + span.SetAttributes(attribute.String(telemetry.WorkerTaskIDKey, workerTaskID)) + } // If we get here it means that task dispatch has occurred locally. // Must wait on result channel to get query result. @@ -2800,7 +2836,7 @@ pollLoop: } } -func (e *matchingEngineImpl) RespondNexusTaskCompleted(ctx context.Context, request *matchingservice.RespondNexusTaskCompletedRequest, opMetrics metrics.Handler) (*matchingservice.RespondNexusTaskCompletedResponse, error) { +func (e *matchingEngineImpl) RespondNexusTaskCompleted(_ context.Context, request *matchingservice.RespondNexusTaskCompletedRequest, opMetrics metrics.Handler) (*matchingservice.RespondNexusTaskCompletedResponse, error) { resultCh, ok := e.nexusResults.Pop(request.GetTaskId()) if !ok { opMetrics.Counter(metrics.RespondNexusTaskFailedPerTaskQueueCounter.Name()).Record(1) @@ -2813,7 +2849,7 @@ func (e *matchingEngineImpl) RespondNexusTaskCompleted(ctx context.Context, requ return &matchingservice.RespondNexusTaskCompletedResponse{}, nil } -func (e *matchingEngineImpl) RespondNexusTaskFailed(ctx context.Context, request *matchingservice.RespondNexusTaskFailedRequest, opMetrics metrics.Handler) (*matchingservice.RespondNexusTaskFailedResponse, error) { +func (e *matchingEngineImpl) RespondNexusTaskFailed(_ context.Context, request *matchingservice.RespondNexusTaskFailedRequest, opMetrics metrics.Handler) (*matchingservice.RespondNexusTaskFailedResponse, error) { resultCh, ok := e.nexusResults.Pop(request.GetTaskId()) if !ok { opMetrics.Counter(metrics.RespondNexusTaskFailedPerTaskQueueCounter.Name()).Record(1) diff --git a/tests/nexus_otel_test.go b/tests/nexus_otel_test.go index 85c61265dad..a56c31b2b81 100644 --- a/tests/nexus_otel_test.go +++ b/tests/nexus_otel_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/chasm/lib/nexusoperation" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/nexus/nexusrpc" + "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/parallelsuite" "go.temporal.io/server/common/testing/testtelemetry" @@ -48,6 +49,12 @@ type nexusHTTPSpan struct { NexusAttrs map[string]any } +type nexusTaskSpan struct { + TraceID int + Name string + TaskID int +} + type NexusOTELSuite struct { parallelsuite.Suite[*NexusOTELSuite] } @@ -321,6 +328,23 @@ func (s *NexusOTELSuite) TestWorkerOperation() { s.Require().True(spanContext.IsValid()) s.Require().Equal(spanContext.TraceID(), httpSpans[0].SpanContext.TraceID()) s.Require().Equal(spanContext.SpanID(), httpSpans[0].SpanContext.SpanID()) + s.requireNexusTaskGRPCSpans(exporter, []nexusTaskSpan{ + { + TraceID: 1, + Name: "temporal.api.workflowservice.v1.WorkflowService/PollNexusTaskQueue", + TaskID: 1, + }, + { + TraceID: 2, + Name: "temporal.api.workflowservice.v1.WorkflowService/RespondNexusTaskCompleted", + TaskID: 1, + }, + { + TraceID: 3, + Name: "temporal.server.api.matchingservice.v1.MatchingService/DispatchNexusTask", + TaskID: 1, + }, + }) } // Verifies the namespace and task queue route propagates tracing and records handler failures without forwarding. @@ -373,6 +397,55 @@ func (s *NexusOTELSuite) TestNamespaceAndTaskQueueDispatch() { }}) s.Require().Equal(traceID, httpSpans[0].SpanContext.TraceID().String()) s.Require().Equal(parentSpanID, httpSpans[0].Parent.SpanID().String()) + s.requireNexusTaskGRPCSpans(exporter, []nexusTaskSpan{ + { + TraceID: 1, + Name: "temporal.api.workflowservice.v1.WorkflowService/PollNexusTaskQueue", + TaskID: 1, + }, + { + TraceID: 2, + Name: "temporal.api.workflowservice.v1.WorkflowService/RespondNexusTaskFailed", + TaskID: 1, + }, + { + TraceID: 3, + Name: "temporal.server.api.matchingservice.v1.MatchingService/DispatchNexusTask", + TaskID: 1, + }, + }) +} + +// requireNexusTaskGRPCSpans compares worker task gRPC spans after assigning stable trace and task IDs. +func (s *NexusOTELSuite) requireNexusTaskGRPCSpans( + exporter *tracetest.InMemoryExporter, + expected []nexusTaskSpan, +) { + s.T().Helper() + requireExportedSpans(s, exporter, expected, s.nexusTaskGRPCSpans) +} + +func (s *NexusOTELSuite) nexusTaskGRPCSpans(spans tracetest.SpanStubs) []nexusTaskSpan { + spans = testtelemetry.FilterSpans(spans, func(span tracetest.SpanStub) bool { + _, ok := testtelemetry.SpanAttribute(span, telemetry.WorkerTaskIDKey) + return ok + }) + slices.SortFunc(spans, func(a, b tracetest.SpanStub) int { + return strings.Compare(a.Name, b.Name) + }) + + localSpanIDs := testtelemetry.LocalSpanIDs(spans) + localTaskIDs := testtelemetry.LocalAttributeIDs(spans, telemetry.WorkerTaskIDKey) + + result := make([]nexusTaskSpan, 0, len(spans)) + for i, span := range spans { + result = append(result, nexusTaskSpan{ + TraceID: localSpanIDs[i].Trace, + Name: span.Name, + TaskID: localTaskIDs[i], + }) + } + return result } // requireNexusHTTPSpans compares all exported HTTP spans and their Nexus attributes after From be0318ac8ff440c51af6250eeb085a122efb451a Mon Sep 17 00:00:00 2001 From: Yichao Yang Date: Tue, 1 Sep 2026 11:04:12 -0700 Subject: [PATCH 105/139] Fix FirstWorkflowTaskBackoff Accumulation upon ContinueAsNew (#11839) ## What changed? - Fix FirstWorkflowTaskBackoff Accumulation upon ContinueAsNew ## Why? - Today if workflow continueAsNew before executionTime, the calculated lifetime will be negative and causing backoff to keep accumulating via continueAsNew. ## How did you test it? - [x] built - [ ] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) --- .../history/workflow/mutable_state_impl.go | 8 +- .../workflow/mutable_state_impl_test.go | 30 ++++ tests/continue_as_new_test.go | 133 ++++++++++++++++++ 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 5e0de6e88f3..091b3985ab2 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -2975,9 +2975,13 @@ func (ms *MutableStateImpl) ContinueAsNewMinBackoff(backoffDuration *durationpb. // lifetime of previous execution // todo@time-skipping: time skipping is naturally supported for continue as new backoff, and need to // make sure the backoff is correctly applied in the time skipping case - lifetime := ms.timeSource.Now().Sub(ms.executionState.StartTime.AsTime().UTC()) + now := ms.timeSource.Now() + lifetime := max(time.Duration(0), now.Sub(ms.executionState.StartTime.AsTime().UTC())) if ms.executionInfo.ExecutionTime != nil { - lifetime = ms.timeSource.Now().Sub(ms.executionInfo.ExecutionTime.AsTime().UTC()) + executionLifetime := now.Sub(ms.executionInfo.ExecutionTime.AsTime().UTC()) + if executionLifetime >= 0 { + lifetime = executionLifetime + } } interval := lifetime diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index ac97ddec479..bb325512270 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -2215,6 +2215,36 @@ func (s *mutableStateSuite) TestContinueAsNewMinBackoff() { s.True(minBackoff == backoff) } +func (s *mutableStateSuite) TestContinueAsNewMinBackoffExecutionCompletesBeforeExecutionTime() { + s.mockConfig.WorkflowIdReuseMinimalInterval = func(namespace string) time.Duration { + return time.Second + } + + now := time.Now() + s.mutableState.timeSource = clock.NewEventTimeSource().Update(now) + + // Guard against clock skew or malformed state making StartTime later than now. + // The lifetime should be clamped at zero, so the full minimal interval is still applied. + s.mutableState.executionState.StartTime = timestamppb.New(now.Add(time.Second)) + s.mutableState.executionInfo.ExecutionTime = nil + + minBackoff := s.mutableState.ContinueAsNewMinBackoff(nil).AsDuration() + s.Equal(time.Second, minBackoff) + + // Simulate a delayed-start run that actually executed and closed before its ExecutionTime. + // In that case lifetime should fall back to close - StartTime, not become negative. + s.mutableState.executionState.StartTime = timestamppb.New(now.Add(-100 * time.Millisecond)) + s.mutableState.executionInfo.ExecutionTime = timestamppb.New(now.Add(time.Second)) + + minBackoff = s.mutableState.ContinueAsNewMinBackoff(nil).AsDuration() + s.Equal(900*time.Millisecond, minBackoff) + + // Existing backoff already satisfies the minimal interval when combined with the fallback lifetime. + backoff := time.Second + minBackoff = s.mutableState.ContinueAsNewMinBackoff(durationpb.New(backoff)).AsDuration() + s.Equal(backoff, minBackoff) +} + func (s *mutableStateSuite) TestEventReapplied() { runID := uuid.NewString() eventID := int64(1) diff --git a/tests/continue_as_new_test.go b/tests/continue_as_new_test.go index ad86d557bf5..c48cc13c0e3 100644 --- a/tests/continue_as_new_test.go +++ b/tests/continue_as_new_test.go @@ -13,6 +13,7 @@ import ( commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" + protocolpb "go.temporal.io/api/protocol/v1" "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" @@ -588,6 +589,138 @@ func (s *ContinueAsNewTestSuite) TestContinueAsNewWithDelayStart() { s.GreaterOrEqual(scheduledTime.Sub(startedTime), delayStart) } +func (s *ContinueAsNewTestSuite) TestContinueAsNewMinBackoffDoesNotAccumulate() { + env := testcore.NewEnv(s.T()) + + const minimalInterval = 10 * time.Second + env.OverrideDynamicConfig(dynamicconfig.WorkflowIdReuseMinimalInterval, minimalInterval) + + id := "functional-continue-as-new-min-backoff-does-not-accumulate-test" + wt := "functional-continue-as-new-min-backoff-does-not-accumulate-test-type" + tl := "functional-continue-as-new-min-backoff-does-not-accumulate-test-taskqueue" + identity := "worker1" + + workflowType := &commonpb.WorkflowType{Name: wt} + taskQueue := &taskqueuepb.TaskQueue{Name: tl, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} + + request := &workflowservice.StartWorkflowExecutionRequest{ + RequestId: uuid.NewString(), + Namespace: env.Namespace().String(), + WorkflowId: id, + WorkflowType: workflowType, + TaskQueue: taskQueue, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + Identity: identity, + } + + we, err := env.FrontendClient().StartWorkflowExecution(s.Context(), request) + s.NoError(err) + + const continueAsNewCount = 3 + var firstWorkflowTaskBackoffs []time.Duration + runIDs := []string{we.RunId} + for i := range continueAsNewCount + 1 { + // Each task is polled after the run has been made executable. For continued runs, + // the admitted update below schedules a workflow task before ExecutionTime. + task, err := env.FrontendClient().PollWorkflowTaskQueue(s.Context(), &workflowservice.PollWorkflowTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: taskQueue, + Identity: identity, + }) + s.NoError(err) + s.NotEmpty(task.GetTaskToken()) + + startAttrs := task.History.Events[0].GetWorkflowExecutionStartedEventAttributes() + firstWorkflowTaskBackoff := startAttrs.GetFirstWorkflowTaskBackoff().AsDuration() + firstWorkflowTaskBackoffs = append(firstWorkflowTaskBackoffs, firstWorkflowTaskBackoff) + + var messages []*protocolpb.Message + var commands []*commandpb.Command + if i > 0 { + s.NotEmpty(task.Messages, "expected update message in task") + updateTV := env.Tv(). + WithWorkflowID(id). + WithRunID(runIDs[i]). + WithUpdateIDNumber(i). + WithMessageIDNumber(i) + messages = env.UpdateAcceptCompleteMessages(updateTV, task.Messages[0]) + commands = env.UpdateAcceptCompleteCommands(updateTV) + } + if i < continueAsNewCount { + // Continue as new without an explicit backoff. The server applies the default + // WorkflowIdReuseMinimalInterval because the previous run closes quickly. + commands = append(commands, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: workflowType, + TaskQueue: taskQueue, + WorkflowRunTimeout: durationpb.New(100 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + }, + }, + }) + } else { + commands = append(commands, &commandpb.Command{ + CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{ + Result: payloads.EncodeString("Done"), + }, + }, + }) + } + + _, err = env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ + Namespace: env.Namespace().String(), + TaskToken: task.GetTaskToken(), + Identity: identity, + Commands: commands, + Messages: messages, + }) + s.NoError(err) + + events := env.GetHistory(env.Namespace().String(), &commonpb.WorkflowExecution{ + WorkflowId: id, + RunId: runIDs[len(runIDs)-1], + }) + lastEvent := events[len(events)-1] + if i < continueAsNewCount { + s.Equal(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW, lastEvent.GetEventType()) + newRunID := lastEvent.GetWorkflowExecutionContinuedAsNewEventAttributes().GetNewExecutionRunId() + runIDs = append(runIDs, newRunID) + // Admit a noop update on the delayed continued run. This schedules the next + // workflow task immediately, so the run can close before its ExecutionTime. + updateTV := env.Tv(). + WithWorkflowID(id). + WithRunID(newRunID). + WithUpdateIDNumber(i + 1). + WithMessageIDNumber(i + 1) + go func(updateTV *testvars.TestVars) { + _, updateErr := env.FrontendClient().UpdateWorkflowExecution( + testcore.NewContext(), + updateWorkflowRequest(env, updateTV, nil), + ) + if updateErr != nil { + s.T().Errorf("Update failed: %v", updateErr) + } + }(updateTV) + waitUpdateAdmitted(env, updateTV) + } else { + s.Equal(enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED, lastEvent.GetEventType()) + } + } + + // The first run has no inherited backoff. Every continued run should stay capped at + // the default minimal interval instead of accumulating across the tight loop. + s.Len(firstWorkflowTaskBackoffs, continueAsNewCount+1) + s.Zero(firstWorkflowTaskBackoffs[0]) + for i, firstWorkflowTaskBackoff := range firstWorkflowTaskBackoffs[1:] { + s.LessOrEqual(firstWorkflowTaskBackoff, minimalInterval, "firstWorkflowTaskBackoff accumulated at run %d: %v", i+1, firstWorkflowTaskBackoff) + } +} + type ( ParentWithChildContinueAsNew struct { suite *ContinueAsNewTestSuite From a5fdb0268cd666ef2a17b8c0c418036037687090 Mon Sep 17 00:00:00 2001 From: David Reiss Date: Tue, 1 Sep 2026 11:39:54 -0700 Subject: [PATCH 106/139] Remove priority/fairness metrics handlers (#11875) ## What changed? Remove priMetricsHandler + fairMetricsHandler in matching service. ## Why? These were used for validation and aren't that useful. This reduces metrics cardinality. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) --- .../matching/physical_task_queue_manager.go | 12 +- service/matching/pri_metrics_handler.go | 147 ------------------ 2 files changed, 6 insertions(+), 153 deletions(-) delete mode 100644 service/matching/pri_metrics_handler.go diff --git a/service/matching/physical_task_queue_manager.go b/service/matching/physical_task_queue_manager.go index 25fb8030f60..c19517be803 100644 --- a/service/matching/physical_task_queue_manager.go +++ b/service/matching/physical_task_queue_manager.go @@ -196,7 +196,7 @@ func newPhysicalTaskQueueManager( pqMgr.logger, pqMgr.throttledLogger, e.matchingRawClient, - newFairMetricsHandler(taggedMetricsHandler), + taggedMetricsHandler, pqMgr.counterFactory, false, ) @@ -217,7 +217,7 @@ func newPhysicalTaskQueueManager( pqMgr.matchingClient, pqMgr.taskValidator, pqMgr.logger, - newFairMetricsHandler(taggedMetricsHandler), + taggedMetricsHandler, partitionMgr.rateLimitManager, pqMgr.onRateLimited, pqMgr.MarkAlive, @@ -237,7 +237,7 @@ func newPhysicalTaskQueueManager( pqMgr.logger, pqMgr.throttledLogger, e.matchingRawClient, - newPriMetricsHandler(taggedMetricsHandler), + taggedMetricsHandler, false, ) var fwdr *priForwarder @@ -257,7 +257,7 @@ func newPhysicalTaskQueueManager( pqMgr.matchingClient, pqMgr.taskValidator, pqMgr.logger, - newPriMetricsHandler(taggedMetricsHandler), + taggedMetricsHandler, partitionMgr.rateLimitManager, pqMgr.onRateLimited, pqMgr.MarkAlive, @@ -393,7 +393,7 @@ func (c *physicalTaskQueueManagerImpl) SetupDraining() { logger, log.With(c.throttledLogger, backlogTagPriorityDrain), c.partitionMgr.engine.matchingRawClient, - newPriMetricsHandler(c.metricsHandler), + c.metricsHandler, true, ) case *priBacklogManagerImpl: @@ -406,7 +406,7 @@ func (c *physicalTaskQueueManagerImpl) SetupDraining() { logger, log.With(c.throttledLogger, backlogTagFairnessDrain), c.partitionMgr.engine.matchingRawClient, - newFairMetricsHandler(c.metricsHandler), + c.metricsHandler, c.counterFactory, true, ) diff --git a/service/matching/pri_metrics_handler.go b/service/matching/pri_metrics_handler.go deleted file mode 100644 index d3d74e96709..00000000000 --- a/service/matching/pri_metrics_handler.go +++ /dev/null @@ -1,147 +0,0 @@ -// nolint:forbidigo,panic // panic for unused metric handler methods -package matching - -import ( - "time" - - "go.temporal.io/server/common/log" - "go.temporal.io/server/common/metrics" -) - -type ( - priMetricHandler struct { - handler metrics.Handler - } - priMetricsTimer struct { - name string - handler metrics.Handler - } - priMetricsCounter struct { - name string - handler metrics.Handler - } - priMetricsGauge struct { - name string - handler metrics.Handler - } - fairMetricHandler struct { - handler metrics.Handler - } - fairMetricsTimer struct { - name string - handler metrics.Handler - } - fairMetricsCounter struct { - name string - handler metrics.Handler - } - fairMetricsGauge struct { - name string - handler metrics.Handler - } -) - -// TODO(pri): cleanup; delete this -func newPriMetricsHandler(handler metrics.Handler) priMetricHandler { - return priMetricHandler{ - handler: handler, - } -} - -func (p priMetricHandler) Stop(logger log.Logger) { - p.handler.Stop(logger) -} - -func (p priMetricHandler) Counter(name string) metrics.CounterIface { - return priMetricsCounter{name: name, handler: p.handler} -} -func (p priMetricHandler) Timer(name string) metrics.TimerIface { - return priMetricsTimer{name: name, handler: p.handler} -} - -func (p priMetricHandler) Gauge(name string) metrics.GaugeIface { - return priMetricsGauge{name: name, handler: p.handler} -} - -func (p priMetricHandler) WithTags(...metrics.Tag) metrics.Handler { - panic("not implemented") -} - -func (p priMetricHandler) Histogram(string, metrics.MetricUnit) metrics.HistogramIface { - panic("not implemented") -} - -func (p priMetricHandler) StartBatch(string) metrics.BatchHandler { - panic("not implemented") -} - -func (c priMetricsCounter) Record(i int64, tag ...metrics.Tag) { - c.handler.Counter(c.name).Record(i, tag...) - c.handler.Counter(withPriPrefix(c.name)).Record(i, tag...) -} - -func (t priMetricsTimer) Record(duration time.Duration, tag ...metrics.Tag) { - t.handler.Timer(t.name).Record(duration, tag...) - t.handler.Timer(withPriPrefix(t.name)).Record(duration, tag...) -} - -func (t priMetricsGauge) Record(v float64, tag ...metrics.Tag) { - t.handler.Gauge(t.name).Record(v, tag...) - t.handler.Gauge(withPriPrefix(t.name)).Record(v, tag...) -} - -func withPriPrefix(name string) string { - return "pri_" + name -} - -func newFairMetricsHandler(handler metrics.Handler) fairMetricHandler { - return fairMetricHandler{ - handler: handler, - } -} - -func (p fairMetricHandler) Stop(logger log.Logger) { - p.handler.Stop(logger) -} - -func (p fairMetricHandler) Counter(name string) metrics.CounterIface { - return fairMetricsCounter{name: name, handler: p.handler} -} -func (p fairMetricHandler) Timer(name string) metrics.TimerIface { - return fairMetricsTimer{name: name, handler: p.handler} -} - -func (p fairMetricHandler) Gauge(name string) metrics.GaugeIface { - return fairMetricsGauge{name: name, handler: p.handler} -} - -func (p fairMetricHandler) WithTags(...metrics.Tag) metrics.Handler { - panic("not implemented") -} - -func (p fairMetricHandler) Histogram(string, metrics.MetricUnit) metrics.HistogramIface { - panic("not implemented") -} - -func (p fairMetricHandler) StartBatch(string) metrics.BatchHandler { - panic("not implemented") -} - -func (c fairMetricsCounter) Record(i int64, tag ...metrics.Tag) { - c.handler.Counter(c.name).Record(i, tag...) - c.handler.Counter(withFairPrefix(c.name)).Record(i, tag...) -} - -func (t fairMetricsTimer) Record(duration time.Duration, tag ...metrics.Tag) { - t.handler.Timer(t.name).Record(duration, tag...) - t.handler.Timer(withFairPrefix(t.name)).Record(duration, tag...) -} - -func (t fairMetricsGauge) Record(v float64, tag ...metrics.Tag) { - t.handler.Gauge(t.name).Record(v, tag...) - t.handler.Gauge(withFairPrefix(t.name)).Record(v, tag...) -} - -func withFairPrefix(name string) string { - return "fair_" + name -} From f61c721da6c0ac52786a557bac425af4373378d1 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Tue, 1 Sep 2026 11:46:03 -0700 Subject: [PATCH 107/139] Add test-only passive replication hook seams (#11790) ## Summary - add one namespace-scoped, `test_dep`-only `HistoryPassiveReplicationTest` hook for the single-cluster passive replication test stack - pass the existing test hook registry through the original workflow-cache, workflow-context, and active/standby executor constructors - split workflow updates into prepare-mutable-state, close-mutable-state, and execute-transaction phases so the test hook can inspect the complete mutation, snapshot, events, tasks, and mutable state before persistence - use an uncached transient workflow context for test-initiated versioned-transition replication, leaving the active cached context and its Workflow Update registry, callbacks, and futures intact - run selected history tasks through standby validation first; `ErrTaskRetry` continues to active execution while every other standby result is preserved - preserve the original production workflow continuation, cache acquisition, persistence transaction, and namespace-based active/standby routing when the hook is absent This PR intentionally excludes the `tests/passivepath` harness package so the core production seams can be reviewed independently. ## Verification - normal and `test_dep` NDC and queue package compilation - focused and full passive-path functional suites - Omes scenarios covering Workflow Updates, retries, heartbeats, timeouts, activities, timers, signals, child workflows, and continue-as-new - 1,447 diverted and passively applied transactions, with zero apply errors - diversion increased from 33.5% to 69.4%; the only remaining harness bailouts are buffered-event staging and cleanup - `git diff --check` --- common/metrics/metric_defs.go | 6 +- common/testing/testhooks/hooks.go | 39 ++-- service/history/api/create_workflow_util.go | 2 + .../api/respondworkflowtaskcompleted/api.go | 2 + .../respondworkflowtaskcompleted/api_test.go | 3 +- .../api_test.go | 3 +- service/history/chasm_engine.go | 2 + service/history/chasm_engine_test.go | 2 + service/history/history_engine.go | 1 + service/history/history_engine2_test.go | 3 +- .../history/history_engine3_eventsv2_test.go | 3 +- service/history/history_engine_test.go | 3 +- .../ndc/activity_state_replicator_test.go | 3 +- service/history/ndc/history_replicator.go | 2 + .../history/ndc/hsm_state_replicator_test.go | 3 +- .../history/ndc/mutable_state_initializer.go | 2 + service/history/ndc/resetter_test.go | 2 + .../history/ndc/transaction_manager_test.go | 3 +- service/history/ndc/workflow_resetter.go | 2 + service/history/ndc/workflow_resetter_test.go | 3 +- .../history/ndc/workflow_state_replicator.go | 62 +++++- .../ndc/workflow_state_replicator_test.go | 11 + ...orkflow_state_replicator_test_hook_test.go | 61 ++++++ service/history/outbound_queue_factory.go | 2 + service/history/queue_factory_base.go | 2 + .../history/queues/active_standby_executor.go | 63 +++++- .../queues/active_standby_executor_test.go | 4 + .../active_standby_executor_test_hook_test.go | 185 ++++++++++++++++ .../history/replication/ack_manager_test.go | 3 +- .../history/statemachine_environment_test.go | 5 +- .../timer_queue_active_task_executor.go | 2 + .../timer_queue_active_task_executor_test.go | 3 +- service/history/timer_queue_factory.go | 2 + .../timer_queue_standby_task_executor_test.go | 3 +- ...ransfer_queue_active_task_executor_test.go | 2 +- service/history/transfer_queue_factory.go | 4 +- ...ansfer_queue_standby_task_executor_test.go | 11 +- .../visibility_queue_task_executor_test.go | 3 +- service/history/workflow/cache/cache.go | 11 +- service/history/workflow/cache/cache_test.go | 32 +-- service/history/workflow/context.go | 200 ++++++++++++++++-- service/history/workflow/context_test.go | 2 + .../workflow/context_test_hook_test.go | 78 +++++++ 43 files changed, 752 insertions(+), 88 deletions(-) create mode 100644 service/history/ndc/workflow_state_replicator_test_hook_test.go create mode 100644 service/history/queues/active_standby_executor_test_hook_test.go create mode 100644 service/history/workflow/context_test_hook_test.go diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index ce57ce21e6c..dd4d74ce96b 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1053,7 +1053,11 @@ var ( DuplicateReplicationEventsCounter = NewCounterDef("duplicate_replication_events") AcquireLockFailedCounter = NewCounterDef("acquire_lock_failed") WorkflowContextCleared = NewCounterDef("workflow_context_cleared") - MutableStateSize = NewBytesHistogramDef( + HistoryPassiveReplicationTestHookCounter = NewCounterDef( + "history_passive_replication_test_hook", + WithDescription("Number of times the test-only passive replication hook executes. This must be zero in production."), + ) + MutableStateSize = NewBytesHistogramDef( "mutable_state_size", WithDescription("The size of an individual Workflow Execution's state, emitted each time a workflow execution is retrieved or updated."), ) diff --git a/common/testing/testhooks/hooks.go b/common/testing/testhooks/hooks.go index 474f2eb1ecb..72cd5f8cac6 100644 --- a/common/testing/testhooks/hooks.go +++ b/common/testing/testhooks/hooks.go @@ -13,22 +13,35 @@ import ( historytasks "go.temporal.io/server/service/history/tasks" ) +// HistoryPassiveReplicationTestHook defines the test-only seams used to replace an +// active workflow update with a passive replication apply and exercise the resulting +// history tasks through standby execution. InterceptUpdate's payload is owned by +// service/history/workflow and is intentionally opaque here. +type HistoryPassiveReplicationTestHook interface { + InterceptUpdate(context.Context, any, func() error) error + UseTransientWorkflowContextForReplication(context.Context) bool + ShouldExecuteTaskAsPassive(historytasks.Task) bool +} + // Test hook keys with their return type and scope. // Try to avoid global scope as it requires a dedicated test cluster. var ( - MatchingDisableSyncMatch = newKey[bool, namespace.ID]() - MatchingLBForceReadPartition = newKey[int, namespace.ID]() - MatchingLBForceWritePartition = newKey[int, namespace.ID]() - UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() - UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() - TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() - MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() - MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() - MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() - HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() - HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() - HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() - HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() + MatchingDisableSyncMatch = newKey[bool, namespace.ID]() + MatchingLBForceReadPartition = newKey[int, namespace.ID]() + MatchingLBForceWritePartition = newKey[int, namespace.ID]() + UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() + UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() + TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() + MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() + MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() + MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() + HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() + HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() + HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() + HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() + // HistoryPassiveReplicationTest enables the single-cluster passive replication + // test stack for one namespace. Production builds always report it as unset. + HistoryPassiveReplicationTest = newKey[HistoryPassiveReplicationTestHook, namespace.ID]() HistoryDLQTaskDeleteInterceptor = newKey[func(context.Context, *historyservice.DeleteDLQTasksRequest, func(context.Context, *historyservice.DeleteDLQTasksRequest) (*historyservice.DeleteDLQTasksResponse, error)) (*historyservice.DeleteDLQTasksResponse, error), global]() NamespaceReplicationTaskInterceptor = newKey[func(context.Context, *replicationspb.NamespaceTaskAttributes, func() error) error, namespace.Name]() GRPCRequestFaultGeneratorByNamespaceID = newKey[grpcfaults.RequestCallback, namespace.ID]() diff --git a/service/history/api/create_workflow_util.go b/service/history/api/create_workflow_util.go index af62e3f2262..7f6504dae35 100644 --- a/service/history/api/create_workflow_util.go +++ b/service/history/api/create_workflow_util.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/retrypolicy" "go.temporal.io/server/common/rpc/interceptor" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" wcache "go.temporal.io/server/service/history/workflow/cache" @@ -153,6 +154,7 @@ func NewWorkflowLeaseAndContext( shardCtx.GetThrottledLogger(), shardCtx.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ), wcache.NoopReleaseFn, ms, diff --git a/service/history/api/respondworkflowtaskcompleted/api.go b/service/history/api/respondworkflowtaskcompleted/api.go index a569e0ecc26..d0881ba7899 100644 --- a/service/history/api/respondworkflowtaskcompleted/api.go +++ b/service/history/api/respondworkflowtaskcompleted/api.go @@ -34,6 +34,7 @@ import ( "go.temporal.io/server/common/persistence/visibility/manager" "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/recordworkflowtaskstarted" @@ -667,6 +668,7 @@ func (handler *WorkflowTaskCompletedHandler) Invoke( handler.shardContext.GetThrottledLogger(), handler.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ), newMutableState, ) diff --git a/service/history/api/respondworkflowtaskcompleted/api_test.go b/service/history/api/respondworkflowtaskcompleted/api_test.go index 5aa650e091c..d3ff780ff61 100644 --- a/service/history/api/respondworkflowtaskcompleted/api_test.go +++ b/service/history/api/respondworkflowtaskcompleted/api_test.go @@ -38,6 +38,7 @@ import ( "go.temporal.io/server/common/testing/historyrequire" "go.temporal.io/server/common/testing/protorequire" "go.temporal.io/server/common/testing/protoutils" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/common/testing/updateutils" "go.temporal.io/server/service/history/api" @@ -120,7 +121,7 @@ func (s *WorkflowTaskCompletedHandlerSuite) SetupSubTest() { s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() s.logger = s.mockShard.GetLogger() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.workflowTaskCompletedHandler = NewWorkflowTaskCompletedHandler( s.mockShard, tasktoken.NewSerializer(), diff --git a/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go b/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go index 548b145b6d3..ef30eedbcbd 100644 --- a/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go +++ b/service/history/api/verifyfirstworkflowtaskscheduled/api_test.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/workflowresend" "go.temporal.io/server/service/history/events" @@ -98,7 +99,7 @@ func (s *VerifyFirstWorkflowTaskScheduledSuite) SetupTest() { s.workflowConsistencyChecker = api.NewWorkflowConsistencyChecker( s.shardContext, - wcache.NewHostLevelCache(s.shardContext.GetConfig(), s.shardContext.GetLogger(), metrics.NoopMetricsHandler)) + wcache.NewHostLevelCache(s.shardContext.GetConfig(), s.shardContext.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{})) s.mockEventsCache = s.shardContext.MockEventsCache s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() s.logger = s.shardContext.GetLogger() diff --git a/service/history/chasm_engine.go b/service/history/chasm_engine.go index 28e3c59d6dc..3900d6a982a 100644 --- a/service/history/chasm_engine.go +++ b/service/history/chasm_engine.go @@ -24,6 +24,7 @@ import ( "go.temporal.io/server/common/primitives" serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -948,6 +949,7 @@ func (e *ChasmEngine) createNewExecutionWithUpdate( shardContext.GetThrottledLogger(), shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ), mutableState: mutableState, snapshot: snapshot, diff --git a/service/history/chasm_engine_test.go b/service/history/chasm_engine_test.go index 8213aa5d362..55330cd278a 100644 --- a/service/history/chasm_engine_test.go +++ b/service/history/chasm_engine_test.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/persistence/serialization" serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/hsm" @@ -89,6 +90,7 @@ func (s *chasmEngineSuite) SetupTest() { s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, + testhooks.TestHooks{}, ) s.namespaceEntry = tests.GlobalNamespaceEntry diff --git a/service/history/history_engine.go b/service/history/history_engine.go index aab1aa4b67b..019638b02b9 100644 --- a/service/history/history_engine.go +++ b/service/history/history_engine.go @@ -312,6 +312,7 @@ func NewEngineWithShardContext( persistenceRateLimiter, logger, shard.GetEventLogger(), + testHooks, ) historyEngImpl.nDCHSMStateReplicator = ndc.NewHSMStateReplicator( shard, diff --git a/service/history/history_engine2_test.go b/service/history/history_engine2_test.go index 9a7a140f93f..5d2f889ff74 100644 --- a/service/history/history_engine2_test.go +++ b/service/history/history_engine2_test.go @@ -52,6 +52,7 @@ import ( "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/protomock" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testvars" "go.temporal.io/server/common/util" "go.temporal.io/server/common/wideevents" @@ -201,7 +202,7 @@ func (s *engine2Suite) SetupTest() { }, ). AnyTimes() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = log.NewMockLogger(s.controller) s.logger.EXPECT().Debug(gomock.Any(), gomock.Any()).AnyTimes() s.logger.EXPECT().Info(gomock.Any(), gomock.Any()).AnyTimes() diff --git a/service/history/history_engine3_eventsv2_test.go b/service/history/history_engine3_eventsv2_test.go index e360539df45..2e6cff29ded 100644 --- a/service/history/history_engine3_eventsv2_test.go +++ b/service/history/history_engine3_eventsv2_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/common/searchattribute" "go.temporal.io/server/common/searchattribute/sadefs" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/events" @@ -120,7 +121,7 @@ func (s *engine3Suite) SetupTest() { s.mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(false, common.EmptyVersion).Return(cluster.TestCurrentClusterName).AnyTimes() s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.mockVisibilityManager.EXPECT().GetIndexName().Return("").AnyTimes() s.mockVisibilityManager.EXPECT(). ValidateCustomSearchAttributes(gomock.Any()). diff --git a/service/history/history_engine_test.go b/service/history/history_engine_test.go index c8b2ff25e0e..7550fdcfad9 100644 --- a/service/history/history_engine_test.go +++ b/service/history/history_engine_test.go @@ -59,6 +59,7 @@ import ( serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/api" "go.temporal.io/server/service/history/api/getworkflowexecutionrawhistoryv2" "go.temporal.io/server/service/history/api/workflowresend" @@ -184,7 +185,7 @@ func (s *engineSuite) SetupTest() { }, s.config, ) - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.eventsCache = events.NewHostLevelEventsCache( s.mockShard.GetExecutionManager(), diff --git a/service/history/ndc/activity_state_replicator_test.go b/service/history/ndc/activity_state_replicator_test.go index 3644c25a92e..32c36956a06 100644 --- a/service/history/ndc/activity_state_replicator_test.go +++ b/service/history/ndc/activity_state_replicator_test.go @@ -25,6 +25,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives/timestamp" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" @@ -81,7 +82,7 @@ func (s *activityReplicatorStateSuite) SetupTest() { tests.NewDynamicConfig(), ) - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.mockNamespaceCache = s.mockShard.Resource.NamespaceCache s.mockExecutionMgr = s.mockShard.Resource.ExecutionMgr diff --git a/service/history/ndc/history_replicator.go b/service/history/ndc/history_replicator.go index f84e09c3085..83830c81d84 100644 --- a/service/history/ndc/history_replicator.go +++ b/service/history/ndc/history_replicator.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/common/primitives/timestamp" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" @@ -600,6 +601,7 @@ func (r *HistoryReplicatorImpl) applyNonStartEventsToCurrentBranch( r.shardContext.GetThrottledLogger(), r.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ) newWorkflow = NewWorkflow( diff --git a/service/history/ndc/hsm_state_replicator_test.go b/service/history/ndc/hsm_state_replicator_test.go index 696f1ba447c..604b7f9bbab 100644 --- a/service/history/ndc/hsm_state_replicator_test.go +++ b/service/history/ndc/hsm_state_replicator_test.go @@ -21,6 +21,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" "go.temporal.io/server/service/history/hsm/hsmtest" @@ -90,7 +91,7 @@ func (s *hsmStateReplicatorSuite) SetupTest() { err = stateMachineRegistry.RegisterTaskSerializer(hsmtest.TaskType, hsmtest.TaskSerializer{}) s.NoError(err) - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.namespaceEntry = tests.GlobalNamespaceEntry s.workflowKey = definition.NewWorkflowKey(s.namespaceEntry.ID().String(), tests.WorkflowID, tests.RunID) diff --git a/service/history/ndc/mutable_state_initializer.go b/service/history/ndc/mutable_state_initializer.go index 56bbf66b7c9..719acc75abd 100644 --- a/service/history/ndc/mutable_state_initializer.go +++ b/service/history/ndc/mutable_state_initializer.go @@ -17,6 +17,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/serialization" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" wcache "go.temporal.io/server/service/history/workflow/cache" @@ -162,6 +163,7 @@ func (r *MutableStateInitializerImpl) InitializeFromToken( r.shardContext.GetThrottledLogger(), r.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ) mutableStateRow, dbRecordVersion, dbHistorySize, existsInDB, err := r.deserializeBackfillToken(token) if err != nil { diff --git a/service/history/ndc/resetter_test.go b/service/history/ndc/resetter_test.go index 69d397b3be3..a0c3de4dcc1 100644 --- a/service/history/ndc/resetter_test.go +++ b/service/history/ndc/resetter_test.go @@ -18,6 +18,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" @@ -94,6 +95,7 @@ func (s *resetterSuite) SetupTest() { s.mockShard.GetThrottledLogger(), s.mockShard.GetMetricsHandler(), nil, + testhooks.TestHooks{}, ) s.newRunID = uuid.NewString() diff --git a/service/history/ndc/transaction_manager_test.go b/service/history/ndc/transaction_manager_test.go index acbdb5c68dc..f47f3cdb1bf 100644 --- a/service/history/ndc/transaction_manager_test.go +++ b/service/history/ndc/transaction_manager_test.go @@ -22,6 +22,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" @@ -83,7 +84,7 @@ func (s *transactionMgrSuite) SetupTest() { s.transactionMgr = NewTransactionManager( s.mockShard, - wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler), + wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}), s.mockEventsReapplier, s.logger, false, diff --git a/service/history/ndc/workflow_resetter.go b/service/history/ndc/workflow_resetter.go index 8266d05288b..1b95952a46b 100644 --- a/service/history/ndc/workflow_resetter.go +++ b/service/history/ndc/workflow_resetter.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/api/updateworkflowoptions" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" @@ -538,6 +539,7 @@ func (r *workflowResetterImpl) replayResetWorkflow( r.shardContext.GetLogger(), r.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ) resetMutableState, resetStats, err := r.stateRebuilder.Rebuild( diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index 126a95058cb..04c786bce2e 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -37,6 +37,7 @@ import ( "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testlogger" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/hsm" @@ -110,7 +111,7 @@ func (s *workflowResetterSuite) SetupTest() { s.workflowResetter = NewWorkflowResetter( s.mockShard, - wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler), + wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}), s.logger, ) s.workflowResetter.stateRebuilder = s.mockStateRebuilder diff --git a/service/history/ndc/workflow_state_replicator.go b/service/history/ndc/workflow_state_replicator.go index 78269aa4aec..df266bbee39 100644 --- a/service/history/ndc/workflow_state_replicator.go +++ b/service/history/ndc/workflow_state_replicator.go @@ -8,6 +8,7 @@ import ( "fmt" "slices" "sort" + "sync" "time" "github.com/google/uuid" @@ -42,6 +43,7 @@ import ( "go.temporal.io/server/common/quotas" serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" @@ -78,6 +80,7 @@ type ( logger log.Logger eventLogger otellog.Logger taskRefresher workflow.TaskRefresher + testHooks testhooks.TestHooks } ) @@ -89,6 +92,7 @@ func NewWorkflowStateReplicator( persistenceRateLimiter quotas.RequestRateLimiter, logger log.Logger, eventLogger otellog.Logger, + testHooks testhooks.TestHooks, ) *WorkflowStateReplicatorImpl { logger = log.With(logger, tag.ComponentWorkflowStateReplicator) @@ -105,6 +109,7 @@ func NewWorkflowStateReplicator( logger: logger, eventLogger: eventLogger, taskRefresher: workflow.NewTaskRefresher(shardContext), + testHooks: testHooks, } } @@ -228,6 +233,53 @@ func (r *WorkflowStateReplicatorImpl) SyncWorkflowState( ) } +func (r *WorkflowStateReplicatorImpl) getWorkflowContext( + ctx context.Context, + namespaceID namespace.ID, + execution *commonpb.WorkflowExecution, + archetypeID chasm.ArchetypeID, +) (historyi.WorkflowContext, historyi.ReleaseWorkflowContextFunc, error) { + hook, ok := testhooks.Get( + r.testHooks, + testhooks.HistoryPassiveReplicationTest, + namespaceID, + ) + if ok && hook.UseTransientWorkflowContextForReplication(ctx) { + metrics.HistoryPassiveReplicationTestHookCounter.With(r.shardContext.GetMetricsHandler()).Record( + 1, + metrics.OperationTag("ReplicateVersionedTransition"), + ) + wfCtx := workflow.NewContext( + r.shardContext.GetConfig(), + definition.NewWorkflowKey(namespaceID.String(), execution.GetWorkflowId(), execution.GetRunId()), + archetypeID, + r.logger, + r.shardContext.GetThrottledLogger(), + r.shardContext.GetMetricsHandler(), + nil, + r.testHooks, + ) + if err := wfCtx.Lock(ctx, locks.PriorityHigh); err != nil { + return nil, nil, err + } + var releaseOnce sync.Once + return wfCtx, func(error) { + releaseOnce.Do(func() { + wfCtx.Clear() + wfCtx.Unlock() + }) + }, nil + } + return r.workflowCache.GetOrCreateChasmExecution( + ctx, + r.shardContext, + namespaceID, + execution, + archetypeID, + locks.PriorityHigh, + ) +} + //nolint:revive // cognitive complexity 37 (> max enabled 25) func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( ctx context.Context, @@ -268,16 +320,11 @@ func (r *WorkflowStateReplicatorImpl) ReplicateVersionedTransition( } }() - wfCtx, releaseFn, err := r.workflowCache.GetOrCreateChasmExecution( + wfCtx, releaseFn, err := r.getWorkflowContext( ctx, - r.shardContext, namespaceID, - &commonpb.WorkflowExecution{ - WorkflowId: wid, - RunId: rid, - }, + &commonpb.WorkflowExecution{WorkflowId: wid, RunId: rid}, archetypeID, - locks.PriorityHigh, ) if err != nil { return err @@ -1253,6 +1300,7 @@ func (r *WorkflowStateReplicatorImpl) getNewRunWorkflow( r.shardContext.GetThrottledLogger(), r.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ) return NewWorkflow( diff --git a/service/history/ndc/workflow_state_replicator_test.go b/service/history/ndc/workflow_state_replicator_test.go index 5b1e29796d9..89f128c161e 100644 --- a/service/history/ndc/workflow_state_replicator_test.go +++ b/service/history/ndc/workflow_state_replicator_test.go @@ -35,6 +35,7 @@ import ( "go.temporal.io/server/common/primitives/timestamp" "go.temporal.io/server/common/quotas" serviceerrors "go.temporal.io/server/common/serviceerror" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" @@ -132,6 +133,7 @@ func (s *workflowReplicatorSuite) SetupTest() { quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) } @@ -723,6 +725,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_SameBranch_S quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -816,6 +819,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_DifferentBra quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -903,6 +907,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_SameBranch_S quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -1001,6 +1006,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_FirstTask_Ca quotas.NoopRequestRateLimiter, s.logger, eventCapture, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -1088,6 +1094,7 @@ func (s *workflowReplicatorSuite) Test_ReplicateVersionedTransition_MutationProv quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -1720,6 +1727,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_WithSnapshot_S quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -1800,6 +1808,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_WithMutation_S quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) @@ -1875,6 +1884,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_InvalidArtifac quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) versionedTransitionArtifact := &replicationspb.VersionedTransitionArtifact{} @@ -1902,6 +1912,7 @@ func (s *workflowReplicatorSuite) Test_handleFirstReplicationTask_CreateWorkflow quotas.NoopRequestRateLimiter, s.logger, nil, + testhooks.NewTestHooks(), ) mockTransactionManager := NewMockTransactionManager(s.controller) mockTaskRefresher := workflow.NewMockTaskRefresher(s.controller) diff --git a/service/history/ndc/workflow_state_replicator_test_hook_test.go b/service/history/ndc/workflow_state_replicator_test_hook_test.go new file mode 100644 index 00000000000..2defde240d6 --- /dev/null +++ b/service/history/ndc/workflow_state_replicator_test_hook_test.go @@ -0,0 +1,61 @@ +//go:build test_dep + +package ndc + +import ( + "context" + + commonpb "go.temporal.io/api/common/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testhooks" + historytasks "go.temporal.io/server/service/history/tasks" +) + +type workflowStateReplicatorPassiveReplicationTestHook struct{} + +func (workflowStateReplicatorPassiveReplicationTestHook) InterceptUpdate( + _ context.Context, + _ any, + next func() error, +) error { + return next() +} + +func (workflowStateReplicatorPassiveReplicationTestHook) UseTransientWorkflowContextForReplication(context.Context) bool { + return true +} + +func (workflowStateReplicatorPassiveReplicationTestHook) ShouldExecuteTaskAsPassive(historytasks.Task) bool { + return false +} + +func (s *workflowReplicatorSuite) Test_GetWorkflowContext_PassiveReplicationHookRecordsMetric() { + namespaceID := namespace.ID("namespace-id") + testHooks := s.workflowStateReplicator.testHooks + s.T().Cleanup(testhooks.Set[testhooks.HistoryPassiveReplicationTestHook]( + testHooks, + testhooks.HistoryPassiveReplicationTest, + workflowStateReplicatorPassiveReplicationTestHook{}, + namespaceID, + )) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + s.T().Cleanup(func() { metricsHandler.StopCapture(capture) }) + s.mockShard.SetMetricsHandler(metricsHandler) + + wfCtx, releaseFn, err := s.workflowStateReplicator.getWorkflowContext( + context.Background(), + namespaceID, + &commonpb.WorkflowExecution{WorkflowId: s.workflowID, RunId: s.runID}, + chasm.WorkflowArchetypeID, + ) + s.Require().NoError(err) + s.Require().NotNil(wfCtx) + releaseFn(nil) + recordings := capture.Snapshot()[metrics.HistoryPassiveReplicationTestHookCounter.Name()] + s.Require().Len(recordings, 1) + s.Require().Equal("ReplicateVersionedTransition", recordings[0].Tags[metrics.OperationTagName]) +} diff --git a/service/history/outbound_queue_factory.go b/service/history/outbound_queue_factory.go index 66e96785b7f..a45d0d33096 100644 --- a/service/history/outbound_queue_factory.go +++ b/service/history/outbound_queue_factory.go @@ -251,6 +251,8 @@ func (f *outboundQueueFactory) CreateQueue( activeExecutor, standbyExecutor, logger, + metricsHandler, + f.TestHooks, ) if f.ExecutorWrapper != nil { diff --git a/service/history/queue_factory_base.go b/service/history/queue_factory_base.go index c58cec8165a..d1f8532df21 100644 --- a/service/history/queue_factory_base.go +++ b/service/history/queue_factory_base.go @@ -17,6 +17,7 @@ import ( "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/quotas" "go.temporal.io/server/common/quotas/calculator" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/circuitbreakerpool" "go.temporal.io/server/service/history/configs" historyi "go.temporal.io/server/service/history/interfaces" @@ -57,6 +58,7 @@ type ( RemoteHistoryFetcher eventhandler.HistoryPaginatedFetcher ChasmEngine chasm.Engine ChasmRegistry *chasm.Registry + TestHooks testhooks.TestHooks } QueueFactoryBase struct { diff --git a/service/history/queues/active_standby_executor.go b/service/history/queues/active_standby_executor.go index 453ba40ba6e..bdb44ad3716 100644 --- a/service/history/queues/active_standby_executor.go +++ b/service/history/queues/active_standby_executor.go @@ -2,11 +2,15 @@ package queues import ( "context" + "errors" "go.temporal.io/server/common/headers" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/service/history/consts" ) type ( @@ -16,6 +20,8 @@ type ( activeExecutor Executor standbyExecutor Executor logger log.Logger + metricsHandler metrics.Handler + testHooks testhooks.TestHooks } ) @@ -25,6 +31,8 @@ func NewActiveStandbyExecutor( activeExecutor Executor, standbyExecutor Executor, logger log.Logger, + metricsHandler metrics.Handler, + testHooks testhooks.TestHooks, ) Executor { return &activeStandbyExecutor{ currentClusterName: currentClusterName, @@ -32,6 +40,8 @@ func NewActiveStandbyExecutor( activeExecutor: activeExecutor, standbyExecutor: standbyExecutor, logger: logger, + metricsHandler: metricsHandler, + testHooks: testHooks, } } @@ -39,10 +49,51 @@ func (e *activeStandbyExecutor) Execute( ctx context.Context, executable Executable, ) ExecuteResponse { - if e.isActiveTask(executable) { + namespaceID := namespace.ID(executable.GetNamespaceID()) + if testHook, ok := testhooks.Get( + e.testHooks, + testhooks.HistoryPassiveReplicationTest, + namespaceID, + ); ok && testHook.ShouldExecuteTaskAsPassive(executable.GetTask()) { + metrics.HistoryPassiveReplicationTestHookCounter.With(e.metricsHandler).Record( + 1, + metrics.OperationTag("ActiveStandbyExecutor"), + ) + return e.executeForPassiveReplicationTest(ctx, executable) + } + return e.executeNormally(ctx, executable, namespaceID) +} + +func (e *activeStandbyExecutor) executeNormally( + ctx context.Context, + executable Executable, + namespaceID namespace.ID, +) ExecuteResponse { + if e.isActiveTask(executable, namespaceID) { return e.activeExecutor.Execute(ctx, executable) } + return e.executeStandby(ctx, executable) +} +func (e *activeStandbyExecutor) executeForPassiveReplicationTest( + ctx context.Context, + executable Executable, +) ExecuteResponse { + response := e.executeStandby(ctx, executable) + if !errors.Is(response.ExecutionErr, consts.ErrTaskRetry) { + return response + } + response = e.activeExecutor.Execute(ctx, executable) + if response.ExecutionErr != nil { + return response + } + return e.executeStandby(ctx, executable) +} + +func (e *activeStandbyExecutor) executeStandby( + ctx context.Context, + executable Executable, +) ExecuteResponse { // for standby tasks, use preemptable callerType to avoid impacting active traffic return e.standbyExecutor.Execute( headers.SetCallerType(ctx, headers.CallerTypePreemptable), @@ -52,21 +103,21 @@ func (e *activeStandbyExecutor) Execute( func (e *activeStandbyExecutor) isActiveTask( executable Executable, + namespaceID namespace.ID, ) bool { // Following is the existing task allocator logic for verifying active task - namespaceID := executable.GetNamespaceID() - entry, err := e.registry.GetNamespaceByID(namespace.ID(namespaceID)) + entry, err := e.registry.GetNamespaceByID(namespaceID) if err != nil { - e.logger.Warn("Unable to find namespace, process task as active.", tag.WorkflowNamespaceID(namespaceID), tag.Value(executable.GetTask())) + e.logger.Warn("Unable to find namespace, process task as active.", tag.WorkflowNamespaceID(namespaceID.String()), tag.Value(executable.GetTask())) return true } if entry.ActiveClusterName(namespace.RoutingKey{ID: executable.GetWorkflowID()}) != e.currentClusterName { - e.logger.Debug("Process task as standby.", tag.WorkflowNamespaceID(namespaceID), tag.Value(executable.GetTask())) + e.logger.Debug("Process task as standby.", tag.WorkflowNamespaceID(namespaceID.String()), tag.Value(executable.GetTask())) return false } - e.logger.Debug("Process task as active.", tag.WorkflowNamespaceID(namespaceID), tag.Value(executable.GetTask())) + e.logger.Debug("Process task as active.", tag.WorkflowNamespaceID(namespaceID.String()), tag.Value(executable.GetTask())) return true } diff --git a/service/history/queues/active_standby_executor_test.go b/service/history/queues/active_standby_executor_test.go index a4caa2d1416..de499c0b7a7 100644 --- a/service/history/queues/active_standby_executor_test.go +++ b/service/history/queues/active_standby_executor_test.go @@ -8,7 +8,9 @@ import ( "github.com/stretchr/testify/suite" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testhooks" "go.uber.org/mock/gomock" ) @@ -48,6 +50,8 @@ func (s *executorSuite) SetupTest() { s.activeExecutor, s.standbyExecutor, log.NewNoopLogger(), + metrics.NoopMetricsHandler, + testhooks.TestHooks{}, ) } diff --git a/service/history/queues/active_standby_executor_test_hook_test.go b/service/history/queues/active_standby_executor_test_hook_test.go new file mode 100644 index 00000000000..9dcd5699db7 --- /dev/null +++ b/service/history/queues/active_standby_executor_test_hook_test.go @@ -0,0 +1,185 @@ +//go:build test_dep + +package queues + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/service/history/consts" + historytasks "go.temporal.io/server/service/history/tasks" + "go.uber.org/mock/gomock" +) + +type activeStandbyPassiveReplicationTestHook struct{} + +func (activeStandbyPassiveReplicationTestHook) InterceptUpdate( + _ context.Context, + _ any, + next func() error, +) error { + return next() +} + +func (activeStandbyPassiveReplicationTestHook) UseTransientWorkflowContextForReplication(context.Context) bool { + return false +} + +func (activeStandbyPassiveReplicationTestHook) ShouldExecuteTaskAsPassive(historytasks.Task) bool { + return true +} + +func TestActiveStandbyExecutor_ForcedStandbyRetryExecutesActiveThenStandby(t *testing.T) { + ctrl := gomock.NewController(t) + registry := namespace.NewMockRegistry(ctrl) + activeExecutor := NewMockExecutor(ctrl) + standbyExecutor := NewMockExecutor(ctrl) + testHooks := testhooks.NewTestHooks() + t.Cleanup(testhooks.Set[testhooks.HistoryPassiveReplicationTestHook]( + testHooks, + testhooks.HistoryPassiveReplicationTest, + activeStandbyPassiveReplicationTestHook{}, + namespace.ID("namespace_id"), + )) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + t.Cleanup(func() { metricsHandler.StopCapture(capture) }) + executor := NewActiveStandbyExecutor( + currentCluster, + registry, + activeExecutor, + standbyExecutor, + log.NewNoopLogger(), + metricsHandler, + testHooks, + ) + + task := historytasks.NewFakeTask( + definition.NewWorkflowKey("namespace_id", "workflow_id", "run_id"), + historytasks.CategoryTransfer, + time.Time{}, + ) + executable := NewMockExecutable(ctrl) + executable.EXPECT().GetNamespaceID().Return("namespace_id") + executable.EXPECT().GetTask().Return(task) + gomock.InOrder( + standbyExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: false, + ExecutionErr: consts.ErrTaskRetry, + }), + activeExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: true, + }), + standbyExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: false, + }), + ) + + response := executor.Execute(context.Background(), executable) + require.False(t, response.ExecutedAsActive) + require.NoError(t, response.ExecutionErr) + recordings := capture.Snapshot()[metrics.HistoryPassiveReplicationTestHookCounter.Name()] + require.Len(t, recordings, 1) + require.Equal(t, "ActiveStandbyExecutor", recordings[0].Tags[metrics.OperationTagName]) +} + +func TestActiveStandbyExecutor_ActiveFailureSkipsSecondStandby(t *testing.T) { + ctrl := gomock.NewController(t) + registry := namespace.NewMockRegistry(ctrl) + activeExecutor := NewMockExecutor(ctrl) + standbyExecutor := NewMockExecutor(ctrl) + testHooks := testhooks.NewTestHooks() + t.Cleanup(testhooks.Set[testhooks.HistoryPassiveReplicationTestHook]( + testHooks, + testhooks.HistoryPassiveReplicationTest, + activeStandbyPassiveReplicationTestHook{}, + namespace.ID("namespace_id"), + )) + executor := NewActiveStandbyExecutor( + currentCluster, + registry, + activeExecutor, + standbyExecutor, + log.NewNoopLogger(), + metrics.NoopMetricsHandler, + testHooks, + ) + + task := historytasks.NewFakeTask( + definition.NewWorkflowKey("namespace_id", "workflow_id", "run_id"), + historytasks.CategoryTransfer, + time.Time{}, + ) + executable := NewMockExecutable(ctrl) + executable.EXPECT().GetNamespaceID().Return("namespace_id") + executable.EXPECT().GetTask().Return(task) + activeErr := errors.New("active execution failed") + gomock.InOrder( + standbyExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: false, + ExecutionErr: consts.ErrTaskRetry, + }), + activeExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: true, + ExecutionErr: activeErr, + }), + ) + + response := executor.Execute(context.Background(), executable) + require.True(t, response.ExecutedAsActive) + require.ErrorIs(t, response.ExecutionErr, activeErr) +} + +func TestActiveStandbyExecutor_ForcedStandbySuccessIsReturned(t *testing.T) { + ctrl := gomock.NewController(t) + registry := namespace.NewMockRegistry(ctrl) + activeExecutor := NewMockExecutor(ctrl) + standbyExecutor := NewMockExecutor(ctrl) + testHooks := testhooks.NewTestHooks() + t.Cleanup(testhooks.Set[testhooks.HistoryPassiveReplicationTestHook]( + testHooks, + testhooks.HistoryPassiveReplicationTest, + activeStandbyPassiveReplicationTestHook{}, + namespace.ID("namespace_id"), + )) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + t.Cleanup(func() { metricsHandler.StopCapture(capture) }) + executor := NewActiveStandbyExecutor( + currentCluster, + registry, + activeExecutor, + standbyExecutor, + log.NewNoopLogger(), + metricsHandler, + testHooks, + ) + + task := historytasks.NewFakeTask( + definition.NewWorkflowKey("namespace_id", "workflow_id", "run_id"), + historytasks.CategoryTransfer, + time.Time{}, + ) + executable := NewMockExecutable(ctrl) + executable.EXPECT().GetNamespaceID().Return("namespace_id") + executable.EXPECT().GetTask().Return(task) + standbyExecutor.EXPECT().Execute(gomock.Any(), executable).Return(ExecuteResponse{ + ExecutedAsActive: false, + }) + + response := executor.Execute(context.Background(), executable) + require.False(t, response.ExecutedAsActive) + require.NoError(t, response.ExecutionErr) + recordings := capture.Snapshot()[metrics.HistoryPassiveReplicationTestHookCounter.Name()] + require.Len(t, recordings, 1) + require.Equal(t, "ActiveStandbyExecutor", recordings[0].Tags[metrics.OperationTagName]) +} diff --git a/service/history/replication/ack_manager_test.go b/service/history/replication/ack_manager_test.go index 8bd511dd3d8..584fb09ecdf 100644 --- a/service/history/replication/ack_manager_test.go +++ b/service/history/replication/ack_manager_test.go @@ -20,6 +20,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" historyi "go.temporal.io/server/service/history/interfaces" @@ -97,7 +98,7 @@ func (s *ackManagerSuite) SetupTest() { s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, gomock.Any()).Return(cluster.TestCurrentClusterName).AnyTimes() s.logger = s.mockShard.GetLogger() - workflowCache := wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + workflowCache := wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) replicationProgressCache := NewProgressCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) s.syncStateRetriever = NewMockSyncStateRetriever(s.controller) s.replicationAckManager = NewAckManager( diff --git a/service/history/statemachine_environment_test.go b/service/history/statemachine_environment_test.go index af66194022f..0bb419f800b 100644 --- a/service/history/statemachine_environment_test.go +++ b/service/history/statemachine_environment_test.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/tasktoken" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" "go.temporal.io/server/service/history/hsm" @@ -87,7 +88,7 @@ func newStateMachineEnvTestContext(t *testing.T, enableTransitionHistory bool) * require.NoError(t, callbacks.RegisterStateMachine(reg)) require.NoError(t, callbacks.RegisterTaskSerializers(reg)) s.mockShard.SetStateMachineRegistry(reg) - s.workflowCache = cache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), s.mockShard.GetMetricsHandler()) + s.workflowCache = cache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), s.mockShard.GetMetricsHandler(), testhooks.TestHooks{}) mockClusterMetadata := s.mockShard.Resource.ClusterMetadata mockClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() @@ -277,7 +278,7 @@ func TestValidateStateMachineRef(t *testing.T) { tc.mutateNode(node) tc.mutateRef(&ref) - workflowContext := workflow.NewContext(s.mockShard.GetConfig(), mutableState.GetWorkflowKey(), chasm.WorkflowArchetypeID, log.NewTestLogger(), log.NewTestLogger(), metrics.NoopMetricsHandler, nil) + workflowContext := workflow.NewContext(s.mockShard.GetConfig(), mutableState.GetWorkflowKey(), chasm.WorkflowArchetypeID, log.NewTestLogger(), log.NewTestLogger(), metrics.NoopMetricsHandler, nil, testhooks.TestHooks{}) if tc.clearTransitionHistory { mutableState.GetExecutionInfo().TransitionHistory = nil } diff --git a/service/history/timer_queue_active_task_executor.go b/service/history/timer_queue_active_task_executor.go index e6fad3009bc..a4ca37bae1c 100644 --- a/service/history/timer_queue_active_task_executor.go +++ b/service/history/timer_queue_active_task_executor.go @@ -27,6 +27,7 @@ import ( "go.temporal.io/server/common/priorities" "go.temporal.io/server/common/resource" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/deletemanager" @@ -801,6 +802,7 @@ func (t *timerQueueActiveTaskExecutor) executeWorkflowRunTimeoutTask( t.shardContext.GetThrottledLogger(), t.shardContext.GetMetricsHandler(), nil, // no pagination buffer limiter as it is a transient context + testhooks.TestHooks{}, ), newMutableState, ) diff --git a/service/history/timer_queue_active_task_executor_test.go b/service/history/timer_queue_active_task_executor_test.go index 88cc7e2bfbc..e7ff8df5966 100644 --- a/service/history/timer_queue_active_task_executor_test.go +++ b/service/history/timer_queue_active_task_executor_test.go @@ -36,6 +36,7 @@ import ( "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/telemetry" pm "go.temporal.io/server/common/testing/protomock" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -158,7 +159,7 @@ func (s *timerQueueActiveTaskExecutorSuite) SetupTest() { s.mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() s.mockClusterMetadata.EXPECT().IsGlobalNamespaceEnabled().Return(true).AnyTimes() s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(s.namespaceEntry.IsGlobalNamespace(), s.version).Return(s.mockClusterMetadata.GetCurrentClusterName()).AnyTimes() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = s.mockShard.GetLogger() s.mockDeleteManager = deletemanager.NewMockDeleteManager(s.controller) diff --git a/service/history/timer_queue_factory.go b/service/history/timer_queue_factory.go index 64c710e8913..cadb038f972 100644 --- a/service/history/timer_queue_factory.go +++ b/service/history/timer_queue_factory.go @@ -150,6 +150,8 @@ func (f *timerQueueFactory) CreateQueue( activeExecutor, standbyExecutor, logger, + metricsHandler, + f.TestHooks, ) if f.ExecutorWrapper != nil { executor = f.ExecutorWrapper.Wrap(executor) diff --git a/service/history/timer_queue_standby_task_executor_test.go b/service/history/timer_queue_standby_task_executor_test.go index 844cbd4eb1d..f5ecf41b897 100644 --- a/service/history/timer_queue_standby_task_executor_test.go +++ b/service/history/timer_queue_standby_task_executor_test.go @@ -36,6 +36,7 @@ import ( "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/protorequire" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" @@ -168,7 +169,7 @@ func (s *timerQueueStandbyTaskExecutorSuite) SetupTest() { s.mockClusterMetadata.EXPECT().GetAllClusterInfo().Return(cluster.TestAllClusterInfo).AnyTimes() s.mockClusterMetadata.EXPECT().IsGlobalNamespaceEnabled().Return(true).AnyTimes() s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(s.namespaceEntry.IsGlobalNamespace(), s.version).Return(s.clusterName).AnyTimes() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = s.mockShard.GetLogger() s.mockDeleteManager = deletemanager.NewMockDeleteManager(s.controller) diff --git a/service/history/transfer_queue_active_task_executor_test.go b/service/history/transfer_queue_active_task_executor_test.go index 7f8ef23f177..7d5c11e95f9 100644 --- a/service/history/transfer_queue_active_task_executor_test.go +++ b/service/history/transfer_queue_active_task_executor_test.go @@ -201,7 +201,7 @@ func (s *transferQueueActiveTaskExecutorSuite) SetupTest() { s.mockArchivalMetadata.SetVisibilityEnabledByDefault() s.mockChasmEngine = chasm.NewMockEngine(s.controller) - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = s.mockShard.GetLogger() h := &historyEngineImpl{ diff --git a/service/history/transfer_queue_factory.go b/service/history/transfer_queue_factory.go index a7f97477db2..f17414cb588 100644 --- a/service/history/transfer_queue_factory.go +++ b/service/history/transfer_queue_factory.go @@ -10,7 +10,6 @@ import ( "go.temporal.io/server/common/sdk" ctasks "go.temporal.io/server/common/tasks" "go.temporal.io/server/common/telemetry" - "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/queues" @@ -34,7 +33,6 @@ type ( MatchingRawClient resource.MatchingRawClient VisibilityManager manager.VisibilityManager VersionMembershipCache worker_versioning.VersionMembershipAndReactivationStatusCache - TestHooks testhooks.TestHooks } transferQueueFactory struct { @@ -150,6 +148,8 @@ func (f *transferQueueFactory) CreateQueue( activeExecutor, standbyExecutor, logger, + metricsHandler, + f.TestHooks, ) if f.ExecutorWrapper != nil { executor = f.ExecutorWrapper.Wrap(executor) diff --git a/service/history/transfer_queue_standby_task_executor_test.go b/service/history/transfer_queue_standby_task_executor_test.go index 457e1d4c4fc..07a4c9fb9b8 100644 --- a/service/history/transfer_queue_standby_task_executor_test.go +++ b/service/history/transfer_queue_standby_task_executor_test.go @@ -43,6 +43,7 @@ import ( "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/mockapi/workflowservicemock/v1" "go.temporal.io/server/common/testing/protomock" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/util" "go.temporal.io/server/common/wideevents" "go.temporal.io/server/service/history/consts" @@ -188,7 +189,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) SetupTest() { s.mockClusterMetadata.EXPECT().IsGlobalNamespaceEnabled().Return(true).AnyTimes() s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(s.namespaceEntry.IsGlobalNamespace(), s.version).Return(s.clusterName).AnyTimes() s.clientBean = client.NewMockBean(s.controller) - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = s.mockShard.GetLogger() s.mockFrontendClient = s.mockShard.Resource.FrontendClient s.mockChasmEngine = chasm.NewMockEngine(s.controller) @@ -947,7 +948,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessCloseExecution() { s.Equal(wideevents.ParentChildOutcomeVerified, ignoredAttributes["outcome"].AsString()) persistenceMutableState.ExecutionInfo.TransitionHistory = nil - s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) s.mockHistoryClient.EXPECT().VerifyChildExecutionCompletionRecorded(gomock.Any(), expectedVerificationWithResendParentRequest).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) @@ -1283,7 +1284,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P mutableState.FlushBufferedEvents() // clear the cache - s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) persistenceMutableState = s.createPersistenceMutableState(mutableState, event.GetEventId(), event.GetVersion()) s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) @@ -1370,7 +1371,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_P s.NoError(resp.ExecutionErr) persistenceMutableState.ExecutionInfo.TransitionHistory = nil - s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), expectedVerificationWithResendChildRequest).Return(nil, nil) resp = s.transferQueueStandbyTaskExecutor.Execute(context.Background(), s.newTaskExecutable(transferTask)) @@ -1456,7 +1457,7 @@ func (s *transferQueueStandbyTaskExecutorSuite) TestProcessStartChildExecution_S ) s.NoError(err) - s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.transferQueueStandbyTaskExecutor.cache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) persistenceMutableState = s.createPersistenceMutableState(mutableState, event.GetEventId(), event.GetVersion()) s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) s.mockHistoryClient.EXPECT().VerifyFirstWorkflowTaskScheduled(gomock.Any(), gomock.Any()).Return(nil, nil) diff --git a/service/history/visibility_queue_task_executor_test.go b/service/history/visibility_queue_task_executor_test.go index e26ebdcd898..4ebedb6131b 100644 --- a/service/history/visibility_queue_task_executor_test.go +++ b/service/history/visibility_queue_task_executor_test.go @@ -38,6 +38,7 @@ import ( "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/telemetry" "go.temporal.io/server/common/testing/protomock" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/events" @@ -161,7 +162,7 @@ func (s *visibilityQueueTaskExecutorSuite) SetupTest() { mockClusterMetadata.EXPECT().IsGlobalNamespaceEnabled().Return(true).AnyTimes() mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(true, s.version).Return(mockClusterMetadata.GetCurrentClusterName()).AnyTimes() - s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.workflowCache = wcache.NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) s.logger = s.mockShard.GetLogger() h := &historyEngineImpl{ diff --git a/service/history/workflow/cache/cache.go b/service/history/workflow/cache/cache.go index 8688a766bb1..7a7d4c146eb 100644 --- a/service/history/workflow/cache/cache.go +++ b/service/history/workflow/cache/cache.go @@ -23,6 +23,7 @@ import ( "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -66,7 +67,9 @@ type ( nonUserContextLockTimeout time.Duration // paginationLimiter limits pagination buffer size across all workflow cache contexts paginationLimiter *limiter.KeyedBytesLimiter + testHooks testhooks.TestHooks } + cacheItem struct { shardId int32 wfContext historyi.WorkflowContext @@ -97,6 +100,7 @@ func NewHostLevelCache( config *configs.Config, logger log.Logger, handler metrics.Handler, + testHooks testhooks.TestHooks, ) Cache { maxSize := config.HistoryHostLevelCacheMaxSize() if config.HistoryCacheLimitSizeBased { @@ -149,11 +153,13 @@ func NewHostLevelCache( taggedHandler := handler.WithTags(metrics.CacheTypeTag(metrics.MutableStateCacheTypeTagValue)) c := cache.NewWithMetrics(maxSize, opts, taggedHandler) - return &cacheImpl{ + impl := &cacheImpl{ Cache: c, nonUserContextLockTimeout: config.HistoryCacheNonUserContextLockTimeout(), paginationLimiter: limiter.NewKeyedBytesLimiter(), + testHooks: testHooks, } + return impl } func (c *cacheImpl) stop() { @@ -298,9 +304,9 @@ func (c *cacheImpl) getOrCreateWorkflowExecutionInternal( shardContext.GetThrottledLogger(), shardContext.GetMetricsHandler(), c.paginationLimiter, + c.testHooks, ) - var err error value := &cacheItem{shardId: shardContext.GetShardID(), wfContext: workflowCtx, finalizer: shardContext.GetFinalizer()} existing, err := c.PutIfNotExist(cacheKey, value) if err != nil { @@ -320,7 +326,6 @@ func (c *cacheImpl) getOrCreateWorkflowExecutionInternal( // TODO This will create a closure on every request. // Consider revisiting this if it causes too much GC activity releaseFunc := c.makeReleaseFunc(cacheKey, shardContext, workflowCtx, forceClearContext, handler, time.Now()) - return workflowCtx, releaseFunc, nil } diff --git a/service/history/workflow/cache/cache_test.go b/service/history/workflow/cache/cache_test.go index 522f6428bea..19155a18cf5 100644 --- a/service/history/workflow/cache/cache_test.go +++ b/service/history/workflow/cache/cache_test.go @@ -29,6 +29,7 @@ import ( "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" @@ -82,7 +83,7 @@ func (s *workflowCacheSuite) TearDownTest() { } func (s *workflowCacheSuite) TestHistoryCacheBasic() { - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution1 := commonpb.WorkflowExecution{ @@ -129,7 +130,7 @@ func (s *workflowCacheSuite) TestHistoryCacheBasic() { } func (s *workflowCacheSuite) TestHistoryCachePanic() { - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution1 := commonpb.WorkflowExecution{ @@ -172,7 +173,7 @@ func (s *workflowCacheSuite) TestHistoryCachePanic() { func (s *workflowCacheSuite) TestHistoryCachePinning() { s.mockShard.GetConfig().HistoryHostLevelCacheMaxSize = dynamicconfig.GetIntPropertyFn(1) namespaceID := namespace.ID("test_namespace_id") - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) we := commonpb.WorkflowExecution{ WorkflowId: "wf-cache-test-pinning", RunId: uuid.NewString(), @@ -238,7 +239,7 @@ func (s *workflowCacheSuite) TestHistoryCacheEvictionReleasesPaginationBuffer() // Production shards always have a finalizer, so give the test shard one too s.mockShard.SetFinalizerForTest(finalizer.New(s.mockShard.GetLogger(), metrics.NoopMetricsHandler)) namespaceID := namespace.ID("test_namespace_id") - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) limiter := s.cache.(*cacheImpl).paginationLimiter we := commonpb.WorkflowExecution{ @@ -303,7 +304,7 @@ func (s *workflowCacheSuite) TestHistoryCacheEvictionReleasesPaginationBuffer() func (s *workflowCacheSuite) TestHistoryCacheClear() { s.mockShard.GetConfig().HistoryHostLevelCacheMaxSize = dynamicconfig.GetIntPropertyFn(20) namespaceID := namespace.ID("test_namespace_id") - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) we := commonpb.WorkflowExecution{ WorkflowId: "wf-cache-test-clear", RunId: uuid.NewString(), @@ -360,7 +361,7 @@ func (s *workflowCacheSuite) TestHistoryCacheConcurrentAccess_Release() { coroutineCount := 50 s.mockShard.GetConfig().HistoryHostLevelCacheMaxSize = dynamicconfig.GetIntPropertyFn(cacheMaxSize) - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) startGroup := &sync.WaitGroup{} stopGroup := &sync.WaitGroup{} @@ -429,7 +430,7 @@ func (s *workflowCacheSuite) TestHistoryCacheConcurrentAccess_Pin() { s.mockShard.GetConfig().HistoryHostLevelCacheMaxSize = dynamicconfig.GetIntPropertyFn(cacheMaxSize) s.mockShard.GetConfig().HistoryCacheTTL = dynamicconfig.GetDurationPropertyFn(time.Nanosecond) - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) startGroup := &sync.WaitGroup{} stopGroup := &sync.WaitGroup{} @@ -485,7 +486,7 @@ func (s *workflowCacheSuite) TestHistoryCacheConcurrentAccess_Pin() { }*/ func (s *workflowCacheSuite) TestHistoryCache_CacheLatencyMetricContext() { - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) ctx := metrics.AddMetricsContext(context.Background()) currentRelease, err := s.cache.GetOrCreateCurrentExecution( @@ -526,7 +527,7 @@ func (s *workflowCacheSuite) TestHistoryCache_CacheHoldTimeMetricContext() { capture := metricsHandler.StartCapture() s.mockShard.SetMetricsHandler(metricsHandler) - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metricsHandler, testhooks.TestHooks{}) release1, err := s.cache.GetOrCreateCurrentExecution( context.Background(), @@ -615,7 +616,7 @@ func (s *workflowCacheSuite) TestCacheImpl_lockWorkflowExecution() { } for _, tt := range testSets { s.Run(tt.name, func() { - c := NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + c := NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution := commonpb.WorkflowExecution{ @@ -635,6 +636,7 @@ func (s *workflowCacheSuite) TestCacheImpl_lockWorkflowExecution() { s.mockShard.GetThrottledLogger(), s.mockShard.GetMetricsHandler(), nil, + testhooks.TestHooks{}, ) ctx := headers.SetCallerType(context.Background(), tt.callerType) ctx, cancel := context.WithTimeout(ctx, 5*time.Second) @@ -665,7 +667,7 @@ func (s *workflowCacheSuite) TestCacheImpl_RejectsRequestWhenAtLimitSimple() { }, config, ) - s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution1 := commonpb.WorkflowExecution{ @@ -731,7 +733,7 @@ func (s *workflowCacheSuite) TestCacheImpl_RejectsRequestWhenAtLimitMultiple() { }, config, ) - s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution1 := commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", @@ -875,7 +877,7 @@ func (s *workflowCacheSuite) TestCacheImpl_CheckCacheLimitSizeBasedFlag() { }, config, ) - s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(config, s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution1 := commonpb.WorkflowExecution{ @@ -909,7 +911,7 @@ func (s *workflowCacheSuite) TestCacheImpl_CheckCacheLimitSizeBasedFlag() { } func (s *workflowCacheSuite) TestCacheImpl_GetCurrentRunID_CurrentRunExists() { - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution := commonpb.WorkflowExecution{ @@ -946,7 +948,7 @@ func (s *workflowCacheSuite) TestCacheImpl_GetCurrentRunID_CurrentRunExists() { } func (s *workflowCacheSuite) TestCacheImpl_GetCurrentRunID_NoCurrentRun() { - s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler) + s.cache = NewHostLevelCache(s.mockShard.GetConfig(), s.mockShard.GetLogger(), metrics.NoopMetricsHandler, testhooks.TestHooks{}) namespaceID := namespace.ID("test_namespace_id") execution := commonpb.WorkflowExecution{ diff --git a/service/history/workflow/context.go b/service/history/workflow/context.go index 04af98b16b2..183e229f635 100644 --- a/service/history/workflow/context.go +++ b/service/history/workflow/context.go @@ -28,6 +28,7 @@ import ( "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/service/history/configs" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" @@ -44,6 +45,7 @@ type ( throttledLogger log.ThrottledLogger metricsHandler metrics.Handler config *configs.Config + testHooks testhooks.TestHooks lock locks.PrioritySemaphore MutableState historyi.MutableState @@ -73,6 +75,30 @@ type ( identity workflowTaskIdentity // the workflow task this buffer belongs to namespace string // namespace name } + + // TestHookUpdateExecutionRequest is passed only to the test-only + // HistoryPassiveReplicationTest. + TestHookUpdateExecutionRequest struct { + ExecutionContext *ContextImpl + ShardContext historyi.ShardContext + UpdateMode persistence.UpdateWorkflowMode + NewContext historyi.WorkflowContext + NewMutableState historyi.MutableState + UpdateExecutionTransactionPolicy historyi.TransactionPolicy + NewExecutionTransactionPolicy *historyi.TransactionPolicy + PrepareMutableStateTransaction func() error + CloseMutableStateTransaction func() (*ExecutionTransactionPayload, error) + ExecuteExecutionTransaction func(*ExecutionTransactionPayload) error + } + + // ExecutionTransactionPayload contains the persistence payload produced by closing + // an update-with-new transaction. + ExecutionTransactionPayload struct { + ExecutionMutation *persistence.WorkflowMutation + ExecutionEvents []*persistence.WorkflowEvents + NewExecutionSnapshot *persistence.WorkflowSnapshot + NewExecutionEvents []*persistence.WorkflowEvents + } ) var _ historyi.WorkflowContext = (*ContextImpl)(nil) @@ -98,6 +124,7 @@ func NewContext( throttledLogger log.ThrottledLogger, metricsHandler metrics.Handler, paginationLimiter *limiter.KeyedBytesLimiter, + testHooks testhooks.TestHooks, ) *ContextImpl { tags := func() []tag.Tag { return []tag.Tag{ @@ -115,6 +142,7 @@ func NewContext( config: config, lock: locks.NewPrioritySemaphore(1), paginationLimiter: paginationLimiter, + testHooks: testHooks, } softassert.That( contextImpl.throttledLogger, @@ -788,6 +816,83 @@ func (c *ContextImpl) UpdateWorkflowExecutionWithNew( newMutableState historyi.MutableState, updateWorkflowTransactionPolicy historyi.TransactionPolicy, newWorkflowTransactionPolicy *historyi.TransactionPolicy, +) error { + if hook, ok := testhooks.Get( + c.testHooks, + testhooks.HistoryPassiveReplicationTest, + namespace.ID(c.workflowKey.NamespaceID), + ); ok { + metrics.HistoryPassiveReplicationTestHookCounter.With(c.metricsHandler).Record( + 1, + metrics.OperationTag("WorkflowContext"), + ) + request := &TestHookUpdateExecutionRequest{ + ExecutionContext: c, + ShardContext: shardContext, + UpdateMode: updateMode, + NewContext: newContext, + NewMutableState: newMutableState, + UpdateExecutionTransactionPolicy: updateWorkflowTransactionPolicy, + NewExecutionTransactionPolicy: newWorkflowTransactionPolicy, + PrepareMutableStateTransaction: func() error { + return c.prepareMutableStateTransaction( + shardContext, + newContext, + newMutableState, + newWorkflowTransactionPolicy, + ) + }, + CloseMutableStateTransaction: func() (*ExecutionTransactionPayload, error) { + return c.closeMutableStateTransaction( + ctx, + newContext, + newMutableState, + updateWorkflowTransactionPolicy, + newWorkflowTransactionPolicy, + ) + }, + ExecuteExecutionTransaction: func(payload *ExecutionTransactionPayload) error { + return c.executeWorkflowTransaction( + ctx, + shardContext, + updateMode, + newMutableState, + payload, + ) + }, + } + return hook.InterceptUpdate(ctx, request, func() error { + return c.updateWorkflowExecutionWithNew( + ctx, + shardContext, + updateMode, + newContext, + newMutableState, + updateWorkflowTransactionPolicy, + newWorkflowTransactionPolicy, + ) + }) + } + + return c.updateWorkflowExecutionWithNew( + ctx, + shardContext, + updateMode, + newContext, + newMutableState, + updateWorkflowTransactionPolicy, + newWorkflowTransactionPolicy, + ) +} + +func (c *ContextImpl) updateWorkflowExecutionWithNew( + ctx context.Context, + shardContext historyi.ShardContext, + updateMode persistence.UpdateWorkflowMode, + newContext historyi.WorkflowContext, + newMutableState historyi.MutableState, + updateWorkflowTransactionPolicy historyi.TransactionPolicy, + newWorkflowTransactionPolicy *historyi.TransactionPolicy, ) (retError error) { defer func() { @@ -795,7 +900,48 @@ func (c *ContextImpl) UpdateWorkflowExecutionWithNew( c.Clear() } }() + if newContext != nil && newMutableState != nil && newWorkflowTransactionPolicy != nil { + defer func() { + if retError != nil { + newContext.Clear() + } + }() + } + + if err := c.prepareMutableStateTransaction( + shardContext, + newContext, + newMutableState, + newWorkflowTransactionPolicy, + ); err != nil { + return err + } + + payload, err := c.closeMutableStateTransaction( + ctx, + newContext, + newMutableState, + updateWorkflowTransactionPolicy, + newWorkflowTransactionPolicy, + ) + if err != nil { + return err + } + return c.executeWorkflowTransaction( + ctx, + shardContext, + updateMode, + newMutableState, + payload, + ) +} +func (c *ContextImpl) prepareMutableStateTransaction( + shardContext historyi.ShardContext, + newContext historyi.WorkflowContext, + newMutableState historyi.MutableState, + newWorkflowTransactionPolicy *historyi.TransactionPolicy, +) error { if newContext != nil && newMutableState != nil && newWorkflowTransactionPolicy != nil { if *newWorkflowTransactionPolicy == historyi.TransactionPolicyActive { execInfo := newMutableState.GetExecutionInfo() @@ -821,42 +967,56 @@ func (c *ContextImpl) UpdateWorkflowExecutionWithNew( // reconcileTaskCompletionBuffer drops an orphaned buffer for the pagination of // RespondWorkflowTaskCompleted requests. c.reconcileTaskCompletionBuffer() + return nil +} +func (c *ContextImpl) closeMutableStateTransaction( + ctx context.Context, + newContext historyi.WorkflowContext, + newMutableState historyi.MutableState, + updateWorkflowTransactionPolicy historyi.TransactionPolicy, + newWorkflowTransactionPolicy *historyi.TransactionPolicy, +) (*ExecutionTransactionPayload, error) { updateWorkflow, updateWorkflowEventsSeq, err := c.MutableState.CloseTransactionAsMutation( ctx, updateWorkflowTransactionPolicy, ) if err != nil { - return err + return nil, err } - var newWorkflow *persistence.WorkflowSnapshot - var newWorkflowEventsSeq []*persistence.WorkflowEvents + payload := &ExecutionTransactionPayload{ + ExecutionMutation: updateWorkflow, + ExecutionEvents: updateWorkflowEventsSeq, + } if newContext != nil && newMutableState != nil && newWorkflowTransactionPolicy != nil { - defer func() { - if retError != nil { - newContext.Clear() - } - }() - - newWorkflow, newWorkflowEventsSeq, err = newMutableState.CloseTransactionAsSnapshot( + payload.NewExecutionSnapshot, payload.NewExecutionEvents, err = newMutableState.CloseTransactionAsSnapshot( ctx, *newWorkflowTransactionPolicy, ) if err != nil { - return err + return nil, err } } + return payload, nil +} +func (c *ContextImpl) executeWorkflowTransaction( + ctx context.Context, + shardContext historyi.ShardContext, + updateMode persistence.UpdateWorkflowMode, + newMutableState historyi.MutableState, + payload *ExecutionTransactionPayload, +) error { if err := c.mergeUpdateWithNewReplicationTasks( - updateWorkflow, - newWorkflow, + payload.ExecutionMutation, + payload.NewExecutionSnapshot, ); err != nil { return err } - eventsToReapply := updateWorkflowEventsSeq - if len(updateWorkflowEventsSeq) == 0 { + eventsToReapply := payload.ExecutionEvents + if len(payload.ExecutionEvents) == 0 { if reapplyCandidateEvents := c.MutableState.GetReapplyCandidateEvents(); len(reapplyCandidateEvents) != 0 { eventsToReapply = []*persistence.WorkflowEvents{ { @@ -876,7 +1036,7 @@ func (c *ContextImpl) UpdateWorkflowExecutionWithNew( eventsToReapply, // The new run is created by applying events so the history builder in newMutableState contains the events be re-applied. // So we can use newWorkflowEventsSeq directly to reapply events. - newWorkflowEventsSeq, + payload.NewExecutionEvents, ); err != nil { return err } @@ -886,11 +1046,11 @@ func (c *ContextImpl) UpdateWorkflowExecutionWithNew( updateMode, c.archetypeID, c.MutableState.GetCurrentVersion(), - updateWorkflow, - updateWorkflowEventsSeq, + payload.ExecutionMutation, + payload.ExecutionEvents, MutableStateFailoverVersion(newMutableState), - newWorkflow, - newWorkflowEventsSeq, + payload.NewExecutionSnapshot, + payload.NewExecutionEvents, c.MutableState.IsWorkflow(), ); err != nil { return err diff --git a/service/history/workflow/context_test.go b/service/history/workflow/context_test.go index ba8317201f2..1ff94078016 100644 --- a/service/history/workflow/context_test.go +++ b/service/history/workflow/context_test.go @@ -25,6 +25,7 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/primitives/timestamp" + "go.temporal.io/server/common/testing/testhooks" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tasks" @@ -82,6 +83,7 @@ func (s *contextSuite) SetupTest() { log.NewNoopLogger(), metrics.NoopMetricsHandler, limiter.NewKeyedBytesLimiter(), + testhooks.TestHooks{}, ) } diff --git a/service/history/workflow/context_test_hook_test.go b/service/history/workflow/context_test_hook_test.go new file mode 100644 index 00000000000..3c07403d44b --- /dev/null +++ b/service/history/workflow/context_test_hook_test.go @@ -0,0 +1,78 @@ +//go:build test_dep + +package workflow + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/testing/testhooks" + historyi "go.temporal.io/server/service/history/interfaces" + historytasks "go.temporal.io/server/service/history/tasks" + "go.temporal.io/server/service/history/tests" +) + +type workflowContextPassiveReplicationTestHook struct { + err error +} + +func (h workflowContextPassiveReplicationTestHook) InterceptUpdate( + context.Context, + any, + func() error, +) error { + return h.err +} + +func (workflowContextPassiveReplicationTestHook) UseTransientWorkflowContextForReplication(context.Context) bool { + return false +} + +func (workflowContextPassiveReplicationTestHook) ShouldExecuteTaskAsPassive(historytasks.Task) bool { + return false +} + +func TestContextImpl_PassiveReplicationHookRecordsMetric(t *testing.T) { + testHooks := testhooks.NewTestHooks() + hookErr := errors.New("hook called") + t.Cleanup(testhooks.Set[testhooks.HistoryPassiveReplicationTestHook]( + testHooks, + testhooks.HistoryPassiveReplicationTest, + workflowContextPassiveReplicationTestHook{err: hookErr}, + tests.NamespaceID, + )) + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + t.Cleanup(func() { metricsHandler.StopCapture(capture) }) + workflowContext := NewContext( + tests.NewDynamicConfig(), + tests.WorkflowKey, + chasm.WorkflowArchetypeID, + log.NewNoopLogger(), + log.NewNoopLogger(), + metricsHandler, + nil, + testHooks, + ) + + err := workflowContext.UpdateWorkflowExecutionWithNew( + context.Background(), + nil, + persistence.UpdateWorkflowModeUpdateCurrent, + nil, + nil, + historyi.TransactionPolicyActive, + nil, + ) + require.ErrorIs(t, err, hookErr) + recordings := capture.Snapshot()[metrics.HistoryPassiveReplicationTestHookCounter.Name()] + require.Len(t, recordings, 1) + require.Equal(t, "WorkflowContext", recordings[0].Tags[metrics.OperationTagName]) +} From a398c80a7d713215b14e3166bd7bc2d07a3e0424 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Tue, 1 Sep 2026 13:53:14 -0700 Subject: [PATCH 108/139] Restore linting for legacy HSM packages (#11847) ## What changed? Removed the temporary golangci exclusion for the relocated legacy HSM packages. ## Why? The exclusion was only needed while the package move itself is the lint diff. --- .github/.golangci.yml | 1 - service/history/hsm/callbacks/request_test.go | 13 +++++++---- service/history/hsm/callbacks/statemachine.go | 23 ++++++++++--------- .../hsm/callbacks/statemachine_test.go | 6 ++--- .../history/hsm/nexusoperations/completion.go | 2 +- service/history/hsm/nexusoperations/events.go | 4 ++-- .../hsm/nexusoperations/events_test.go | 2 ++ .../history/hsm/nexusoperations/executors.go | 7 ++++-- .../hsm/nexusoperations/executors_test.go | 6 ++--- .../hsm/nexusoperations/statemachine.go | 16 ++++++------- .../hsm/nexusoperations/statemachine_test.go | 8 +++++-- .../hsm/nexusoperations/workflow/commands.go | 18 +++++++-------- .../nexusoperations/workflow/commands_test.go | 2 ++ 13 files changed, 62 insertions(+), 46 deletions(-) diff --git a/.github/.golangci.yml b/.github/.golangci.yml index e82b7d31edf..5c9c7fa4c67 100644 --- a/.github/.golangci.yml +++ b/.github/.golangci.yml @@ -171,7 +171,6 @@ linters: - ^api - ^proto - ^.git - - ^\.\./service/history/hsm/(callbacks|dummy|nexusoperations)/ rules: - path-except: _test\.go|tests/.+\.go text: "time.Sleep" diff --git a/service/history/hsm/callbacks/request_test.go b/service/history/hsm/callbacks/request_test.go index dac01cef5ab..0ddef488e02 100644 --- a/service/history/hsm/callbacks/request_test.go +++ b/service/history/hsm/callbacks/request_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/nexus-rpc/sdk-go/nexus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.temporal.io/api/serviceerror" persistencespb "go.temporal.io/server/api/persistence/v1" @@ -130,7 +129,9 @@ func TestRouteRequest_SourceHeaderUnknownCluster(t *testing.T) { func TestRouteSystemCallbackRequest_NilHeaders(t *testing.T) { // When the request has nil headers, it should fall back to the local client. ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, commonnexus.PathCompletionCallbackNoIdentifier, r.URL.Path) + if r.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { + t.Errorf("unexpected request path: %q", r.URL.Path) + } w.WriteHeader(http.StatusOK) })) defer ts.Close() @@ -239,7 +240,9 @@ func TestRouteSystemCallbackRequest_NamespaceNotFound(t *testing.T) { func TestRouteSystemCallbackRequest_Success(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, commonnexus.PathCompletionCallbackNoIdentifier, r.URL.Path) + if r.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { + t.Errorf("unexpected request path: %q", r.URL.Path) + } w.WriteHeader(http.StatusOK) })) defer ts.Close() @@ -295,7 +298,9 @@ func TestRouteSystemCallbackRequest_Success(t *testing.T) { func TestRouteRequest_SystemCallback(t *testing.T) { // Verify that routeRequest delegates to routeSystemCallbackRequest for system callback URLs. ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, commonnexus.PathCompletionCallbackNoIdentifier, r.URL.Path) + if r.URL.Path != commonnexus.PathCompletionCallbackNoIdentifier { + t.Errorf("unexpected request path: %q", r.URL.Path) + } w.WriteHeader(http.StatusOK) })) defer ts.Close() diff --git a/service/history/hsm/callbacks/statemachine.go b/service/history/hsm/callbacks/statemachine.go index 69c730bacc4..9a1435b5605 100644 --- a/service/history/hsm/callbacks/statemachine.go +++ b/service/history/hsm/callbacks/statemachine.go @@ -47,7 +47,7 @@ func NewWorkflowClosedTrigger() *persistencespb.CallbackInfo_Trigger { // NewCallback creates a new callback in the STANDBY state from given params. func NewCallback( - requestId string, + requestID string, registrationTime *timestamppb.Timestamp, trigger *persistencespb.CallbackInfo_Trigger, cb *persistencespb.Callback, @@ -58,7 +58,7 @@ func NewCallback( Callback: cb, State: enumsspb.CALLBACK_STATE_STANDBY, RegistrationTime: registrationTime, - RequestId: requestId, + RequestId: requestID, }, } } @@ -71,9 +71,9 @@ func (c Callback) SetState(state enumsspb.CallbackState) { c.CallbackInfo.State = state } -func (c Callback) recordAttempt(ts time.Time) { - c.CallbackInfo.Attempt++ - c.CallbackInfo.LastAttemptCompleteTime = timestamppb.New(ts) +func (c *Callback) recordAttempt(ts time.Time) { + c.Attempt++ + c.LastAttemptCompleteTime = timestamppb.New(ts) } func (c Callback) RegenerateTasks(*hsm.Node) ([]hsm.Task, error) { @@ -95,8 +95,9 @@ func (c Callback) RegenerateTasks(*hsm.Node) ([]hsm.Task, error) { default: return nil, fmt.Errorf("unsupported callback variant %v", v) } + default: + return nil, nil } - return nil, nil } func (c Callback) output() (hsm.TransitionOutput, error) { @@ -201,7 +202,7 @@ var TransitionRescheduled = hsm.NewTransition( []enumsspb.CallbackState{enumsspb.CALLBACK_STATE_BACKING_OFF}, enumsspb.CALLBACK_STATE_SCHEDULED, func(cb Callback, event EventRescheduled) (hsm.TransitionOutput, error) { - cb.CallbackInfo.NextAttemptScheduleTime = nil + cb.NextAttemptScheduleTime = nil return cb.output() }, ) @@ -221,8 +222,8 @@ var TransitionAttemptFailed = hsm.NewTransition( // Use 0 for elapsed time as we don't limit the retry by time (for now). nextDelay := event.RetryPolicy.ComputeNextDelay(0, int(cb.Attempt), event.Err) nextAttemptScheduleTime := event.Time.Add(nextDelay) - cb.CallbackInfo.NextAttemptScheduleTime = timestamppb.New(nextAttemptScheduleTime) - cb.CallbackInfo.LastAttemptFailure = &failurepb.Failure{ + cb.NextAttemptScheduleTime = timestamppb.New(nextAttemptScheduleTime) + cb.LastAttemptFailure = &failurepb.Failure{ Message: event.Err.Error(), FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ @@ -245,7 +246,7 @@ var TransitionFailed = hsm.NewTransition( enumsspb.CALLBACK_STATE_FAILED, func(cb Callback, event EventFailed) (hsm.TransitionOutput, error) { cb.recordAttempt(event.Time) - cb.CallbackInfo.LastAttemptFailure = &failurepb.Failure{ + cb.LastAttemptFailure = &failurepb.Failure{ Message: event.Err.Error(), FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ @@ -267,7 +268,7 @@ var TransitionSucceeded = hsm.NewTransition( enumsspb.CALLBACK_STATE_SUCCEEDED, func(cb Callback, event EventSucceeded) (hsm.TransitionOutput, error) { cb.recordAttempt(event.Time) - cb.CallbackInfo.LastAttemptFailure = nil + cb.LastAttemptFailure = nil return cb.output() }, ) diff --git a/service/history/hsm/callbacks/statemachine_test.go b/service/history/hsm/callbacks/statemachine_test.go index eb300a580c8..3713a70a4bd 100644 --- a/service/history/hsm/callbacks/statemachine_test.go +++ b/service/history/hsm/callbacks/statemachine_test.go @@ -1,7 +1,7 @@ package callbacks_test import ( - "fmt" + "errors" "testing" "time" @@ -32,7 +32,7 @@ func TestValidTransitions(t *testing.T) { // AttemptFailed out, err := callbacks.TransitionAttemptFailed.Apply(callback, callbacks.EventAttemptFailed{ Time: currentTime, - Err: fmt.Errorf("test"), + Err: errors.New("test"), RetryPolicy: backoff.NewExponentialRetryPolicy(time.Second), }) require.NoError(t, err) @@ -92,7 +92,7 @@ func TestValidTransitions(t *testing.T) { currentTime = currentTime.Add(time.Second) // failed - out, err = callbacks.TransitionFailed.Apply(callback, callbacks.EventFailed{Time: currentTime, Err: fmt.Errorf("failed")}) + out, err = callbacks.TransitionFailed.Apply(callback, callbacks.EventFailed{Time: currentTime, Err: errors.New("failed")}) require.NoError(t, err) // Assert info object is updated only where needed diff --git a/service/history/hsm/nexusoperations/completion.go b/service/history/hsm/nexusoperations/completion.go index 1541d5ea3cb..fb56c6c539f 100644 --- a/service/history/hsm/nexusoperations/completion.go +++ b/service/history/hsm/nexusoperations/completion.go @@ -151,7 +151,7 @@ func fabricateStartedEventIfMissing( ScheduledEventId: eventID, OperationToken: operationToken, // TODO(bergundy): Remove this fallback after the 1.27 release. - OperationId: operationToken, + OperationId: operationToken, //nolint:staticcheck // Retained for compatibility with older servers. RequestId: requestID, }, } diff --git a/service/history/hsm/nexusoperations/events.go b/service/history/hsm/nexusoperations/events.go index 91ac686d3f3..0724ac6cd70 100644 --- a/service/history/hsm/nexusoperations/events.go +++ b/service/history/hsm/nexusoperations/events.go @@ -329,14 +329,14 @@ func findOperationNode(root *hsm.Node, event *historypb.HistoryEvent) (*hsm.Node // Attributes is always a struct with a single field (e.g: HistoryEvent_NexusOperationScheduledEventAttributes) if attrs.Kind() != reflect.Struct || attrs.NumField() != 1 { - panic("invalid event, expected Attributes field with a single field struct") + panic("invalid event, expected Attributes field with a single field struct") //nolint:forbidigo // The history event shape is an HSM invariant. } f := attrs.Field(0).Interface() eventIDGetter, ok := f.(interface{ GetScheduledEventId() int64 }) if !ok { - panic("Event does not have a ScheduledEventId field") + panic("Event does not have a ScheduledEventId field") //nolint:forbidigo // The history event shape is an HSM invariant. } coll := MachineCollection(root) nodeID := strconv.FormatInt(eventIDGetter.GetScheduledEventId(), 10) diff --git a/service/history/hsm/nexusoperations/events_test.go b/service/history/hsm/nexusoperations/events_test.go index 75e0838b106..96d05645817 100644 --- a/service/history/hsm/nexusoperations/events_test.go +++ b/service/history/hsm/nexusoperations/events_test.go @@ -180,6 +180,8 @@ func TestTerminalStatesDeletion(t *testing.T) { a.ScheduledEventId = eventID case *historypb.NexusOperationTimedOutEventAttributes: a.ScheduledEventId = eventID + default: + require.FailNowf(t, "unexpected attributes type", "%T", a) } event := &historypb.HistoryEvent{ diff --git a/service/history/hsm/nexusoperations/executors.go b/service/history/hsm/nexusoperations/executors.go index 0436d9c880e..9b57b1be3e3 100644 --- a/service/history/hsm/nexusoperations/executors.go +++ b/service/history/hsm/nexusoperations/executors.go @@ -165,6 +165,7 @@ func buildCallbackFromTemplate(callbackTemplate string, ns *namespace.Namespace) return builder.String(), nil } +//nolint:revive // The legacy executor is kept intact while existing HSM operations remain loadable. func (e taskExecutor) executeInvocationTask(ctx context.Context, env hsm.Environment, ref hsm.Ref, task InvocationTask) error { ns, err := e.NamespaceRegistry.GetNamespaceByID(namespace.ID(ref.WorkflowKey.NamespaceID)) if err != nil { @@ -467,7 +468,7 @@ func (e taskExecutor) saveStartedResult(env hsm.Environment, node *hsm.Node, ope ScheduledEventId: eventID, OperationToken: result.Pending.Token, // TODO(bergundy): Remove this fallback after the 1.27 release. - OperationId: result.Pending.Token, + OperationId: result.Pending.Token, //nolint:staticcheck // Retained for compatibility with older servers. RequestId: operation.RequestId, }, } @@ -679,6 +680,7 @@ func (e taskExecutor) recordOperationTimeout(node *hsm.Node, timeoutType enumspb }) } +//nolint:revive // The legacy executor is kept intact while existing HSM operations remain loadable. func (e taskExecutor) executeCancelationTask(ctx context.Context, env hsm.Environment, ref hsm.Ref, task CancelationTask) error { ns, err := e.NamespaceRegistry.GetNamespaceByID(namespace.ID(ref.WorkflowKey.NamespaceID)) if err != nil { @@ -861,6 +863,7 @@ func (e taskExecutor) loadArgsForCancelation(ctx context.Context, env hsm.Enviro return } +//nolint:revive // The legacy executor is kept intact while existing HSM operations remain loadable. func (e taskExecutor) saveCancelationResult(ctx context.Context, env hsm.Environment, ref hsm.Ref, callErr error, scheduledEventID int64) error { return env.Access(ctx, ref, hsm.AccessWrite, func(n *hsm.Node) error { return hsm.MachineTransition(n, func(c Cancelation) (hsm.TransitionOutput, error) { @@ -957,7 +960,7 @@ func createNexusOperationFailure(operation Operation, scheduledEventID int64, ca Operation: operation.Operation, OperationToken: operation.OperationToken, // TODO(bergundy): This field is deprecated, remove it after the 1.27 release. - OperationId: operation.OperationToken, + OperationId: operation.OperationToken, //nolint:staticcheck // Retained for compatibility with older servers. ScheduledEventId: scheduledEventID, }, }, diff --git a/service/history/hsm/nexusoperations/executors_test.go b/service/history/hsm/nexusoperations/executors_test.go index bf0510e789a..b4c84f7d7b9 100644 --- a/service/history/hsm/nexusoperations/executors_test.go +++ b/service/history/hsm/nexusoperations/executors_test.go @@ -153,7 +153,7 @@ func TestProcessInvocationTask(t *testing.T) { protorequire.ProtoEqual(t, &historypb.NexusOperationStartedEventAttributes{ ScheduledEventId: 1, OperationToken: "op-token", - OperationId: "op-token", + OperationId: "op-token", //nolint:staticcheck // Verify compatibility with older servers. RequestId: op.RequestId, }, events[0].GetNexusOperationStartedEventAttributes()) require.Len(t, events[0].Links, 1) @@ -407,7 +407,7 @@ func TestProcessInvocationTask(t *testing.T) { destinationDown: true, expectedMetricOutcome: "request-timeout", onStartOperation: func(ctx context.Context, service, operation string, input *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Millisecond * 100) //nolint:forbidigo // Allow time.Sleep for timeout tests return &nexus.HandlerStartOperationResultAsync{OperationToken: "op-token"}, nil }, checkOutcome: func(t *testing.T, op nexusoperations.Operation, events []*historypb.HistoryEvent) { @@ -1701,7 +1701,7 @@ func TestProcessInvocationTask_SystemEndpoint(t *testing.T) { protorequire.ProtoEqual(t, &historypb.NexusOperationStartedEventAttributes{ ScheduledEventId: 1, OperationToken: "system-op-token", - OperationId: "system-op-token", + OperationId: "system-op-token", //nolint:staticcheck // Verify compatibility with older servers. RequestId: op.RequestId, }, events[0].GetNexusOperationStartedEventAttributes()) require.Len(t, events[0].Links, 1) diff --git a/service/history/hsm/nexusoperations/statemachine.go b/service/history/hsm/nexusoperations/statemachine.go index a8fe1ef5fe2..ea159324872 100644 --- a/service/history/hsm/nexusoperations/statemachine.go +++ b/service/history/hsm/nexusoperations/statemachine.go @@ -88,10 +88,10 @@ func (o Operation) SetState(state enumsspb.NexusOperationState) { o.NexusOperationInfo.State = state } -func (o Operation) recordAttempt(ts time.Time) { - o.NexusOperationInfo.Attempt++ - o.NexusOperationInfo.LastAttemptCompleteTime = timestamppb.New(ts) - o.NexusOperationInfo.LastAttemptFailure = nil +func (o *Operation) recordAttempt(ts time.Time) { + o.Attempt++ + o.LastAttemptCompleteTime = timestamppb.New(ts) + o.LastAttemptFailure = nil } func (o Operation) cancelRequested(node *hsm.Node) (bool, error) { @@ -534,10 +534,10 @@ func (c Cancelation) SetState(state enumspb.NexusOperationCancellationState) { c.NexusOperationCancellationInfo.State = state } -func (c Cancelation) recordAttempt(ts time.Time) { - c.NexusOperationCancellationInfo.Attempt++ - c.NexusOperationCancellationInfo.LastAttemptCompleteTime = timestamppb.New(ts) - c.NexusOperationCancellationInfo.LastAttemptFailure = nil +func (c *Cancelation) recordAttempt(ts time.Time) { + c.Attempt++ + c.LastAttemptCompleteTime = timestamppb.New(ts) + c.LastAttemptFailure = nil } func (c Cancelation) RegenerateTasks(node *hsm.Node) ([]hsm.Task, error) { diff --git a/service/history/hsm/nexusoperations/statemachine_test.go b/service/history/hsm/nexusoperations/statemachine_test.go index 2a4b0dd9e53..124263d7d9d 100644 --- a/service/history/hsm/nexusoperations/statemachine_test.go +++ b/service/history/hsm/nexusoperations/statemachine_test.go @@ -297,7 +297,8 @@ func TestRegenerateTasks(t *testing.T) { } node := newOperationNode(t, &hsmtest.NodeBackend{}, event) - if tc.state == enumsspb.NEXUS_OPERATION_STATE_BACKING_OFF { + switch tc.state { + case enumsspb.NEXUS_OPERATION_STATE_BACKING_OFF: require.NoError(t, hsm.MachineTransition(node, func(op nexusoperations.Operation) (hsm.TransitionOutput, error) { return nexusoperations.TransitionAttemptFailed.Apply(op, nexusoperations.EventAttemptFailed{ Time: time.Now(), @@ -306,7 +307,7 @@ func TestRegenerateTasks(t *testing.T) { RetryPolicy: backoff.NewExponentialRetryPolicy(time.Second), }) })) - } else if tc.state == enumsspb.NEXUS_OPERATION_STATE_STARTED { + case enumsspb.NEXUS_OPERATION_STATE_STARTED: require.NoError(t, hsm.MachineTransition(node, func(op nexusoperations.Operation) (hsm.TransitionOutput, error) { return nexusoperations.TransitionStarted.Apply(op, nexusoperations.EventStarted{ Time: time.Now(), @@ -316,6 +317,9 @@ func TestRegenerateTasks(t *testing.T) { }, }) })) + case enumsspb.NEXUS_OPERATION_STATE_SCHEDULED: + default: + require.FailNowf(t, "unexpected operation state", "%v", tc.state) } op, err := hsm.MachineData[nexusoperations.Operation](node) diff --git a/service/history/hsm/nexusoperations/workflow/commands.go b/service/history/hsm/nexusoperations/workflow/commands.go index b39a5f3a6bf..92703cf539c 100644 --- a/service/history/hsm/nexusoperations/workflow/commands.go +++ b/service/history/hsm/nexusoperations/workflow/commands.go @@ -31,6 +31,7 @@ type commandHandler struct { nexusProcessor *chasm.NexusEndpointProcessor } +//nolint:revive // The legacy handler is kept intact while existing HSM operations remain loadable. func (ch *commandHandler) HandleScheduleCommand( ctx context.Context, ms historyi.MutableState, @@ -260,17 +261,16 @@ func (ch *commandHandler) HandleCancelCommand( node, err := coll.Node(nodeID) hasBufferedEvent := ms.HasAnyBufferedEvent(makeNexusOperationTerminalEventFilter(attrs.ScheduledEventId)) if err != nil { - if errors.Is(err, hsm.ErrStateMachineNotFound) { - if !hasBufferedEvent { - return chasmworkflow.FailWorkflowTaskError{ - Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_NEXUS_OPERATION_ATTRIBUTES, - Message: fmt.Sprintf("requested cancelation for a non-existing or already completed operation with scheduled event ID of %d", attrs.ScheduledEventId), - } - } - // Fallthrough and apply the event, there's special logic that will handle state machine not found below. - } else { + if !errors.Is(err, hsm.ErrStateMachineNotFound) { return err } + if !hasBufferedEvent { + return chasmworkflow.FailWorkflowTaskError{ + Cause: enumspb.WORKFLOW_TASK_FAILED_CAUSE_BAD_REQUEST_CANCEL_NEXUS_OPERATION_ATTRIBUTES, + Message: fmt.Sprintf("requested cancelation for a non-existing or already completed operation with scheduled event ID of %d", attrs.ScheduledEventId), + } + } + // Fallthrough and apply the event, there's special logic that will handle state machine not found below. } if node != nil { diff --git a/service/history/hsm/nexusoperations/workflow/commands_test.go b/service/history/hsm/nexusoperations/workflow/commands_test.go index d7dca109390..29f18823ea5 100644 --- a/service/history/hsm/nexusoperations/workflow/commands_test.go +++ b/service/history/hsm/nexusoperations/workflow/commands_test.go @@ -975,6 +975,8 @@ func TestOperationNodeDeletionOnTerminalEvents(t *testing.T) { a.ScheduledEventId = scheduledEvent.EventId case *historypb.NexusOperationTimedOutEventAttributes: a.ScheduledEventId = scheduledEvent.EventId + default: + require.FailNowf(t, "unexpected event attributes type", "%T", a) } applyTerminalEventAndAssertDeletion(t, tcx, scheduledEvent.EventId, tc.eventType, tc.eventAttr, tc.eventDef) From 30e8ba922904d48e223c27f5de5a9447a6dfdcd8 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Tue, 1 Sep 2026 13:54:20 -0700 Subject: [PATCH 109/139] Refactor Nexus dispatch result classification (#11852) I had the AI rewrite the changes to make them easier to review, consider looking at the changes commit-by-commit to see how things all come together. --- ## What changed? Refactors the frontend's Nexus code relating to how it classifies `DispatchNexusTaskResponse` values into a reusable `DispatchOutcome`/`DispatchResult` types. Exposing the exact behavior we have today would carry forward some bugs and deprecated types. So there are several intentional behavior changes with this PR (covered below.) ### Context When making a Nexus request, the last step of the invocation is done via the Matching Service's `DispatchNexusTask` operation. The returned `DispatchNexusTaskResponse` proto has an `Outcome` field with 4x variants (`Failure`, `HandlerError`, `RequestTimeout`, and `Response`). And those variants then expand into 9x(!) distinct outcomes that each need to be handled differently. For example, the Nexus task can complete successful, with either a sync or async result. Or if the handler fails, there are different formats the client can use to send back error responses: handler error or operation error? And both `nexuspb.HandlerError` and `nexuspb.UnsuccessfulOperationError` are both deprecated, and have been transitioned to use `failurepb.Failure` for newer clients! It is sufficient to say that this isn't the type of code we want to be duplicated anywhere, given how rickety it is today. This refactoring will allow the worker callbacks feature to reuse the same `DispatchNexusTaskResponse` classification logic, so that there won't be any discrepancies in how we extract errors arising from Nexus handlers getting invoked. e.g. now there is a single, canonical way that "outcome metric tags" will be labeled. > Note that the metric tags we emit today are a little off. A successful _cancelation_ of a Nexus operation is labeled as `"success"`, whereas a successful synchronous operation is labeled with `"sync_success"`. And an uninterpretable response reports `handler_error:EMPTY_OUTCOME` despite they not coming from a Nexus handler. > > All the existing labels were kept as-is except for addressing existing bugs. ### Functional Changes There are a few behavior changes in this PR: **Metric Tags** - The metric tag used for a worker failure without a handler failure info changed from `"handler_error:"` (empty suffix) to `"handler_error:UNKNOWN"` instead. - We now map all non-spec handler error types to `handler_error:UNKNOWN`, instead of previously introducing their own `handler_error:${userText}`. Without the cap, the cardinality of the outcome metric could be unbounded. **Error Conversion** `commonnexus.DispatchResultToError` now properly handles the "deprecated" errors from clients sending older response Nexus protos, instead of treating them as `nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty or unknown dispatch outcome")` today. The deprecated `matchingservice.DispatchNexusTaskResponse_HandlerError` and nexuspb.StartOperationResponse_OperationError` types are now converted into `failurepb.Failure` protos. This does change what the Temporal server expects to receive from workers sending the deprecated errors. A client emitting a failed operation is now seen as `ApplicationFailure{Type: "OperationError", NonRetryable: true}` wrapping the worker's failure. Whereas today, we'd just see the worker's failure alone. (This however aligns with what the current generation of Workers send now.) **Everything Else** Everything else should be just shuffling the code around and simplifying existing function signatures. In `service/frontend/nexus_handler.go`, the `DispatchNexusTaskResponse` objects were handled by two methods: - `func (h *nexusHandler) StartOperation(...) (...)` - `func (h *nexusHandler) CancelOperation(...) (...)` The logic has been moved into methods in `service/frontend/nexus_dispatch_result.go`, like `func (c *operationContext) startOperationOutcome(...)`. ## Why? By exposing this classification logic, other parts of the codebase that deal with dispatching Nexus tasks (e.g. worker callbacks) will produce the same results. And emit the same outcome metrics. Also, having a clearer set of possible outcomes from a `DispatchNexusTaskResponse` makes it easier to understand at a glance if the code processing the Nexus task's result is handling all the relevant cases. (e.g. multiple sources of failures, deprecated failure types, etc.) ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential Risks If there is some change to how Nexus results are classified or how errors are handled, it could break Nexus. That would be very bad. And AI code reviewers should confirm that there is no behavior changes outside of how metric tags are derived. --- common/nexus/dispatch_outcome.go | 325 ++++++++ common/nexus/dispatch_outcome_test.go | 599 ++++++++++++++ common/nexus/dispatch_response.go | 55 +- common/nexus/dispatch_response_test.go | 312 +++++--- service/frontend/nexus_dispatch_result.go | 150 ++++ .../frontend/nexus_dispatch_result_test.go | 749 ++++++++++++++++++ service/frontend/nexus_handler.go | 170 +--- tests/nexus_api_test.go | 52 +- tests/xdc/nexus_request_forwarding_test.go | 24 +- 9 files changed, 2092 insertions(+), 344 deletions(-) create mode 100644 common/nexus/dispatch_outcome.go create mode 100644 common/nexus/dispatch_outcome_test.go create mode 100644 service/frontend/nexus_dispatch_result.go create mode 100644 service/frontend/nexus_dispatch_result_test.go diff --git a/common/nexus/dispatch_outcome.go b/common/nexus/dispatch_outcome.go new file mode 100644 index 00000000000..eb2378da5e2 --- /dev/null +++ b/common/nexus/dispatch_outcome.go @@ -0,0 +1,325 @@ +package nexus + +import ( + "github.com/nexus-rpc/sdk-go/nexus" + commonpb "go.temporal.io/api/common/v1" + failurepb "go.temporal.io/api/failure/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/metrics" +) + +// DispatchOutcome names the arm of matching's DispatchNexusTaskResponse that came back from a +// DispatchNexusTask call. The nested oneofs and the deprecated variants collapse into one flat set of +// cases. (Including those from a worker sending the deprecated failure responses, those will get +// silently converted into the newer forms.) +// +// The zero value is the empty string and is not a valid outcome, so a switch over a DispatchOutcome +// needs a default clause. For metric tags use DispatchResult.OutcomeTag, not the string value. +type DispatchOutcome string + +const ( + // DispatchOutcomeUnrecognized is a response this build cannot interpret: no outcome set, an + // outcome variant added after this build, or a worker answer that carries nothing usable. + DispatchOutcomeUnrecognized DispatchOutcome = "unrecognized-outcome" + + // DispatchOutcomeSyncSuccess means the worker ran the operation to completion inline. + DispatchOutcomeSyncSuccess DispatchOutcome = "sync-success" + + // DispatchOutcomeAsyncSuccess means the worker started the operation; the result arrives later, + // out of band. + DispatchOutcomeAsyncSuccess DispatchOutcome = "async-success" + + // DispatchOutcomeCancelAccepted means the worker accepted the cancellation request. + DispatchOutcomeCancelAccepted DispatchOutcome = "cancel-accepted" + + // DispatchOutcomeOperationFailure means the worker ran the operation and it resolved as failed + // or canceled. The task was delivered and answered; just the operation itself did not succeed. + // + // e.g. the Nexus handler fails with: + // nexus.NewOperationFailedError("insufficient funds") + // nexus.NewOperationCanceledError("already canceled upstream") + DispatchOutcomeOperationFailure DispatchOutcome = "operation-failure" + + // DispatchOutcomeHandlerFailure means the worker answered with a Nexus handler error, whose retry + // behavior says whether another attempt is worthwhile. The handler can return one itself, or the + // worker can fail the task before the handler runs. + // + // e.g. + // nexus.NewHandlerErrorf(nexus.HandlerErrorTypeNotFound, "no such workflow") + // nexus.NewHandlerErrorf(nexus.HandlerErrorTypeBadRequest, "cannot deserialize input") + DispatchOutcomeHandlerFailure DispatchOutcome = "nexus-handler-failure" + + // DispatchOutcomeWorkerFailure means the worker failed the task with a failure carrying no Nexus + // handler failure info. RespondNexusTaskFailed rejects such a response, so matching cannot produce + // this outcome today. It exists so that a malformed response is classified rather than read as a + // handler error with no type. + DispatchOutcomeWorkerFailure DispatchOutcome = "worker-failure" + + // DispatchOutcomeRequestTimeout means matching gave up before the task was answered: no worker + // was polling the task queue, or a worker took the task and never responded. + DispatchOutcomeRequestTimeout DispatchOutcome = "request-timeout" +) + +// Succeeded reports whether the worker accepted the request. An asynchronous start counts: the worker +// took responsibility for the operation, even though it has not finished it. +func (o DispatchOutcome) Succeeded() bool { + switch o { + case DispatchOutcomeSyncSuccess, DispatchOutcomeAsyncSuccess, DispatchOutcomeCancelAccepted: + return true + default: + return false + } +} + +// DispatchResult is the classified form of a DispatchNexusTaskResponse: the outcome, plus whatever +// that arm of the response carried, hoisted out of the nested oneofs. +type DispatchResult struct { + Outcome DispatchOutcome + + // OperationResult is the operation's result. Set for DispatchOutcomeSyncSuccess, where it may still + // be nil: an operation is allowed to succeed with no value. + OperationResult *commonpb.Payload + + // OperationToken is set for DispatchOutcomeAsyncSuccess. + OperationToken string + + // Links are the handler links the worker attached to a successful start. Set for + // DispatchOutcomeSyncSuccess and DispatchOutcomeAsyncSuccess. + Links []*nexuspb.Link + + // Failure is the Temporal failure the worker reported. Set for DispatchOutcomeHandlerFailure, + // DispatchOutcomeWorkerFailure and DispatchOutcomeOperationFailure. + // + // IMPORTANT: For the current response formats this aliases the proto inside the response rather + // than copying it, so callers that convert it in place mutate the response too. + Failure *failurepb.Failure + + // usedDeprecatedFormat records that the worker answered in a format predating Temporal failure + // responses. Outcome and Failure are normalized either way; only the metric tag differs. + usedDeprecatedFormat bool +} + +// baseClassifyDispatchNexusTaskResponse converts a DispatchNexusTaskResponse into a DispatchResult. +// +// onResponseFn classifies a Response outcome. The DispatchNexusTaskResponse proto is shared by every +// kind of Nexus task, so only the caller knows how to interpret a succesful response. +func baseClassifyDispatchNexusTaskResponse( + resp *matchingservice.DispatchNexusTaskResponse, + onResponseFn func(*nexuspb.StartOperationResponse) DispatchResult, +) DispatchResult { + switch t := resp.GetOutcome().(type) { + case *matchingservice.DispatchNexusTaskResponse_Failure: + // A handler error is a Nexus-level refusal whose retry behavior is meaningful; anything else + // is an arbitrary failure the worker chose to report. + outcome := DispatchOutcomeWorkerFailure + if t.Failure.GetNexusHandlerFailureInfo() != nil { + outcome = DispatchOutcomeHandlerFailure + } + return DispatchResult{Outcome: outcome, Failure: t.Failure} + + case *matchingservice.DispatchNexusTaskResponse_HandlerError: //nolint:staticcheck // Deprecated, still sent by older workers. + return DispatchResult{ + Outcome: DispatchOutcomeHandlerFailure, + //nolint:staticcheck // Deprecated field on a deprecated variant. + Failure: deprecatedHandlerErrorToFailure(t.HandlerError), + usedDeprecatedFormat: true, + } + + case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: + return DispatchResult{Outcome: DispatchOutcomeRequestTimeout} + + case *matchingservice.DispatchNexusTaskResponse_Response: + // Only the caller knows how to interpret a successful response, so defer to onResponseFn. + return onResponseFn(t.Response.GetStartOperation()) + + default: + return DispatchResult{Outcome: DispatchOutcomeUnrecognized} + } +} + +// classifyStartOperationResponse classifies the response a worker gave to a StartOperation request. +func classifyStartOperationResponse(resp *nexuspb.StartOperationResponse) DispatchResult { + switch t := resp.GetVariant().(type) { + case *nexuspb.StartOperationResponse_SyncSuccess: + return DispatchResult{ + Outcome: DispatchOutcomeSyncSuccess, + OperationResult: t.SyncSuccess.GetPayload(), + Links: t.SyncSuccess.GetLinks(), + } + + case *nexuspb.StartOperationResponse_AsyncSuccess: + token := t.AsyncSuccess.GetOperationToken() + if token == "" { + // Workers predating the operation-token rename only set the operation ID. + //nolint:staticcheck // Deprecated, still sent by older workers. + token = t.AsyncSuccess.GetOperationId() + } + return DispatchResult{ + Outcome: DispatchOutcomeAsyncSuccess, + OperationToken: token, + Links: t.AsyncSuccess.GetLinks(), + } + + case *nexuspb.StartOperationResponse_Failure: + return DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: t.Failure, + } + + case *nexuspb.StartOperationResponse_OperationError: //nolint:staticcheck // Deprecated, still sent by older workers. + return DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + //nolint:staticcheck // Deprecated field on a deprecated variant. + Failure: deprecatedOperationErrorToFailure(t.OperationError), + usedDeprecatedFormat: true, + } + + default: + return DispatchResult{Outcome: DispatchOutcomeUnrecognized} + } +} + +// ClassifyStartOperationDispatch classifies matching's response to a dispatched StartOperation task. +func ClassifyStartOperationDispatch(resp *matchingservice.DispatchNexusTaskResponse) DispatchResult { + return baseClassifyDispatchNexusTaskResponse(resp, classifyStartOperationResponse) +} + +// ClassifyCancelOperationDispatch classifies matching's response to a dispatched CancelOperation task. +func ClassifyCancelOperationDispatch(resp *matchingservice.DispatchNexusTaskResponse) DispatchResult { + return baseClassifyDispatchNexusTaskResponse( + resp, + func(*nexuspb.StartOperationResponse) DispatchResult { + // A cancel response carries no fields, so any response means the worker accepted. + return DispatchResult{Outcome: DispatchOutcomeCancelAccepted} + }) +} + +// deprecatedHandlerErrorToFailure converts the deprecated nexuspb.HandlerError into the new format, +// a Temporal failurepb.Failure. +func deprecatedHandlerErrorToFailure(handlerErr *nexuspb.HandlerError) *failurepb.Failure { + failure := &failurepb.Failure{ + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: handlerErr.GetErrorType(), + RetryBehavior: handlerErr.GetRetryBehavior(), + }, + }, + } + if handlerErr.GetFailure() != nil { + failure.Cause = convertNexusFailureToTemporalFailure(handlerErr.GetFailure()) + } + return failure +} + +// deprecatedOperationErrorToFailure converts the deprecated nexuspb.UnsuccessfulOperationError into +// the new format, a Temporal failurepb.Failure proto. +// +// The deprecated variant reports the operation state in a field of its own and sends the handler's +// failure bare. The current format has no state field: the failure that wraps the handler's failure is +// what reports the state, canceled or failed. So the wrapper is rebuilt here from the state field. +// Workers converting in the other direction drop the wrapper and keep its cause. +func deprecatedOperationErrorToFailure(opErr *nexuspb.UnsuccessfulOperationError) *failurepb.Failure { + cause := convertNexusFailureToTemporalFailure(opErr.GetFailure()) + // The deprecated variant carries no message of its own, so the wrapper repeats the cause's. + failure := &failurepb.Failure{Message: cause.GetMessage(), Cause: cause} + + if nexus.OperationState(opErr.GetOperationState()) == nexus.OperationStateCanceled { + failure.FailureInfo = &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + } + } else { + failure.FailureInfo = &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "OperationError", + NonRetryable: true, + }, + } + } + return failure +} + +// convertNexusFailureToTemporalFailure converts the deprecated nexuspb.Failure into the new format, +// a Temporal failurepb.Failure. +func convertNexusFailureToTemporalFailure(nexusFailure *nexuspb.Failure) *failurepb.Failure { + //nolint:staticcheck // Deprecated function still in use for backward compatibility. + converted, err := NexusFailureToTemporalFailure(ProtoFailureToNexusFailure(nexusFailure)) + // A failure that cannot be re-encoded falls back to its message. We know the operation failed, + // we just don't recognize the format of the data in its cause/details. + if err != nil { + return &failurepb.Failure{ + Message: nexusFailure.GetMessage(), + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "NexusFailure", + NonRetryable: false, + }, + }, + } + } + return converted +} + +// OutcomeTag returns the metrics outcome tag for a dispatch. +// +// A handler failure reports its error type as the tag's suffix. A type outside the Nexus spec, or a +// failure with no type at all, reports "handler_error:UNKNOWN" so that a worker cannot mint new time +// series. +func (r DispatchResult) OutcomeTag() metrics.Tag { + return metrics.OutcomeTag(r.metricOutcome()) +} + +func (r DispatchResult) metricOutcome() string { + // NOTE: Some of these are confusing (e.g. "success" for CancelAccepted), but + // changing these would break existing dashboards. + switch r.Outcome { + case DispatchOutcomeSyncSuccess: + return "sync_success" + case DispatchOutcomeAsyncSuccess: + return "async_success" + case DispatchOutcomeCancelAccepted: + return "success" + case DispatchOutcomeOperationFailure: + if r.usedDeprecatedFormat { + return "operation_error" + } + return "failure" + case DispatchOutcomeHandlerFailure, + DispatchOutcomeWorkerFailure: + // A worker failure has no handler error type to report and will map to UNKNOWN. + hErrType := r.Failure.GetNexusHandlerFailureInfo().GetType() + return "handler_error:" + boundHandlerErrorType(hErrType) + case DispatchOutcomeRequestTimeout: + return "handler_timeout" + case DispatchOutcomeUnrecognized: + fallthrough + default: + return "handler_error:EMPTY_OUTCOME" + } +} + +// handlerErrorTypes are the handler error types that may appear verbatim in a metric tag. +// Keep in sync with the HandlerErrorType consts in nexus-rpc/sdk-go/nexus/errors.go. +var handlerErrorTypes = map[string]struct{}{ + string(nexus.HandlerErrorTypeBadRequest): {}, + string(nexus.HandlerErrorTypeUnauthenticated): {}, + string(nexus.HandlerErrorTypeUnauthorized): {}, + string(nexus.HandlerErrorTypeNotFound): {}, + string(nexus.HandlerErrorTypeRequestTimeout): {}, + string(nexus.HandlerErrorTypeConflict): {}, + string(nexus.HandlerErrorTypeResourceExhausted): {}, + string(nexus.HandlerErrorTypeInternal): {}, + string(nexus.HandlerErrorTypeNotImplemented): {}, + string(nexus.HandlerErrorTypeUnavailable): {}, + string(nexus.HandlerErrorTypeUpstreamTimeout): {}, +} + +// boundHandlerErrorType bounds the metric cardinality a worker can introduce through a handler error +// type. Types in the Nexus spec pass through; anything else, including the empty string, collapses to +// UNKNOWN. +func boundHandlerErrorType(errType string) string { + if _, ok := handlerErrorTypes[errType]; ok { + return errType + } + return "UNKNOWN" +} diff --git a/common/nexus/dispatch_outcome_test.go b/common/nexus/dispatch_outcome_test.go new file mode 100644 index 00000000000..27ffe9d0ab2 --- /dev/null +++ b/common/nexus/dispatch_outcome_test.go @@ -0,0 +1,599 @@ +package nexus + +import ( + "testing" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/testing/protorequire" +) + +// Response fixtures, shared with dispatch_response_test.go. + +func startResponse(sor *nexuspb.StartOperationResponse) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_StartOperation{StartOperation: sor}, + }, + }, + } +} + +func syncSuccessResponse(sync *nexuspb.StartOperationResponse_Sync) *matchingservice.DispatchNexusTaskResponse { + return startResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_SyncSuccess{SyncSuccess: sync}, + }) +} + +func asyncSuccessResponse(async *nexuspb.StartOperationResponse_Async) *matchingservice.DispatchNexusTaskResponse { + return startResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_AsyncSuccess{AsyncSuccess: async}, + }) +} + +func operationFailureResponse(f *failurepb.Failure) *matchingservice.DispatchNexusTaskResponse { + return startResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_Failure{Failure: f}, + }) +} + +func cancelResponse() *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_CancelOperation{ + CancelOperation: &nexuspb.CancelOperationResponse{}, + }, + }, + }, + } +} + +// taskFailureResponse is the arm a worker reaches through RespondNexusTaskFailed. +func taskFailureResponse(f *failurepb.Failure) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{Failure: f}, + } +} + +func requestTimeoutResponse() *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ + RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + }, + } +} + +func handlerFailureResponse(errType string, retry enumspb.NexusHandlerErrorRetryBehavior) *matchingservice.DispatchNexusTaskResponse { + return taskFailureResponse(&failurepb.Failure{ + Message: "handler said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: errType, + RetryBehavior: retry, + }, + }, + }) +} + +func applicationFailure(msg, errType string) *failurepb.Failure { + return &failurepb.Failure{ + Message: msg, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: errType}, + }, + } +} + +func canceledFailure(msg string) *failurepb.Failure { + return &failurepb.Failure{ + Message: msg, + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, + } +} + +func deprecatedHandlerErrorResponse(he *nexuspb.HandlerError) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{HandlerError: he}, + } +} + +func deprecatedOperationErrorResponse(opErr *nexuspb.UnsuccessfulOperationError) *matchingservice.DispatchNexusTaskResponse { + return startResponse(&nexuspb.StartOperationResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Variant: &nexuspb.StartOperationResponse_OperationError{OperationError: opErr}, + }) +} + +// Tests for *BOTH* ClassifyStartOperationDispatch and ClassifyCancelOperationDispatch. +func TestClassifyOperationDispatch(t *testing.T) { + failure := &failurepb.Failure{Message: "op failed"} + links := []*nexuspb.Link{{Url: "http://a.test/l", Type: "t"}} + payload := &commonpb.Payload{Data: []byte("xxx")} + + nexusCanceledFailure, err := TemporalFailureToNexusFailure(canceledFailure("canceled within Nexus")) + require.NoError(t, err) + + cases := []struct { + Name string + Response *matchingservice.DispatchNexusTaskResponse + Want DispatchResult + }{ + // DispatchOutcomeSyncSuccess + { + Name: "sync success", + Response: syncSuccessResponse(&nexuspb.StartOperationResponse_Sync{Payload: payload, Links: links}), + Want: DispatchResult{ + Outcome: DispatchOutcomeSyncSuccess, + OperationResult: payload, + Links: links, + }, + }, + { + Name: "sync success (no payload, links)", + Response: syncSuccessResponse(&nexuspb.StartOperationResponse_Sync{}), + Want: DispatchResult{ + Outcome: DispatchOutcomeSyncSuccess, + OperationResult: nil, + Links: nil, + }, + }, + + // DispatchOutcomeAsyncSuccess + { + Name: "async (pick op token over token id)", + //nolint:staticcheck // Exercising the deprecated field on purpose. + Response: asyncSuccessResponse(&nexuspb.StartOperationResponse_Async{OperationToken: "op-token", OperationId: "old-and-busted"}), + Want: DispatchResult{ + Outcome: DispatchOutcomeAsyncSuccess, + OperationToken: "op-token", + }, + }, + { + Name: "async (falls back to deprecated ID)", + //nolint:staticcheck // Exercising the deprecated field on purpose. + Response: asyncSuccessResponse(&nexuspb.StartOperationResponse_Async{OperationId: "operation-id"}), + Want: DispatchResult{ + Outcome: DispatchOutcomeAsyncSuccess, + OperationToken: "operation-id", + }, + }, + { + Name: "async (no token set)", + Response: asyncSuccessResponse(&nexuspb.StartOperationResponse_Async{}), + Want: DispatchResult{ + Outcome: DispatchOutcomeAsyncSuccess, + OperationToken: "", + }, + }, + + // DispatchOutcomeOperationFailure + { + Name: "failure", + Response: operationFailureResponse(failure), + Want: DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: failure, + }, + }, + + // DispatchOutcomeOperationFailure (using deprecated proto) + { + // The deprecated error response does some extra work in the conversion, + // setting the ApplicationFailureInfo to Canceled based on the OperationState. + Name: "failure (deprecated proto, canceled)", + Response: deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateCanceled), + Failure: &nexuspb.Failure{Message: "operation canceled"}, + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: &failurepb.Failure{ + Message: "operation canceled", + Cause: &failurepb.Failure{ + Message: "operation canceled", + }, + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, + }, + usedDeprecatedFormat: true, + }, + }, + { + // The OperationStateFailed becomes an unretryable "OperationError". + Name: "failure (deprecated proto, failed)", + Response: deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{Message: "operation failed"}, + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: &failurepb.Failure{ + Message: "operation failed", + Cause: &failurepb.Failure{ + Message: "operation failed", + }, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "OperationError", + NonRetryable: true, + }, + }, + }, + usedDeprecatedFormat: true, + }, + }, + { + // The operation state is failed, but the cause is due to a cancellation. + // Set both ApplicationFailureInfos, for both levels of the failurepb.Failure. + Name: "failure (deprecated proto, failed surfaces canceled info)", + Response: deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: NexusFailureToProtoFailure(nexusCanceledFailure), + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: &failurepb.Failure{ + Message: "canceled within Nexus", + Cause: &failurepb.Failure{ + Message: "canceled within Nexus", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, + }, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "OperationError", + NonRetryable: true, + }, + }, + }, + usedDeprecatedFormat: true, + }, + }, + { + // A worker can send a failure the server cannot re-encode. The dispatch is still classified by what + // happened to the operation -- calling the whole response unrecognized would turn a settled operation + // into a retryable internal error -- so the failure degrades to its message. + Name: "failure (deprecated proto, unrecognized)", + Response: deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{ + Message: "unrecognized error message", + Metadata: map[string]string{"type": failureTypeString}, + Details: []byte("clearly not a JSON blob!"), + }, + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeOperationFailure, + Failure: &failurepb.Failure{ + Message: "unrecognized error message", + Cause: &failurepb.Failure{ + Message: "unrecognized error message", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "NexusFailure", + }, + }, + }, + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "OperationError", + NonRetryable: true, + }, + }, + }, + usedDeprecatedFormat: true, + }, + }, + + // DispatchOutcomeHandlerFailure + { + Name: "handler failure", + Response: handlerFailureResponse(string(nexus.HandlerErrorTypeBadRequest), enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_UNSPECIFIED), + Want: DispatchResult{ + Outcome: DispatchOutcomeHandlerFailure, + Failure: &failurepb.Failure{ + Message: "handler said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: "BAD_REQUEST", + }, + }, + }, + }, + }, + + // DispatchOutcomeHandlerFailure (using deprecated protos) + { + // A worker predating Temporal failure responses answers with the deprecated handler error. The + // classifier re-encodes it as the Nexus handler failure the current variant carries, keeping the type + // and retry behavior callers act on and nesting the worker's failure as the cause. + Name: "handler failure (deprecated proto, custom retry behavior)", + Response: deprecatedHandlerErrorResponse(&nexuspb.HandlerError{ + ErrorType: string(nexus.HandlerErrorTypeResourceExhausted), + RetryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + Failure: &nexuspb.Failure{Message: "slow down"}, + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeHandlerFailure, + Failure: &failurepb.Failure{ + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: "RESOURCE_EXHAUSTED", + RetryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + }, + }, + Cause: &failurepb.Failure{ + Message: "slow down", + }, + }, + usedDeprecatedFormat: true, + }, + }, + { + // A handler error with no failure of its own still classifies correctly. + Name: "handler failure (deprecated proto, no message)", + Response: deprecatedHandlerErrorResponse(&nexuspb.HandlerError{ + ErrorType: string(nexus.HandlerErrorTypeNotFound), + }), + Want: DispatchResult{ + Outcome: DispatchOutcomeHandlerFailure, + Failure: &failurepb.Failure{ + Message: "", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: "NOT_FOUND", + }, + }, + }, + usedDeprecatedFormat: true, + }, + }, + + // DispatchOutcomeWorkerFailure + { + Name: "worker error", + Response: taskFailureResponse(applicationFailure("worker exploded", "ErrTypeStr")), + Want: DispatchResult{ + Outcome: DispatchOutcomeWorkerFailure, + Failure: &failurepb.Failure{ + Message: "worker exploded", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ + Type: "ErrTypeStr", + }, + }, + }, + }, + }, + + // DispatchOutcomeUnrecognized + { + Name: "unrecognized (no outcome)", + Response: &matchingservice.DispatchNexusTaskResponse{}, + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + Name: "unrecognized (nil)", + Response: nil, + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + Name: "unrecognized (cancel answer to start request)", + Response: cancelResponse(), + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + Name: "unrecognized (no start variant)", + Response: startResponse(&nexuspb.StartOperationResponse{}), + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + Name: "unrecognized (nil start operation)", + Response: startResponse(nil), + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + Name: "unrecognized (empty response envelope)", + Response: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{}, + }, + }, + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + { + // The only fixture that reaches the start classifier with a nil Response inside a set + // outcome arm. On the cancel side it is also the case proving the outcome arm is what + // marks the task answered, not the message inside it. + Name: "unrecognized (nil response envelope)", + Response: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{}, + }, + Want: DispatchResult{ + Outcome: DispatchOutcomeUnrecognized, + }, + }, + + // DispatchOutcomeRequestTimeout + { + Name: "request-timeout", + Response: requestTimeoutResponse(), + Want: DispatchResult{ + Outcome: DispatchOutcomeRequestTimeout, + }, + }, + } + for _, tc := range cases { + t.Run(tc.Name, func(t *testing.T) { + assertClassificationsMatch := func(t *testing.T, want, got DispatchResult) { + require.Equal(t, want.Outcome, got.Outcome) + require.Equal(t, want.OperationToken, got.OperationToken) + require.Equal(t, want.usedDeprecatedFormat, got.usedDeprecatedFormat) + + // TIP: To get a better error message, use require.Equal(...) for the + // specific testcase that is producing the wrong proto result. + protorequire.DeepEqual(t, want.OperationResult, got.OperationResult) + protorequire.DeepEqual(t, want.Failure, got.Failure) + protorequire.DeepEqual(t, want.Links, got.Links) + } + + t.Run("ClassifyStartOperationDispatch", func(t *testing.T) { + got := ClassifyStartOperationDispatch(tc.Response) + assertClassificationsMatch(t, tc.Want, got) + }) + + t.Run("ClassifyCancelOperationDispatch", func(t *testing.T) { + // Testing the Cancel- variant is simple. Any response that sets the + // successful Outcome variant is considered success. + // + // Any other type of response uses the same error handling as before. + got := ClassifyCancelOperationDispatch(tc.Response) + if _, ok := tc.Response.GetOutcome().(*matchingservice.DispatchNexusTaskResponse_Response); ok { + want := DispatchResult{ + Outcome: DispatchOutcomeCancelAccepted, + } + assertClassificationsMatch(t, want, got) + } else { + assertClassificationsMatch(t, tc.Want, got) + } + }) + }) + } +} + +// The classifier aliases the response's failure proto rather than copying it, which callers that +// convert failures in place rely on. DispatchResult.Failure documents this as part of its contract, +// and service/frontend's TemporalFailureToNexusFailureInPlace call depends on it. The table above +// compares protos by value, so only an identity assertion catches a change to cloning. +func TestClassifyStartOperationDispatch_FailureAliasesTheResponse(t *testing.T) { + failure := applicationFailure("worker exploded", "SomeError") + r := ClassifyStartOperationDispatch(taskFailureResponse(failure)) + require.Same(t, failure, r.Failure) +} + +// The outcome tag values are what existing dashboards query, so they are pinned here rather than only +// asserted end-to-end through a handler. +func TestDispatchResultOutcomeTag(t *testing.T) { + for _, tc := range []struct { + name string + result DispatchResult + want string + }{ + { + name: "sync success", + result: DispatchResult{Outcome: DispatchOutcomeSyncSuccess}, + want: "sync_success", + }, + { + name: "async success", + result: DispatchResult{Outcome: DispatchOutcomeAsyncSuccess}, + want: "async_success", + }, + { + name: "cancel accepted", + result: DispatchResult{Outcome: DispatchOutcomeCancelAccepted}, + want: "success", + }, + { + name: "operation failure", + result: DispatchResult{Outcome: DispatchOutcomeOperationFailure}, + want: "failure", + }, + { + // Old-format operation errors have always had their own tag value, so normalizing the + // outcome must not merge them into "failure". + name: "deprecated operation error", + result: ClassifyStartOperationDispatch(deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{Message: "op failed"}, + })), + want: "operation_error", + }, + { + name: "request timeout", + result: DispatchResult{Outcome: DispatchOutcomeRequestTimeout}, + want: "handler_timeout", + }, + { + name: "unrecognized", + result: DispatchResult{Outcome: DispatchOutcomeUnrecognized}, + want: "handler_error:EMPTY_OUTCOME", + }, + { + // The zero value is not a named outcome; it must still land somewhere sensible. + name: "zero value", + result: DispatchResult{}, + want: "handler_error:EMPTY_OUTCOME", + }, + { + name: "handler failure reports its type", + result: ClassifyStartOperationDispatch(handlerFailureResponse( + string(nexus.HandlerErrorTypeBadRequest), + enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_UNSPECIFIED, + )), + want: "handler_error:BAD_REQUEST", + }, + { + name: "deprecated handler error reports its type", + result: ClassifyStartOperationDispatch(deprecatedHandlerErrorResponse( + &nexuspb.HandlerError{ErrorType: string(nexus.HandlerErrorTypeNotFound)})), + want: "handler_error:NOT_FOUND", + }, + { + // Not a handler error, so there is no type to report and the suffix bounds to UNKNOWN. + name: "worker failure", + result: ClassifyStartOperationDispatch( + taskFailureResponse(applicationFailure("worker exploded", "SomeError"))), + want: "handler_error:UNKNOWN", + }, + } { + t.Run(tc.name, func(t *testing.T) { + tag := tc.result.OutcomeTag() + require.Equal(t, "outcome", tag.Key) + require.Equal(t, tc.want, tag.Value) + }) + } +} + +func TestBoundHandlerErrorType(t *testing.T) { + for _, spec := range []string{ + "BAD_REQUEST", "UNAUTHENTICATED", "UNAUTHORIZED", "NOT_FOUND", "REQUEST_TIMEOUT", + "CONFLICT", "RESOURCE_EXHAUSTED", "INTERNAL", "NOT_IMPLEMENTED", "UNAVAILABLE", + "UPSTREAM_TIMEOUT", + } { + require.Equal(t, spec, boundHandlerErrorType(spec), "spec types pass through verbatim") + } + // A worker picks this string, so it must not be able to mint new time series. + require.Equal(t, "UNKNOWN", boundHandlerErrorType("whatever-the-worker-said")) + require.Equal(t, "UNKNOWN", boundHandlerErrorType("")) + require.Equal(t, "UNKNOWN", boundHandlerErrorType("bad_request"), "matching is case sensitive") +} diff --git a/common/nexus/dispatch_response.go b/common/nexus/dispatch_response.go index f5218f89c08..54c38911501 100644 --- a/common/nexus/dispatch_response.go +++ b/common/nexus/dispatch_response.go @@ -2,7 +2,6 @@ package nexus import ( "github.com/nexus-rpc/sdk-go/nexus" - nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/sdk/temporal" "go.temporal.io/server/api/matchingservice/v1" ) @@ -11,35 +10,43 @@ import ( // Returns nil if the response indicates success. // // For failure cases (worker explicitly returned an error), the Temporal SDK's failure -// converter is used to produce standard Go errors (ApplicationError, CanceledError). +// converter is used to produce standard Go errors (temporal.ApplicationError, temporal.CanceledError). // For transport-level issues (timeout, internal), a nexus.HandlerError is returned // so the caller can check Retryable(). func MatchingDispatchResponseToError(resp *matchingservice.DispatchNexusTaskResponse) error { - switch t := resp.GetOutcome().(type) { - case *matchingservice.DispatchNexusTaskResponse_Failure: - // Worker received the task and explicitly failed it (via RespondNexusTaskFailed). - return temporal.GetDefaultFailureConverter().FailureToError(t.Failure) - case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") - case *matchingservice.DispatchNexusTaskResponse_Response: - return StartOperationResponseToError(t.Response.GetStartOperation()) - default: - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty or unknown dispatch outcome") - } + return DispatchResultToError(ClassifyStartOperationDispatch(resp)) } -// StartOperationResponseToError converts a StartOperationResponse proto into a Go error. -// Returns nil for success variants (SyncSuccess, AsyncSuccess). -func StartOperationResponseToError(resp *nexuspb.StartOperationResponse) error { - switch t := resp.GetVariant().(type) { - case *nexuspb.StartOperationResponse_SyncSuccess: - return nil - case *nexuspb.StartOperationResponse_AsyncSuccess: +// DispatchResultToError converts a classified dispatch into a Go error for server-internal +// consumption. Returns nil for the success outcomes. +// +// The errors this produces are SDK-shaped: a failure the worker reported becomes a +// *temporal.ApplicationError or *temporal.CanceledError, and a Nexus handler error becomes a +// *nexus.HandlerError. +// +// Wire-facing paths convert through nexusrpc instead. The Nexus HTTP handler re-serializes a returned +// error with nexusrpc's failure converter, which flattens an SDK-shaped cause to just its message and +// drops the failure type and details. +func DispatchResultToError(result DispatchResult) error { + if result.Outcome.Succeeded() { return nil - case *nexuspb.StartOperationResponse_Failure: - // Operation processed but failed — the worker returned an explicit failure. - return temporal.GetDefaultFailureConverter().FailureToError(t.Failure) + } + + switch result.Outcome { + case DispatchOutcomeHandlerFailure, + DispatchOutcomeWorkerFailure, + DispatchOutcomeOperationFailure: + // The worker either failed the task (via RespondNexusTaskFailed) or answered with a failed + // operation. Either way the failure reports what the worker said. + return temporal.GetDefaultFailureConverter().FailureToError(result.Failure) + + case DispatchOutcomeRequestTimeout: + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") + default: - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty or unknown start operation response variant") + return nexus.NewHandlerErrorf( + nexus.HandlerErrorTypeInternal, + "unsupported or unrecognized dispatch outcome: %s", result.Outcome, + ) } } diff --git a/common/nexus/dispatch_response_test.go b/common/nexus/dispatch_response_test.go index d73daa14118..6499343448d 100644 --- a/common/nexus/dispatch_response_test.go +++ b/common/nexus/dispatch_response_test.go @@ -1,160 +1,220 @@ package nexus import ( + "errors" "testing" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" - failurepb "go.temporal.io/api/failure/v1" + enumspb "go.temporal.io/api/enums/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/sdk/temporal" "go.temporal.io/server/api/matchingservice/v1" ) -func TestMatchingDispatchResponseToError_SyncSuccess(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ - Response: &nexuspb.Response{ - Variant: &nexuspb.Response_StartOperation{ - StartOperation: &nexuspb.StartOperationResponse{ - Variant: &nexuspb.StartOperationResponse_SyncSuccess{ - SyncSuccess: &nexuspb.StartOperationResponse_Sync{}, - }, - }, - }, - }, +// DispatchResultToError serves server-internal callers, which branch on *nexus.HandlerError -- a +// delivery problem whose Retryable() says whether to re-deliver the task -- versus any other error, +// which is the worker's own answer and therefore permanent. These tests pin that split for every +// outcome, and pin that a worker answering in a deprecated format lands on the same error as one +// answering in the current format. +// +// Response fixtures live in dispatch_outcome_test.go. + +// An async start counts as success: the worker took responsibility for the operation even though it +// has not finished it. +func TestDispatchResultToError_SuccessOutcomes(t *testing.T) { + require.NoError(t, MatchingDispatchResponseToError( + syncSuccessResponse(&nexuspb.StartOperationResponse_Sync{}))) + require.NoError(t, MatchingDispatchResponseToError(startResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_AsyncSuccess{ + AsyncSuccess: &nexuspb.StartOperationResponse_Async{OperationToken: "tok"}, }, - } - err := MatchingDispatchResponseToError(resp) - require.NoError(t, err) + }))) + require.NoError(t, DispatchResultToError(ClassifyCancelOperationDispatch(cancelResponse()))) } -func TestMatchingDispatchResponseToError_AsyncSuccess(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ - Response: &nexuspb.Response{ - Variant: &nexuspb.Response_StartOperation{ - StartOperation: &nexuspb.StartOperationResponse{ - Variant: &nexuspb.StartOperationResponse_AsyncSuccess{ - AsyncSuccess: &nexuspb.StartOperationResponse_Async{ - OperationId: "test-op-id", - }, - }, - }, - }, - }, +// A task that never reached a handler, or an answer this build cannot interpret, is a delivery problem +// rather than an answer, so it has to surface as a handler error. +func TestDispatchResultToError_DeliveryProblems(t *testing.T) { + for _, tc := range []struct { + name string + resp *matchingservice.DispatchNexusTaskResponse + wantType nexus.HandlerErrorType + }{ + { + name: "request timeout", + resp: requestTimeoutResponse(), + wantType: nexus.HandlerErrorTypeUpstreamTimeout, }, - } - err := MatchingDispatchResponseToError(resp) - require.NoError(t, err) -} - -func TestMatchingDispatchResponseToError_RequestTimeout(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ - RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + { + name: "no outcome", + resp: &matchingservice.DispatchNexusTaskResponse{}, + wantType: nexus.HandlerErrorTypeInternal, }, - } - err := MatchingDispatchResponseToError(resp) - require.Error(t, err) - - var handlerErr *nexus.HandlerError - require.ErrorAs(t, err, &handlerErr) - require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, handlerErr.Type) -} - -func TestMatchingDispatchResponseToError_WorkerFailure(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ - Failure: &failurepb.Failure{ - Message: "bad request from worker", - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ - ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: "SomeError", - }, - }, - }, + { + name: "no start operation variant", + resp: startResponse(&nexuspb.StartOperationResponse{}), + wantType: nexus.HandlerErrorTypeInternal, }, + } { + t.Run(tc.name, func(t *testing.T) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, MatchingDispatchResponseToError(tc.resp), &handlerErr) + require.Equal(t, tc.wantType, handlerErr.Type) + }) } - err := MatchingDispatchResponseToError(resp) - require.Error(t, err) - - var appErr *temporal.ApplicationError - require.ErrorAs(t, err, &appErr) - require.Equal(t, "SomeError", appErr.Type()) } -func TestMatchingDispatchResponseToError_OperationFailure_ApplicationError(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ - Response: &nexuspb.Response{ - Variant: &nexuspb.Response_StartOperation{ - StartOperation: &nexuspb.StartOperationResponse{ - Variant: &nexuspb.StartOperationResponse_Failure{ - Failure: &failurepb.Failure{ - Message: "activity failed", - FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ - ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{ - Type: "SomeError", - }, - }, - }, - }, - }, - }, - }, +// A failure the worker reported becomes an SDK-shaped error and never a handler error, so callers +// treat it as the worker's answer instead of re-delivering the task. +func TestDispatchResultToError_WorkerReportedFailures(t *testing.T) { + for _, tc := range []struct { + name string + resp *matchingservice.DispatchNexusTaskResponse + wantMessage string + wantType string + }{ + { + name: "failed task", + resp: taskFailureResponse(applicationFailure("bad request from worker", "SomeError")), + wantMessage: "bad request from worker", + wantType: "SomeError", }, + { + name: "failed operation", + resp: operationFailureResponse(applicationFailure("activity failed", "SomeError")), + wantMessage: "activity failed", + wantType: "SomeError", + }, + { + // The classifier rebuilds the wrapper the current variant sends, so a failed operation + // reports the same failure type either way. + name: "deprecated operation error", + resp: deprecatedOperationErrorResponse(&nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{Message: "operation failed"}, + }), + wantMessage: "operation failed", + wantType: "OperationError", + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := MatchingDispatchResponseToError(tc.resp) + var appErr *temporal.ApplicationError + require.ErrorAs(t, err, &appErr) + require.Equal(t, tc.wantMessage, appErr.Message()) + require.Equal(t, tc.wantType, appErr.Type()) + var handlerErr *nexus.HandlerError + require.NotErrorAs(t, err, &handlerErr) + }) } - err := MatchingDispatchResponseToError(resp) - require.Error(t, err) +} - var appErr *temporal.ApplicationError - require.ErrorAs(t, err, &appErr) - require.Equal(t, "SomeError", appErr.Type()) +// A canceled operation has to reach the caller as a canceled error however the worker encoded it, +// since callers branch on the error type rather than on the response format. +func TestDispatchResultToError_CanceledOutcomes(t *testing.T) { + t.Run("a canceled failure", func(t *testing.T) { + var canceledErr *temporal.CanceledError + require.ErrorAs(t, MatchingDispatchResponseToError( + operationFailureResponse(canceledFailure("canceled"))), &canceledErr) + }) + + t.Run("a canceled failure in the deprecated variant", func(t *testing.T) { + nf, err := TemporalFailureToNexusFailure(canceledFailure("operation canceled")) + require.NoError(t, err) + + var canceledErr *temporal.CanceledError + require.ErrorAs(t, MatchingDispatchResponseToError(deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateCanceled), + Failure: NexusFailureToProtoFailure(nf), + })), &canceledErr) + }) + + // A handler outside Temporal sends a failure with no Temporal failure info, so the deprecated state + // field is the only signal that the operation was canceled. The worker's own failure still has to + // survive as the cause, or the caller loses why the operation was canceled. + t.Run("the deprecated state field alone", func(t *testing.T) { + err := MatchingDispatchResponseToError(deprecatedOperationErrorResponse( + &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateCanceled), + Failure: &nexuspb.Failure{Message: "operation canceled"}, + })) + + var canceledErr *temporal.CanceledError + require.ErrorAs(t, err, &canceledErr) + var appErr *temporal.ApplicationError + require.ErrorAs(t, errors.Unwrap(canceledErr), &appErr) + require.Equal(t, "operation canceled", appErr.Message()) + }) } -func TestMatchingDispatchResponseToError_OperationFailure_CanceledError(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{ - Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ - Response: &nexuspb.Response{ - Variant: &nexuspb.Response_StartOperation{ - StartOperation: &nexuspb.StartOperationResponse{ - Variant: &nexuspb.StartOperationResponse_Failure{ - Failure: &failurepb.Failure{ - Message: "canceled", - FailureInfo: &failurepb.Failure_CanceledFailureInfo{ - CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, - }, - }, - }, - }, - }, - }, +// Callers decide whether re-delivering the task is worthwhile from Retryable(), so both response +// formats have to preserve the worker's error type and retry behavior. +func TestDispatchResultToError_HandlerFailureRetryBehavior(t *testing.T) { + for _, tc := range []struct { + name string + errType nexus.HandlerErrorType + retryBehavior enumspb.NexusHandlerErrorRetryBehavior + wantRetryable bool + }{ + { + name: "non-retryable by type", + errType: nexus.HandlerErrorTypeBadRequest, + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_UNSPECIFIED, + wantRetryable: false, + }, + { + name: "retryable by type", + errType: nexus.HandlerErrorTypeUnavailable, + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_UNSPECIFIED, + wantRetryable: true, }, + { + name: "explicit override wins over the type default", + errType: nexus.HandlerErrorTypeBadRequest, + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + wantRetryable: true, + }, + } { + for _, format := range []struct { + name string + resp *matchingservice.DispatchNexusTaskResponse + }{ + { + name: "current", + resp: handlerFailureResponse(string(tc.errType), tc.retryBehavior), + }, + { + name: "deprecated", + resp: deprecatedHandlerErrorResponse(&nexuspb.HandlerError{ + ErrorType: string(tc.errType), + RetryBehavior: tc.retryBehavior, + Failure: &nexuspb.Failure{Message: "worker said no"}, + }), + }, + } { + t.Run(format.name+"/"+tc.name, func(t *testing.T) { + var handlerErr *nexus.HandlerError + require.ErrorAs(t, MatchingDispatchResponseToError(format.resp), &handlerErr) + require.Equal(t, tc.errType, handlerErr.Type) + require.Equal(t, tc.wantRetryable, handlerErr.Retryable()) + }) + } } - err := MatchingDispatchResponseToError(resp) - require.Error(t, err) - - var cancelErr *temporal.CanceledError - require.ErrorAs(t, err, &cancelErr) -} - -func TestMatchingDispatchResponseToError_EmptyOutcome(t *testing.T) { - resp := &matchingservice.DispatchNexusTaskResponse{} - err := MatchingDispatchResponseToError(resp) - require.Error(t, err) - - var handlerErr *nexus.HandlerError - require.ErrorAs(t, err, &handlerErr) - require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) } -func TestStartOperationResponseToError_EmptyVariant(t *testing.T) { - resp := &nexuspb.StartOperationResponse{} - err := StartOperationResponseToError(resp) - require.Error(t, err) +// The deprecated variant carries the worker's failure in the Nexus encoding. It has to come back as +// the handler error's cause in the same SDK shape the current variant produces. +func TestDispatchResultToError_DeprecatedHandlerErrorCause(t *testing.T) { + err := MatchingDispatchResponseToError(deprecatedHandlerErrorResponse(&nexuspb.HandlerError{ + ErrorType: string(nexus.HandlerErrorTypeInternal), + Failure: &nexuspb.Failure{Message: "worker said no"}, + })) var handlerErr *nexus.HandlerError require.ErrorAs(t, err, &handlerErr) - require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) + var appErr *temporal.ApplicationError + require.ErrorAs(t, handlerErr.Cause, &appErr) + require.Equal(t, "worker said no", appErr.Message()) } diff --git a/service/frontend/nexus_dispatch_result.go b/service/frontend/nexus_dispatch_result.go new file mode 100644 index 00000000000..ad76571d891 --- /dev/null +++ b/service/frontend/nexus_dispatch_result.go @@ -0,0 +1,150 @@ +package frontend + +import ( + "github.com/nexus-rpc/sdk-go/nexus" + failurepb "go.temporal.io/api/failure/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/log/tag" + commonnexus "go.temporal.io/server/common/nexus" + "go.temporal.io/server/common/nexus/nexusrpc" +) + +// handleStartOperationResponse converts matching's response to a StartOperation dispatch into the result the +// Nexus SDK expects, recording the metrics outcome tag and the failure-source response header along +// the way. +// +// Links are returned to the caller instead of being attached here: nexus.AddHandlerLinks requires the +// SDK's handler context. +func (c *operationContext) handleStartOperationResponse( + resp *matchingservice.DispatchNexusTaskResponse, + operation string, +) (nexus.HandlerStartOperationResult[any], []nexus.Link, error) { + result := commonnexus.ClassifyStartOperationDispatch(resp) + c.recordDispatchOutcome(result) + + switch result.Outcome { + case commonnexus.DispatchOutcomeSyncSuccess: + return &nexus.HandlerStartOperationResultSync[any]{ + Value: result.OperationResult, + }, parseLinks(result.Links, c.logger), nil + + case commonnexus.DispatchOutcomeAsyncSuccess: + return &nexus.HandlerStartOperationResultAsync{ + OperationToken: result.OperationToken, + }, parseLinks(result.Links, c.logger), nil + + case commonnexus.DispatchOutcomeOperationFailure: + // The worker ran the operation and it came back failed or canceled. That is a legitimate + // answer, reported to the caller as a Nexus operation error rather than a handler error. + cause, internalErr := c.convertWorkerFailure(result.Failure, operation) + if internalErr != nil { + return nil, nil, internalErr + } + state := nexus.OperationStateFailed + if result.Failure.GetCanceledFailureInfo() != nil { + state = nexus.OperationStateCanceled + } + return nil, nil, c.operationError(state, cause, operation) + + default: + return nil, nil, c.failedDispatchToNexusError(result, operation) + } +} + +// handleCancelOperationResponse converts matching's response to a CancelOperation dispatch into the error the +// Nexus SDK expects, recording the metrics outcome tag and the failure-source response header along +// the way. A nil error means the cancel was accepted. +func (c *operationContext) handleCancelOperationResponse( + resp *matchingservice.DispatchNexusTaskResponse, + operation string, +) error { + result := commonnexus.ClassifyCancelOperationDispatch(resp) + c.recordDispatchOutcome(result) + + if result.Outcome == commonnexus.DispatchOutcomeCancelAccepted { + return nil + } + return c.failedDispatchToNexusError(result, operation) +} + +// failedDispatchToNexusError converts the outcomes that mean the task was never handled, or was +// refused outright, into the error to report to the caller. These arms are identical for every kind of +// dispatched request, so both handler methods share them. +func (c *operationContext) failedDispatchToNexusError( + result commonnexus.DispatchResult, + operation string, +) error { + switch result.Outcome { + case commonnexus.DispatchOutcomeHandlerFailure, commonnexus.DispatchOutcomeWorkerFailure: + // A handler error round-trips as a handler error. Anything else the worker used to fail the + // task converts to an opaque failure error, which the SDK reports to the caller as internal. + // Neither is wrapped, so both errors are reported to the caller unchanged. + cause, internalErr := c.convertWorkerFailure(result.Failure, operation) + if internalErr != nil { + return internalErr + } + return cause + + case commonnexus.DispatchOutcomeRequestTimeout: + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") + + default: + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty outcome") + } +} + +// convertWorkerFailure converts a Temporal failure a worker reported into its Nexus-shaped +// equivalent, preserving the cause chain in the form the wire needs. +// +// Exactly one of the returned errors is non-nil. cause is the worker's own failure, which the caller +// may return directly or use as the cause of an OperationError. internalErr means the worker sent a +// failure the server could not parse; the caller must return it as-is, since the operation's outcome +// is unknown at that point. +func (c *operationContext) convertWorkerFailure( + failure *failurepb.Failure, + operation string, +) (cause error, internalErr error) { + // Converting in place is safe here: the failure belongs to a response this request owns. + nf, err := commonnexus.TemporalFailureToNexusFailureInPlace(failure) + if err != nil { + c.logger.Error("error converting Temporal failure to Nexus failure", + tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(c.namespaceName)) + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") + } + cause, err = nexusrpc.DefaultFailureConverter().FailureToError(nf) + if err != nil { + c.logger.Error("error converting Nexus failure to Nexus error", + tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(c.namespaceName)) + return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") + } + return cause, nil +} + +// operationError builds the envelope that tells a Nexus caller the operation itself failed, marked so +// that a Temporal caller unwraps to the real cause instead of recording the synthetic envelope. +func (c *operationContext) operationError( + state nexus.OperationState, + cause error, + operation string, +) error { + opErr := &nexus.OperationError{ + State: state, + Message: "operation error", + Cause: cause, + } + if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { + c.logger.Error("error converting OperationError to Nexus failure", + tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(c.namespaceName)) + return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") + } + return opErr +} + +// recordDispatchOutcome tags the request's metrics with the dispatch outcome and, when the dispatch +// did not succeed, attributes the failure to the worker in the response header. +func (c *operationContext) recordDispatchOutcome(result commonnexus.DispatchResult) { + c.metricsHandler = c.metricsHandler.WithTags(result.OutcomeTag()) + if !result.Outcome.Succeeded() { + c.setFailureSource(commonnexus.FailureSourceWorker) + } +} diff --git a/service/frontend/nexus_dispatch_result_test.go b/service/frontend/nexus_dispatch_result_test.go new file mode 100644 index 00000000000..093e038d3b4 --- /dev/null +++ b/service/frontend/nexus_dispatch_result_test.go @@ -0,0 +1,749 @@ +package frontend + +import ( + "encoding/json" + "testing" + + "github.com/nexus-rpc/sdk-go/nexus" + "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + failurepb "go.temporal.io/api/failure/v1" + nexuspb "go.temporal.io/api/nexus/v1" + "go.temporal.io/server/api/matchingservice/v1" + "go.temporal.io/server/common/metrics/metricstest" + commonnexus "go.temporal.io/server/common/nexus" +) + +// These tests pin down how the frontend turns matching's DispatchNexusTaskResponse into the result the +// Nexus SDK serializes back to the caller. Every arm of the response oneof is wire-visible: the error +// type decides the HTTP status, and the outcome tag and failure-source header are consumed by +// dashboards and by interceptRequest's error-reporting cleanup. They are asserted here so the shared +// classifier introduced alongside them cannot silently change any of it. + +// outcomeTagOf reads the outcome tag accumulated on the context's metrics handler. +func outcomeTagOf(t *testing.T, oc *operationContext) string { + t.Helper() + mh, ok := oc.metricsHandler.(*metricstest.CaptureHandler) + require.True(t, ok, "expected a capture handler") + capture := mh.StartCapture() + oc.metricsHandler.Counter("test").Record(1) + mh.StopCapture(capture) + snap := capture.Snapshot() + require.Len(t, snap["test"], 1) + return snap["test"][0].Tags["outcome"] +} + +func failureSourceOf(oc *operationContext) string { + return oc.responseHeaders[commonnexus.FailureSourceHeaderName] +} + +func testOperationContext() *operationContext { + return newOperationContext(contextOptions{ + namespaceState: enumspb.NAMESPACE_STATE_REGISTERED, + quota: 1, + namespaceRateLimitAllow: true, + rateLimitAllow: true, + }) +} + +// startOperationResponse wraps a StartOperationResponse in the matching response envelope. The oneof +// variant interface is unexported, so callers pass a fully built StartOperationResponse. +func startOperationResponse(sor *nexuspb.StartOperationResponse) *matchingservice.DispatchNexusTaskResponse { + return &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_StartOperation{StartOperation: sor}, + }, + }, + } +} + +func TestHandleStartOperationResponse_SyncSuccess(t *testing.T) { + oc := testOperationContext() + payload := &commonpb.Payload{Data: []byte("hello")} + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_SyncSuccess{ + SyncSuccess: &nexuspb.StartOperationResponse_Sync{ + Payload: payload, + Links: []*nexuspb.Link{ + {Url: "http://links.test/valid", Type: "some.Type"}, + {Url: "http://not\na/url", Type: "bad"}, // dropped, non-fatal + }, + }, + }, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.NoError(t, err) + sync, ok := result.(*nexus.HandlerStartOperationResultSync[any]) + require.True(t, ok, "expected a sync result, got %T", result) + require.Same(t, payload, sync.Value) + require.Len(t, links, 1) + require.Equal(t, "http://links.test/valid", links[0].URL.String()) + require.Equal(t, "some.Type", links[0].Type) + require.Equal(t, "sync_success", outcomeTagOf(t, oc)) + require.Empty(t, failureSourceOf(oc), "success must not be attributed to the worker") +} + +func TestHandleStartOperationResponse_SyncSuccess_NoPayloadNoLinks(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_SyncSuccess{ + SyncSuccess: &nexuspb.StartOperationResponse_Sync{}, + }, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.NoError(t, err) + sync, ok := result.(*nexus.HandlerStartOperationResultSync[any]) + require.True(t, ok) + require.Nil(t, sync.Value) + require.Empty(t, links) + require.Equal(t, "sync_success", outcomeTagOf(t, oc)) +} + +func TestHandleStartOperationResponse_AsyncSuccess_PrefersOperationToken(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_AsyncSuccess{ + AsyncSuccess: &nexuspb.StartOperationResponse_Async{ + OperationToken: "token", + //nolint:staticcheck // Exercising the deprecated field on purpose. + OperationId: "id", + Links: []*nexuspb.Link{{Url: "http://links.test/a", Type: "t"}}, + }, + }, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.NoError(t, err) + async, ok := result.(*nexus.HandlerStartOperationResultAsync) + require.True(t, ok, "expected an async result, got %T", result) + require.Equal(t, "token", async.OperationToken) + require.Len(t, links, 1) + require.Equal(t, "async_success", outcomeTagOf(t, oc)) + require.Empty(t, failureSourceOf(oc)) +} + +// Workers older than the operation-token rename only set the deprecated operation ID. +func TestHandleStartOperationResponse_AsyncSuccess_FallsBackToOperationID(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_AsyncSuccess{ + //nolint:staticcheck // Exercising the deprecated field on purpose. + AsyncSuccess: &nexuspb.StartOperationResponse_Async{OperationId: "id-only"}, + }, + }) + + result, _, err := oc.handleStartOperationResponse(resp, "op") + require.NoError(t, err) + async, ok := result.(*nexus.HandlerStartOperationResultAsync) + require.True(t, ok) + require.Equal(t, "id-only", async.OperationToken) +} + +func TestHandleStartOperationResponse_HandlerFailure(t *testing.T) { + for _, tc := range []struct { + name string + retryBehavior enumspb.NexusHandlerErrorRetryBehavior + wantRetryable bool + }{ + { + name: "explicitly retryable", + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + wantRetryable: true, + }, + { + // BAD_REQUEST is non-retryable by default in the Nexus spec. + name: "unspecified defers to the error type", + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_UNSPECIFIED, + wantRetryable: false, + }, + { + name: "explicitly non-retryable", + retryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE, + wantRetryable: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "handler said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeBadRequest), + RetryBehavior: tc.retryBehavior, + }, + }, + }, + }, + } + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeBadRequest, handlerErr.Type) + require.Equal(t, "handler said no", handlerErr.Message) + require.Equal(t, tc.wantRetryable, handlerErr.Retryable()) + require.NoError(t, handlerErr.Cause, "no cause on the wire means no cause on the error") + require.Equal(t, "handler_error:BAD_REQUEST", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) + }) + } +} + +// The cause chain must stay Nexus-shaped all the way down: the HTTP layer re-serializes this error +// with the same failure converter, and an SDK-shaped cause would be flattened to just its message. +func TestHandleStartOperationResponse_HandlerFailure_CauseChainStaysNexusShaped(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "handler said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeInternal), + }, + }, + Cause: &failurepb.Failure{ + Message: "the real problem", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: "RootCause"}, + }, + }, + }, + }, + } + + _, _, err := oc.handleStartOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + cause, ok := handlerErr.Cause.(*nexus.FailureError) + require.True(t, ok, "expected a Nexus FailureError cause, got %T", handlerErr.Cause) + require.Equal(t, "the real problem", cause.Failure.Message) + // The application failure info survives as structured details, not as a flattened string. + require.Equal(t, "temporal.api.failure.v1.Failure", cause.Failure.Metadata["type"]) + require.Contains(t, string(cause.Failure.Details), "RootCause") +} + +// A worker that fails the task with something other than a handler error (a plain +// RespondNexusTaskFailed) produces an opaque FailureError, which the SDK reports to the caller as an +// internal error. Documented here because it is current behavior, not because it is desirable. +func TestHandleStartOperationResponse_WorkerFailure_NotAHandlerError(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "worker exploded", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: "SomeError"}, + }, + }, + }, + } + + _, _, err := oc.handleStartOperationResponse(resp, "op") + require.Error(t, err) + var failureErr *nexus.FailureError + require.ErrorAs(t, err, &failureErr) + require.Equal(t, "worker exploded", failureErr.Failure.Message) + var handlerErr *nexus.HandlerError + require.NotErrorAs(t, err, &handlerErr, "not reported as a handler error today") + // There is no handler error type to report, so the tag bounds to UNKNOWN. + require.Equal(t, "handler_error:UNKNOWN", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleStartOperationResponse_DeprecatedHandlerError(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{ + HandlerError: &nexuspb.HandlerError{ + ErrorType: string(nexus.HandlerErrorTypeResourceExhausted), + RetryBehavior: enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE, + Failure: &nexuspb.Failure{Message: "slow down"}, + }, + }, + } + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeResourceExhausted, handlerErr.Type) + require.True(t, handlerErr.Retryable()) + deprecatedCause, ok := handlerErr.Cause.(*nexus.FailureError) + require.True(t, ok, "expected a Nexus FailureError cause, got %T", handlerErr.Cause) + require.Equal(t, "slow down", deprecatedCause.Failure.Message) + require.Equal(t, "handler_error:RESOURCE_EXHAUSTED", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleStartOperationResponse_RequestTimeout(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ + RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + }, + } + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, handlerErr.Type) + require.Equal(t, "upstream timeout", handlerErr.Message) + require.Equal(t, "handler_timeout", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleStartOperationResponse_OperationFailure(t *testing.T) { + for _, tc := range []struct { + name string + info *failurepb.Failure + wantState nexus.OperationState + }{ + { + name: "application failure is a failed operation", + info: &failurepb.Failure{ + Message: "activity failed", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: "SomeError"}, + }, + }, + wantState: nexus.OperationStateFailed, + }, + { + name: "canceled failure is a canceled operation", + info: &failurepb.Failure{ + Message: "canceled", + FailureInfo: &failurepb.Failure_CanceledFailureInfo{ + CanceledFailureInfo: &failurepb.CanceledFailureInfo{}, + }, + }, + wantState: nexus.OperationStateCanceled, + }, + } { + t.Run(tc.name, func(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_Failure{Failure: tc.info}, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var opErr *nexus.OperationError + require.ErrorAs(t, err, &opErr) + require.Equal(t, tc.wantState, opErr.State) + require.Equal(t, "operation error", opErr.Message) + require.IsType(t, &nexus.FailureError{}, opErr.Cause) //nolint:testifylint // Asserting the concrete wire shape, not error identity. + // The envelope must be marked so a Temporal caller unwraps to the real cause instead of + // recording the synthetic "operation error" wrapper. + require.NotNil(t, opErr.OriginalFailure) + require.Equal(t, "true", opErr.OriginalFailure.Metadata["unwrap-error"]) + require.NotNil(t, opErr.OriginalFailure.Cause) + require.Equal(t, "failure", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) + }) + } +} + +// unconvertibleFailure is a Temporal failure the server cannot put on the wire: protojson rejects the +// invalid UTF-8 in Type, so TemporalFailureToNexusFailureInPlace fails while serializing it. Used to +// reach the conversion-error arms below without stubbing the failure converter. +func unconvertibleFailure() *failurepb.Failure { + return &failurepb.Failure{ + Message: "worker sent something unserializable", + FailureInfo: &failurepb.Failure_ApplicationFailureInfo{ + ApplicationFailureInfo: &failurepb.ApplicationFailureInfo{Type: string([]byte{0xff})}, + }, + } +} + +// A failure the server cannot convert means the operation's real outcome is unknown, so it must not be +// reported as a legitimate operation error with a missing cause. +func TestHandleStartOperationResponse_OperationFailure_UnconvertibleFailureIsInternal(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + Variant: &nexuspb.StartOperationResponse_Failure{Failure: unconvertibleFailure()}, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) + var opErr *nexus.OperationError + require.NotErrorAs(t, err, &opErr, "an unreadable failure is not a legitimate operation error") + // The outcome was still classified as an operation failure, so the tag and header stand. + require.Equal(t, "failure", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +// The cause is converted before the handler error wrapping it, so an unconvertible cause fails the +// whole conversion. The caller gets the internal error rather than a handler error with no cause. +func TestHandleStartOperationResponse_HandlerFailure_UnconvertibleCauseIsInternal(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "handler said no", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeBadRequest), + }, + }, + Cause: unconvertibleFailure(), + }, + }, + } + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type, + "the worker's own BAD_REQUEST must not survive a failed conversion") + require.Equal(t, "handler_error:BAD_REQUEST", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +// The conversion arm is shared with the start path, but cancel returns a bare error, where a dropped +// internal error would surface as a nil error, i.e. an accepted cancel. +func TestHandleCancelOperationResponse_UnconvertibleFailureIsInternal(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: unconvertibleFailure(), + }, + } + + err := oc.handleCancelOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) +} + +func TestHandleStartOperationResponse_DeprecatedOperationError(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Variant: &nexuspb.StartOperationResponse_OperationError{ + OperationError: &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateCanceled), + Failure: &nexuspb.Failure{Message: "worker canceled it"}, + }, + }, + }) + + result, links, err := oc.handleStartOperationResponse(resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var opErr *nexus.OperationError + require.ErrorAs(t, err, &opErr) + require.Equal(t, nexus.OperationStateCanceled, opErr.State) + require.Equal(t, "operation error", opErr.Message) + cause, ok := opErr.Cause.(*nexus.FailureError) + require.True(t, ok, "expected a FailureError cause, got %T", opErr.Cause) + require.Equal(t, "worker canceled it", cause.Failure.Message) + require.NotNil(t, opErr.OriginalFailure) + require.Equal(t, "true", opErr.OriginalFailure.Metadata["unwrap-error"]) + require.Equal(t, "operation_error", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +// The deprecated operation error carries the worker's failure in the Nexus encoding, and reports the +// operation state in a field of its own. The classifier re-encodes both, so the caller receives what a +// current-format worker would have sent: the state as the wrapping failure, and the worker's metadata +// and details underneath it as the details of a NexusFailure application failure. +func TestHandleStartOperationResponse_DeprecatedOperationErrorReEncodesWorkerFailure(t *testing.T) { + oc := testOperationContext() + resp := startOperationResponse(&nexuspb.StartOperationResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Variant: &nexuspb.StartOperationResponse_OperationError{ + OperationError: &nexuspb.UnsuccessfulOperationError{ + OperationState: string(nexus.OperationStateFailed), + Failure: &nexuspb.Failure{ + Message: "deliberate test failure", + Metadata: map[string]string{"k": "v"}, + Details: []byte(`"details"`), + }, + }, + }, + }) + + _, _, err := oc.handleStartOperationResponse(resp, "op") + var opErr *nexus.OperationError + require.ErrorAs(t, err, &opErr) + require.Equal(t, nexus.OperationStateFailed, opErr.State) + cause, ok := opErr.Cause.(*nexus.FailureError) + require.True(t, ok, "expected a FailureError cause, got %T", opErr.Cause) + require.Equal(t, "deliberate test failure", cause.Failure.Message) + + // Decoding the wire failure the way a Temporal caller does recovers the wrapper the current format + // sends for a failed operation, with the worker's failure whole underneath it. + tFailure, convErr := commonnexus.NexusFailureToTemporalFailure(cause.Failure) + require.NoError(t, convErr) + require.Equal(t, "OperationError", tFailure.GetApplicationFailureInfo().GetType()) + details := tFailure.GetCause().GetApplicationFailureInfo().GetDetails().GetPayloads() + require.Len(t, details, 1) + var workerFailure nexus.Failure + require.NoError(t, json.Unmarshal(details[0].GetData(), &workerFailure)) + require.Equal(t, map[string]string{"k": "v"}, workerFailure.Metadata) + require.JSONEq(t, `"details"`, string(workerFailure.Details)) + require.Equal(t, "operation_error", outcomeTagOf(t, oc)) +} + +// Anything the frontend cannot interpret is blamed on the worker and reported as an internal error. +func TestHandleStartOperationResponse_UnrecognizedOutcomes(t *testing.T) { + for _, tc := range []struct { + name string + resp *matchingservice.DispatchNexusTaskResponse + }{ + { + name: "no outcome set", + resp: &matchingservice.DispatchNexusTaskResponse{}, + }, + { + name: "nil response", + resp: nil, + }, + { + name: "response with no start operation variant", + resp: startOperationResponse(&nexuspb.StartOperationResponse{}), + }, + { + name: "response carrying a cancel outcome instead of a start outcome", + resp: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_CancelOperation{ + CancelOperation: &nexuspb.CancelOperationResponse{}, + }, + }, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + oc := testOperationContext() + result, links, err := oc.handleStartOperationResponse(tc.resp, "op") + require.Nil(t, result) + require.Nil(t, links) + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) + require.Equal(t, "empty outcome", handlerErr.Message) + require.Equal(t, "handler_error:EMPTY_OUTCOME", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) + }) + } +} + +// CancelOperation accepts any Response outcome without inspecting its variant. +func TestHandleCancelOperationResponse_Success(t *testing.T) { + for _, tc := range []struct { + name string + resp *matchingservice.DispatchNexusTaskResponse + }{ + { + name: "cancel response", + resp: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{ + Variant: &nexuspb.Response_CancelOperation{ + CancelOperation: &nexuspb.CancelOperationResponse{}, + }, + }, + }, + }, + }, + { + name: "response with no variant is still accepted", + resp: &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Response{ + Response: &nexuspb.Response{}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + oc := testOperationContext() + require.NoError(t, oc.handleCancelOperationResponse(tc.resp, "op")) + require.Equal(t, "success", outcomeTagOf(t, oc)) + require.Empty(t, failureSourceOf(oc)) + }) + } +} + +func TestHandleCancelOperationResponse_HandlerFailure(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "cannot cancel", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeNotFound), + }, + }, + }, + }, + } + + err := oc.handleCancelOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeNotFound, handlerErr.Type) + require.Equal(t, "cannot cancel", handlerErr.Message) + require.Equal(t, "handler_error:NOT_FOUND", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleCancelOperationResponse_DeprecatedHandlerError(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{ + HandlerError: &nexuspb.HandlerError{ + ErrorType: string(nexus.HandlerErrorTypeNotImplemented), + Failure: &nexuspb.Failure{Message: "nope"}, + }, + }, + } + + err := oc.handleCancelOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeNotImplemented, handlerErr.Type) + require.Equal(t, "handler_error:NOT_IMPLEMENTED", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleCancelOperationResponse_RequestTimeout(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_RequestTimeout{ + RequestTimeout: &matchingservice.DispatchNexusTaskResponse_Timeout{}, + }, + } + + err := oc.handleCancelOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, handlerErr.Type) + require.Equal(t, "upstream timeout", handlerErr.Message) + require.Equal(t, "handler_timeout", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +func TestHandleCancelOperationResponse_UnrecognizedOutcome(t *testing.T) { + oc := testOperationContext() + err := oc.handleCancelOperationResponse(&matchingservice.DispatchNexusTaskResponse{}, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorTypeInternal, handlerErr.Type) + require.Equal(t, "empty outcome", handlerErr.Message) + require.Equal(t, "handler_error:EMPTY_OUTCOME", outcomeTagOf(t, oc)) + require.Equal(t, commonnexus.FailureSourceWorker, failureSourceOf(oc)) +} + +// The details blob of a converted handler failure has to stay parseable by the Nexus failure +// converter on the other side of the wire. +func TestHandleStartOperationResponse_HandlerFailureDetailsAreWellFormed(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "bad input", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{ + Type: string(nexus.HandlerErrorTypeBadRequest), + }, + }, + }, + }, + } + + _, _, err := oc.handleStartOperationResponse(resp, "op") + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.NotNil(t, handlerErr.OriginalFailure) + require.Equal(t, "nexus.HandlerError", handlerErr.OriginalFailure.Metadata["type"]) + var details map[string]any + require.NoError(t, json.Unmarshal(handlerErr.OriginalFailure.Details, &details)) + require.Equal(t, "BAD_REQUEST", details["type"]) +} + +// The handler error type in the outcome tag is a string the worker chose, so it must not be able to +// mint new time series. +func TestHandleStartOperationResponse_HandlerErrorTypeTagIsBounded(t *testing.T) { + for _, tc := range []struct { + name string + errType string + wantTag string + }{ + { + name: "a spec type passes through", + errType: string(nexus.HandlerErrorTypeUnavailable), + wantTag: "handler_error:UNAVAILABLE", + }, + { + name: "an arbitrary worker string is collapsed", + errType: "MY_CUSTOM_ERROR_a1b2c3", + wantTag: "handler_error:UNKNOWN", + }, + { + name: "an empty type is collapsed", + errType: "", + wantTag: "handler_error:UNKNOWN", + }, + } { + t.Run(tc.name, func(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + Outcome: &matchingservice.DispatchNexusTaskResponse_Failure{ + Failure: &failurepb.Failure{ + Message: "nope", + FailureInfo: &failurepb.Failure_NexusHandlerFailureInfo{ + NexusHandlerFailureInfo: &failurepb.NexusHandlerFailureInfo{Type: tc.errType}, + }, + }, + }, + } + + _, _, err := oc.handleStartOperationResponse(resp, "op") + require.Error(t, err) + require.Equal(t, tc.wantTag, outcomeTagOf(t, oc)) + // The error itself still carries the worker's real type; only the metric is bounded. + var handlerErr *nexus.HandlerError + require.ErrorAs(t, err, &handlerErr) + require.Equal(t, nexus.HandlerErrorType(tc.errType), handlerErr.Type) + }) + } +} + +// The deprecated handler error variant is bounded the same way. +func TestHandleCancelOperationResponse_DeprecatedHandlerErrorTypeTagIsBounded(t *testing.T) { + oc := testOperationContext() + resp := &matchingservice.DispatchNexusTaskResponse{ + //nolint:staticcheck // Exercising the deprecated variant on purpose. + Outcome: &matchingservice.DispatchNexusTaskResponse_HandlerError{ + HandlerError: &nexuspb.HandlerError{ErrorType: "something-a-worker-made-up"}, + }, + } + + require.Error(t, oc.handleCancelOperationResponse(resp, "op")) + require.Equal(t, "handler_error:UNKNOWN", outcomeTagOf(t, oc)) +} diff --git a/service/frontend/nexus_handler.go b/service/frontend/nexus_handler.go index 17d60adad85..356014f64f7 100644 --- a/service/frontend/nexus_handler.go +++ b/service/frontend/nexus_handler.go @@ -484,114 +484,12 @@ func (h *nexusHandler) StartOperation( return nil, commonnexus.ConvertGRPCError(err, false) } // Convert to standard Nexus SDK response. - switch t := response.GetOutcome().(type) { - case *matchingservice.DispatchNexusTaskResponse_Failure: - // Set the failure source to "worker" if we've reached this case. - // Failure conversions errors below are the user's fault, as it implies that malformed completions were sent from - // the worker. - oc.setFailureSource(commonnexus.FailureSourceWorker) - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:" + t.Failure.GetNexusHandlerFailureInfo().GetType())) - nf, err := commonnexus.TemporalFailureToNexusFailureInPlace(t.Failure) - if err != nil { - oc.logger.Error("error converting Temporal failure to Nexus failure", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - he, err := nexusrpc.DefaultFailureConverter().FailureToError(nf) - if err != nil { - oc.logger.Error("error converting Nexus failure to Nexus HandlerError", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - return nil, he - - case *matchingservice.DispatchNexusTaskResponse_HandlerError: - // Deprecated case. Replaced with DispatchNexusTaskResponse_Failure - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:" + t.HandlerError.GetErrorType())) - oc.setFailureSource(commonnexus.FailureSourceWorker) - err := convertOutcomeToNexusHandlerError(t) + result, handlerLinks, err := oc.handleStartOperationResponse(response, operation) + if err != nil { return nil, err - - case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_timeout")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") - - case *matchingservice.DispatchNexusTaskResponse_Response: - switch t := t.Response.GetStartOperation().GetVariant().(type) { - case *nexuspb.StartOperationResponse_SyncSuccess: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("sync_success")) - links := parseLinks(t.SyncSuccess.GetLinks(), oc.logger) - nexus.AddHandlerLinks(ctx, links...) - return &nexus.HandlerStartOperationResultSync[any]{ - Value: t.SyncSuccess.GetPayload(), - }, nil - - case *nexuspb.StartOperationResponse_AsyncSuccess: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("async_success")) - token := t.AsyncSuccess.GetOperationToken() - if token == "" { - token = t.AsyncSuccess.GetOperationId() - } - links := parseLinks(t.AsyncSuccess.GetLinks(), oc.logger) - nexus.AddHandlerLinks(ctx, links...) - return &nexus.HandlerStartOperationResultAsync{ - OperationToken: token, - }, nil - - case *nexuspb.StartOperationResponse_OperationError: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("operation_error")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - opErr := &nexus.OperationError{ - Message: "operation error", - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - State: nexus.OperationState(t.OperationError.GetOperationState()), - Cause: &nexus.FailureError{ - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - Failure: commonnexus.ProtoFailureToNexusFailure(t.OperationError.GetFailure()), - }, - } - if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { - oc.logger.Error("error converting OperationError to Nexus failure", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - return nil, opErr - - case *nexuspb.StartOperationResponse_Failure: - // Set the failure source to "worker" if we've reached this case. - // Failure conversions errors below are the user's fault, as it implies that malformed completions were sent from - // the worker. - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("failure")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - nf, err := commonnexus.TemporalFailureToNexusFailureInPlace(t.Failure) - if err != nil { - oc.logger.Error("error converting Temporal failure to Nexus failure", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - cause, err := nexusrpc.DefaultFailureConverter().FailureToError(nf) - if err != nil { - oc.logger.Error("error converting Nexus failure to Nexus OperationError", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - state := nexus.OperationStateFailed - if t.Failure.GetCanceledFailureInfo() != nil { - state = nexus.OperationStateCanceled - } - opErr := &nexus.OperationError{ - State: state, - Message: "operation error", - Cause: cause, - } - if err := nexusrpc.MarkAsWrapperError(nexusrpc.DefaultFailureConverter(), opErr); err != nil { - oc.logger.Error("error converting OperationError to Nexus failure", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - return nil, opErr - } } - // This is the worker's fault. - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:EMPTY_OUTCOME")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - - return nil, nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty outcome") + nexus.AddHandlerLinks(ctx, handlerLinks...) + return result, nil } func parseLinks(links []*nexuspb.Link, logger log.Logger) []nexus.Link { @@ -704,46 +602,7 @@ func (h *nexusHandler) CancelOperation(ctx context.Context, service, operation, return commonnexus.ConvertGRPCError(err, false) } // Convert to standard Nexus SDK response. - switch t := response.GetOutcome().(type) { - case *matchingservice.DispatchNexusTaskResponse_Failure: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:" + t.Failure.GetNexusHandlerFailureInfo().GetType())) - // Set the failure source to "worker" if we've reached this case. - // Failure conversions errors below are the user's fault, as it implies that malformed completions were sent from - // the worker. - oc.setFailureSource(commonnexus.FailureSourceWorker) - nf, err := commonnexus.TemporalFailureToNexusFailureInPlace(t.Failure) - if err != nil { - oc.logger.Error("error converting Temporal failure to Nexus failure", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - he, err := nexusrpc.DefaultFailureConverter().FailureToError(nf) - if err != nil { - oc.logger.Error("error converting Nexus failure to Nexus HandlerError", tag.Error(err), tag.Operation(operation), tag.WorkflowNamespace(oc.namespaceName)) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "internal error") - } - return he - - case *matchingservice.DispatchNexusTaskResponse_HandlerError: - // Deprecated case. Replaced with DispatchNexusTaskResponse_Failure - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:" + t.HandlerError.GetErrorType())) - oc.setFailureSource(commonnexus.FailureSourceWorker) - err := convertOutcomeToNexusHandlerError(t) - return err - - case *matchingservice.DispatchNexusTaskResponse_RequestTimeout: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_timeout")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") - - case *matchingservice.DispatchNexusTaskResponse_Response: - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("success")) - return nil - } - // This is the worker's fault. - oc.metricsHandler = oc.metricsHandler.WithTags(metrics.OutcomeTag("handler_error:EMPTY_OUTCOME")) - oc.setFailureSource(commonnexus.FailureSourceWorker) - - return nexus.NewHandlerErrorf(nexus.HandlerErrorTypeInternal, "empty outcome") + return oc.handleCancelOperationResponse(response, operation) } func (h *nexusHandler) forwardCancelOperation( @@ -840,25 +699,6 @@ func (h *nexusHandler) nexusClientForActiveCluster(oc *operationContext, service }) } -func convertOutcomeToNexusHandlerError(resp *matchingservice.DispatchNexusTaskResponse_HandlerError) *nexus.HandlerError { - var retryBehavior nexus.HandlerErrorRetryBehavior - // nolint:exhaustive // unspecified is the default - switch resp.HandlerError.RetryBehavior { - case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_RETRYABLE: - retryBehavior = nexus.HandlerErrorRetryBehaviorRetryable - case enumspb.NEXUS_HANDLER_ERROR_RETRY_BEHAVIOR_NON_RETRYABLE: - retryBehavior = nexus.HandlerErrorRetryBehaviorNonRetryable - } - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - cause := commonnexus.ProtoFailureToNexusFailure(resp.HandlerError.GetFailure()) - return &nexus.HandlerError{ - // nolint:staticcheck // Deprecated function still in use for backward compatibility. - Type: nexus.HandlerErrorType(resp.HandlerError.GetErrorType()), - RetryBehavior: retryBehavior, - Cause: &nexus.FailureError{Failure: cause}, - } -} - func (nc *nexusContext) setFailureSource(source string) { nc.responseHeadersMutex.Lock() defer nc.responseHeadersMutex.Unlock() diff --git a/tests/nexus_api_test.go b/tests/nexus_api_test.go index 42e47dee0f4..ea60b85d28a 100644 --- a/tests/nexus_api_test.go +++ b/tests/nexus_api_test.go @@ -166,32 +166,36 @@ func (s *NexusApiTestSuite) TestNexusStartOperation_Outcomes(useTemporalFailures var operationError *nexus.OperationError s.ErrorAs(err, &operationError) s.Equal(nexus.OperationStateFailed, operationError.State) - if useTemporalFailures { - // Through the Temporal failure round-trip, the cause chain has an extra wrapper - // for the OperationError's ApplicationFailureInfo. - var failureErr *nexus.FailureError - s.ErrorAs(operationError.Cause, &failureErr) - var innerErr *nexus.FailureError - s.ErrorAs(failureErr.Cause, &innerErr) - tFailure, err := commonnexus.NexusFailureToTemporalFailure(innerErr.Failure) - s.NoError(err) - convErr := temporal.GetDefaultFailureConverter().FailureToError(tFailure) - var appErr *temporal.ApplicationError - s.ErrorAs(convErr, &appErr) - s.Equal("deliberate test failure", appErr.Message()) - var details nexus.Failure - s.NoError(appErr.Details(&details)) - s.Equal("v", details.Metadata["k"]) - } else { + + if !useTemporalFailures { + // The deprecated variant carries no message of its own, so the wrapper the server + // rebuilds from it repeats the worker's message. s.Equal("deliberate test failure", operationError.Cause.Error()) - var failureErr *nexus.FailureError - s.ErrorAs(operationError.Cause, &failureErr) - s.Equal(map[string]string{"k": "v"}, failureErr.Failure.Metadata) - var details string - err = json.Unmarshal(failureErr.Failure.Details, &details) - s.NoError(err) - s.Equal("details", details) } + + // Both response formats reach the caller as the same failure: an operation-error + // wrapper whose cause is the worker's own failure. The legacy variant reports the state + // in a field of its own, and the server rebuilds the wrapper from it. + var wrapper *nexus.FailureError + s.ErrorAs(operationError.Cause, &wrapper) + + var wrapperCause *nexus.FailureError + s.ErrorAs(wrapper.Cause, &wrapperCause) + wrapperCauseTFailure, err := commonnexus.NexusFailureToTemporalFailure(wrapperCause.Failure) + s.NoError(err) + wrapperCauseErr := temporal.GetDefaultFailureConverter().FailureToError(wrapperCauseTFailure) + + var appErr *temporal.ApplicationError + s.ErrorAs(wrapperCauseErr, &appErr) + s.Equal("deliberate test failure", appErr.Message()) + + // The worker's own metadata and details survive the re-encoding. + var appErrDetails nexus.Failure + s.NoError(appErr.Details(&appErrDetails)) + s.Equal(map[string]string{"k": "v"}, appErrDetails.Metadata) + var details string + s.NoError(json.Unmarshal(appErrDetails.Details, &details)) + s.Equal("details", details) }, }, { diff --git a/tests/xdc/nexus_request_forwarding_test.go b/tests/xdc/nexus_request_forwarding_test.go index e73a4ab33bf..d3544598782 100644 --- a/tests/xdc/nexus_request_forwarding_test.go +++ b/tests/xdc/nexus_request_forwarding_test.go @@ -23,6 +23,7 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" "go.temporal.io/api/workflowservice/v1" "go.temporal.io/sdk/client" + "go.temporal.io/sdk/temporal" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/common/authorization" "go.temporal.io/server/common/dynamicconfig" @@ -158,12 +159,25 @@ func (s *NexusRequestForwardingSuite) TestStartOperationForwardedFromStandbyToAc require.ErrorAs(t, retErr, &operationError) require.Equal(t, nexus.OperationStateFailed, operationError.State) require.Equal(t, "deliberate test failure", operationError.Cause.Error()) - var failureError *nexus.FailureError - require.ErrorAs(t, operationError.Cause, &failureError) - require.Equal(t, map[string]string{"k": "v"}, failureError.Failure.Metadata) - var details string - err := json.Unmarshal(failureError.Failure.Details, &details) + + // The server rebuilds the failure a current-format worker would have sent: an + // operation-error wrapper carrying the worker's own failure as its cause. + var wrapperErr *nexus.FailureError + require.ErrorAs(t, operationError.Cause, &wrapperErr) + var wrapperErrCause *nexus.FailureError + require.ErrorAs(t, wrapperErr.Cause, &wrapperErrCause) + tFailure, err := cnexus.NexusFailureToTemporalFailure(wrapperErrCause.Failure) require.NoError(t, err) + convErr := temporal.GetDefaultFailureConverter().FailureToError(tFailure) + + var appErr *temporal.ApplicationError + require.ErrorAs(t, convErr, &appErr) + require.Equal(t, "deliberate test failure", appErr.Message()) + var appErrDetails nexus.Failure + require.NoError(t, appErr.Details(&appErrDetails)) + require.Equal(t, map[string]string{"k": "v"}, appErrDetails.Metadata) + var details string + require.NoError(t, json.Unmarshal(appErrDetails.Details, &details)) require.Equal(t, "details", details) requireExpectedMetricsCaptured(t, activeSnap, ns, "StartNexusOperation", "operation_error") requireExpectedMetricsCaptured(t, passiveSnap, ns, "StartNexusOperation", "forwarded_request_error") From c51464f9b470eee2d248a8ceaa63d90fdf14af30 Mon Sep 17 00:00:00 2001 From: Kannan Date: Tue, 1 Sep 2026 15:31:59 -0700 Subject: [PATCH 110/139] Make worker commands timeout and max attempts configurable (#11863) ## What Convert `DispatchTimeout` and `MaxTaskAttempts` from compile-time constants to dynamic config settings (`WorkerCommandsDispatchTimeout` and `WorkerCommandsMaxAttempts`). ## Why Makes these values tunable at runtime without redeploying. Default values are unchanged (10s timeout, 3 max attempts). ## How did you test it? - Updated unit tests --------- Co-authored-by: Claude Opus 4.6 --- common/dynamicconfig/constants.go | 10 ++++++++++ common/workercommands/dispatcher.go | 9 ++------- common/workercommands/dispatcher_test.go | 6 ++++++ service/history/configs/config.go | 4 ++++ service/history/outbound_queue_active_task_executor.go | 2 +- .../outbound_queue_active_task_executor_test.go | 4 +++- 6 files changed, 26 insertions(+), 9 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 1e6594b7e42..38dce2f0142 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -210,6 +210,16 @@ in the consistent hash ring used by ringpop. Changing it may cause service disru false, `EnableCancelActivityWorkerCommand enables pushing activity cancellation to workers via Nexus worker commands`, ) + WorkerCommandsDispatchTimeout = NewGlobalDurationSetting( + "system.workerCommandsDispatchTimeout", + 10*time.Second*debug.TimeoutMultiplier, + `WorkerCommandsDispatchTimeout is the timeout for dispatching worker commands to a worker via Nexus.`, + ) + WorkerCommandsMaxAttempts = NewGlobalIntSetting( + "system.workerCommandsMaxAttempts", + 3, + `WorkerCommandsMaxAttempts is the maximum number of dispatch attempts for a worker commands task before dropping it.`, + ) NamespaceMinRetentionGlobal = NewGlobalDurationSetting( "system.namespaceMinRetentionGlobal", 24*time.Hour, diff --git a/common/workercommands/dispatcher.go b/common/workercommands/dispatcher.go index fcd80fde65c..947f1a26068 100644 --- a/common/workercommands/dispatcher.go +++ b/common/workercommands/dispatcher.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "time" "github.com/nexus-rpc/sdk-go/nexus" commonpb "go.temporal.io/api/common/v1" @@ -14,7 +13,6 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" workerpb "go.temporal.io/api/worker/v1" "go.temporal.io/server/api/matchingservice/v1" - "go.temporal.io/server/common/debug" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -26,9 +24,6 @@ import ( ) const ( - DispatchTimeout = time.Second * 10 * debug.TimeoutMultiplier - MaxTaskAttempts = 3 - // Nexus service and operation names for worker commands. // TODO: Replace with workerservicepb.WorkerService.ServiceName and // workerservicepb.WorkerService.ExecuteCommands.Name() once the Nexus service @@ -51,7 +46,7 @@ const ( // *temporal.CanceledError. Permanent — the worker contract requires success for all // defined commands, so this indicates a bug or version incompatibility. // -// Callers are responsible for enforcing retry limits (see MaxTaskAttempts). +// Callers are responsible for enforcing retry limits (see WorkerCommandsMaxAttempts dynamic config). // These commands are best-effort — the activity will eventually time out anyway — // so excessive retries waste resources. type Dispatcher struct { @@ -95,7 +90,7 @@ func (d *Dispatcher) Execute( return nil } - ctx, cancel := context.WithTimeout(ctx, DispatchTimeout) + ctx, cancel := context.WithTimeout(ctx, d.config.WorkerCommandsDispatchTimeout()) defer cancel() return d.dispatchToWorker(ctx, task, namespaceName) diff --git a/common/workercommands/dispatcher_test.go b/common/workercommands/dispatcher_test.go index da7159cbc17..5dc4c74ab7a 100644 --- a/common/workercommands/dispatcher_test.go +++ b/common/workercommands/dispatcher_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "github.com/nexus-rpc/sdk-go/nexus" "github.com/stretchr/testify/require" @@ -47,6 +48,7 @@ func TestExecute_FeatureFlagOff_DropsTask(t *testing.T) { d := &Dispatcher{ config: &configs.Config{ EnableCancelActivityWorkerCommand: func(string) bool { return false }, + WorkerCommandsDispatchTimeout: func() time.Duration { return 10 * time.Second }, }, logger: log.NewNoopLogger(), } @@ -60,6 +62,7 @@ func TestExecute_EmptyCommands_DropsTask(t *testing.T) { d := &Dispatcher{ config: &configs.Config{ EnableCancelActivityWorkerCommand: func(string) bool { return true }, + WorkerCommandsDispatchTimeout: func() time.Duration { return 10 * time.Second }, }, logger: log.NewNoopLogger(), } @@ -81,6 +84,7 @@ func TestExecute_DispatchSuccess(t *testing.T) { matchingClient: mockClient, config: &configs.Config{ EnableCancelActivityWorkerCommand: func(string) bool { return true }, + WorkerCommandsDispatchTimeout: func() time.Duration { return 10 * time.Second }, }, metricsHandler: metricsHandler, logger: log.NewNoopLogger(), @@ -128,6 +132,7 @@ func TestExecute_DispatchRPCError(t *testing.T) { matchingClient: mockClient, config: &configs.Config{ EnableCancelActivityWorkerCommand: func(string) bool { return true }, + WorkerCommandsDispatchTimeout: func() time.Duration { return 10 * time.Second }, }, metricsHandler: metricsHandler, logger: log.NewNoopLogger(), @@ -155,6 +160,7 @@ func TestExecute_UpstreamTimeout(t *testing.T) { matchingClient: mockClient, config: &configs.Config{ EnableCancelActivityWorkerCommand: func(string) bool { return true }, + WorkerCommandsDispatchTimeout: func() time.Duration { return 10 * time.Second }, }, metricsHandler: metricsHandler, logger: log.NewNoopLogger(), diff --git a/service/history/configs/config.go b/service/history/configs/config.go index 2e197e0e43c..44220a7aa1c 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -397,6 +397,8 @@ type Config struct { EnableActivityEagerExecution dynamicconfig.BoolPropertyFnWithNamespaceFilter EnableActivityRetryStampIncrement dynamicconfig.BoolPropertyFn EnableCancelActivityWorkerCommand dynamicconfig.BoolPropertyFnWithNamespaceFilter + WorkerCommandsDispatchTimeout dynamicconfig.DurationPropertyFn + WorkerCommandsMaxAttempts dynamicconfig.IntPropertyFn EnableEagerWorkflowStart dynamicconfig.BoolPropertyFnWithNamespaceFilter NamespaceCacheRefreshInterval dynamicconfig.DurationPropertyFn @@ -808,6 +810,8 @@ func NewConfig( EnableActivityEagerExecution: dynamicconfig.EnableActivityEagerExecution.Get(dc), EnableActivityRetryStampIncrement: dynamicconfig.EnableActivityRetryStampIncrement.Get(dc), EnableCancelActivityWorkerCommand: dynamicconfig.EnableCancelActivityWorkerCommand.Get(dc), + WorkerCommandsDispatchTimeout: dynamicconfig.WorkerCommandsDispatchTimeout.Get(dc), + WorkerCommandsMaxAttempts: dynamicconfig.WorkerCommandsMaxAttempts.Get(dc), EnableEagerWorkflowStart: dynamicconfig.EnableEagerWorkflowStart.Get(dc), NamespaceCacheRefreshInterval: dynamicconfig.NamespaceCacheRefreshInterval.Get(dc), diff --git a/service/history/outbound_queue_active_task_executor.go b/service/history/outbound_queue_active_task_executor.go index 49ef3e21834..d7a76d79fad 100644 --- a/service/history/outbound_queue_active_task_executor.go +++ b/service/history/outbound_queue_active_task_executor.go @@ -107,7 +107,7 @@ func (e *outboundQueueActiveTaskExecutor) Execute( task.Attempt = executable.Attempt() return respond(e.executeChasmSideEffectTask(ctx, task)) case *tasks.WorkerCommandsTask: - if executable.Attempt() > workercommands.MaxTaskAttempts { + if executable.Attempt() > e.shardContext.GetConfig().WorkerCommandsMaxAttempts() { e.logger.Info("Worker commands task exceeded max attempts, dropping", tag.WorkflowID(task.WorkflowID), tag.WorkflowRunID(task.RunID), diff --git a/service/history/outbound_queue_active_task_executor_test.go b/service/history/outbound_queue_active_task_executor_test.go index 048838aae36..b31c9831648 100644 --- a/service/history/outbound_queue_active_task_executor_test.go +++ b/service/history/outbound_queue_active_task_executor_test.go @@ -12,6 +12,7 @@ import ( "go.temporal.io/server/api/matchingservicemock/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" @@ -292,8 +293,9 @@ func (s *outboundQueueActiveTaskExecutorSuite) TestExecute_WorkerCommandsTask_Ex }, } + s.mockShard.GetConfig().WorkerCommandsMaxAttempts = dynamicconfig.GetIntPropertyFn(3) s.mockExecutable.EXPECT().GetTask().Return(task).AnyTimes() - s.mockExecutable.EXPECT().Attempt().Return(workercommands.MaxTaskAttempts + 1).AnyTimes() + s.mockExecutable.EXPECT().Attempt().Return(4).AnyTimes() s.mockExecutable.EXPECT().GetWorkflowID().Return("").AnyTimes() result := s.executor.Execute(ctx, s.mockExecutable) From a3b9b99bb56db0e938afdf77491638b80120525c Mon Sep 17 00:00:00 2001 From: Will Duan Date: Tue, 1 Sep 2026 16:17:17 -0700 Subject: [PATCH 111/139] Add single-cluster passive replication test harness (#11887) ## Summary - add a single-cluster harness that diverts active workflow mutations through versioned-transition replication and persists only the passive result - execute generated history tasks through standby validation around normal active execution - compare active and passive task payloads, track diversion and bailout counters, and provide unit coverage, a no-hook control, and an opt-in server for external load scenarios ## Testing - go test -tags test_dep ./tests/passivepath -count=1 -timeout 5m - make lint-code with LINT_CODE_TARGETS=./tests/passivepath --- tests/passivepath/cluster.go | 88 ++++++ tests/passivepath/context.go | 290 +++++++++++++++++++ tests/passivepath/harness.go | 443 ++++++++++++++++++++++++++++++ tests/passivepath/harness_test.go | 154 +++++++++++ tests/passivepath/server_test.go | 116 ++++++++ 5 files changed, 1091 insertions(+) create mode 100644 tests/passivepath/cluster.go create mode 100644 tests/passivepath/context.go create mode 100644 tests/passivepath/harness.go create mode 100644 tests/passivepath/harness_test.go create mode 100644 tests/passivepath/server_test.go diff --git a/tests/passivepath/cluster.go b/tests/passivepath/cluster.go new file mode 100644 index 00000000000..b79ab7689dd --- /dev/null +++ b/tests/passivepath/cluster.go @@ -0,0 +1,88 @@ +package passivepath + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/operatorservice/v1" + replicationpb "go.temporal.io/api/replication/v1" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/common" + "go.temporal.io/server/common/cluster" + "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/searchattribute" + "go.temporal.io/server/tests/testcore" + "google.golang.org/protobuf/types/known/durationpb" +) + +func newSingleClusterWithGlobalNamespace(t *testing.T, logger log.Logger) *testcore.TestCluster { + clusterName := "passivepath_" + common.GenerateRandomString(5) + + persistenceDefaults := testcore.GetPersistenceTestDefaults() + persistenceDefaults.DBName += "_" + clusterName + + config := &testcore.TestClusterConfig{ + ClusterMetadata: cluster.Config{ + EnableGlobalNamespace: true, + FailoverVersionIncrement: 10, + MasterClusterName: clusterName, + CurrentClusterName: clusterName, + ClusterInformation: map[string]cluster.ClusterInformation{ + clusterName: { + Enabled: true, + InitialFailoverVersion: 1, + }, + }, + }, + HistoryConfig: testcore.HistoryConfig{NumHistoryShards: 1}, + Persistence: persistenceDefaults, + DynamicConfigOverrides: map[dynamicconfig.Key]any{ + dynamicconfig.EnableTransitionHistory.Key(): true, + }, + EnableHistoryTaskRecorder: true, + } + + tc, err := testcore.NewTestClusterFactory().NewCluster(t, config, logger) + require.NoError(t, err) + t.Cleanup(func() { _ = tc.TearDownCluster() }) + return tc +} + +func registerGlobalNamespace(t *testing.T, tc *testcore.TestCluster, name string) namespace.ID { + clusterName := tc.ClusterName() + _, err := tc.FrontendClient().RegisterNamespace( + testcore.NewContext(), + &workflowservice.RegisterNamespaceRequest{ + Namespace: name, + IsGlobalNamespace: true, + ActiveClusterName: clusterName, + Clusters: []*replicationpb.ClusterReplicationConfig{{ClusterName: clusterName}}, + WorkflowExecutionRetentionPeriod: durationpb.New(24 * time.Hour), + }) + require.NoError(t, err) + + describeResponse, err := tc.FrontendClient().DescribeNamespace( + testcore.NewContext(), + &workflowservice.DescribeNamespaceRequest{Namespace: name}, + ) + require.NoError(t, err) + namespaceID := namespace.ID(describeResponse.GetNamespaceInfo().GetId()) + + time.Sleep(2 * testcore.NamespaceCacheRefreshInterval) //nolint:forbidigo + + searchAttributes := searchattribute.TestSearchAttributesToRegister() + searchAttributes["SimulatedFailure"] = enumspb.INDEXED_VALUE_TYPE_BOOL + _, err = tc.OperatorClient().AddSearchAttributes( + testcore.NewContext(), + &operatorservice.AddSearchAttributesRequest{ + Namespace: name, + SearchAttributes: searchAttributes, + }, + ) + require.NoError(t, err) + return namespaceID +} diff --git a/tests/passivepath/context.go b/tests/passivepath/context.go new file mode 100644 index 00000000000..0b6ca027b53 --- /dev/null +++ b/tests/passivepath/context.go @@ -0,0 +1,290 @@ +package passivepath + +import ( + "context" + "errors" + "fmt" + "time" + + commonpb "go.temporal.io/api/common/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + replicationspb "go.temporal.io/server/api/replication/v1" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/persistence" + "go.temporal.io/server/common/persistence/transitionhistory" + "go.temporal.io/server/common/testing/testhooks" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/workflow" + wcache "go.temporal.io/server/service/history/workflow/cache" +) + +const applyTimeout = 30 * time.Second + +type replicationApplyContextKey struct{} + +type replicationApplyContext struct{} + +var _ testhooks.HistoryPassiveReplicationTestHook = (*Harness)(nil) + +// InterceptUpdate diverts supported active updates into a replication artifact and +// applies it synchronously while the caller's workflow cache lease remains held. +func (h *Harness) InterceptUpdate( + ctx context.Context, + payload any, + next func() error, +) (retErr error) { + h.recordIntercepted() + + request, ok := payload.(*workflow.TestHookUpdateExecutionRequest) + if !ok { + return fmt.Errorf("passivepath: unexpected update hook payload %T", payload) + } + defer func() { + if retErr != nil { + request.ExecutionContext.Clear() + if request.NewContext != nil { + request.NewContext.Clear() + } + } + }() + delegate := func(reason BailReason) error { + h.recordBailout(reason) + return next() + } + + if request.UpdateExecutionTransactionPolicy != historyi.TransactionPolicyActive { + h.recordBailout(BailPassivePolicy) + if err := request.PrepareMutableStateTransaction(); err != nil { + return err + } + transactionPayload, err := request.CloseMutableStateTransaction() + if err != nil { + return err + } + if request.ExecutionContext != nil { + if err := h.comparePassiveTasks( + request.ExecutionContext.GetWorkflowKey(), + request.ExecutionContext.MutableState, + transactionPayload.ExecutionMutation.Tasks, + ); err != nil { + return err + } + } + if request.NewMutableState != nil && transactionPayload.NewExecutionSnapshot != nil { + if err := h.comparePassiveTasks( + request.NewMutableState.GetWorkflowKey(), + request.NewMutableState, + transactionPayload.NewExecutionSnapshot.Tasks, + ); err != nil { + return err + } + } + return request.ExecuteExecutionTransaction(transactionPayload) + } + newRun := request.NewContext != nil || request.NewMutableState != nil || request.NewExecutionTransactionPolicy != nil + if newRun && (request.NewContext == nil || request.NewMutableState == nil || + request.NewExecutionTransactionPolicy == nil || + *request.NewExecutionTransactionPolicy != historyi.TransactionPolicyActive) { + return delegate(BailNewRun) + } + if request.UpdateMode != persistence.UpdateWorkflowModeUpdateCurrent { + return delegate(BailUpdateMode) + } + + mutableState := request.ExecutionContext.MutableState + if mutableState == nil { + return delegate(BailNoMutableState) + } + if len(mutableState.GetExecutionInfo().TransitionHistory) == 0 { + return delegate(BailNoTransitionHistory) + } + if mutableState.HasBufferedEvents() { + return delegate(BailBufferedEvents) + } + + exclusiveStart := transitionhistory.CopyVersionedTransition(mutableState.CurrentVersionedTransition()) + if exclusiveStart == nil { + return delegate(BailNoTransitionHistory) + } + + if err := request.PrepareMutableStateTransaction(); err != nil { + return err + } + transactionPayload, err := request.CloseMutableStateTransaction() + if err != nil { + return err + } + activeMutation := transactionPayload.ExecutionMutation + eventsSeq := transactionPayload.ExecutionEvents + if mutableState.HasBufferedEvents() { + return errors.New("passivepath: mutable state has buffered events after close") + } + if activeMutation.ClearBufferedEvents { + h.recordBailout(BailClearBufferedEvents) + return request.ExecuteExecutionTransaction(transactionPayload) + } + + artifact, err := h.buildArtifact( + ctx, + request.ShardContext, + request.ExecutionContext.GetWorkflowKey(), + mutableState, + exclusiveStart, + eventsSeq, + ) + if err != nil { + return err + } + if newRun { + newRunEventBatches, err := h.serializeEvents(transactionPayload.NewExecutionEvents) + if err != nil { + return err + } + if len(newRunEventBatches) == 0 { + return errors.New("passivepath: new run has no initial event batch") + } + artifact.NewRunInfo = &replicationspb.NewRunInfo{ + RunId: request.NewMutableState.GetExecutionState().GetRunId(), + EventBatch: newRunEventBatches[0], + } + h.expectPassiveTasks(request.NewMutableState.GetWorkflowKey(), transactionPayload.NewExecutionSnapshot.Tasks) + } + h.expectPassiveTasks(request.ExecutionContext.GetWorkflowKey(), activeMutation.Tasks) + + workflowKeys := []definition.WorkflowKey{request.ExecutionContext.GetWorkflowKey()} + if newRun { + workflowKeys = append(workflowKeys, definition.NewWorkflowKey( + request.ExecutionContext.GetWorkflowKey().NamespaceID, + request.ExecutionContext.GetWorkflowKey().WorkflowID, + request.NewMutableState.GetExecutionState().GetRunId(), + )) + } + h.recordDiverted(workflowKeys...) + err = h.apply( + ctx, + request.ShardContext, + request.ExecutionContext.GetArchetypeID(), + artifact, + ) + if err != nil { + h.recordApplyError(err) + return err + } + h.recordApplied() + return nil +} + +func (h *Harness) buildArtifact( + ctx context.Context, + shardContext historyi.ShardContext, + workflowKey definition.WorkflowKey, + mutableState historyi.MutableState, + exclusiveStart *persistencespb.VersionedTransition, + eventsSeq []*persistence.WorkflowEvents, +) (*replicationspb.VersionedTransitionArtifact, error) { + // SyncStateRetriever normally loads the already-persisted successor run when this + // field is set. In this test hook the successor is still only in memory, and its + // first event batch is supplied by InterceptUpdate below instead. Hide the ID from + // that lookup, then restore it on both mutable state and the generated mutation. + successorRunID := mutableState.GetExecutionInfo().GetSuccessorRunId() + mutableState.GetExecutionInfo().SuccessorRunId = "" + defer func() { + mutableState.GetExecutionInfo().SuccessorRunId = successorRunID + }() + result, err := h.newRetriever(shardContext).GetSyncWorkflowStateArtifactFromMutableState( + ctx, + workflowKey.NamespaceID, + &commonpb.WorkflowExecution{WorkflowId: workflowKey.WorkflowID, RunId: workflowKey.RunID}, + mutableState, + exclusiveStart, + nil, + wcache.NoopReleaseFn, + ) + if err != nil { + return nil, err + } + artifact := result.VersionedTransitionArtifact + if artifact.GetSyncWorkflowStateMutationAttributes() == nil { + return nil, fmt.Errorf("passivepath: expected mutation artifact for %s, got snapshot", workflowKey.String()) + } + if successorRunID != "" { + mutation := artifact.GetSyncWorkflowStateMutationAttributes().GetStateMutation() + if mutation.GetExecutionInfo() == nil { + return nil, fmt.Errorf("passivepath: mutation for %s has no execution info", workflowKey.String()) + } + mutation.ExecutionInfo.SuccessorRunId = successorRunID + } + eventBatches, err := h.serializeEvents(eventsSeq) + if err != nil { + return nil, err + } + artifact.EventBatches = eventBatches + + var firstID, lastID int64 + var count int + var eventTypes []string + for _, batch := range eventsSeq { + for _, event := range batch.Events { + if firstID == 0 || event.GetEventId() < firstID { + firstID = event.GetEventId() + } + if event.GetEventId() > lastID { + lastID = event.GetEventId() + } + count++ + eventTypes = append(eventTypes, event.GetEventType().String()) + } + } + h.logArtifactEvents(workflowKey, count, firstID, lastID, mutableState.GetNextEventID(), eventTypes) + return artifact, nil +} + +func (h *Harness) serializeEvents(eventsSeq []*persistence.WorkflowEvents) ([]*commonpb.DataBlob, error) { + var blobs []*commonpb.DataBlob + for _, events := range eventsSeq { + if len(events.Events) == 0 { + continue + } + blob, err := h.serializer.SerializeEvents(events.Events) + if err != nil { + return nil, err + } + blobs = append(blobs, blob) + } + return blobs, nil +} + +// UseTransientWorkflowContextForReplication scopes the uncached passive context to +// the synchronous replication call made by this harness. +func (h *Harness) UseTransientWorkflowContextForReplication(ctx context.Context) bool { + _, ok := ctx.Value(replicationApplyContextKey{}).(replicationApplyContext) + return ok +} + +func (h *Harness) apply( + ctx context.Context, + shardContext historyi.ShardContext, + archetypeID chasm.ArchetypeID, + artifact *replicationspb.VersionedTransitionArtifact, +) error { + ctx, cancel := context.WithTimeout(ctx, applyTimeout) + defer cancel() + ctx = context.WithValue(ctx, replicationApplyContextKey{}, replicationApplyContext{}) + + engine, err := shardContext.GetEngine(ctx) + if err != nil { + return err + } + err = engine.ReplicateVersionedTransition( + ctx, + archetypeID, + artifact, + shardContext.GetClusterMetadata().GetCurrentClusterName(), + ) + if errors.Is(err, consts.ErrDuplicate) { + return nil + } + return err +} diff --git a/tests/passivepath/harness.go b/tests/passivepath/harness.go new file mode 100644 index 00000000000..c2717f3bd36 --- /dev/null +++ b/tests/passivepath/harness.go @@ -0,0 +1,443 @@ +// Package passivepath provides a test harness that makes a single-cluster Temporal +// server persist workflow state exclusively through the *passive* replication apply +// path, so that passive task-refresh logic can be exercised by ordinary workflows. +// +// Normally an active mutation is committed by workflow.ContextImpl: +// CloseTransactionAsMutation -> UpdateWorkflowExecution, which writes both the mutable +// state and the active-generated transfer/timer tasks. This harness intercepts that +// commit, converts the just-closed mutable state into a VersionedTransitionArtifact, +// discards the active write, and applies the artifact via +// Engine.ReplicateVersionedTransition. The passive apply path then writes the mutable +// state together with tasks produced by workflow.TaskRefresher. +// +// Because there is exactly one cluster and it is active for the namespace, the +// harness can also exercise each refresher-generated task in standby mode before +// its normal active execution. ErrTaskRetry hands the task to the active executor; +// a nil standby result acknowledges it without active execution. If standby wrongly +// drops a task that the workflow needs, the workflow stalls and the test catches it. +package passivepath + +import ( + "encoding/json" + "fmt" + "maps" + "slices" + "strings" + "sync" + + enumspb "go.temporal.io/api/enums/v1" + enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/persistence/serialization" + historyi "go.temporal.io/server/service/history/interfaces" + "go.temporal.io/server/service/history/replication" + historytasks "go.temporal.io/server/service/history/tasks" +) + +// BailReason records why a write was not diverted through the passive path. +type BailReason string + +const ( + // BailPassivePolicy is a non-active transaction, including the replicator's own + // passive write. Expected and required: diverting it would recurse forever. + BailPassivePolicy BailReason = "passive-policy" + // BailNewRun is an incomplete or non-active update-with-new request. Supported + // active continue-as-new requests are diverted together with their successor run. + BailNewRun BailReason = "new-run" + // BailUpdateMode is an update mode other than UpdateCurrent (e.g. zombie workflows). + BailUpdateMode BailReason = "update-mode" + // BailNoTransitionHistory means transition history is empty, so there is nothing to + // anchor an artifact to. Requires dynamicconfig.EnableTransitionHistory. + BailNoTransitionHistory BailReason = "no-transition-history" + // BailBufferedEvents means the mutable state has buffered events. These cannot be + // represented in an artifact at all -- neither WorkflowMutableState nor + // WorkflowMutableStateMutation has a buffered_events field -- which is why the real + // SyncStateRetriever rejects them outright. They are a transient active-side staging + // area that gets folded into a real history batch at the next workflow task, after + // which they replicate normally. Real XDC waits for that flush; so do we. + BailBufferedEvents BailReason = "buffered-events" + // BailClearBufferedEvents persists the transaction that removes buffered events + // written by an earlier active bailout. The replication mutation does not carry + // this database-local cleanup because a real passive cluster never stores the + // active cluster's transient buffer. + BailClearBufferedEvents BailReason = "clear-buffered-events" + // BailNoMutableState means no mutable state was loaded, so there is nothing to + // convert into an artifact. + BailNoMutableState BailReason = "no-mutable-state" +) + +// Harness is shared by workflow-update hooks in one cluster. It owns artifact +// construction, synchronous passive application, and assertion counters. +// +// The counters matter more than they look. A bail-out is safe, but it is also the +// primary way this harness can lie: a bailed-out write +// commits *active*-generated tasks, which can mask exactly the refresher gap under +// test, while the suite still passes green. So tests assert on these, per reason, +// rather than merely logging them. +type Harness struct { + logger log.Logger + serializer serialization.Serializer + + mu sync.Mutex + intercepted int + diverted int + applied int + bailouts map[BailReason]int + applyErrs []error + + workflows map[definition.WorkflowKey]struct{} + expectedTasks map[definition.WorkflowKey]map[historytasks.Category][]historytasks.Task + allowedExtraTasks map[string]struct{} + standbyExecutions int + compareTasks bool +} + +// Intercepted is the number of workflow updates observed by the test hook. +func (h *Harness) Intercepted() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.intercepted +} + +// ActiveAttempts is the number of classified active writes: diverted writes plus +// active bailouts. It excludes the passive commit's recursive interception. +func (h *Harness) ActiveAttempts() int { + h.mu.Lock() + defer h.mu.Unlock() + count := h.diverted + for reason, bailouts := range h.bailouts { + if reason != BailPassivePolicy { + count += bailouts + } + } + return count +} + +func NewHarness(logger log.Logger) *Harness { + return &Harness{ + logger: logger, + serializer: serialization.NewSerializer(), + bailouts: make(map[BailReason]int), + workflows: make(map[definition.WorkflowKey]struct{}), + expectedTasks: make(map[definition.WorkflowKey]map[historytasks.Category][]historytasks.Task), + allowedExtraTasks: make(map[string]struct{}), + compareTasks: true, + } +} + +func (h *Harness) AllowPassiveOnlyTaskTypes(taskTypes ...string) { + h.mu.Lock() + defer h.mu.Unlock() + for _, taskType := range taskTypes { + h.allowedExtraTasks[taskType] = struct{}{} + } +} + +func (h *Harness) DisableTaskComparison() { + h.mu.Lock() + defer h.mu.Unlock() + h.compareTasks = false +} + +func (h *Harness) expectPassiveTasks( + workflowKey definition.WorkflowKey, + tasks map[historytasks.Category][]historytasks.Task, +) { + h.mu.Lock() + defer h.mu.Unlock() + h.expectedTasks[workflowKey] = tasks +} + +// comparePassiveTasks verifies exact task-generation parity at the recursive passive +// commit. Both missing and passive-only tasks are failures, including differences in +// multiplicity. +func (h *Harness) comparePassiveTasks( + workflowKey definition.WorkflowKey, + mutableState historyi.MutableState, + passiveTasks map[historytasks.Category][]historytasks.Task, +) error { + h.mu.Lock() + expected, ok := h.expectedTasks[workflowKey] + if ok { + delete(h.expectedTasks, workflowKey) + } + compareTasks := h.compareTasks + allowedExtraTasks := make(map[string]struct{}, len(h.allowedExtraTasks)) + for taskType := range h.allowedExtraTasks { + allowedExtraTasks[taskType] = struct{}{} + } + h.mu.Unlock() + if !ok || !compareTasks { + return nil + } + + stickyWorkflowTaskEvents := stickyWorkflowTaskEventIDs(expected) + expectedFingerprints, err := taskFingerprintsForComparison(expected, taskFingerprintOptions{ + stickyWorkflowTaskEvents: stickyWorkflowTaskEvents, + ignoreStickyTimeouts: true, + }) + if err != nil { + return fmt.Errorf("passivepath: fingerprint active tasks for %s: %w", workflowKey.String(), err) + } + actualFingerprints, err := taskFingerprintsForComparison(passiveTasks, taskFingerprintOptions{ + stickyWorkflowTaskEvents: stickyWorkflowTaskEvents, + }) + if err != nil { + return fmt.Errorf("passivepath: fingerprint passive tasks for %s: %w", workflowKey.String(), err) + } + missing := missingTaskFingerprints(expectedFingerprints, actualFingerprints) + extra := missingTaskFingerprints(actualFingerprints, expectedFingerprints) + extra = slices.DeleteFunc(extra, func(fingerprint string) bool { + payloadSeparator := strings.IndexByte(fingerprint, '/') + if payloadSeparator == -1 { + return false + } + payloadSeparator += 1 + strings.IndexByte(fingerprint[payloadSeparator+1:], '/') + if payloadSeparator == 0 { + return false + } + _, allowed := allowedExtraTasks[fingerprint[:payloadSeparator]] + return allowed + }) + if len(missing) != 0 || len(extra) != 0 { + executionInfo := mutableState.GetExecutionInfo() + return fmt.Errorf("passivepath: active/passive task generation differs for %s: missing=%v extra=%v active=%v passive=%v "+ + "currentTransition=%v executionStateTransition=%v visibilityTransition=%v workflowTaskTransition=%v", + workflowKey.String(), missing, extra, expectedFingerprints, actualFingerprints, + mutableState.CurrentVersionedTransition(), + mutableState.GetExecutionState().GetLastUpdateVersionedTransition(), + executionInfo.GetVisibilityLastUpdateVersionedTransition(), + executionInfo.GetWorkflowTaskLastUpdateVersionedTransition()) + } + return nil +} + +func missingTaskFingerprints(expected, actual []string) []string { + actualCounts := make(map[string]int, len(actual)) + for _, fingerprint := range actual { + actualCounts[fingerprint]++ + } + var missing []string + for _, fingerprint := range expected { + if actualCounts[fingerprint] == 0 { + missing = append(missing, fingerprint) + continue + } + actualCounts[fingerprint]-- + } + return missing +} + +func taskFingerprints(tasksByCategory map[historytasks.Category][]historytasks.Task) ([]string, error) { + return taskFingerprintsForComparison(tasksByCategory, taskFingerprintOptions{}) +} + +type taskFingerprintOptions struct { + stickyWorkflowTaskEvents map[int64]struct{} + ignoreStickyTimeouts bool +} + +func stickyWorkflowTaskEventIDs(tasksByCategory map[historytasks.Category][]historytasks.Task) map[int64]struct{} { + eventIDs := make(map[int64]struct{}) + for _, task := range tasksByCategory[historytasks.CategoryTimer] { + workflowTaskTimeout, ok := task.(*historytasks.WorkflowTaskTimeoutTask) + if ok && workflowTaskTimeout.TimeoutType == enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START { + eventIDs[workflowTaskTimeout.EventID] = struct{}{} + } + } + return eventIDs +} + +func taskFingerprintsForComparison( + tasksByCategory map[historytasks.Category][]historytasks.Task, + options taskFingerprintOptions, +) ([]string, error) { + var fingerprints []string + for category, taskList := range tasksByCategory { + if category == historytasks.CategoryReplication { + continue + } + for _, task := range taskList { + if workflowTaskTimeout, ok := task.(*historytasks.WorkflowTaskTimeoutTask); ok && + options.ignoreStickyTimeouts && + workflowTaskTimeout.TimeoutType == enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START { + continue + } + payload, err := json.Marshal(task) + if err != nil { + return nil, err + } + var normalized any + if err := json.Unmarshal(payload, &normalized); err != nil { + return nil, err + } + removeTaskIDs(normalized) + if workflowTask, ok := task.(*historytasks.WorkflowTask); ok { + if _, sticky := options.stickyWorkflowTaskEvents[workflowTask.ScheduledEventID]; sticky { + removeField(normalized, "TaskQueue") + } + } + // These task deadlines include independently generated jitter. The rest of + // their payload must match, while deterministic timer deadlines remain exact. + if task.GetType() == enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT || + task.GetType() == enumsspb.TASK_TYPE_ARCHIVAL_ARCHIVE_EXECUTION { + removeField(normalized, "VisibilityTimestamp") + } + payload, err = json.Marshal(normalized) + if err != nil { + return nil, err + } + fingerprints = append(fingerprints, fmt.Sprintf("%s/%s/%s", category.Name(), task.GetType(), payload)) + } + } + slices.Sort(fingerprints) + return fingerprints, nil +} + +func removeTaskIDs(value any) { + removeField(value, "TaskID") +} + +func removeField(value any, field string) { + switch value := value.(type) { + case map[string]any: + delete(value, field) + for _, child := range value { + removeField(child, field) + } + case []any: + for _, child := range value { + removeField(child, field) + } + default: + return + } +} + +func (h *Harness) recordIntercepted() { + h.mu.Lock() + h.intercepted++ + h.mu.Unlock() +} + +// newRetriever builds a SyncStateRetriever for artifact construction. +// +// workflowCache, workflowConsistencyChecker and eventBlobCache are nil because the +// mutation-only path neither takes another lease nor reads already-persisted events. +func (h *Harness) newRetriever(shardContext historyi.ShardContext) replication.SyncStateRetriever { + return replication.NewSyncStateRetriever(shardContext, nil, nil, nil, h.logger) +} + +func (h *Harness) recordDiverted(workflowKeys ...definition.WorkflowKey) { + h.mu.Lock() + defer h.mu.Unlock() + h.diverted++ + for _, workflowKey := range workflowKeys { + h.workflows[workflowKey] = struct{}{} + } +} + +// ShouldExecuteTaskAsPassive selects tasks belonging to workflows whose active writes were +// diverted. The router first runs the standby executor: ErrTaskRetry proceeds to the +// active executor, while nil means standby validated the task as safe to drop. +func (h *Harness) ShouldExecuteTaskAsPassive(task historytasks.Task) bool { + workflowKey := definition.NewWorkflowKey( + task.GetNamespaceID(), + task.GetWorkflowID(), + task.GetRunID(), + ) + + h.mu.Lock() + defer h.mu.Unlock() + if _, ok := h.workflows[workflowKey]; !ok { + return false + } + h.standbyExecutions++ + return true +} + +// StandbyExecutions returns the number of tasks selected for standby-first execution. +func (h *Harness) StandbyExecutions() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.standbyExecutions +} + +func (h *Harness) logArtifactEvents( + workflowKey definition.WorkflowKey, + count int, + firstID int64, + lastID int64, + nextEventID int64, + eventTypes []string, +) { + h.logger.Info(fmt.Sprintf("PPEVT workflow=%s count=%d range=[%d..%d] nextEventID=%d types=%v", + workflowKey.String(), count, firstID, lastID, nextEventID, eventTypes)) +} + +func (h *Harness) recordApplied() { + h.mu.Lock() + defer h.mu.Unlock() + h.applied++ +} + +func (h *Harness) recordBailout(reason BailReason) { + h.mu.Lock() + defer h.mu.Unlock() + h.bailouts[reason]++ +} + +// recordApplyError captures a failure from the synchronous passive apply. +func (h *Harness) recordApplyError(err error) { + h.mu.Lock() + defer h.mu.Unlock() + h.applyErrs = append(h.applyErrs, err) + h.logger.Error(fmt.Sprintf("passivepath: applying replication artifact failed: %v", err)) +} + +// Diverted is the number of active writes routed through the passive apply path. +func (h *Harness) Diverted() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.diverted +} + +// Applied is the number of artifacts successfully applied. +func (h *Harness) Applied() int { + h.mu.Lock() + defer h.mu.Unlock() + return h.applied +} + +// Bailouts returns the per-reason bail-out counts, excluding BailPassivePolicy, which +// is normal and expected (it is how the replicator's own write gets committed). +func (h *Harness) Bailouts() map[BailReason]int { + h.mu.Lock() + defer h.mu.Unlock() + out := make(map[BailReason]int, len(h.bailouts)) + for reason, count := range h.bailouts { + if reason == BailPassivePolicy { + continue + } + out[reason] = count + } + return out +} + +// AllBailouts returns every bail-out count including BailPassivePolicy. Diagnostic only; +// assertions should use Bailouts. +func (h *Harness) AllBailouts() map[BailReason]int { + h.mu.Lock() + defer h.mu.Unlock() + out := make(map[BailReason]int, len(h.bailouts)) + maps.Copy(out, h.bailouts) + return out +} + +// ApplyErrors returns failures encountered while applying artifacts. +func (h *Harness) ApplyErrors() []error { + h.mu.Lock() + defer h.mu.Unlock() + return append([]error(nil), h.applyErrs...) +} diff --git a/tests/passivepath/harness_test.go b/tests/passivepath/harness_test.go new file mode 100644 index 00000000000..0ebc2288644 --- /dev/null +++ b/tests/passivepath/harness_test.go @@ -0,0 +1,154 @@ +package passivepath + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/log" + historyi "go.temporal.io/server/service/history/interfaces" + historytasks "go.temporal.io/server/service/history/tasks" + "go.temporal.io/server/service/history/workflow" +) + +func TestMissingTaskFingerprints(t *testing.T) { + require.Empty(t, missingTaskFingerprints( + []string{"activity", "activity"}, + []string{"activity", "activity", "passive-only"}, + )) + require.Equal(t, []string{"activity"}, missingTaskFingerprints( + []string{"activity", "activity"}, + []string{"activity"}, + )) + require.Equal(t, []string{"passive-only"}, missingTaskFingerprints( + []string{"activity", "activity", "passive-only"}, + []string{"activity", "activity"}, + )) +} + +func TestTaskFingerprintsNormalizeOnlyPersistenceAndJitterFields(t *testing.T) { + workflowKey := definition.NewWorkflowKey("namespace", "workflow", "run") + activity := func(taskID int64) map[historytasks.Category][]historytasks.Task { + return map[historytasks.Category][]historytasks.Task{ + historytasks.CategoryTransfer: {&historytasks.ActivityTask{ + WorkflowKey: workflowKey, + TaskID: taskID, + TaskQueue: "queue", + ScheduledEventID: 7, + }}, + } + } + first, err := taskFingerprints(activity(1)) + require.NoError(t, err) + second, err := taskFingerprints(activity(2)) + require.NoError(t, err) + require.Equal(t, first, second) + + deleteTask := func(visibilityTime time.Time) map[historytasks.Category][]historytasks.Task { + return map[historytasks.Category][]historytasks.Task{ + historytasks.CategoryTimer: {&historytasks.DeleteHistoryEventTask{ + WorkflowKey: workflowKey, + VisibilityTimestamp: visibilityTime, + }}, + } + } + first, err = taskFingerprints(deleteTask(time.Unix(1, 0))) + require.NoError(t, err) + second, err = taskFingerprints(deleteTask(time.Unix(2, 0))) + require.NoError(t, err) + require.Equal(t, first, second) + + userTimer := func(visibilityTime time.Time) map[historytasks.Category][]historytasks.Task { + return map[historytasks.Category][]historytasks.Task{ + historytasks.CategoryTimer: {&historytasks.UserTimerTask{ + WorkflowKey: workflowKey, + VisibilityTimestamp: visibilityTime, + }}, + } + } + first, err = taskFingerprints(userTimer(time.Unix(1, 0))) + require.NoError(t, err) + second, err = taskFingerprints(userTimer(time.Unix(2, 0))) + require.NoError(t, err) + require.NotEqual(t, first, second) +} + +func TestTaskFingerprintsNormalizeStickyWorkflowTaskForPassive(t *testing.T) { + workflowKey := definition.NewWorkflowKey("namespace", "workflow", "run") + eventID := int64(7) + active := map[historytasks.Category][]historytasks.Task{ + historytasks.CategoryTransfer: {&historytasks.WorkflowTask{ + WorkflowKey: workflowKey, + TaskQueue: "sticky-queue", + ScheduledEventID: eventID, + }}, + historytasks.CategoryTimer: {&historytasks.WorkflowTaskTimeoutTask{ + WorkflowKey: workflowKey, + EventID: eventID, + TimeoutType: enumspb.TIMEOUT_TYPE_SCHEDULE_TO_START, + }}, + } + passive := map[historytasks.Category][]historytasks.Task{ + historytasks.CategoryTransfer: {&historytasks.WorkflowTask{ + WorkflowKey: workflowKey, + TaskQueue: "normal-queue", + ScheduledEventID: eventID, + }}, + } + + stickyEvents := stickyWorkflowTaskEventIDs(active) + activeFingerprints, err := taskFingerprintsForComparison(active, taskFingerprintOptions{ + stickyWorkflowTaskEvents: stickyEvents, + ignoreStickyTimeouts: true, + }) + require.NoError(t, err) + passiveFingerprints, err := taskFingerprintsForComparison(passive, taskFingerprintOptions{ + stickyWorkflowTaskEvents: stickyEvents, + }) + require.NoError(t, err) + require.Equal(t, activeFingerprints, passiveFingerprints) +} + +func TestPassiveUpdateRunsClosedTransaction(t *testing.T) { + harness := NewHarness(log.NewNoopLogger()) + prepared := false + closed := false + executed := false + payload := &workflow.ExecutionTransactionPayload{} + err := harness.InterceptUpdate( + context.Background(), + &workflow.TestHookUpdateExecutionRequest{ + UpdateExecutionTransactionPolicy: historyi.TransactionPolicyPassive, + PrepareMutableStateTransaction: func() error { + prepared = true + return nil + }, + CloseMutableStateTransaction: func() (*workflow.ExecutionTransactionPayload, error) { + closed = true + return payload, nil + }, + ExecuteExecutionTransaction: func(actual *workflow.ExecutionTransactionPayload) error { + executed = true + require.Same(t, payload, actual) + return nil + }, + }, + func() error { return nil }, + ) + require.NoError(t, err) + require.True(t, prepared) + require.True(t, closed) + require.True(t, executed) + require.Equal(t, map[BailReason]int{BailPassivePolicy: 1}, harness.AllBailouts()) +} + +func TestTransientWorkflowContextForReplicationIsScopedToMarkedContext(t *testing.T) { + harness := NewHarness(log.NewNoopLogger()) + require.False(t, harness.UseTransientWorkflowContextForReplication(context.Background())) + + ctx := context.WithValue(context.Background(), replicationApplyContextKey{}, replicationApplyContext{}) + require.True(t, harness.UseTransientWorkflowContextForReplication(ctx)) +} diff --git a/tests/passivepath/server_test.go b/tests/passivepath/server_test.go new file mode 100644 index 00000000000..eb72b3259b6 --- /dev/null +++ b/tests/passivepath/server_test.go @@ -0,0 +1,116 @@ +package passivepath + +import ( + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/testing/testhooks" +) + +// TestPassivePathServer runs a long-lived single-cluster server with the passive-path +// diversion installed, so external load generators (bench-go, omes, the temporal CLI) +// can be pointed at it. It is not an assertion-style test; it is a server. +// +// Skipped unless PASSIVEPATH_SERVER=1, so it never runs in a normal package run. +// +// PASSIVEPATH_SERVER=1 PASSIVEPATH_DURATION=10m \ +// go test -tags test_dep -count=1 -timeout 0 -v \ +// -run TestPassivePathServer ./tests/passivepath/... +// +// Env: +// +// PASSIVEPATH_SERVER must be "1" +// PASSIVEPATH_DURATION how long to stay up (default 10m) +// PASSIVEPATH_NAMESPACE global namespace to pre-create (default "default") +// PASSIVEPATH_ADDR_FILE file to write the frontend address to (default +// /tmp/passivepath_frontend_addr) +// +// The namespace is pre-created as *global* because the NDC stack -- and therefore +// ReplicateVersionedTransition -- only exists on a cluster with global namespaces +// enabled. bench-go's own RegisterNamespace call tolerates AlreadyExists, so it will +// simply reuse this one. +func TestPassivePathServer(t *testing.T) { + if os.Getenv("PASSIVEPATH_SERVER") != "1" { + t.Skip("set PASSIVEPATH_SERVER=1 to run the passive-path server") + } + + duration := 10 * time.Minute + if v := os.Getenv("PASSIVEPATH_DURATION"); v != "" { + d, err := time.ParseDuration(v) + require.NoError(t, err) + duration = d + } + ns := os.Getenv("PASSIVEPATH_NAMESPACE") + if ns == "" { + ns = "default" + } + addrFile := os.Getenv("PASSIVEPATH_ADDR_FILE") + if addrFile == "" { + addrFile = "/tmp/passivepath_frontend_addr" + } + logger := log.NewNoopLogger() // server logs would drown the stats output + harness := NewHarness(logger) + // Active execution can return the next workflow task inline and deliberately skip + // persisting its transfer task. Passive apply cannot use that active-side delivery, + // so TaskRefresher must persist the transfer task instead. + harness.AllowPassiveOnlyTaskTypes("transfer/TransferWorkflowTask") + tc := newSingleClusterWithGlobalNamespace(t, logger) + namespaceID := registerGlobalNamespace(t, tc, ns) + t.Cleanup(tc.InjectHook(t, testhooks.NewHook[testhooks.HistoryPassiveReplicationTestHook]( + testhooks.HistoryPassiveReplicationTest, + harness, + ), namespaceID)) + + addr := tc.Host().FrontendGRPCAddress() + require.NoError(t, os.WriteFile(addrFile, []byte(addr), 0o644)) + t.Cleanup(func() { _ = os.Remove(addrFile) }) + + fmt.Printf("PASSIVEPATH_READY addr=%s namespace=%s duration=%s\n", addr, ns, duration) + + deadline := time.Now().Add(duration) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + var lastApplied int + lastAt := time.Now() + for range ticker.C { + applied := harness.Applied() + now := time.Now() + rate := float64(applied-lastApplied) / now.Sub(lastAt).Seconds() + lastApplied, lastAt = applied, now + activeAttempts := harness.ActiveAttempts() + diversionPercent := 0.0 + if activeAttempts != 0 { + diversionPercent = 100 * float64(harness.Diverted()) / float64(activeAttempts) + } + + fmt.Printf("PASSIVEPATH_STATS intercepted=%d activeAttempts=%d diverted=%d diversion=%.1f%% applied=%d applies/s=%.1f "+ + "standbyExecutions=%d bailouts=%v applyErrs=%d\n", + harness.Intercepted(), activeAttempts, harness.Diverted(), diversionPercent, applied, rate, + harness.StandbyExecutions(), + harness.Bailouts(), len(harness.ApplyErrors())) + if applyErrs := harness.ApplyErrors(); len(applyErrs) != 0 { + t.Fatalf("passivepath: passive apply failed: %v", applyErrs[0]) + } + + if now.After(deadline) { + break + } + } + + fmt.Printf("PASSIVEPATH_FINAL intercepted=%d activeAttempts=%d diverted=%d applied=%d standbyExecutions=%d bailouts=%v allBailouts=%v applyErrs=%d\n", + harness.Intercepted(), harness.ActiveAttempts(), harness.Diverted(), harness.Applied(), + harness.StandbyExecutions(), + harness.Bailouts(), harness.AllBailouts(), len(harness.ApplyErrors())) + for i, err := range harness.ApplyErrors() { + if i >= 10 { + fmt.Printf("PASSIVEPATH_ERR ... and %d more\n", len(harness.ApplyErrors())-10) + break + } + fmt.Printf("PASSIVEPATH_ERR %v\n", err) + } +} From a108773969d5ec002b133a518ea0a915c6f6d215 Mon Sep 17 00:00:00 2001 From: Kannan Date: Tue, 1 Sep 2026 22:01:29 -0700 Subject: [PATCH 112/139] Stop retrying timeout error when dispatching worker command (#11851) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Stop retrying worker command dispatch when the failure is a poller timeout (UpstreamTimeout). Transport errors (gRPC unreachable, connection refused) are still retried. ## Why When a worker is gone, each `DispatchNexusTask` attempt blocks a goroutine for the full 10s dispatch timeout waiting for a poller that will never arrive. With 3 max attempts, that's ~32s of blocked resources per command. At scale this adds up: if many workers go away simultaneously (e.g., deployment rollout), each dead worker can accumulate pending cancel commands. With per-destination concurrency of 100, this means up to 100 goroutines × N dead workers × 10s per attempt × 3 attempts — thousands of goroutines blocked on matching RPCs, causing memory pressure and connection buildup. Since worker commands are best-effort, there's no value in retrying after the first timeout. If no poller appeared in 10s, the worker is likely gone. ## How did you test it? - Unit tests: updated `TestExecute_UpstreamTimeout` and `TestHandleError_UpstreamTimeout_ReturnNil` to verify no retry on timeout while transport errors still retry. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 --- common/workercommands/dispatcher.go | 10 +++++----- common/workercommands/dispatcher_test.go | 14 +++----------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/common/workercommands/dispatcher.go b/common/workercommands/dispatcher.go index 947f1a26068..04ce3b669c2 100644 --- a/common/workercommands/dispatcher.go +++ b/common/workercommands/dispatcher.go @@ -36,11 +36,9 @@ const ( // // Failure scenarios: // - No worker polling: matching returns RequestTimeout -> *nexus.HandlerError{Type: UpstreamTimeout}. -// Retryable -- worker may come up later. +// Not retried — if no poller appeared within the dispatch timeout, the worker is likely gone. // - Worker crashes after receiving the task: matching blocks waiting for a response until // context deadline, then returns RequestTimeout. Indistinguishable from "no worker polling". -// Safe to retry because commands are idempotent (e.g., cancelling a missing activity is a -// no-op success per the worker contract). // - Transport/RPC failure: *nexus.HandlerError. Retryable. // - Worker failure (worker explicitly returns error): *temporal.ApplicationError or // *temporal.CanceledError. Permanent — the worker contract requires success for all @@ -158,10 +156,12 @@ func (d *Dispatcher) handleError(nexusErr error, task *tasks.WorkerCommandsTask, // Handler-level error (transport, timeout, internal). These are constructed by // MatchingDispatchResponseToError for non-worker-returned failures. if handlerErr.Type == nexus.HandlerErrorTypeUpstreamTimeout { - d.logger.Warn("No worker polling control queue", + d.logger.Debug("No worker polling control queue, dropping command", tag.NewStringTag("control_queue", task.Destination)) d.recordCommandMetrics(task.Commands, namespaceName, "no_poller") - return nexusErr + // Don't retry — if no poller appeared within the dispatch timeout, the worker + // is likely gone. + return nil } if !handlerErr.Retryable() { diff --git a/common/workercommands/dispatcher_test.go b/common/workercommands/dispatcher_test.go index 5dc4c74ab7a..4aa74459ab4 100644 --- a/common/workercommands/dispatcher_test.go +++ b/common/workercommands/dispatcher_test.go @@ -175,11 +175,7 @@ func TestExecute_UpstreamTimeout(t *testing.T) { task := testWorkerCommandsTask() err := d.Execute(context.Background(), task, "test-namespace") - require.Error(t, err) - - var he *nexus.HandlerError - require.ErrorAs(t, err, &he) - require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, he.Type) + require.NoError(t, err, "upstream timeout should not be retried — worker is likely gone") requireMetricValue(t, capture.Snapshot(), "no_poller") } @@ -203,7 +199,7 @@ func TestHandleError_WorkerError_ReturnNil(t *testing.T) { requireMetricValue(t, capture.Snapshot(), "worker_error") } -func TestHandleError_UpstreamTimeout_ReturnRetryable(t *testing.T) { +func TestHandleError_UpstreamTimeout_ReturnNil(t *testing.T) { metricsHandler := metricstest.NewCaptureHandler() capture := metricsHandler.StartCapture() defer metricsHandler.StopCapture(capture) @@ -216,11 +212,7 @@ func TestHandleError_UpstreamTimeout_ReturnRetryable(t *testing.T) { handlerErr := nexus.NewHandlerErrorf(nexus.HandlerErrorTypeUpstreamTimeout, "upstream timeout") task := testWorkerCommandsTask() err := d.handleError(handlerErr, task, "test-namespace") - require.Error(t, err, "upstream timeout should be retried") - - var he *nexus.HandlerError - require.ErrorAs(t, err, &he) - require.Equal(t, nexus.HandlerErrorTypeUpstreamTimeout, he.Type) + require.NoError(t, err, "upstream timeout should not be retried — worker is likely gone") requireMetricValue(t, capture.Snapshot(), "no_poller") } From 899f4010b809873dc3b96ccd4c733b975af761d6 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Wed, 2 Sep 2026 09:28:39 -0700 Subject: [PATCH 113/139] Avoid refreshing workflow start tasks on later transitions (#11853) ## What changed? Workflow start task refresh now uses the workflow start versioned transition instead of the latest execution-state transition. This limits workflow start task regeneration to replication ranges that include the initial workflow transition. ## Why? Later execution-state changes can advance LastUpdateVersionedTransition while the workflow remains running. Partial task refresh then incorrectly treats the workflow as newly started and creates another WorkflowRunTimeoutTask. ## How did you test it? - [ ] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) Ran the focused TaskRefresher unit test and the single-cluster passive replication workflow and parallel-activity scenarios with exact active/passive task comparison enabled. ## Potential risks This relies on workflow start being transition count 1, matching transition-history initialization. Full refresh still includes the initial transition and continues regenerating required workflow start tasks. --- service/history/workflow/task_refresher.go | 8 +++----- service/history/workflow/task_refresher_test.go | 6 +++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/service/history/workflow/task_refresher.go b/service/history/workflow/task_refresher.go index fc8920de62c..d884c4422c6 100644 --- a/service/history/workflow/task_refresher.go +++ b/service/history/workflow/task_refresher.go @@ -224,11 +224,9 @@ func RefreshTasksForWorkflowStart( return nil } - // Skip task generation if workflow state has not been updated since minVersionedTransition. - if transitionhistory.Compare( - executionState.LastUpdateVersionedTransition, - minVersionedTransition, - ) < 0 { + // Workflow start tasks belong only to the first state transition. A transition + // count of zero represents a full refresh. + if minVersionedTransition.GetTransitionCount() > 1 { return nil } diff --git a/service/history/workflow/task_refresher_test.go b/service/history/workflow/task_refresher_test.go index 05cc3a7fa6d..0bce6709851 100644 --- a/service/history/workflow/task_refresher_test.go +++ b/service/history/workflow/task_refresher_test.go @@ -112,7 +112,7 @@ func (s *taskRefresherSuite) TestRefreshWorkflowStartTasks() { State: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, LastUpdateVersionedTransition: &persistencespb.VersionedTransition{ - TransitionCount: 1, + TransitionCount: 3, NamespaceFailoverVersion: common.EmptyVersion, }, }, @@ -164,8 +164,8 @@ func (s *taskRefresherSuite) TestRefreshWorkflowStartTasks() { s.Equal(int32(TimerTaskStatusCreated), mutableState.GetExecutionInfo().WorkflowExecutionTimerTaskStatus) err = RefreshTasksForWorkflowStart(context.Background(), mutableState, s.mockTaskGenerator, &persistencespb.VersionedTransition{ - // TransitionCount is higher than workflow state's last update versioned transition, - // no task should be generated and no call to task generator should be made. + // The replicated range starts after the workflow-start transition. A later + // execution-state update must not regenerate workflow-start tasks. TransitionCount: 2, NamespaceFailoverVersion: common.EmptyVersion, }) From acab8b1aa5fdeb7fd62bab5aa53539856bf7550e Mon Sep 17 00:00:00 2001 From: Will Duan Date: Wed, 2 Sep 2026 09:29:07 -0700 Subject: [PATCH 114/139] Avoid redundant visibility upserts during passive refresh (#11854) ## What changed? The task refresher no longer creates an UpsertExecutionVisibilityTask for the initial visibility transition, which is already represented by StartExecutionVisibilityTask. Internal workflow state changes now mark visibility as updated only when the externally visible workflow status also changes. ## Why? Scheduling the first workflow task changes the internal execution state from CREATED to RUNNING while its public status remains RUNNING. That advanced VisibilityLastUpdateVersionedTransition without producing an active visibility task, causing passive refresh to create a passive-only visibility upsert. ## How did you test it? - [ ] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) Ran focused mutable-state and TaskRefresher unit tests and the single-cluster passive replication workflow and parallel-activity scenarios with exact active/passive task comparison enabled. ## Potential risks Internal execution-state-only changes no longer advance the visibility transition marker. Public workflow status changes still do, and continue to generate the corresponding visibility work. --- .../history/workflow/mutable_state_impl.go | 5 ++++- .../workflow/mutable_state_impl_test.go | 21 +++++++++++++++++++ service/history/workflow/task_refresher.go | 8 ++++++- .../history/workflow/task_refresher_test.go | 10 ++++++++- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 091b3985ab2..f5eaf1b06ad 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -7515,7 +7515,10 @@ func (ms *MutableStateImpl) UpdateWorkflowStateStatus( ms.executionState.State != enumsspb.WORKFLOW_EXECUTION_STATE_ZOMBIE { // Suppress and Revive workflows are cluster local operations. ms.executionStateUpdated = true - ms.visibilityUpdated = true // workflow status & state change triggers visibility change as well + // Internal state changes do not affect visibility. Status changes do. + if status != ms.executionState.Status { + ms.visibilityUpdated = true + } } return true, setStateStatus(ms.executionState, state, status) } diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index bb325512270..ba74313aa81 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -1937,6 +1937,27 @@ func (s *mutableStateSuite) TestUpdateWorkflowStateStatus_Table() { } } +func (s *mutableStateSuite) TestUpdateWorkflowStateStatus_VisibilityTracking() { + s.SetupSubTest() + s.mutableState.executionState.State = enumsspb.WORKFLOW_EXECUTION_STATE_CREATED + s.mutableState.executionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + + _, err := s.mutableState.UpdateWorkflowStateStatus( + enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + ) + s.NoError(err) + s.True(s.mutableState.executionStateUpdated) + s.False(s.mutableState.visibilityUpdated) + + _, err = s.mutableState.UpdateWorkflowStateStatus( + enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + ) + s.NoError(err) + s.True(s.mutableState.visibilityUpdated) +} + func (s *mutableStateSuite) TestAddWorkflowExecutionPausedEvent() { s.SetupSubTest() s.mockEventsCache.EXPECT().PutEvent(gomock.Any(), gomock.Any()).AnyTimes() diff --git a/service/history/workflow/task_refresher.go b/service/history/workflow/task_refresher.go index d884c4422c6..902e2acfb26 100644 --- a/service/history/workflow/task_refresher.go +++ b/service/history/workflow/task_refresher.go @@ -632,10 +632,16 @@ func (r *TaskRefresherImpl) refreshTasksForWorkflowSearchAttr( return nil } + visibilityVersionedTransition := mutableState.GetExecutionInfo().VisibilityLastUpdateVersionedTransition + // The first visibility transition is handled by StartExecutionVisibilityTask. + // Generating an upsert for it would create a redundant visibility task. + if visibilityVersionedTransition.GetTransitionCount() == 1 { + return nil + } // Skip task generation if no transition since minVersionedTransition requires // an update in the visibility record. if transitionhistory.Compare( - mutableState.GetExecutionInfo().VisibilityLastUpdateVersionedTransition, + visibilityVersionedTransition, minVersionedTransition, ) < 0 { return nil diff --git a/service/history/workflow/task_refresher_test.go b/service/history/workflow/task_refresher_test.go index 0bce6709851..d93259c5a68 100644 --- a/service/history/workflow/task_refresher_test.go +++ b/service/history/workflow/task_refresher_test.go @@ -1323,7 +1323,7 @@ func (s *taskRefresherSuite) TestRefreshWorkflowSearchAttributesTasks() { NamespaceId: tests.NamespaceID.String(), WorkflowId: tests.WorkflowID, VisibilityLastUpdateVersionedTransition: &persistencespb.VersionedTransition{ - TransitionCount: 3, + TransitionCount: 1, NamespaceFailoverVersion: common.EmptyVersion, }, }, @@ -1346,6 +1346,14 @@ func (s *taskRefresherSuite) TestRefreshWorkflowSearchAttributesTasks() { s.mockTaskGenerator.EXPECT().GenerateUpsertVisibilityTask().Return(nil).Times(1) + // Workflow start uses StartExecutionVisibilityTask and must not also produce an upsert. + err = s.taskRefresher.refreshTasksForWorkflowSearchAttr(mutableState, s.mockTaskGenerator, EmptyVersionedTransition) + s.NoError(err) + + mutableState.GetExecutionInfo().VisibilityLastUpdateVersionedTransition = &persistencespb.VersionedTransition{ + TransitionCount: 3, + NamespaceFailoverVersion: common.EmptyVersion, + } err = s.taskRefresher.refreshTasksForWorkflowSearchAttr(mutableState, s.mockTaskGenerator, &persistencespb.VersionedTransition{ TransitionCount: 2, NamespaceFailoverVersion: common.EmptyVersion, From 446e22d8105e4034db93067cce9978dbe19c741a Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 2 Sep 2026 10:19:54 -0700 Subject: [PATCH 115/139] Dedupe the gRPC status unwrap behind common.GetRPCStatus (#11893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I'm trying to reduce the size of a monstrous PR I'm working on, this was one of the changes that could be teased out independently. ## Problem Four places independently unwrapped a `*status.Status` from an error with the same 12-line dance: `common.IsRetryableRPCError`, `commonnexus.ConvertGRPCError`, and a private `isRetryableRPCResponse` in both `chasm/lib/callback` and `service/history/hsm/callbacks`. ## Fix Extract it as `common.GetRPCStatus(err) (*status.Status, bool)` and delete the copies. No behavior change. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- chasm/lib/callback/invocable_internal.go | 32 ++--------------- common/nexus/failure.go | 16 +++------ common/util.go | 35 ++++++++++++------- .../history/hsm/callbacks/chasm_invocation.go | 32 ++--------------- 4 files changed, 32 insertions(+), 83 deletions(-) diff --git a/chasm/lib/callback/invocable_internal.go b/chasm/lib/callback/invocable_internal.go index b0249324475..7fd3c40d9e4 100644 --- a/chasm/lib/callback/invocable_internal.go +++ b/chasm/lib/callback/invocable_internal.go @@ -12,13 +12,12 @@ import ( tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" callbackspb "go.temporal.io/server/chasm/lib/callback/gen/callbackpb/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -93,7 +92,7 @@ func (c invocableInternal) Invoke( _, err = h.historyClient.CompleteNexusOperationChasm(ctx, request) if err != nil { msg := logInternalError(h.logger, "failed to complete Nexus operation", err) - if isRetryableRPCResponse(err) { + if common.IsRetryableRPCError(err) { return invocationResultRetry{err: msg} } return invocationResultFail{msg} @@ -102,33 +101,6 @@ func (c invocableInternal) Invoke( return invocationResultOK{} } -func isRetryableRPCResponse(err error) bool { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } - } - // nolint:exhaustive - switch st.Code() { - case codes.Canceled, - codes.Unknown, - codes.Unavailable, - codes.DeadlineExceeded, - codes.ResourceExhausted, - codes.Aborted, - codes.Internal: - return true - default: - return false - } -} - func (c invocableInternal) getHistoryRequest( refBytes []byte, requestID string, diff --git a/common/nexus/failure.go b/common/nexus/failure.go index 0b5dfb7a427..1dbc25e31b9 100644 --- a/common/nexus/failure.go +++ b/common/nexus/failure.go @@ -14,8 +14,8 @@ import ( failurepb "go.temporal.io/api/failure/v1" nexuspb "go.temporal.io/api/nexus/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common" "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" ) @@ -323,16 +323,10 @@ func nexusFailureMetadataToApplicationFailureInfo(failure nexus.Failure) (*failu // and // https://github.com/grpc-ecosystem/grpc-gateway/blob/a7cf811e6ffabeaddcfb4ff65602c12671ff326e/runtime/errors.go#L56. func ConvertGRPCError(err error, exposeDetails bool) error { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // The Nexus SDK will translate this into an internal server error and will not expose the error details. - return err - } + st, ok := common.GetRPCStatus(err) + if !ok { + // The Nexus SDK will translate this into an internal server error and will not expose the error details. + return err } errMessage := err.Error() diff --git a/common/util.go b/common/util.go index 87e02bbc63a..412a2892e14 100644 --- a/common/util.go +++ b/common/util.go @@ -762,21 +762,32 @@ func getFieldNameFromStruct(structPtr any, fieldPtr any) (string, error) { return "", serviceerror.NewInternal("field not found in the struct") } +// GetRPCStatus returns the gRPC status carried by err, or false if err is not a gRPC-induced +// error. +// +// Wrapped gRPC status errors are supported, but wrapped errors implementing +// Status() must be unwrapped before calling GetRPCStatus. +func GetRPCStatus(err error) (*status.Status, bool) { + // This isn't correct, but is to maintain existing behavior. + // + // Exposing gRPC errors via `Status()` was a convention that existed for several years, + // but the canonical way to expose error status (from google.golang.org/grpc/status) is + // by an `GRPCStatus() *status.Status` method. [status.FromError] below does unwrapping + // and checks for that. + if stGetter, ok := err.(interface{ Status() *status.Status }); ok { + return stGetter.Status(), true + } + return status.FromError(err) +} + // IsRetryableRPCError checks if the error is a retryable gRPC error. // -// This does not unwrap err: If err may be wrapped (e.g. by an HTTP client, which wraps -// RoundTripper errors in *url.Error), unwrap it yourself and pass the unwrapped error here. +// This does not unwrap err, see [GetRPCStatus]. func IsRetryableRPCError(err error) bool { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } + st, ok := GetRPCStatus(err) + if !ok { + // Not a gRPC induced error + return false } // nolint:exhaustive switch st.Code() { diff --git a/service/history/hsm/callbacks/chasm_invocation.go b/service/history/hsm/callbacks/chasm_invocation.go index bb7db429f12..db2adf9361c 100644 --- a/service/history/hsm/callbacks/chasm_invocation.go +++ b/service/history/hsm/callbacks/chasm_invocation.go @@ -11,13 +11,12 @@ import ( persistencespb "go.temporal.io/server/api/persistence/v1" tokenspb "go.temporal.io/server/api/token/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" commonnexus "go.temporal.io/server/common/nexus" "go.temporal.io/server/common/nexus/nexusrpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -71,7 +70,7 @@ func (c chasmInvocation) Invoke(ctx context.Context, ns *namespace.Namespace, e _, err = e.HistoryClient.CompleteNexusOperationChasm(ctx, request) if err != nil { redactedErr := logInternalError(e.Logger, "failed to complete Nexus operation", err) - if isRetryableRPCResponse(err) { + if common.IsRetryableRPCError(err) { return invocationResultRetry{redactedErr} } return invocationResultFail{redactedErr} @@ -133,30 +132,3 @@ func (c chasmInvocation) getHistoryRequest( return req, nil } - -func isRetryableRPCResponse(err error) bool { - var st *status.Status - stGetter, ok := err.(interface{ Status() *status.Status }) - if ok { - st = stGetter.Status() - } else { - st, ok = status.FromError(err) - if !ok { - // Not a gRPC induced error - return false - } - } - // nolint:exhaustive - switch st.Code() { - case codes.Canceled, - codes.Unknown, - codes.Unavailable, - codes.DeadlineExceeded, - codes.ResourceExhausted, - codes.Aborted, - codes.Internal: - return true - default: - return false - } -} From e4fd5c47b4f5f6d8bc13ae4959c2edad39c992ac Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 2 Sep 2026 10:26:26 -0700 Subject: [PATCH 116/139] Guard that ValidatorConfig.Validate checks every field (#11898) ## What changed? Add a new unit test for `common/callbacks.ValidatorConfig`. ## Why? While working on a PR that extended `ValidatorConfig` and adding new fields, some bugs crept in because the new fields weren't checked in the `Validate` function. This unit test will now catch those automatically, by running `Validate` on an empty `ValidatorConfig`, and then confirming that every field of the type is found in the error message. (Reporting that it is uninitialized.) This PR also allows me to remove this tiny change from an otherwise monstrous PR I am trying to slim down. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks None --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- common/callbacks/validator_test.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/common/callbacks/validator_test.go b/common/callbacks/validator_test.go index a27f19830f4..a103f33a175 100644 --- a/common/callbacks/validator_test.go +++ b/common/callbacks/validator_test.go @@ -2,6 +2,8 @@ package callbacks import ( "context" + "fmt" + "reflect" "regexp" "testing" @@ -27,6 +29,19 @@ func TestValidatorConfigValidate(t *testing.T) { require.EqualError(t, err, "missing required fields: [URLMaxLength EndpointRules]") } +// Every field of ValidatorConfig is required, and Validate names each missing one explicitly. A +// field added to the struct but not to Validate stays nil and panics at request time instead of +// failing at startup. Optional or non-nilable fields would need a different check. +func TestValidatorConfigValidateNamesEveryField(t *testing.T) { + var fields []string + for field := range reflect.TypeFor[ValidatorConfig]().Fields() { + fields = append(fields, field.Name) + } + + _, err := NewValidator(ValidatorConfig{}) + require.EqualError(t, err, fmt.Sprintf("missing required fields: %v", fields)) +} + func TestValidateCallbacks(t *testing.T) { ctx := context.Background() From 45a36d11b0f2efbd7a06d41eee04d230e5e5d76d Mon Sep 17 00:00:00 2001 From: Vladyslav Simonenko Date: Wed, 2 Sep 2026 10:45:29 -0700 Subject: [PATCH 117/139] Preserve the original workflow start time on RebuildMutableState (#11668) ## What changed? `AdminRebuildMutableState`, invoked by `tdbg workflow rebuild`, no longer replaces the mutable state start time with the current time. It reuses the run's recorded start time, falling back to `ExecutionInfo.StartTime`. Only if both `ExecutionState.StartTime` and `ExecutionInfo.StartTime` are empty, the current time will be used. The rebuild now also refreshes the timeout timer tasks so the run and execution deadlines are anchoraed at the time of the call, which is what reset already does. Without it, preserving the start time would make a workflow rebuild after its deadline timeout immediately. It also records ExecutionInfo.MutableStateRebuildTime so that a repaired run could be identify by operator. ## Why? Moving the start time to now, often results in workflow with a negative duration, since the end time is kept as is. Downstream systems may handle this incorrectly. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks 1) Some downstream systems may prefer the workflow start time to move. 2) Admin may intent for the workflow to have a new timeout instead of getting a timeout --- api/persistence/v1/executions.pb.go | 256 +++++++++--------- .../api/persistence/v1/executions.proto | 4 + service/history/ndc/conflict_resolver.go | 1 + service/history/ndc/state_rebuilder.go | 20 +- service/history/ndc/state_rebuilder_mock.go | 16 +- service/history/workflow_rebuilder.go | 38 ++- service/history/workflow_rebuilder_test.go | 69 +++++ tests/admin_test.go | 208 +++++++++++++- 8 files changed, 457 insertions(+), 155 deletions(-) diff --git a/api/persistence/v1/executions.pb.go b/api/persistence/v1/executions.pb.go index ce4ad9f364a..27072a441fd 100644 --- a/api/persistence/v1/executions.pb.go +++ b/api/persistence/v1/executions.pb.go @@ -375,8 +375,11 @@ type WorkflowExecutionInfo struct { DeclinedTargetVersionUpgrade *v17.DeclinedTargetVersionUpgrade `protobuf:"bytes,114,opt,name=declined_target_version_upgrade,json=declinedTargetVersionUpgrade,proto3" json:"declined_target_version_upgrade,omitempty"` // Time skipping info that contains the config and runtime history of the time skipping for the workflow. TimeSkippingInfo *TimeSkippingInfo `protobuf:"bytes,115,opt,name=time_skipping_info,json=timeSkippingInfo,proto3" json:"time_skipping_info,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Time of the most recent AdminRebuildMutableState call for this run, unset if the run + // was never rebuilt. + MutableStateRebuildTime *timestamppb.Timestamp `protobuf:"bytes,116,opt,name=mutable_state_rebuild_time,json=mutableStateRebuildTime,proto3" json:"mutable_state_rebuild_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkflowExecutionInfo) Reset() { @@ -1148,6 +1151,13 @@ func (x *WorkflowExecutionInfo) GetTimeSkippingInfo() *TimeSkippingInfo { return nil } +func (x *WorkflowExecutionInfo) GetMutableStateRebuildTime() *timestamppb.Timestamp { + if x != nil { + return x.MutableStateRebuildTime + } + return nil +} + type isWorkflowExecutionInfo_LastWorkflowTaskFailure interface { isWorkflowExecutionInfo_LastWorkflowTaskFailure() } @@ -4972,7 +4982,7 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\x05R\x03key\x12D\n" + "\x05value\x18\x02 \x01(\v2..temporal.server.api.persistence.v1.QueueStateR\x05value:\x028\x01J\x04\b\x04\x10\x05J\x04\b\x05\x10\x06J\x04\b\b\x10\tJ\x04\b\t\x10\n" + "J\x04\b\n" + - "\x10\vJ\x04\b\v\x10\fJ\x04\b\f\x10\rJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11\"\xd3B\n" + + "\x10\vJ\x04\b\v\x10\fJ\x04\b\f\x10\rJ\x04\b\x0e\x10\x0fJ\x04\b\x0f\x10\x10J\x04\b\x10\x10\x11\"\xacC\n" + "\x15WorkflowExecutionInfo\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12\x1f\n" + "\vworkflow_id\x18\x02 \x01(\tR\n" + @@ -5084,7 +5094,8 @@ const file_temporal_server_api_persistence_v1_executions_proto_rawDesc = "" + "!last_workflow_task_timed_out_type\x18l \x01(\x0e2\".temporal.api.enums.v1.TimeoutTypeH\x00R\x1clastWorkflowTaskTimedOutType\x12~\n" + "\x1clast_notified_target_version\x18q \x01(\v2=.temporal.server.api.persistence.v1.LastNotifiedTargetVersionR\x19lastNotifiedTargetVersion\x12|\n" + "\x1fdeclined_target_version_upgrade\x18r \x01(\v25.temporal.api.history.v1.DeclinedTargetVersionUpgradeR\x1cdeclinedTargetVersionUpgrade\x12b\n" + - "\x12time_skipping_info\x18s \x01(\v24.temporal.server.api.persistence.v1.TimeSkippingInfoR\x10timeSkippingInfo\x1ad\n" + + "\x12time_skipping_info\x18s \x01(\v24.temporal.server.api.persistence.v1.TimeSkippingInfoR\x10timeSkippingInfo\x12W\n" + + "\x1amutable_state_rebuild_time\x18t \x01(\v2\x1a.google.protobuf.TimestampR\x17mutableStateRebuildTime\x1ad\n" + "\x15SearchAttributesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x125\n" + "\x05value\x18\x02 \x01(\v2\x1f.temporal.api.common.v1.PayloadR\x05value:\x028\x01\x1aX\n" + @@ -5609,124 +5620,125 @@ var file_temporal_server_api_persistence_v1_executions_proto_depIdxs = []int32{ 4, // 45: temporal.server.api.persistence.v1.WorkflowExecutionInfo.last_notified_target_version:type_name -> temporal.server.api.persistence.v1.LastNotifiedTargetVersion 63, // 46: temporal.server.api.persistence.v1.WorkflowExecutionInfo.declined_target_version_upgrade:type_name -> temporal.api.history.v1.DeclinedTargetVersionUpgrade 2, // 47: temporal.server.api.persistence.v1.WorkflowExecutionInfo.time_skipping_info:type_name -> temporal.server.api.persistence.v1.TimeSkippingInfo - 64, // 48: temporal.server.api.persistence.v1.TimeSkippingInfo.config:type_name -> temporal.api.common.v1.TimeSkippingConfig - 48, // 49: temporal.server.api.persistence.v1.TimeSkippingInfo.accumulated_skipped_duration:type_name -> google.protobuf.Duration - 3, // 50: temporal.server.api.persistence.v1.TimeSkippingInfo.fast_forward_info:type_name -> temporal.server.api.persistence.v1.FastForwardInfo - 56, // 51: temporal.server.api.persistence.v1.TimeSkippingInfo.fast_forward_info_last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 56, // 52: temporal.server.api.persistence.v1.TimeSkippingInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 47, // 53: temporal.server.api.persistence.v1.FastForwardInfo.target_time:type_name -> google.protobuf.Timestamp - 65, // 54: temporal.server.api.persistence.v1.LastNotifiedTargetVersion.deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion - 66, // 55: temporal.server.api.persistence.v1.WorkflowExecutionState.state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState - 67, // 56: temporal.server.api.persistence.v1.WorkflowExecutionState.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 56, // 57: temporal.server.api.persistence.v1.WorkflowExecutionState.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 47, // 58: temporal.server.api.persistence.v1.WorkflowExecutionState.start_time:type_name -> google.protobuf.Timestamp - 37, // 59: temporal.server.api.persistence.v1.WorkflowExecutionState.request_ids:type_name -> temporal.server.api.persistence.v1.WorkflowExecutionState.RequestIdsEntry - 68, // 60: temporal.server.api.persistence.v1.RequestIDInfo.event_type:type_name -> temporal.api.enums.v1.EventType - 47, // 61: temporal.server.api.persistence.v1.RequestIDInfo.attach_time:type_name -> google.protobuf.Timestamp - 69, // 62: temporal.server.api.persistence.v1.TransferTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 47, // 63: temporal.server.api.persistence.v1.TransferTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 38, // 64: temporal.server.api.persistence.v1.TransferTaskInfo.close_execution_task_details:type_name -> temporal.server.api.persistence.v1.TransferTaskInfo.CloseExecutionTaskDetails - 70, // 65: temporal.server.api.persistence.v1.TransferTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo - 69, // 66: temporal.server.api.persistence.v1.ReplicationTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 47, // 67: temporal.server.api.persistence.v1.ReplicationTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 71, // 68: temporal.server.api.persistence.v1.ReplicationTaskInfo.priority:type_name -> temporal.server.api.enums.v1.TaskPriority - 56, // 69: temporal.server.api.persistence.v1.ReplicationTaskInfo.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 9, // 70: temporal.server.api.persistence.v1.ReplicationTaskInfo.task_equivalents:type_name -> temporal.server.api.persistence.v1.ReplicationTaskInfo - 72, // 71: temporal.server.api.persistence.v1.ReplicationTaskInfo.last_version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 69, // 72: temporal.server.api.persistence.v1.VisibilityTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 47, // 73: temporal.server.api.persistence.v1.VisibilityTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 47, // 74: temporal.server.api.persistence.v1.VisibilityTaskInfo.close_time:type_name -> google.protobuf.Timestamp - 47, // 75: temporal.server.api.persistence.v1.VisibilityTaskInfo.start_time:type_name -> google.protobuf.Timestamp - 70, // 76: temporal.server.api.persistence.v1.VisibilityTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo - 69, // 77: temporal.server.api.persistence.v1.TimerTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 62, // 78: temporal.server.api.persistence.v1.TimerTaskInfo.timeout_type:type_name -> temporal.api.enums.v1.TimeoutType - 73, // 79: temporal.server.api.persistence.v1.TimerTaskInfo.workflow_backoff_type:type_name -> temporal.server.api.enums.v1.WorkflowBackoffType - 47, // 80: temporal.server.api.persistence.v1.TimerTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 70, // 81: temporal.server.api.persistence.v1.TimerTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo - 56, // 82: temporal.server.api.persistence.v1.TimerTaskInfo.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 69, // 83: temporal.server.api.persistence.v1.ArchivalTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 47, // 84: temporal.server.api.persistence.v1.ArchivalTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 69, // 85: temporal.server.api.persistence.v1.OutboundTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType - 47, // 86: temporal.server.api.persistence.v1.OutboundTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp - 74, // 87: temporal.server.api.persistence.v1.OutboundTaskInfo.state_machine_info:type_name -> temporal.server.api.persistence.v1.StateMachineTaskInfo - 70, // 88: temporal.server.api.persistence.v1.OutboundTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo - 14, // 89: temporal.server.api.persistence.v1.OutboundTaskInfo.worker_commands_task:type_name -> temporal.server.api.persistence.v1.WorkerCommandsTask - 75, // 90: temporal.server.api.persistence.v1.WorkerCommandsTask.commands:type_name -> temporal.api.worker.v1.WorkerCommand - 47, // 91: temporal.server.api.persistence.v1.ActivityInfo.scheduled_time:type_name -> google.protobuf.Timestamp - 47, // 92: temporal.server.api.persistence.v1.ActivityInfo.started_time:type_name -> google.protobuf.Timestamp - 48, // 93: temporal.server.api.persistence.v1.ActivityInfo.schedule_to_start_timeout:type_name -> google.protobuf.Duration - 48, // 94: temporal.server.api.persistence.v1.ActivityInfo.schedule_to_close_timeout:type_name -> google.protobuf.Duration - 48, // 95: temporal.server.api.persistence.v1.ActivityInfo.start_to_close_timeout:type_name -> google.protobuf.Duration - 48, // 96: temporal.server.api.persistence.v1.ActivityInfo.heartbeat_timeout:type_name -> google.protobuf.Duration - 48, // 97: temporal.server.api.persistence.v1.ActivityInfo.retry_initial_interval:type_name -> google.protobuf.Duration - 48, // 98: temporal.server.api.persistence.v1.ActivityInfo.retry_maximum_interval:type_name -> google.protobuf.Duration - 47, // 99: temporal.server.api.persistence.v1.ActivityInfo.retry_expiration_time:type_name -> google.protobuf.Timestamp - 76, // 100: temporal.server.api.persistence.v1.ActivityInfo.retry_last_failure:type_name -> temporal.api.failure.v1.Failure - 77, // 101: temporal.server.api.persistence.v1.ActivityInfo.last_heartbeat_details:type_name -> temporal.api.common.v1.Payloads - 47, // 102: temporal.server.api.persistence.v1.ActivityInfo.last_heartbeat_update_time:type_name -> google.protobuf.Timestamp - 78, // 103: temporal.server.api.persistence.v1.ActivityInfo.activity_type:type_name -> temporal.api.common.v1.ActivityType - 39, // 104: temporal.server.api.persistence.v1.ActivityInfo.use_workflow_build_id_info:type_name -> temporal.server.api.persistence.v1.ActivityInfo.UseWorkflowBuildIdInfo - 55, // 105: temporal.server.api.persistence.v1.ActivityInfo.last_worker_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp - 56, // 106: temporal.server.api.persistence.v1.ActivityInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 47, // 107: temporal.server.api.persistence.v1.ActivityInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp - 47, // 108: temporal.server.api.persistence.v1.ActivityInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 79, // 109: temporal.server.api.persistence.v1.ActivityInfo.last_started_deployment:type_name -> temporal.api.deployment.v1.Deployment - 65, // 110: temporal.server.api.persistence.v1.ActivityInfo.last_deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion - 60, // 111: temporal.server.api.persistence.v1.ActivityInfo.priority:type_name -> temporal.api.common.v1.Priority - 40, // 112: temporal.server.api.persistence.v1.ActivityInfo.pause_info:type_name -> temporal.server.api.persistence.v1.ActivityInfo.PauseInfo - 53, // 113: temporal.server.api.persistence.v1.ActivityInfo.started_clock:type_name -> temporal.server.api.clock.v1.VectorClock - 47, // 114: temporal.server.api.persistence.v1.TimerInfo.expiry_time:type_name -> google.protobuf.Timestamp - 56, // 115: temporal.server.api.persistence.v1.TimerInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 80, // 116: temporal.server.api.persistence.v1.ChildExecutionInfo.parent_close_policy:type_name -> temporal.api.enums.v1.ParentClosePolicy - 53, // 117: temporal.server.api.persistence.v1.ChildExecutionInfo.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 56, // 118: temporal.server.api.persistence.v1.ChildExecutionInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 60, // 119: temporal.server.api.persistence.v1.ChildExecutionInfo.priority:type_name -> temporal.api.common.v1.Priority - 56, // 120: temporal.server.api.persistence.v1.RequestCancelInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 56, // 121: temporal.server.api.persistence.v1.SignalInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 81, // 122: temporal.server.api.persistence.v1.Checksum.flavor:type_name -> temporal.server.api.enums.v1.ChecksumFlavor - 42, // 123: temporal.server.api.persistence.v1.Callback.nexus:type_name -> temporal.server.api.persistence.v1.Callback.Nexus - 43, // 124: temporal.server.api.persistence.v1.Callback.hsm:type_name -> temporal.server.api.persistence.v1.Callback.HSM - 82, // 125: temporal.server.api.persistence.v1.Callback.links:type_name -> temporal.api.common.v1.Link - 83, // 126: temporal.server.api.persistence.v1.HSMCompletionCallbackArg.last_event:type_name -> temporal.api.history.v1.HistoryEvent - 23, // 127: temporal.server.api.persistence.v1.CallbackInfo.callback:type_name -> temporal.server.api.persistence.v1.Callback - 46, // 128: temporal.server.api.persistence.v1.CallbackInfo.trigger:type_name -> temporal.server.api.persistence.v1.CallbackInfo.Trigger - 47, // 129: temporal.server.api.persistence.v1.CallbackInfo.registration_time:type_name -> google.protobuf.Timestamp - 84, // 130: temporal.server.api.persistence.v1.CallbackInfo.state:type_name -> temporal.server.api.enums.v1.CallbackState - 47, // 131: temporal.server.api.persistence.v1.CallbackInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 76, // 132: temporal.server.api.persistence.v1.CallbackInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 47, // 133: temporal.server.api.persistence.v1.CallbackInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 48, // 134: temporal.server.api.persistence.v1.NexusOperationInfo.schedule_to_close_timeout:type_name -> google.protobuf.Duration - 47, // 135: temporal.server.api.persistence.v1.NexusOperationInfo.scheduled_time:type_name -> google.protobuf.Timestamp - 85, // 136: temporal.server.api.persistence.v1.NexusOperationInfo.state:type_name -> temporal.server.api.enums.v1.NexusOperationState - 47, // 137: temporal.server.api.persistence.v1.NexusOperationInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 76, // 138: temporal.server.api.persistence.v1.NexusOperationInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 47, // 139: temporal.server.api.persistence.v1.NexusOperationInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 48, // 140: temporal.server.api.persistence.v1.NexusOperationInfo.schedule_to_start_timeout:type_name -> google.protobuf.Duration - 48, // 141: temporal.server.api.persistence.v1.NexusOperationInfo.start_to_close_timeout:type_name -> google.protobuf.Duration - 47, // 142: temporal.server.api.persistence.v1.NexusOperationInfo.started_time:type_name -> google.protobuf.Timestamp - 47, // 143: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.requested_time:type_name -> google.protobuf.Timestamp - 86, // 144: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.state:type_name -> temporal.api.enums.v1.NexusOperationCancellationState - 47, // 145: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 76, // 146: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure - 47, // 147: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp - 47, // 148: temporal.server.api.persistence.v1.WorkflowPauseInfo.pause_time:type_name -> google.protobuf.Timestamp - 87, // 149: temporal.server.api.persistence.v1.ShardInfo.QueueStatesEntry.value:type_name -> temporal.server.api.persistence.v1.QueueState - 88, // 150: temporal.server.api.persistence.v1.WorkflowExecutionInfo.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload - 88, // 151: temporal.server.api.persistence.v1.WorkflowExecutionInfo.MemoEntry.value:type_name -> temporal.api.common.v1.Payload - 89, // 152: temporal.server.api.persistence.v1.WorkflowExecutionInfo.UpdateInfosEntry.value:type_name -> temporal.server.api.persistence.v1.UpdateInfo - 90, // 153: temporal.server.api.persistence.v1.WorkflowExecutionInfo.SubStateMachinesByTypeEntry.value:type_name -> temporal.server.api.persistence.v1.StateMachineMap - 28, // 154: temporal.server.api.persistence.v1.WorkflowExecutionInfo.ChildrenInitializedPostResetPointEntry.value:type_name -> temporal.server.api.persistence.v1.ResetChildInfo - 7, // 155: temporal.server.api.persistence.v1.WorkflowExecutionState.RequestIdsEntry.value:type_name -> temporal.server.api.persistence.v1.RequestIDInfo - 47, // 156: temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.pause_time:type_name -> google.protobuf.Timestamp - 41, // 157: temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.manual:type_name -> temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.Manual - 44, // 158: temporal.server.api.persistence.v1.Callback.Nexus.header:type_name -> temporal.server.api.persistence.v1.Callback.Nexus.HeaderEntry - 91, // 159: temporal.server.api.persistence.v1.Callback.HSM.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef - 45, // 160: temporal.server.api.persistence.v1.CallbackInfo.Trigger.workflow_closed:type_name -> temporal.server.api.persistence.v1.CallbackInfo.WorkflowClosed - 161, // [161:161] is the sub-list for method output_type - 161, // [161:161] is the sub-list for method input_type - 161, // [161:161] is the sub-list for extension type_name - 161, // [161:161] is the sub-list for extension extendee - 0, // [0:161] is the sub-list for field type_name + 47, // 48: temporal.server.api.persistence.v1.WorkflowExecutionInfo.mutable_state_rebuild_time:type_name -> google.protobuf.Timestamp + 64, // 49: temporal.server.api.persistence.v1.TimeSkippingInfo.config:type_name -> temporal.api.common.v1.TimeSkippingConfig + 48, // 50: temporal.server.api.persistence.v1.TimeSkippingInfo.accumulated_skipped_duration:type_name -> google.protobuf.Duration + 3, // 51: temporal.server.api.persistence.v1.TimeSkippingInfo.fast_forward_info:type_name -> temporal.server.api.persistence.v1.FastForwardInfo + 56, // 52: temporal.server.api.persistence.v1.TimeSkippingInfo.fast_forward_info_last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 56, // 53: temporal.server.api.persistence.v1.TimeSkippingInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 47, // 54: temporal.server.api.persistence.v1.FastForwardInfo.target_time:type_name -> google.protobuf.Timestamp + 65, // 55: temporal.server.api.persistence.v1.LastNotifiedTargetVersion.deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion + 66, // 56: temporal.server.api.persistence.v1.WorkflowExecutionState.state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState + 67, // 57: temporal.server.api.persistence.v1.WorkflowExecutionState.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 56, // 58: temporal.server.api.persistence.v1.WorkflowExecutionState.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 47, // 59: temporal.server.api.persistence.v1.WorkflowExecutionState.start_time:type_name -> google.protobuf.Timestamp + 37, // 60: temporal.server.api.persistence.v1.WorkflowExecutionState.request_ids:type_name -> temporal.server.api.persistence.v1.WorkflowExecutionState.RequestIdsEntry + 68, // 61: temporal.server.api.persistence.v1.RequestIDInfo.event_type:type_name -> temporal.api.enums.v1.EventType + 47, // 62: temporal.server.api.persistence.v1.RequestIDInfo.attach_time:type_name -> google.protobuf.Timestamp + 69, // 63: temporal.server.api.persistence.v1.TransferTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 47, // 64: temporal.server.api.persistence.v1.TransferTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 38, // 65: temporal.server.api.persistence.v1.TransferTaskInfo.close_execution_task_details:type_name -> temporal.server.api.persistence.v1.TransferTaskInfo.CloseExecutionTaskDetails + 70, // 66: temporal.server.api.persistence.v1.TransferTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo + 69, // 67: temporal.server.api.persistence.v1.ReplicationTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 47, // 68: temporal.server.api.persistence.v1.ReplicationTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 71, // 69: temporal.server.api.persistence.v1.ReplicationTaskInfo.priority:type_name -> temporal.server.api.enums.v1.TaskPriority + 56, // 70: temporal.server.api.persistence.v1.ReplicationTaskInfo.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 9, // 71: temporal.server.api.persistence.v1.ReplicationTaskInfo.task_equivalents:type_name -> temporal.server.api.persistence.v1.ReplicationTaskInfo + 72, // 72: temporal.server.api.persistence.v1.ReplicationTaskInfo.last_version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 69, // 73: temporal.server.api.persistence.v1.VisibilityTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 47, // 74: temporal.server.api.persistence.v1.VisibilityTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 47, // 75: temporal.server.api.persistence.v1.VisibilityTaskInfo.close_time:type_name -> google.protobuf.Timestamp + 47, // 76: temporal.server.api.persistence.v1.VisibilityTaskInfo.start_time:type_name -> google.protobuf.Timestamp + 70, // 77: temporal.server.api.persistence.v1.VisibilityTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo + 69, // 78: temporal.server.api.persistence.v1.TimerTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 62, // 79: temporal.server.api.persistence.v1.TimerTaskInfo.timeout_type:type_name -> temporal.api.enums.v1.TimeoutType + 73, // 80: temporal.server.api.persistence.v1.TimerTaskInfo.workflow_backoff_type:type_name -> temporal.server.api.enums.v1.WorkflowBackoffType + 47, // 81: temporal.server.api.persistence.v1.TimerTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 70, // 82: temporal.server.api.persistence.v1.TimerTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo + 56, // 83: temporal.server.api.persistence.v1.TimerTaskInfo.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 69, // 84: temporal.server.api.persistence.v1.ArchivalTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 47, // 85: temporal.server.api.persistence.v1.ArchivalTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 69, // 86: temporal.server.api.persistence.v1.OutboundTaskInfo.task_type:type_name -> temporal.server.api.enums.v1.TaskType + 47, // 87: temporal.server.api.persistence.v1.OutboundTaskInfo.visibility_time:type_name -> google.protobuf.Timestamp + 74, // 88: temporal.server.api.persistence.v1.OutboundTaskInfo.state_machine_info:type_name -> temporal.server.api.persistence.v1.StateMachineTaskInfo + 70, // 89: temporal.server.api.persistence.v1.OutboundTaskInfo.chasm_task_info:type_name -> temporal.server.api.persistence.v1.ChasmTaskInfo + 14, // 90: temporal.server.api.persistence.v1.OutboundTaskInfo.worker_commands_task:type_name -> temporal.server.api.persistence.v1.WorkerCommandsTask + 75, // 91: temporal.server.api.persistence.v1.WorkerCommandsTask.commands:type_name -> temporal.api.worker.v1.WorkerCommand + 47, // 92: temporal.server.api.persistence.v1.ActivityInfo.scheduled_time:type_name -> google.protobuf.Timestamp + 47, // 93: temporal.server.api.persistence.v1.ActivityInfo.started_time:type_name -> google.protobuf.Timestamp + 48, // 94: temporal.server.api.persistence.v1.ActivityInfo.schedule_to_start_timeout:type_name -> google.protobuf.Duration + 48, // 95: temporal.server.api.persistence.v1.ActivityInfo.schedule_to_close_timeout:type_name -> google.protobuf.Duration + 48, // 96: temporal.server.api.persistence.v1.ActivityInfo.start_to_close_timeout:type_name -> google.protobuf.Duration + 48, // 97: temporal.server.api.persistence.v1.ActivityInfo.heartbeat_timeout:type_name -> google.protobuf.Duration + 48, // 98: temporal.server.api.persistence.v1.ActivityInfo.retry_initial_interval:type_name -> google.protobuf.Duration + 48, // 99: temporal.server.api.persistence.v1.ActivityInfo.retry_maximum_interval:type_name -> google.protobuf.Duration + 47, // 100: temporal.server.api.persistence.v1.ActivityInfo.retry_expiration_time:type_name -> google.protobuf.Timestamp + 76, // 101: temporal.server.api.persistence.v1.ActivityInfo.retry_last_failure:type_name -> temporal.api.failure.v1.Failure + 77, // 102: temporal.server.api.persistence.v1.ActivityInfo.last_heartbeat_details:type_name -> temporal.api.common.v1.Payloads + 47, // 103: temporal.server.api.persistence.v1.ActivityInfo.last_heartbeat_update_time:type_name -> google.protobuf.Timestamp + 78, // 104: temporal.server.api.persistence.v1.ActivityInfo.activity_type:type_name -> temporal.api.common.v1.ActivityType + 39, // 105: temporal.server.api.persistence.v1.ActivityInfo.use_workflow_build_id_info:type_name -> temporal.server.api.persistence.v1.ActivityInfo.UseWorkflowBuildIdInfo + 55, // 106: temporal.server.api.persistence.v1.ActivityInfo.last_worker_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp + 56, // 107: temporal.server.api.persistence.v1.ActivityInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 47, // 108: temporal.server.api.persistence.v1.ActivityInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp + 47, // 109: temporal.server.api.persistence.v1.ActivityInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 79, // 110: temporal.server.api.persistence.v1.ActivityInfo.last_started_deployment:type_name -> temporal.api.deployment.v1.Deployment + 65, // 111: temporal.server.api.persistence.v1.ActivityInfo.last_deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion + 60, // 112: temporal.server.api.persistence.v1.ActivityInfo.priority:type_name -> temporal.api.common.v1.Priority + 40, // 113: temporal.server.api.persistence.v1.ActivityInfo.pause_info:type_name -> temporal.server.api.persistence.v1.ActivityInfo.PauseInfo + 53, // 114: temporal.server.api.persistence.v1.ActivityInfo.started_clock:type_name -> temporal.server.api.clock.v1.VectorClock + 47, // 115: temporal.server.api.persistence.v1.TimerInfo.expiry_time:type_name -> google.protobuf.Timestamp + 56, // 116: temporal.server.api.persistence.v1.TimerInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 80, // 117: temporal.server.api.persistence.v1.ChildExecutionInfo.parent_close_policy:type_name -> temporal.api.enums.v1.ParentClosePolicy + 53, // 118: temporal.server.api.persistence.v1.ChildExecutionInfo.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 56, // 119: temporal.server.api.persistence.v1.ChildExecutionInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 60, // 120: temporal.server.api.persistence.v1.ChildExecutionInfo.priority:type_name -> temporal.api.common.v1.Priority + 56, // 121: temporal.server.api.persistence.v1.RequestCancelInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 56, // 122: temporal.server.api.persistence.v1.SignalInfo.last_update_versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 81, // 123: temporal.server.api.persistence.v1.Checksum.flavor:type_name -> temporal.server.api.enums.v1.ChecksumFlavor + 42, // 124: temporal.server.api.persistence.v1.Callback.nexus:type_name -> temporal.server.api.persistence.v1.Callback.Nexus + 43, // 125: temporal.server.api.persistence.v1.Callback.hsm:type_name -> temporal.server.api.persistence.v1.Callback.HSM + 82, // 126: temporal.server.api.persistence.v1.Callback.links:type_name -> temporal.api.common.v1.Link + 83, // 127: temporal.server.api.persistence.v1.HSMCompletionCallbackArg.last_event:type_name -> temporal.api.history.v1.HistoryEvent + 23, // 128: temporal.server.api.persistence.v1.CallbackInfo.callback:type_name -> temporal.server.api.persistence.v1.Callback + 46, // 129: temporal.server.api.persistence.v1.CallbackInfo.trigger:type_name -> temporal.server.api.persistence.v1.CallbackInfo.Trigger + 47, // 130: temporal.server.api.persistence.v1.CallbackInfo.registration_time:type_name -> google.protobuf.Timestamp + 84, // 131: temporal.server.api.persistence.v1.CallbackInfo.state:type_name -> temporal.server.api.enums.v1.CallbackState + 47, // 132: temporal.server.api.persistence.v1.CallbackInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 76, // 133: temporal.server.api.persistence.v1.CallbackInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 47, // 134: temporal.server.api.persistence.v1.CallbackInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 48, // 135: temporal.server.api.persistence.v1.NexusOperationInfo.schedule_to_close_timeout:type_name -> google.protobuf.Duration + 47, // 136: temporal.server.api.persistence.v1.NexusOperationInfo.scheduled_time:type_name -> google.protobuf.Timestamp + 85, // 137: temporal.server.api.persistence.v1.NexusOperationInfo.state:type_name -> temporal.server.api.enums.v1.NexusOperationState + 47, // 138: temporal.server.api.persistence.v1.NexusOperationInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 76, // 139: temporal.server.api.persistence.v1.NexusOperationInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 47, // 140: temporal.server.api.persistence.v1.NexusOperationInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 48, // 141: temporal.server.api.persistence.v1.NexusOperationInfo.schedule_to_start_timeout:type_name -> google.protobuf.Duration + 48, // 142: temporal.server.api.persistence.v1.NexusOperationInfo.start_to_close_timeout:type_name -> google.protobuf.Duration + 47, // 143: temporal.server.api.persistence.v1.NexusOperationInfo.started_time:type_name -> google.protobuf.Timestamp + 47, // 144: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.requested_time:type_name -> google.protobuf.Timestamp + 86, // 145: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.state:type_name -> temporal.api.enums.v1.NexusOperationCancellationState + 47, // 146: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 76, // 147: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.last_attempt_failure:type_name -> temporal.api.failure.v1.Failure + 47, // 148: temporal.server.api.persistence.v1.NexusOperationCancellationInfo.next_attempt_schedule_time:type_name -> google.protobuf.Timestamp + 47, // 149: temporal.server.api.persistence.v1.WorkflowPauseInfo.pause_time:type_name -> google.protobuf.Timestamp + 87, // 150: temporal.server.api.persistence.v1.ShardInfo.QueueStatesEntry.value:type_name -> temporal.server.api.persistence.v1.QueueState + 88, // 151: temporal.server.api.persistence.v1.WorkflowExecutionInfo.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload + 88, // 152: temporal.server.api.persistence.v1.WorkflowExecutionInfo.MemoEntry.value:type_name -> temporal.api.common.v1.Payload + 89, // 153: temporal.server.api.persistence.v1.WorkflowExecutionInfo.UpdateInfosEntry.value:type_name -> temporal.server.api.persistence.v1.UpdateInfo + 90, // 154: temporal.server.api.persistence.v1.WorkflowExecutionInfo.SubStateMachinesByTypeEntry.value:type_name -> temporal.server.api.persistence.v1.StateMachineMap + 28, // 155: temporal.server.api.persistence.v1.WorkflowExecutionInfo.ChildrenInitializedPostResetPointEntry.value:type_name -> temporal.server.api.persistence.v1.ResetChildInfo + 7, // 156: temporal.server.api.persistence.v1.WorkflowExecutionState.RequestIdsEntry.value:type_name -> temporal.server.api.persistence.v1.RequestIDInfo + 47, // 157: temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.pause_time:type_name -> google.protobuf.Timestamp + 41, // 158: temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.manual:type_name -> temporal.server.api.persistence.v1.ActivityInfo.PauseInfo.Manual + 44, // 159: temporal.server.api.persistence.v1.Callback.Nexus.header:type_name -> temporal.server.api.persistence.v1.Callback.Nexus.HeaderEntry + 91, // 160: temporal.server.api.persistence.v1.Callback.HSM.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef + 45, // 161: temporal.server.api.persistence.v1.CallbackInfo.Trigger.workflow_closed:type_name -> temporal.server.api.persistence.v1.CallbackInfo.WorkflowClosed + 162, // [162:162] is the sub-list for method output_type + 162, // [162:162] is the sub-list for method input_type + 162, // [162:162] is the sub-list for extension type_name + 162, // [162:162] is the sub-list for extension extendee + 0, // [0:162] is the sub-list for field type_name } func init() { file_temporal_server_api_persistence_v1_executions_proto_init() } diff --git a/proto/internal/temporal/server/api/persistence/v1/executions.proto b/proto/internal/temporal/server/api/persistence/v1/executions.proto index f28631a455f..8ce61de8173 100644 --- a/proto/internal/temporal/server/api/persistence/v1/executions.proto +++ b/proto/internal/temporal/server/api/persistence/v1/executions.proto @@ -319,6 +319,10 @@ message WorkflowExecutionInfo { // Time skipping info that contains the config and runtime history of the time skipping for the workflow. TimeSkippingInfo time_skipping_info = 115; + + // Time of the most recent AdminRebuildMutableState call for this run, unset if the run + // was never rebuilt. + google.protobuf.Timestamp mutable_state_rebuild_time = 116; } message TimeSkippingInfo { diff --git a/service/history/ndc/conflict_resolver.go b/service/history/ndc/conflict_resolver.go index 354ff5ba93f..7209ea89cd3 100644 --- a/service/history/ndc/conflict_resolver.go +++ b/service/history/ndc/conflict_resolver.go @@ -158,6 +158,7 @@ func (r *ConflictResolverImpl) rebuild( rebuildVersionHistory, err := versionhistory.GetCurrentVersionHistory(rebuildVersionHistories) rebuildMutableState.GetExecutionInfo().PreviousTransitionHistory = r.mutableState.GetExecutionInfo().PreviousTransitionHistory rebuildMutableState.GetExecutionInfo().LastTransitionHistoryBreakPoint = r.mutableState.GetExecutionInfo().LastTransitionHistoryBreakPoint + rebuildMutableState.GetExecutionInfo().MutableStateRebuildTime = r.mutableState.GetExecutionInfo().MutableStateRebuildTime if err != nil { return nil, err } diff --git a/service/history/ndc/state_rebuilder.go b/service/history/ndc/state_rebuilder.go index 84b136a5d91..77fd17fa227 100644 --- a/service/history/ndc/state_rebuilder.go +++ b/service/history/ndc/state_rebuilder.go @@ -31,7 +31,7 @@ type ( StateRebuilder interface { Rebuild( ctx context.Context, - now time.Time, + startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, @@ -42,7 +42,7 @@ type ( ) (historyi.MutableState, RebuildStats, error) RebuildWithCurrentMutableState( ctx context.Context, - now time.Time, + startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, @@ -102,7 +102,7 @@ func NewStateRebuilder( func (r *StateRebuilderImpl) Rebuild( ctx context.Context, - now time.Time, + startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, @@ -113,7 +113,7 @@ func (r *StateRebuilderImpl) Rebuild( ) (historyi.MutableState, RebuildStats, error) { rebuiltMutableState, lastTxnId, err := r.buildMutableStateFromEvent( ctx, - now, + startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, @@ -151,7 +151,7 @@ func (r *StateRebuilderImpl) Rebuild( func (r *StateRebuilderImpl) RebuildWithCurrentMutableState( ctx context.Context, - now time.Time, + startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, @@ -163,7 +163,7 @@ func (r *StateRebuilderImpl) RebuildWithCurrentMutableState( // Use the original start request ID handlers can still correlate rebuilt callbacks to the correct BufferedStart entry. rebuiltMutableState, lastTxnId, err := r.buildMutableStateFromEvent( ctx, - now, + startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, @@ -223,7 +223,7 @@ func copyToRebuildMutableState( func (r *StateRebuilderImpl) buildMutableStateFromEvent( ctx context.Context, - now time.Time, + startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, @@ -248,7 +248,7 @@ func (r *StateRebuilderImpl) buildMutableStateFromEvent( rebuiltMutableState, stateBuilder := r.initializeBuilders( namespaceEntry, targetWorkflowIdentifier, - now, + startTime, ) var lastTxnId int64 @@ -307,7 +307,7 @@ func (r *StateRebuilderImpl) buildMutableStateFromEvent( func (r *StateRebuilderImpl) initializeBuilders( namespaceEntry *namespace.Namespace, workflowIdentifier definition.WorkflowKey, - now time.Time, + startTime time.Time, ) (historyi.MutableState, workflow.MutableStateRebuilder) { resetMutableState := workflow.NewMutableState( r.shard, @@ -316,7 +316,7 @@ func (r *StateRebuilderImpl) initializeBuilders( namespaceEntry, workflowIdentifier.GetWorkflowID(), workflowIdentifier.GetRunID(), - now, + startTime, ) stateBuilder := workflow.NewMutableStateRebuilder( r.shard, diff --git a/service/history/ndc/state_rebuilder_mock.go b/service/history/ndc/state_rebuilder_mock.go index fe55b8b2092..4b15658cf61 100644 --- a/service/history/ndc/state_rebuilder_mock.go +++ b/service/history/ndc/state_rebuilder_mock.go @@ -45,9 +45,9 @@ func (m *MockStateRebuilder) EXPECT() *MockStateRebuilderMockRecorder { } // Rebuild mocks base method. -func (m *MockStateRebuilder) Rebuild(ctx context.Context, now time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, baseLastEventVersion *int64, targetWorkflowIdentifier definition.WorkflowKey, targetBranchToken []byte, requestID string) (interfaces.MutableState, RebuildStats, error) { +func (m *MockStateRebuilder) Rebuild(ctx context.Context, startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, baseLastEventVersion *int64, targetWorkflowIdentifier definition.WorkflowKey, targetBranchToken []byte, requestID string) (interfaces.MutableState, RebuildStats, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Rebuild", ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID) + ret := m.ctrl.Call(m, "Rebuild", ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID) ret0, _ := ret[0].(interfaces.MutableState) ret1, _ := ret[1].(RebuildStats) ret2, _ := ret[2].(error) @@ -55,15 +55,15 @@ func (m *MockStateRebuilder) Rebuild(ctx context.Context, now time.Time, baseWor } // Rebuild indicates an expected call of Rebuild. -func (mr *MockStateRebuilderMockRecorder) Rebuild(ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID any) *gomock.Call { +func (mr *MockStateRebuilderMockRecorder) Rebuild(ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rebuild", reflect.TypeOf((*MockStateRebuilder)(nil).Rebuild), ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Rebuild", reflect.TypeOf((*MockStateRebuilder)(nil).Rebuild), ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, requestID) } // RebuildWithCurrentMutableState mocks base method. -func (m *MockStateRebuilder) RebuildWithCurrentMutableState(ctx context.Context, now time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, baseLastEventVersion *int64, targetWorkflowIdentifier definition.WorkflowKey, targetBranchToken []byte, currentMutableState *persistence.WorkflowMutableState) (interfaces.MutableState, RebuildStats, error) { +func (m *MockStateRebuilder) RebuildWithCurrentMutableState(ctx context.Context, startTime time.Time, baseWorkflowIdentifier definition.WorkflowKey, baseBranchToken []byte, baseLastEventID int64, baseLastEventVersion *int64, targetWorkflowIdentifier definition.WorkflowKey, targetBranchToken []byte, currentMutableState *persistence.WorkflowMutableState) (interfaces.MutableState, RebuildStats, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RebuildWithCurrentMutableState", ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState) + ret := m.ctrl.Call(m, "RebuildWithCurrentMutableState", ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState) ret0, _ := ret[0].(interfaces.MutableState) ret1, _ := ret[1].(RebuildStats) ret2, _ := ret[2].(error) @@ -71,7 +71,7 @@ func (m *MockStateRebuilder) RebuildWithCurrentMutableState(ctx context.Context, } // RebuildWithCurrentMutableState indicates an expected call of RebuildWithCurrentMutableState. -func (mr *MockStateRebuilderMockRecorder) RebuildWithCurrentMutableState(ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState any) *gomock.Call { +func (mr *MockStateRebuilderMockRecorder) RebuildWithCurrentMutableState(ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebuildWithCurrentMutableState", reflect.TypeOf((*MockStateRebuilder)(nil).RebuildWithCurrentMutableState), ctx, now, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RebuildWithCurrentMutableState", reflect.TypeOf((*MockStateRebuilder)(nil).RebuildWithCurrentMutableState), ctx, startTime, baseWorkflowIdentifier, baseBranchToken, baseLastEventID, baseLastEventVersion, targetWorkflowIdentifier, targetBranchToken, currentMutableState) } diff --git a/service/history/workflow_rebuilder.go b/service/history/workflow_rebuilder.go index 60b1d77f958..0bbc8ae0b8a 100644 --- a/service/history/workflow_rebuilder.go +++ b/service/history/workflow_rebuilder.go @@ -5,6 +5,7 @@ package history import ( "context" "math" + "time" commonpb "go.temporal.io/api/common/v1" "go.temporal.io/api/serviceerror" @@ -13,6 +14,7 @@ import ( "go.temporal.io/server/common/definition" "go.temporal.io/server/common/locks" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" "go.temporal.io/server/common/persistence/versionhistory" @@ -21,6 +23,7 @@ import ( "go.temporal.io/server/service/history/ndc" "go.temporal.io/server/service/history/workflow" wcache "go.temporal.io/server/service/history/workflow/cache" + "google.golang.org/protobuf/types/known/timestamppb" ) type ( @@ -100,6 +103,14 @@ func (r *workflowRebuilderImpl) rebuild( if err != nil { return err } + rebuildMutableState.GetExecutionInfo().MutableStateRebuildTime = timestamppb.New(r.shard.GetTimeSource().Now()) + // The run deadline is recomputed from the original start time and the execution deadline is + // copied from the started event, so either can already be in the past and its refreshed + // timer task would fire at once. Re-anchor both deadlines at the current time and regenerate + // the timeout timer tasks from them, similarly to what reset does. + if err := rebuildMutableState.RefreshExpirationTimeoutTask(ctx); err != nil { + return err + } return r.overwriteToDB(ctx, rebuildMutableState) } @@ -198,6 +209,31 @@ func (r *workflowRebuilderImpl) getRebuildSpecFromMutableState( }, nil } +// rebuildStartTime returns the start time the rebuilt mutable state should carry. +func (r *workflowRebuilderImpl) rebuildStartTime( + workflowKey definition.WorkflowKey, + mutableState *persistencespb.WorkflowMutableState, +) time.Time { + if startTime := mutableState.GetExecutionState().GetStartTime(); startTime != nil { + return startTime.AsTime() + } + // Fallback to ExecutionInfo.StartTime if ExecutionState.StartTime is unexpectedly nil. + if startTime := mutableState.GetExecutionInfo().GetStartTime(); startTime != nil { + r.logger.Warn("workflowRebuilder unable to find a recorded start time, falling back to ExecutionInfo.StartTime", + tag.WorkflowNamespaceID(workflowKey.NamespaceID), + tag.WorkflowID(workflowKey.WorkflowID), + tag.WorkflowRunID(workflowKey.RunID), + ) + return startTime.AsTime() + } + r.logger.Warn("workflowRebuilder unable to find a recorded start time, falling back to current time", + tag.WorkflowNamespaceID(workflowKey.NamespaceID), + tag.WorkflowID(workflowKey.WorkflowID), + tag.WorkflowRunID(workflowKey.RunID), + ) + return r.shard.GetTimeSource().Now() +} + func (r *workflowRebuilderImpl) replayResetWorkflow( ctx context.Context, workflowKey definition.WorkflowKey, @@ -208,7 +244,7 @@ func (r *workflowRebuilderImpl) replayResetWorkflow( ) (historyi.MutableState, error) { rebuildMutableState, rebuildStats, err := ndc.NewStateRebuilder(r.shard, r.logger).RebuildWithCurrentMutableState( ctx, - r.shard.GetTimeSource().Now(), + r.rebuildStartTime(workflowKey, mutableState), workflowKey, branchToken, math.MaxInt64-1, // NOTE: this is last event ID, layer below will +1 to calculate the next event ID diff --git a/service/history/workflow_rebuilder_test.go b/service/history/workflow_rebuilder_test.go index b841487ab94..89ab10b4751 100644 --- a/service/history/workflow_rebuilder_test.go +++ b/service/history/workflow_rebuilder_test.go @@ -2,13 +2,20 @@ package history import ( "testing" + "time" "github.com/stretchr/testify/require" "go.temporal.io/api/serviceerror" historyspb "go.temporal.io/server/api/history/v1" persistencespb "go.temporal.io/server/api/persistence/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/common/clock" + "go.temporal.io/server/common/definition" "go.temporal.io/server/common/log" + "go.temporal.io/server/service/history/shard" + "go.temporal.io/server/service/history/tests" + "go.uber.org/mock/gomock" + "google.golang.org/protobuf/types/known/timestamppb" ) func TestWorkflowRebuilderImpl_RebuildableCheck(t *testing.T) { @@ -215,3 +222,65 @@ func TestWorkflowRebuilderImpl_RebuildableCheck(t *testing.T) { }) } } + +func TestWorkflowRebuilderImpl_RebuildStartTime(t *testing.T) { + recordedStartTime := time.Date(2026, 7, 15, 10, 0, 0, 0, time.UTC) + rebuildTime := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) + + testCases := []struct { + name string + mutableState *persistencespb.WorkflowMutableState + expected time.Time + }{ + { + name: "start time on execution state", + mutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{ + StartTime: timestamppb.New(recordedStartTime), + }, + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{}, + }, + expected: recordedStartTime, + }, + { + name: "fallback to start time on execution info", + mutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{}, + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{ + StartTime: timestamppb.New(recordedStartTime), + }, + }, + expected: recordedStartTime, + }, + { + name: "no start time recorded anywhere", + mutableState: &persistencespb.WorkflowMutableState{ + ExecutionState: &persistencespb.WorkflowExecutionState{}, + ExecutionInfo: &persistencespb.WorkflowExecutionInfo{}, + }, + expected: rebuildTime, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockShard := shard.NewTestContextWithTimeSource( + ctrl, + &persistencespb.ShardInfo{ShardId: 0, RangeId: 1}, + tests.NewDynamicConfig(), + clock.NewEventTimeSource().Update(rebuildTime), + ) + rebuilder := &workflowRebuilderImpl{ + shard: mockShard, + logger: log.NewTestLogger(), + } + + startTime := rebuilder.rebuildStartTime( + definition.NewWorkflowKey(tests.NamespaceID.String(), tests.WorkflowID, tests.RunID), + tc.mutableState, + ) + require.Equal(t, tc.expected, startTime.UTC()) + }) + } +} diff --git a/tests/admin_test.go b/tests/admin_test.go index f9b79deff1d..66dd06bdbda 100644 --- a/tests/admin_test.go +++ b/tests/admin_test.go @@ -7,6 +7,7 @@ import ( "github.com/google/uuid" commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" sdkclient "go.temporal.io/sdk/client" "go.temporal.io/sdk/workflow" "go.temporal.io/server/api/adminservice/v1" @@ -32,18 +33,7 @@ func TestAdminRebuildMutableState_ChasmEnabled(t *testing.T) { } func (s *AdminTestSuite) TestAdminRebuildMutableState(testWithChasm bool) { - var opts []testcore.TestOption - if testWithChasm { - opts = append(opts, testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true)) - } - env := testcore.NewEnv(s.T(), opts...) - - if testWithChasm { - configValues := env.GetTestCluster().Host().DcClient().GetValue(dynamicconfig.EnableChasm.Key()) - s.NotEmpty(configValues, "EnableChasm config should be set") - configValue, _ := configValues[0].Value.(bool) - s.True(configValue, "EnableChasm config should be true") - } + env := s.newRebuildEnv(testWithChasm) tv := testvars.New(s.T()) workflowFn := func(ctx workflow.Context) error { @@ -138,11 +128,201 @@ func (s *AdminTestSuite) TestAdminRebuildMutableState(testWithChasm bool) { TransitionCount: response1.DatabaseMutableState.ExecutionInfo.StateTransitionCount + 1, }, response2.DatabaseMutableState.ExecutionState.LastUpdateVersionedTransition) - // Rebuild explicitly sets start time, thus start time will change after rebuild. + // Rebuild recreates mutable state for the same run, so the recorded start time must survive it. s.NotNil(response1.DatabaseMutableState.ExecutionState.StartTime) s.NotNil(response2.DatabaseMutableState.ExecutionState.StartTime) timeBefore := timestamp.TimeValue(response1.DatabaseMutableState.ExecutionState.StartTime) timeAfter := timestamp.TimeValue(response2.DatabaseMutableState.ExecutionState.StartTime) - s.False(timeAfter.Before(timeBefore)) + s.Equal(timeBefore, timeAfter) + + s.Equal( + timestamp.TimeValue(response1.DatabaseMutableState.ExecutionInfo.ExecutionTime), + timestamp.TimeValue(response2.DatabaseMutableState.ExecutionInfo.ExecutionTime), + ) + s.Nil(response1.DatabaseMutableState.ExecutionInfo.MutableStateRebuildTime) + s.NotNil(response2.DatabaseMutableState.ExecutionInfo.MutableStateRebuildTime) +} + +// TestAdminRebuildMutableStateRunTimeout checks that the rebuild re-anchors the run timeout +// deadline at the rebuild time, the way reset does, and that the run still times out there. +func (s *AdminTestSuite) TestAdminRebuildMutableStateRunTimeout(testWithChasm bool) { + const runTimeout = 10 * time.Second + + env := s.newRebuildEnv(testWithChasm) + ctx, cancel := context.WithTimeout(s.Context(), 90*time.Second) + defer cancel() + + workflowFn := func(ctx workflow.Context) error { + return workflow.Sleep(ctx, time.Hour) + } + env.SdkWorker().RegisterWorkflow(workflowFn) + + workflowID := testvars.New(s.T()).Any().String() + _, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: workflowID, + TaskQueue: env.WorkerTaskQueue(), + // No execution timeout, so the run deadline is purely start time plus run timeout. + WorkflowRunTimeout: runTimeout, + }, workflowFn) + s.NoError(err) + + before := s.awaitRebuildableMutableState(ctx, env, workflowID) + originalRunExpiration := timestamp.TimeValue(before.ExecutionInfo.WorkflowRunExpirationTime) + s.False(originalRunExpiration.IsZero()) + s.Zero(timestamp.TimeValue(before.ExecutionInfo.WorkflowExecutionExpirationTime)) + + s.rebuildMutableState(ctx, env, workflowID) + + after := s.describeMutableState(ctx, env, workflowID) + runExpiration := timestamp.TimeValue(after.ExecutionInfo.WorkflowRunExpirationTime) + s.True(runExpiration.After(originalRunExpiration)) + s.NotNil(after.ExecutionInfo.MutableStateRebuildTime) + s.Zero( + timestamp.TimeValue(after.ExecutionInfo.WorkflowExecutionExpirationTime), + ) + + s.assertTimesOutAt(ctx, env, workflowID, runExpiration) +} + +// TestAdminRebuildMutableStateExecutionTimeout checks that the rebuild re-anchors the execution +// timeout deadline at the rebuild time and that the workflow still times out there, the same thing +// TestAdminRebuildMutableStateRunTimeout checks for the run timeout deadline. +// +// The continue-as-new is what puts the execution timeout timer under test. The server skips the +// WorkflowExecutionTimeoutTask on a chain's first run and lets the run +// timeout timer enforce the deadline there, so a first run would exercise the same path as +// TestAdminRebuildMutableStateRunTimeout. On the second run the opposite holds: the run timeout +// defaults to the execution timeout, which makes both deadlines the same instant, and +// GenerateWorkflowStartTasks then skips the run timeout task, leaving the execution timeout timer +// as the only thing that can end the workflow. +func (s *AdminTestSuite) TestAdminRebuildMutableStateExecutionTimeout(testWithChasm bool) { + const executionTimeout = 15 * time.Second + + env := s.newRebuildEnv(testWithChasm) + ctx, cancel := context.WithTimeout(s.Context(), 90*time.Second) + defer cancel() + + var workflowFn func(ctx workflow.Context) error + workflowFn = func(ctx workflow.Context) error { + if workflow.GetInfo(ctx).ContinuedExecutionRunID == "" { + return workflow.NewContinueAsNewError(ctx, workflowFn) + } + return workflow.Sleep(ctx, time.Hour) + } + env.SdkWorker().RegisterWorkflow(workflowFn) + + workflowID := testvars.New(s.T()).Any().String() + run, err := env.SdkClient().ExecuteWorkflow(ctx, sdkclient.StartWorkflowOptions{ + ID: workflowID, + TaskQueue: env.WorkerTaskQueue(), + WorkflowExecutionTimeout: executionTimeout, + }, workflowFn) + s.NoError(err) + firstRunID := run.GetRunID() + + s.Await(func(s *AdminTestSuite) { + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, workflowID, "") + s.NoError(err) + s.NotEqual(firstRunID, desc.GetWorkflowExecutionInfo().GetExecution().GetRunId(), + "workflow should continue as new") + }, 20*time.Second, 100*time.Millisecond) + + before := s.awaitRebuildableMutableState(ctx, env, workflowID) + originalExecutionExpiration := timestamp.TimeValue(before.ExecutionInfo.WorkflowExecutionExpirationTime) + originalRunExpiration := timestamp.TimeValue(before.ExecutionInfo.WorkflowRunExpirationTime) + s.False(originalExecutionExpiration.IsZero(), "execution timeout deadline should be set before the rebuild") + s.NotZero(before.ExecutionInfo.WorkflowExecutionTimerTaskStatus, + "the second run should carry an execution timeout timer task") + + s.rebuildMutableState(ctx, env, workflowID) + + after := s.describeMutableState(ctx, env, workflowID) + executionExpiration := timestamp.TimeValue(after.ExecutionInfo.WorkflowExecutionExpirationTime) + runExpiration := timestamp.TimeValue(after.ExecutionInfo.WorkflowRunExpirationTime) + s.True(executionExpiration.After(originalExecutionExpiration), + "rebuild should re-anchor the execution deadline") + s.True(runExpiration.After(originalRunExpiration), "rebuild should re-anchor the run deadline") + s.NotZero(after.ExecutionInfo.WorkflowExecutionTimerTaskStatus, + "the execution timeout timer task should survive the rebuild") + + s.assertTimesOutAt(ctx, env, workflowID, executionExpiration) +} + +func (s *AdminTestSuite) newRebuildEnv(testWithChasm bool) *testcore.TestEnv { + var opts []testcore.TestOption + if testWithChasm { + opts = append(opts, testcore.WithDynamicConfig(dynamicconfig.EnableChasm, true)) + } + env := testcore.NewEnv(s.T(), opts...) + + if testWithChasm { + configValues := env.GetTestCluster().Host().DcClient().GetValue(dynamicconfig.EnableChasm.Key()) + s.NotEmpty(configValues, "EnableChasm config should be set") + configValue, _ := configValues[0].Value.(bool) + s.True(configValue, "EnableChasm config should be true") + } + return env +} + +func (s *AdminTestSuite) describeMutableState( + ctx context.Context, + env *testcore.TestEnv, + workflowID string, +) *persistencespb.WorkflowMutableState { + resp, err := env.AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID}, + Archetype: chasm.WorkflowArchetype, + }) + s.NoError(err) + return resp.DatabaseMutableState +} + +// awaitRebuildableMutableState waits until the current run has completed its first workflow task, +// so that the rebuild replays more than just the start event. +func (s *AdminTestSuite) awaitRebuildableMutableState( + ctx context.Context, + env *testcore.TestEnv, + workflowID string, +) *persistencespb.WorkflowMutableState { + var mutableState *persistencespb.WorkflowMutableState + s.Await(func(s *AdminTestSuite) { + mutableState = s.describeMutableState(ctx, env, workflowID) + s.Positive(mutableState.ExecutionInfo.LastCompletedWorkflowTaskStartedEventId, + "workflow should complete its first workflow task") + }, 20*time.Second, 50*time.Millisecond) + return mutableState +} + +func (s *AdminTestSuite) rebuildMutableState( + ctx context.Context, + env *testcore.TestEnv, + workflowID string, +) { + _, err := env.AdminClient().RebuildMutableState(ctx, &adminservice.RebuildMutableStateRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID}, + }) + s.NoError(err) +} + +// assertTimesOutAt asserts that the workflow survives the rebuild and then times out at deadline. +func (s *AdminTestSuite) assertTimesOutAt( + ctx context.Context, + env *testcore.TestEnv, + workflowID string, + deadline time.Time, +) { + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, workflowID, "") + s.NoError(err) + s.NotEqual(enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT, desc.GetWorkflowExecutionInfo().GetStatus(), + "workflow should not time out immediately after the rebuild") + + s.Await(func(s *AdminTestSuite) { + desc, err := env.SdkClient().DescribeWorkflowExecution(ctx, workflowID, "") + s.NoError(err) + s.Equal(enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT, desc.GetWorkflowExecutionInfo().GetStatus(), + "workflow did not time out at its deadline") + }, time.Until(deadline)+5*time.Second, 250*time.Millisecond) } From e4efc679dd34345dd672d125d329f1df56a2789f Mon Sep 17 00:00:00 2001 From: Shivam <57200924+Shivs11@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:46:28 -0400 Subject: [PATCH 118/139] Fix matching dropped metric test race (#11849) ## What changed? - Reuse one capture handler across each dropped-task metric table test. - Install the handler before any task queue partition is created. ## Why? Each table case replaced `matchingEngine.metricsHandler` while a prior case's task queue could still be initializing asynchronously. The queue goroutine reads the handler while emitting physical task queue gauges, producing the race reported in https://github.com/temporalio/temporal/actions/runs/33111032921/job/98654644755. The capture handler already synchronizes capture sessions, so keeping the engine handler stable removes the unsafe write without changing production matching behavior. ## How did you test it? - [x] Priority and fairness variants: all four affected tests passed 50 iterations under the race detector. - [x] Classic variant: no race was reported; a repeated batch hit an unrelated task-queue-close teardown log, and the required isolated rerun passed. - [x] Adversarial review completed with no findings. ## Potential risks Low. This only changes test setup and introduces no imports, dependencies, or production behavior. --- > [!NOTE] > **Low Risk** > Test-only changes to metric capture setup; no production matching behavior. > > **Overview** > Fixes a **data race** in matching engine dropped-task metric tests by keeping `matchingEngine.metricsHandler` stable for the whole table-driven test instead of swapping it in every subtest. > > `captureDroppedOnEngine` is now documented and enforced to run **once** before any task queue partition exists (`getTaskQueuePartitions` must be empty). The four `tasks_dropped` poll-path tests install a single `metricstest.CaptureHandler` at the start and reuse it across subtests; each subtest only starts/stops a capture session (`defer StopCapture`). That avoids concurrent reads of the handler from async partition initialization while another subtest overwrites it. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit a2f545f3af33443a46ae52f30b2db238f273b559. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --- service/matching/matching_engine_test.go | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/service/matching/matching_engine_test.go b/service/matching/matching_engine_test.go index 93e82d09dca..c2dacf4faf3 100644 --- a/service/matching/matching_engine_test.go +++ b/service/matching/matching_engine_test.go @@ -767,9 +767,10 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_NamespaceHandover() { } // captureDroppedOnEngine points the engine's root metrics handler at a capture handler -// so drops emitted down the partition handler chain are recorded. Must be called before -// the task queue partition is created (i.e. before AddActivity/WorkflowTask). +// so drops emitted down the partition handler chain are recorded. Must be called once +// before any task queue partitions are created. func (s *matchingEngineSuite) captureDroppedOnEngine() *metricstest.CaptureHandler { + s.Empty(s.matchingEngine.getTaskQueuePartitions(1), "captureDroppedOnEngine must run before any partition is loaded") capture := metricstest.NewCaptureHandler() s.matchingEngine.metricsHandler = capture return capture @@ -778,6 +779,8 @@ func (s *matchingEngineSuite) captureDroppedOnEngine() *metricstest.CaptureHandl // TestPollActivityTaskQueues_DroppedTaskMetric asserts each error path that drops an // activity task emits tasks_dropped with the right `reason` tag. func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric() { + capture := s.captureDroppedOnEngine() + cases := []struct { name string err error @@ -799,8 +802,6 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric() { s.logger.Expect(testlogger.Error, "dropping task due to non-nonretryable errors") } - capture := s.captureDroppedOnEngine() - namespaceID := uuid.NewString() taskQueue := &taskqueuepb.TaskQueue{Name: "queue-" + tc.name, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} @@ -817,6 +818,7 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric() { Return(nil, tc.err).Times(1) c := capture.StartCapture() + defer capture.StopCapture(c) resp, err := s.matchingEngine.PollActivityTaskQueue(context.Background(), &matchingservice.PollActivityTaskQueueRequest{ NamespaceId: namespaceID, @@ -831,7 +833,6 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric() { recordings := c.Snapshot()[metrics.DroppedTasksCounter.Name()] s.Len(recordings, 1, "expected one tasks_dropped emission for %s", tc.name) s.Equal(tc.wantReason, recordings[0].Tags["reason"]) - capture.StopCapture(c) }) } } @@ -839,6 +840,8 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric() { // TestPollWorkflowTaskQueues_DroppedTaskMetric is the workflow counterpart of // TestPollActivityTaskQueues_DroppedTaskMetric (no ActivityStartDuringTransition arm). func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric() { + capture := s.captureDroppedOnEngine() + cases := []struct { name string err error @@ -859,8 +862,6 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric() { s.logger.Expect(testlogger.Error, "dropping task due to non-nonretryable errors") } - capture := s.captureDroppedOnEngine() - namespaceID := uuid.NewString() taskQueue := &taskqueuepb.TaskQueue{Name: "wf-queue-" + tc.name, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} @@ -877,6 +878,7 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric() { Return(nil, tc.err).Times(1) c := capture.StartCapture() + defer capture.StopCapture(c) resp, err := s.matchingEngine.PollWorkflowTaskQueue(context.Background(), &matchingservice.PollWorkflowTaskQueueRequest{ NamespaceId: namespaceID, @@ -891,7 +893,6 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric() { recordings := c.Snapshot()[metrics.DroppedTasksCounter.Name()] s.Len(recordings, 1, "expected one tasks_dropped emission for %s", tc.name) s.Equal(tc.wantReason, recordings[0].Tags["reason"]) - capture.StopCapture(c) }) } } @@ -900,6 +901,8 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric() { // asserts errors propagated back to the caller (ResourceExhausted, NamespaceHandover) do // not increment tasks_dropped. func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric_NoEmissionOnPropagatedErrors() { + capture := s.captureDroppedOnEngine() + cases := []struct { name string err error @@ -910,8 +913,6 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric_NoEmi for _, tc := range cases { s.Run(tc.name, func() { - capture := s.captureDroppedOnEngine() - namespaceID := uuid.NewString() taskQueue := &taskqueuepb.TaskQueue{Name: "queue-noemit-" + tc.name, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} @@ -928,6 +929,7 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric_NoEmi Return(nil, tc.err).Times(1) c := capture.StartCapture() + defer capture.StopCapture(c) _, err = s.matchingEngine.PollActivityTaskQueue(context.Background(), &matchingservice.PollActivityTaskQueueRequest{ NamespaceId: namespaceID, @@ -940,12 +942,13 @@ func (s *matchingEngineSuite) TestPollActivityTaskQueues_DroppedTaskMetric_NoEmi recordings := c.Snapshot()[metrics.DroppedTasksCounter.Name()] s.Empty(recordings, "tasks_dropped must not fire when the error is returned to the caller (%s)", tc.name) - capture.StopCapture(c) }) } } func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric_NoEmissionOnPropagatedErrors() { + capture := s.captureDroppedOnEngine() + cases := []struct { name string err error @@ -956,8 +959,6 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric_NoEmi for _, tc := range cases { s.Run(tc.name, func() { - capture := s.captureDroppedOnEngine() - namespaceID := uuid.NewString() taskQueue := &taskqueuepb.TaskQueue{Name: "wf-queue-noemit-" + tc.name, Kind: enumspb.TASK_QUEUE_KIND_NORMAL} @@ -974,6 +975,7 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric_NoEmi Return(nil, tc.err).Times(1) c := capture.StartCapture() + defer capture.StopCapture(c) _, err = s.matchingEngine.PollWorkflowTaskQueue(context.Background(), &matchingservice.PollWorkflowTaskQueueRequest{ NamespaceId: namespaceID, @@ -986,7 +988,6 @@ func (s *matchingEngineSuite) TestPollWorkflowTaskQueues_DroppedTaskMetric_NoEmi recordings := c.Snapshot()[metrics.DroppedTasksCounter.Name()] s.Empty(recordings, "tasks_dropped must not fire when the error is returned to the caller (%s)", tc.name) - capture.StopCapture(c) }) } } From 985f19596b18f0a67a1781c7e18ff563892537ee Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Wed, 2 Sep 2026 13:33:38 -0700 Subject: [PATCH 119/139] Fixes to Visibility SQL query converter (#11801) ## What changed? - Fix parsing ExecutionStatus in tuple (eg: `ExecutionStatus IN ('Running', 'Completed')`) - Fix parsing negative double values (eg: `CustomDouble > -1.5` was returning an error) - Return error when comparing non-bool types with boolean value (eg: `StartTime = true` was not returning an error). - Return error when comparing Text type search attribute with an empty string/no tokens (eg: `CustomText = ' '`). It was already returning an error with PostgreSQL and SQLite, now also returning an error with MySQL and Elasticsearch. - Added unit tests covering all Visibility stores to make sure they all behave the same. One exception: `ExecutionStatus STARTS_WITH` works with Elasticsearch, but not with SQL. That's because in Elasticsearch we store as string, while in SQL we store as int. ## Why? Bug fixes, and comprehensive unit tests to ensure uniform behavior. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../store/elasticsearch/query_converter.go | 47 +- .../elasticsearch/query_converter_test.go | 22 +- .../store/elasticsearch/visibility_store.go | 10 +- .../visibility/store/query/converter.go | 65 +- .../visibility/store/sql/query_converter.go | 14 +- .../store/tests/query_converter_test.go | 1293 +++++++++++++++++ 6 files changed, 1408 insertions(+), 43 deletions(-) create mode 100644 common/persistence/visibility/store/tests/query_converter_test.go diff --git a/common/persistence/visibility/store/elasticsearch/query_converter.go b/common/persistence/visibility/store/elasticsearch/query_converter.go index 0e67cc5dd37..4d9c7c78819 100644 --- a/common/persistence/visibility/store/elasticsearch/query_converter.go +++ b/common/persistence/visibility/store/elasticsearch/query_converter.go @@ -6,25 +6,29 @@ import ( "github.com/olivere/elastic/v7" "github.com/temporalio/sqlparser" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/visibility/store/query" + "go.temporal.io/server/common/searchattribute" ) -type queryConverter struct{} +type esQueryConverter struct{} -var _ query.StoreQueryConverter[elastic.Query] = (*queryConverter)(nil) +var _ query.StoreQueryConverter[elastic.Query] = (*esQueryConverter)(nil) -func (c *queryConverter) GetDatetimeFormat() string { +func (c *esQueryConverter) GetDatetimeFormat() string { return time.RFC3339Nano } -func (c *queryConverter) BuildParenExpr(expr elastic.Query) (elastic.Query, error) { +func (c *esQueryConverter) BuildParenExpr(expr elastic.Query) (elastic.Query, error) { if expr == nil { return nil, nil } return expr, nil } -func (c *queryConverter) BuildNotExpr(expr elastic.Query) (elastic.Query, error) { +func (c *esQueryConverter) BuildNotExpr(expr elastic.Query) (elastic.Query, error) { if expr == nil { return nil, nil } @@ -37,7 +41,7 @@ func (c *queryConverter) BuildNotExpr(expr elastic.Query) (elastic.Query, error) return newBoolQuery().MustNot(expr), nil } -func (c *queryConverter) BuildAndExpr(exprs ...elastic.Query) (elastic.Query, error) { +func (c *esQueryConverter) BuildAndExpr(exprs ...elastic.Query) (elastic.Query, error) { var reusableBoolQuery *boolQuery validExprs := make([]elastic.Query, 0, len(exprs)) for _, e := range exprs { @@ -65,7 +69,7 @@ func (c *queryConverter) BuildAndExpr(exprs ...elastic.Query) (elastic.Query, er return newBoolQuery().Filter(validExprs...), nil } -func (c *queryConverter) BuildOrExpr(exprs ...elastic.Query) (elastic.Query, error) { +func (c *esQueryConverter) BuildOrExpr(exprs ...elastic.Query) (elastic.Query, error) { var reusableBoolQuery *boolQuery validExprs := make([]elastic.Query, 0, len(exprs)) for _, e := range exprs { @@ -93,7 +97,7 @@ func (c *queryConverter) BuildOrExpr(exprs ...elastic.Query) (elastic.Query, err return newBoolQuery().Should(validExprs...).MinimumNumberShouldMatch(1), nil } -func (c *queryConverter) ConvertComparisonExpr( +func (c *esQueryConverter) ConvertComparisonExpr( operator string, col *query.SAColumn, value any, @@ -126,7 +130,7 @@ func (c *queryConverter) ConvertComparisonExpr( return res, nil } -func (c *queryConverter) ConvertKeywordComparisonExpr( +func (c *esQueryConverter) ConvertKeywordComparisonExpr( operator string, col *query.SAColumn, value any, @@ -152,7 +156,7 @@ func (c *queryConverter) ConvertKeywordComparisonExpr( } } -func (c *queryConverter) ConvertKeywordListComparisonExpr( +func (c *esQueryConverter) ConvertKeywordListComparisonExpr( operator string, col *query.SAColumn, value any, @@ -160,7 +164,7 @@ func (c *queryConverter) ConvertKeywordListComparisonExpr( return c.ConvertKeywordComparisonExpr(operator, col, value) } -func (c *queryConverter) ConvertTextComparisonExpr( +func (c *esQueryConverter) ConvertTextComparisonExpr( operator string, col *query.SAColumn, value any, @@ -176,7 +180,7 @@ func (c *queryConverter) ConvertTextComparisonExpr( } } -func (c *queryConverter) ConvertRangeExpr( +func (c *esQueryConverter) ConvertRangeExpr( operator string, col *query.SAColumn, from, to any, @@ -198,7 +202,7 @@ func (c *queryConverter) ConvertRangeExpr( } } -func (c *queryConverter) ConvertIsExpr( +func (c *esQueryConverter) ConvertIsExpr( operator string, col *query.SAColumn, ) (elastic.Query, error) { @@ -217,3 +221,20 @@ func (c *queryConverter) ConvertIsExpr( ) } } + +func NewQueryConverter( + namespaceName namespace.Name, + saTypeMap searchattribute.NameTypeMap, + saMapper searchattribute.Mapper, + metricsHandler metrics.Handler, + logger log.Logger, +) *query.QueryConverter[elastic.Query] { + return query.NewQueryConverter( + &esQueryConverter{}, + namespaceName, + saTypeMap, + saMapper, + metricsHandler, + logger, + ) +} diff --git a/common/persistence/visibility/store/elasticsearch/query_converter_test.go b/common/persistence/visibility/store/elasticsearch/query_converter_test.go index 4bff8a616c2..958360a714c 100644 --- a/common/persistence/visibility/store/elasticsearch/query_converter_test.go +++ b/common/persistence/visibility/store/elasticsearch/query_converter_test.go @@ -13,7 +13,7 @@ import ( ) func TestQueryConverter_GetDatetimeFormat(t *testing.T) { - qc := &queryConverter{} + qc := &esQueryConverter{} require.Equal(t, time.RFC3339Nano, qc.GetDatetimeFormat()) } @@ -43,7 +43,7 @@ func TestQueryConverter_BuildParenExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.BuildParenExpr(tc.in) r.NoError(err) r.Equal(tc.out, out) @@ -88,7 +88,7 @@ func TestQueryConverter_BuildNotExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.BuildNotExpr(tc.in) r.NoError(err) r.Equal(tc.out, out) @@ -169,7 +169,7 @@ func TestQueryConverter_BuildAndExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.BuildAndExpr(tc.in...) r.NoError(err) r.Equal(tc.out, out) @@ -250,7 +250,7 @@ func TestQueryConverter_BuildOrExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.BuildOrExpr(tc.in...) r.NoError(err) r.Equal(tc.out, out) @@ -346,7 +346,7 @@ func TestQueryConverter_ConvertComparisonExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertComparisonExpr(tc.operator, tc.col, tc.value) if tc.err != "" { r.Error(err) @@ -483,7 +483,7 @@ func TestQueryConverter_ConvertKeywordComparisonExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertKeywordComparisonExpr(tc.operator, tc.col, tc.value) if tc.err != "" { r.Error(err) @@ -558,7 +558,7 @@ func TestQueryConverter_ConvertKeywordListComparisonExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertKeywordListComparisonExpr(tc.operator, tc.col, tc.value) if tc.err != "" { r.Error(err) @@ -617,7 +617,7 @@ func TestQueryConverter_ConvertTextComparisonExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertTextComparisonExpr(tc.operator, tc.col, tc.value) if tc.err != "" { r.Error(err) @@ -681,7 +681,7 @@ func TestQueryConverter_ConvertRangeExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertRangeExpr(tc.operator, tc.col, tc.from, tc.to) if tc.err != "" { r.Error(err) @@ -736,7 +736,7 @@ func TestQueryConverter_ConvertIsExpr(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { r := require.New(t) - qc := &queryConverter{} + qc := &esQueryConverter{} out, err := qc.ConvertIsExpr(tc.operator, tc.col) if tc.err != "" { r.Error(err) diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store.go b/common/persistence/visibility/store/elasticsearch/visibility_store.go index f72695a162e..81af81b7670 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store.go @@ -763,14 +763,8 @@ func (s *VisibilityStore) convertQuery( return nil, err } - c := query.NewQueryConverter( - &queryConverter{}, - namespaceName, - saTypeMap, - saMapper, - s.metricsHandler, - s.logger, - ).WithChasmMapper(chasmMapper). + c := NewQueryConverter(namespaceName, saTypeMap, saMapper, s.metricsHandler, s.logger). + WithChasmMapper(chasmMapper). WithArchetypeID(archetypeID) queryParams, err := c.Convert(queryString) diff --git a/common/persistence/visibility/store/query/converter.go b/common/persistence/visibility/store/query/converter.go index 33c957eea62..2394061d435 100644 --- a/common/persistence/visibility/store/query/converter.go +++ b/common/persistence/visibility/store/query/converter.go @@ -595,8 +595,7 @@ func (c *QueryConverter[ExprT]) parseValueExpr( } return value, nil case sqlparser.BoolVal: - // no-op: no validation needed - return bool(e), nil + return c.validateValueType(saName, saType, bool(e)) case sqlparser.ValTuple: // This is "in (1,2,3)" case. values := make([]any, 0, len(e)) @@ -608,6 +607,44 @@ func (c *QueryConverter[ExprT]) parseValueExpr( values = append(values, item) } return values, nil + case *sqlparser.UnaryExpr: + // Negative value may be parsed as UnaryExpr + if e.Operator != sqlparser.UPlusStr && e.Operator != sqlparser.UMinusStr { + return nil, NewConverterError( + "%s: unary operator %q", + NotSupportedErrMessage, + e.Operator, + ) + } + if value, ok := e.Expr.(*sqlparser.SQLVal); !ok || value.Type == sqlparser.StrVal { + return nil, NewConverterError( + "%s: unary operator not supported in %q", + InvalidExpressionErrMessage, + sqlparser.String(expr), + ) + } + value, err := c.parseValueExpr(e.Expr, saName, saFieldName, saType) + if err != nil { + return nil, err + } + switch v := value.(type) { + case int64: + if e.Operator == sqlparser.UMinusStr { + value = -v + } + case float64: + if e.Operator == sqlparser.UMinusStr { + value = -v + } + default: + // This should never happen, but here to catch any unexpected case. + return nil, NewConverterError( + "%s: unary expression %q", + InvalidExpressionErrMessage, + sqlparser.String(expr), + ) + } + return value, nil case *sqlparser.GroupConcatExpr: return nil, NewConverterError("%s: 'group_concat'", NotSupportedErrMessage) case *sqlparser.FuncExpr: @@ -666,6 +703,16 @@ func (c *QueryConverter[ExprT]) parseSQLVal( } } +// validateValueType validates a single value type against the search attribute type, +// and returns the value formatted if needed. +// If search attribute type is: +// - Int, Double: value type must be int64 or float64 and returns the input value; +// - Bool: value type must be bool and returns the input value; +// - Keyword, KeywordList: value type must be string and returns the input value; +// - Text: value type must be string with a valid token (empty string or string +// with only spaces are not valid) and returns the input value; +// - Datetime: value type must be int64 (Unix nanos) or string and returns datetime +// formated based on storeQC.GetDatetimeFormat() value. func (c *QueryConverter[ExprT]) validateValueType( saName string, saType enumspb.IndexedValueType, @@ -713,12 +760,22 @@ func (c *QueryConverter[ExprT]) validateValueType( } return tm.UTC().Format(c.storeQC.GetDatetimeFormat()), nil case enumspb.INDEXED_VALUE_TYPE_KEYWORD, - enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, - enumspb.INDEXED_VALUE_TYPE_TEXT: + enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST: if _, ok := value.(string); !ok { return nil, valueTypeErr } return value, nil + case enumspb.INDEXED_VALUE_TYPE_TEXT: + if v, ok := value.(string); !ok { + return nil, valueTypeErr + } else if strings.TrimSpace(v) == "" { + return nil, NewConverterError( + "%s: no tokens found filtering on Text type search attribute %s", + InvalidExpressionErrMessage, + saName, + ) + } + return value, nil default: return nil, NewConverterError( "%s: unknown search attribute type %s for %s", diff --git a/common/persistence/visibility/store/sql/query_converter.go b/common/persistence/visibility/store/sql/query_converter.go index e0f8f2ca1a7..0bcbba9778f 100644 --- a/common/persistence/visibility/store/sql/query_converter.go +++ b/common/persistence/visibility/store/sql/query_converter.go @@ -263,15 +263,15 @@ func (c *SQLQueryConverter) ConvertIsExpr( } func (c *SQLQueryConverter) buildValueExpr(name string, value any) (sqlparser.Expr, error) { - // ExecutionStatus is stored as an integer in SQL database. - if name == sadefs.ExecutionStatus { - // Query converter already validates the value, so there should not be any errors here. - status, _ := enumspb.WorkflowExecutionStatusFromString(value.(string)) - return sqlparser.NewIntVal([]byte(strconv.FormatInt(int64(status), 10))), nil - } - switch v := value.(type) { case string: + // ExecutionStatus is stored as an integer in SQL database. + if name == sadefs.ExecutionStatus { + // Query converter already validates the value, so there should not be any errors here. + status, _ := enumspb.WorkflowExecutionStatusFromString(value.(string)) + return sqlparser.NewIntVal([]byte(strconv.FormatInt(int64(status), 10))), nil + } + // escape strings for safety replacer := strings.NewReplacer(escapeCharMap...) return query.NewUnsafeSQLString(replacer.Replace(v)), nil diff --git a/common/persistence/visibility/store/tests/query_converter_test.go b/common/persistence/visibility/store/tests/query_converter_test.go new file mode 100644 index 00000000000..12875c1da33 --- /dev/null +++ b/common/persistence/visibility/store/tests/query_converter_test.go @@ -0,0 +1,1293 @@ +package tests + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/olivere/elastic/v7" + "github.com/stretchr/testify/require" + "github.com/temporalio/sqlparser" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/persistence/sql" + "go.temporal.io/server/common/persistence/visibility/store/elasticsearch" + "go.temporal.io/server/common/persistence/visibility/store/query" + vissql "go.temporal.io/server/common/persistence/visibility/store/sql" + "go.temporal.io/server/common/searchattribute" +) + +const ( + testNamespaceName = namespace.Name("test-namespace") + + mysqlStore = "mysql8" + postgresStore = "postgres12" + sqliteStore = "sqlite" + esStore = "elasticsearch" +) + +var sqlPlugins = []string{mysqlStore, postgresStore, sqliteStore} + +type queryConverterTestCase struct { + name string + in string + + // out is the expected converted query: `sql` is the WHERE clause expected for every SQL + // plugin, and `mysql`, `postgres` and `sqlite` override it for the plugins whose output + // differs (mostly Text and KeywordList types); `es` is the expected Elasticsearch query. + sql string + mysql string + postgres string + sqlite string + es string + + // err is the expected error message for every store: most errors come from the shared + // front-end converter. When only some stores fail, use the store specific fields + // instead: sqlErr for every SQL plugin, and mysqlErr, postgresErr, sqliteErr and esErr + // for a single store. + err string + sqlErr string + mysqlErr string + postgresErr string + sqliteErr string + esErr string + + // groupBy is the expected list of GROUP BY search attribute field names. + groupBy []string + // orderBy is the expected ORDER BY clause. + orderBy string +} + +func (tc *queryConverterTestCase) expected(store string) (out string, errMsg string) { + if tc.err != "" { + return "", tc.err + } + if store == esStore { + return tc.es, tc.esErr + } + + var outOverride, errOverride string + switch store { + case mysqlStore: + outOverride, errOverride = tc.mysql, tc.mysqlErr + case postgresStore: + outOverride, errOverride = tc.postgres, tc.postgresErr + case sqliteStore: + outOverride, errOverride = tc.sqlite, tc.sqliteErr + default: + // no-op + } + + if errOverride != "" { + return "", errOverride + } + if outOverride != "" { + return outOverride, "" + } + return tc.sql, tc.sqlErr +} + +var queryConverterTestCases = []queryConverterTestCase{ + { + name: "empty query", + in: "", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "blank query", + in: " ", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + // Keyword type search attributes. + { + name: "Keyword equal", + in: "WorkflowId = 'wid'", + sql: "TemporalNamespaceDivision is null and workflow_id = 'wid'", + es: `{"bool":{"filter":{"term":{"WorkflowId":"wid"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword not equal", + in: "WorkflowId != 'wid'", + sql: "TemporalNamespaceDivision is null and workflow_id != 'wid'", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"WorkflowId":"wid"}}]}}`, + }, + { + name: "Keyword in", + in: "WorkflowType IN ('foo', 'bar')", + sql: "TemporalNamespaceDivision is null and workflow_type_name in ('foo', 'bar')", + es: `{"bool":{"filter":{"terms":{"WorkflowType":["foo","bar"]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword not in", + in: "WorkflowType NOT IN ('foo', 'bar')", + sql: "TemporalNamespaceDivision is null and workflow_type_name not in ('foo', 'bar')", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"terms":{"WorkflowType":["foo","bar"]}}]}}`, + }, + { + name: "Keyword starts with", + in: "TaskQueue STARTS_WITH 'foo'", + sql: "TemporalNamespaceDivision is null and task_queue like 'foo%' escape '!'", + es: `{"bool":{"filter":{"prefix":{"TaskQueue":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword not starts with", + in: "TaskQueue NOT STARTS_WITH 'foo'", + sql: "TemporalNamespaceDivision is null and task_queue not like 'foo%' escape '!'", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"prefix":{"TaskQueue":"foo"}}]}}`, + }, + { + name: "Keyword starts with special chars", + in: "AliasForKeyword01 STARTS_WITH 'a%b_c!d'", + sql: "TemporalNamespaceDivision is null and Keyword01 like 'a!%b!_c!!d%' escape '!'", + es: `{"bool":{"filter":{"prefix":{"Keyword01":"a%b_c!d"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword starts with non string value", + in: "AliasForKeyword01 STARTS_WITH 123", + err: "invalid expression: invalid value type for search attribute AliasForKeyword01 of type Keyword: 123 (type: int64)", + }, + { + name: "Keyword greater than", + in: "AliasForKeyword01 > 'foo'", + sql: "TemporalNamespaceDivision is null and Keyword01 > 'foo'", + es: `{"bool":{"filter":{"range":{"Keyword01":{"from":"foo","include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword between", + in: "AliasForKeyword01 BETWEEN 'a' AND 'b'", + sql: "TemporalNamespaceDivision is null and Keyword01 between 'a' and 'b'", + es: `{"bool":{"filter":{"range":{"Keyword01":{"from":"a","include_lower":true,"include_upper":true,"to":"b"}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword is null", + in: "AliasForKeyword01 IS NULL", + sql: "TemporalNamespaceDivision is null and Keyword01 is null", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"exists":{"field":"Keyword01"}}]}}`, + }, + { + name: "Keyword is not null", + in: "AliasForKeyword01 IS NOT NULL", + sql: "TemporalNamespaceDivision is null and Keyword01 is not null", + es: `{"bool":{"filter":{"exists":{"field":"Keyword01"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword empty string", + in: "AliasForKeyword01 = ''", + sql: "TemporalNamespaceDivision is null and Keyword01 = ''", + es: `{"bool":{"filter":{"term":{"Keyword01":""}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword value with quote", + in: "AliasForKeyword01 = 'foo''s bar'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo''s bar'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo's bar"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword value with backslash", + in: "AliasForKeyword01 = 'foo\\\\bar'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo\\\\bar'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo\\bar"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword invalid value type", + in: "AliasForKeyword01 = 123", + err: "invalid expression: invalid value type for search attribute AliasForKeyword01 of type Keyword: 123 (type: int64)", + }, + { + name: "Keyword with bool value", + in: "AliasForKeyword01 = true", + err: "invalid expression: invalid value type for search attribute AliasForKeyword01 of type Keyword: true (type: bool)", + }, + { + name: "Keyword with negative value", + in: "AliasForKeyword01 = -'foo'", + err: `invalid expression: unary operator not supported in "-'foo'"`, + }, + { + name: "Keyword with positive value", + in: "AliasForKeyword01 = +'foo'", + err: `invalid expression: unary operator not supported in "+'foo'"`, + }, + { + name: "Keyword unsupported operator", + in: "AliasForKeyword01 LIKE 'foo%'", + err: "operation is not supported: operator 'LIKE' not supported for Keyword type search attribute 'AliasForKeyword01'", + }, + { + name: "Keyword value with escaped chars", + in: "AliasForKeyword01 = 'foo\\nbar\\ttail'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo\\nbar\\ttail'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo\nbar\ttail"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword single value in list", + in: "WorkflowId IN ('wid')", + sql: "TemporalNamespaceDivision is null and workflow_id in ('wid')", + es: `{"bool":{"filter":{"terms":{"WorkflowId":["wid"]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Keyword empty list", + in: "WorkflowId IN ()", + err: "malformed SQL query: syntax error at position 44", + }, + { + name: "search attribute name is case sensitive", + in: "workflowid = 'wid'", + err: "invalid expression: column name 'workflowid' is not a valid search attribute", + }, + + // Int type search attributes. + { + name: "Int equal", + in: "HistoryLength = 10", + sql: "TemporalNamespaceDivision is null and history_length = 10", + es: `{"bool":{"filter":{"term":{"HistoryLength":10}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int negative value", + in: "AliasForInt01 = -10", + sql: "TemporalNamespaceDivision is null and Int01 = -10", + es: `{"bool":{"filter":{"term":{"Int01":-10}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int positive sign value", + in: "AliasForInt01 = +10", + sql: "TemporalNamespaceDivision is null and Int01 = 10", + es: `{"bool":{"filter":{"term":{"Int01":10}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // Only a literal value can be negated, not an arbitrary expression. + name: "Int negative parenthesized value", + in: "AliasForInt01 = -(10)", + err: `invalid expression: unary operator not supported in "-(10)"`, + }, + { + name: "Int greater than", + in: "HistoryLength > 10", + sql: "TemporalNamespaceDivision is null and history_length > 10", + es: `{"bool":{"filter":{"range":{"HistoryLength":{"from":10,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int greater equal", + in: "HistoryLength >= 10", + sql: "TemporalNamespaceDivision is null and history_length >= 10", + es: `{"bool":{"filter":{"range":{"HistoryLength":{"from":10,"include_lower":true,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int less than", + in: "StateTransitionCount < 10", + sql: "TemporalNamespaceDivision is null and state_transition_count < 10", + es: `{"bool":{"filter":{"range":{"StateTransitionCount":{"from":null,"include_lower":true,"include_upper":false,"to":10}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int less equal", + in: "StateTransitionCount <= 10", + sql: "TemporalNamespaceDivision is null and state_transition_count <= 10", + es: `{"bool":{"filter":{"range":{"StateTransitionCount":{"from":null,"include_lower":true,"include_upper":true,"to":10}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int in", + in: "HistorySizeBytes IN (1, 2, 3)", + sql: "TemporalNamespaceDivision is null and history_size_bytes in (1, 2, 3)", + es: `{"bool":{"filter":{"terms":{"HistorySizeBytes":[1,2,3]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int between", + in: "HistoryLength BETWEEN 1 AND 10", + sql: "TemporalNamespaceDivision is null and history_length between 1 and 10", + es: `{"bool":{"filter":{"range":{"HistoryLength":{"from":1,"include_lower":true,"include_upper":true,"to":10}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int not between", + in: "HistoryLength NOT BETWEEN 1 AND 10", + sql: "TemporalNamespaceDivision is null and history_length not between 1 and 10", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"range":{"HistoryLength":{"from":1,"include_lower":true,"include_upper":true,"to":10}}}]}}`, + }, + { + name: "Int with float value", + in: "AliasForInt01 = 1.5", + sql: "TemporalNamespaceDivision is null and Int01 = 1.5", + es: `{"bool":{"filter":{"term":{"Int01":1.5}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int with negative float value", + in: "AliasForInt01 = -1.5", + sql: "TemporalNamespaceDivision is null and Int01 = -1.5", + es: `{"bool":{"filter":{"term":{"Int01":-1.5}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Int with positive sign float value", + in: "AliasForInt01 = +1.5", + sql: "TemporalNamespaceDivision is null and Int01 = 1.5", + es: `{"bool":{"filter":{"term":{"Int01":1.5}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // negative int is parsed as SQLVal, and sqlparser reduces inline + name: "Int negative negative int value", + in: "AliasForInt01 = - -10", + sql: "TemporalNamespaceDivision is null and Int01 = 10", + es: `{"bool":{"filter":{"term":{"Int01":10}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // negative float is parsed as UnaryExpr, and nested unary expressions is not allowed + name: "Int negative negative float value", + in: "AliasForInt01 = - -1.0", + err: `invalid expression: unary operator not supported in "- -1.0"`, + }, + { + name: "Int invalid value type", + in: "HistoryLength = 'foo'", + err: `invalid expression: invalid value type for search attribute HistoryLength of type Int: "foo" (type: string)`, + }, + { + name: "Int unsupported operator", + in: "HistoryLength STARTS_WITH 1", + err: "operation is not supported: operator 'STARTS_WITH' not supported for Int type search attribute 'HistoryLength'", + }, + { + name: "Int mixed types in list", + in: "AliasForInt01 IN (1, 'foo')", + err: `invalid expression: invalid value type for search attribute AliasForInt01 of type Int: "foo" (type: string)`, + }, + + // Double type search attributes. + { + name: "Double equal", + in: "AliasForDouble01 = 1.5", + sql: "TemporalNamespaceDivision is null and Double01 = 1.5", + es: `{"bool":{"filter":{"term":{"Double01":1.5}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // Unlike negative integers, negative floats are parsed as an unary expression. + name: "Double negative value", + in: "AliasForDouble01 >= -1.5", + sql: "TemporalNamespaceDivision is null and Double01 >= -1.5", + es: `{"bool":{"filter":{"range":{"Double01":{"from":-1.5,"include_lower":true,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Double positive sign value", + in: "AliasForDouble01 = +1.5", + sql: "TemporalNamespaceDivision is null and Double01 = 1.5", + es: `{"bool":{"filter":{"term":{"Double01":1.5}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Double with int value", + in: "AliasForDouble01 = 1", + sql: "TemporalNamespaceDivision is null and Double01 = 1", + es: `{"bool":{"filter":{"term":{"Double01":1}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Double with negative int value", + in: "AliasForDouble01 = -1", + sql: "TemporalNamespaceDivision is null and Double01 = -1", + es: `{"bool":{"filter":{"term":{"Double01":-1}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Double with positive sign int value", + in: "AliasForDouble01 = +1", + sql: "TemporalNamespaceDivision is null and Double01 = 1", + es: `{"bool":{"filter":{"term":{"Double01":1}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // negative int is parsed as SQLVal, and sqlparser reduces inline + name: "Double negative negative int value", + in: "AliasForDouble01 = - -10", + sql: "TemporalNamespaceDivision is null and Double01 = 10", + es: `{"bool":{"filter":{"term":{"Double01":10}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // negative float is parsed as UnaryExpr, and nested unary expressions is not allowed + name: "Double negative negative float value", + in: "AliasForDouble01 = - -1.0", + err: `invalid expression: unary operator not supported in "- -1.0"`, + }, + { + name: "Double invalid value type", + in: "AliasForDouble01 = 'foo'", + err: `invalid expression: invalid value type for search attribute AliasForDouble01 of type Double: "foo" (type: string)`, + }, + + // Bool type search attributes. + { + name: "Bool true", + in: "AliasForBool01 = true", + sql: "TemporalNamespaceDivision is null and Bool01 = true", + es: `{"bool":{"filter":{"term":{"Bool01":true}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Bool false", + in: "TemporalSchedulePaused = false", + sql: "TemporalNamespaceDivision is null and TemporalSchedulePaused = false", + es: `{"bool":{"filter":{"term":{"TemporalSchedulePaused":false}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Bool not equal", + in: "AliasForBool01 != true", + sql: "TemporalNamespaceDivision is null and Bool01 != true", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"Bool01":true}}]}}`, + }, + { + name: "Bool invalid value type", + in: "AliasForBool01 = 'true'", + err: `invalid expression: invalid value type for search attribute AliasForBool01 of type Bool: "true" (type: string)`, + }, + { + name: "Bool range condition not supported", + in: "AliasForBool01 BETWEEN false AND true", + err: "invalid expression: cannot do range condition on search attribute 'AliasForBool01' of type Bool", + }, + { + name: "Bool invalid unary negative value type", + in: "AliasForBool01 = -true", + err: `invalid expression: unary operator not supported in "-true"`, + }, + { + name: "Bool invalid unary positive value type", + in: "AliasForBool01 = +true", + err: `invalid expression: unary operator not supported in "+true"`, + }, + + // Datetime type search attributes. + { + name: "Datetime equal", + in: "StartTime = '2020-01-02T15:04:05Z'", + mysql: "TemporalNamespaceDivision is null and start_time = '2020-01-02 15:04:05'", + postgres: "TemporalNamespaceDivision is null and start_time = '2020-01-02 15:04:05'", + sqlite: "TemporalNamespaceDivision is null and start_time = '2020-01-02 15:04:05+00:00'", + es: `{"bool":{"filter":{"term":{"StartTime":"2020-01-02T15:04:05Z"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime with nanoseconds and offset", + in: "StartTime > '2020-01-02T15:04:05.123456789-07:00'", + mysql: "TemporalNamespaceDivision is null and start_time > '2020-01-02 22:04:05.123456'", + postgres: "TemporalNamespaceDivision is null and start_time > '2020-01-02 22:04:05.123456'", + sqlite: "TemporalNamespaceDivision is null and start_time > '2020-01-02 22:04:05.123456+00:00'", + es: `{"bool":{"filter":{"range":{"StartTime":{"from":"2020-01-02T22:04:05.123456789Z","include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime with unix nanoseconds", + in: "StartTime > 1577977445000000000", + mysql: "TemporalNamespaceDivision is null and start_time > '2020-01-02 15:04:05'", + postgres: "TemporalNamespaceDivision is null and start_time > '2020-01-02 15:04:05'", + sqlite: "TemporalNamespaceDivision is null and start_time > '2020-01-02 15:04:05+00:00'", + es: `{"bool":{"filter":{"range":{"StartTime":{"from":"2020-01-02T15:04:05Z","include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime between", + in: "CloseTime BETWEEN '2020-01-02T15:04:05Z' AND '2020-01-03T15:04:05Z'", + mysql: "TemporalNamespaceDivision is null and close_time between '2020-01-02 15:04:05' and '2020-01-03 15:04:05'", + postgres: "TemporalNamespaceDivision is null and close_time between '2020-01-02 15:04:05' and '2020-01-03 15:04:05'", + sqlite: "TemporalNamespaceDivision is null and close_time between '2020-01-02 15:04:05+00:00' and '2020-01-03 15:04:05+00:00'", + es: `{"bool":{"filter":{"range":{"CloseTime":{"from":"2020-01-02T15:04:05Z","include_lower":true,"include_upper":true,"to":"2020-01-03T15:04:05Z"}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime is null", + in: "CloseTime IS NULL", + sql: "TemporalNamespaceDivision is null and close_time is null", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"exists":{"field":"CloseTime"}}]}}`, + }, + { + name: "Datetime is not null", + in: "CloseTime IS NOT NULL", + sql: "TemporalNamespaceDivision is null and close_time is not null", + es: `{"bool":{"filter":{"exists":{"field":"CloseTime"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime custom search attribute", + in: "AliasForDatetime01 <= '2020-01-02T15:04:05Z'", + mysql: "TemporalNamespaceDivision is null and Datetime01 <= '2020-01-02 15:04:05'", + postgres: "TemporalNamespaceDivision is null and Datetime01 <= '2020-01-02 15:04:05'", + sqlite: "TemporalNamespaceDivision is null and Datetime01 <= '2020-01-02 15:04:05+00:00'", + es: `{"bool":{"filter":{"range":{"Datetime01":{"from":null,"include_lower":true,"include_upper":true,"to":"2020-01-02T15:04:05Z"}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Datetime invalid value", + in: "StartTime = 'foo'", + err: "invalid expression: unable to parse datetime 'foo'", + }, + { + name: "Datetime with bool value", + in: "StartTime = true", + err: "invalid expression: invalid value type for search attribute StartTime of type Datetime: true (type: bool)", + }, + { + name: "Datetime invalid unary negative value type", + in: "StartTime = -'2020-01-02T15:04:05Z'", + err: `invalid expression: unary operator not supported in "-'2020-01-02T15:04:05Z'"`, + }, + { + name: "Datetime invalid unary positive value type", + in: "StartTime = +'2020-01-02T15:04:05Z'", + err: `invalid expression: unary operator not supported in "+'2020-01-02T15:04:05Z'"`, + }, + + // ExecutionStatus search attribute. + { + name: "ExecutionStatus equal", + in: "ExecutionStatus = 'Running'", + sql: "TemporalNamespaceDivision is null and status = 1", + es: `{"bool":{"filter":{"term":{"ExecutionStatus":"Running"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionStatus equal code", + in: "ExecutionStatus = 1", + sql: "TemporalNamespaceDivision is null and status = 1", + es: `{"bool":{"filter":{"term":{"ExecutionStatus":"Running"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionStatus not equal", + in: "ExecutionStatus != 'Completed'", + sql: "TemporalNamespaceDivision is null and status != 2", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"ExecutionStatus":"Completed"}}]}}`, + }, + { + name: "ExecutionStatus in", + in: "ExecutionStatus IN ('Running', 'Completed')", + sql: "TemporalNamespaceDivision is null and status in (1, 2)", + es: `{"bool":{"filter":{"terms":{"ExecutionStatus":["Running","Completed"]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionStatus in code", + in: "ExecutionStatus IN (1, 2)", + sql: "TemporalNamespaceDivision is null and status in (1, 2)", + es: `{"bool":{"filter":{"terms":{"ExecutionStatus":["Running","Completed"]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionStatus not in", + in: "ExecutionStatus NOT IN ('Running', 'Completed')", + sql: "TemporalNamespaceDivision is null and status not in (1, 2)", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"terms":{"ExecutionStatus":["Running","Completed"]}}]}}`, + }, + { + name: "ExecutionStatus unspecified", + in: "ExecutionStatus = 'Unspecified'", + sql: "TemporalNamespaceDivision is null and status = 0", + es: `{"bool":{"filter":{"term":{"ExecutionStatus":"Unspecified"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionStatus invalid name", + in: "ExecutionStatus = 'Foo'", + err: "invalid expression: invalid ExecutionStatus value 'Foo'", + }, + { + name: "ExecutionStatus invalid code", + in: "ExecutionStatus = 42", + err: "invalid expression: invalid ExecutionStatus value 42", + }, + { + name: "ExecutionStatus invalid value type", + in: "ExecutionStatus = 1.5", + err: "invalid expression: unexpected value type float64 for search attribute ExecutionStatus", + }, + { + name: "ExecutionStatus invalid value in list", + in: "ExecutionStatus IN ('Running', 'Foo')", + err: "invalid expression: invalid ExecutionStatus value 'Foo'", + }, + { + name: "ExecutionStatus is null", + in: "ExecutionStatus IS NULL", + sql: "TemporalNamespaceDivision is null and status is null", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"exists":{"field":"ExecutionStatus"}}]}}`, + }, + { + // ExecutionStatus is stored as an integer in SQL databases, thus a prefix search + // is only possible in Elasticsearch. + name: "ExecutionStatus starts with", + in: "ExecutionStatus STARTS_WITH 'Running'", + sqlErr: `invalid expression: right-hand side of "starts_with" operator must be a literal string (got: Running)`, + es: `{"bool":{"filter":{"prefix":{"ExecutionStatus":"Running"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + + // ExecutionDuration search attribute. + { + name: "ExecutionDuration nanoseconds", + in: "ExecutionDuration > 1000000", + sql: "TemporalNamespaceDivision is null and execution_duration > 1000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":1000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration golang duration", + in: "ExecutionDuration > '10s'", + sql: "TemporalNamespaceDivision is null and execution_duration > 10000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":10000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration days", + in: "ExecutionDuration > '2d'", + sql: "TemporalNamespaceDivision is null and execution_duration > 172800000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":172800000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration hh:mm:ss", + in: "ExecutionDuration > '00:10:30'", + sql: "TemporalNamespaceDivision is null and execution_duration > 630000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":630000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + // negative zero hours works because -0 = 0, it does not negate the duration (bug?) + name: "ExecutionDuration negative 00:mm:ss", + in: "ExecutionDuration > '-00:10:30'", + sql: "TemporalNamespaceDivision is null and execution_duration > 630000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":630000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration between", + in: "ExecutionDuration BETWEEN '1s' AND '1m'", + sql: "TemporalNamespaceDivision is null and execution_duration between 1000000000 and 60000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":1000000000,"include_lower":true,"include_upper":true,"to":60000000000}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration invalid value", + in: "ExecutionDuration > 'foo'", + err: "invalid expression: invalid duration value for search attribute ExecutionDuration: foo", + }, + { + name: "ExecutionDuration negative value", + in: "ExecutionDuration > -1000", + sql: "TemporalNamespaceDivision is null and execution_duration > -1000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":-1000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration positive sign value", + in: "ExecutionDuration > +1000", + sql: "TemporalNamespaceDivision is null and execution_duration > 1000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":1000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration golang negative duration", + in: "ExecutionDuration > '-10s'", + sql: "TemporalNamespaceDivision is null and execution_duration > -10000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":-10000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration golang positive sign duration", + in: "ExecutionDuration > '+10s'", + sql: "TemporalNamespaceDivision is null and execution_duration > 10000000000", + es: `{"bool":{"filter":{"range":{"ExecutionDuration":{"from":10000000000,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ExecutionDuration negative golang duration", + in: "ExecutionDuration > -'10s'", + err: `invalid expression: unary operator not supported in "-'10s'"`, + }, + { + name: "ExecutionDuration positive sign golang duration", + in: "ExecutionDuration > +'10s'", + err: `invalid expression: unary operator not supported in "+'10s'"`, + }, + { + name: "ExecutionDuration negative hh:mm:ss", + in: "ExecutionDuration > '-01:10:30'", + err: "invalid expression: invalid duration value for search attribute ExecutionDuration: -01:10:30", + }, + { + name: "ExecutionDuration unary negative hh:mm:ss", + in: "ExecutionDuration > -'00:10:30'", + err: `invalid expression: unary operator not supported in "-'00:10:30'"`, + }, + { + name: "ExecutionDuration unary positive hh:mm:ss", + in: "ExecutionDuration > +'00:10:30'", + err: `invalid expression: unary operator not supported in "+'00:10:30'"`, + }, + { + name: "ExecutionDuration with bool value", + in: "ExecutionDuration > true", + err: "invalid expression: invalid value type for search attribute ExecutionDuration of type Int: true (type: bool)", + }, + + // KeywordList type search attributes. + { + name: "KeywordList equal", + in: "AliasForKeywordList01 = 'foo'", + mysql: "TemporalNamespaceDivision is null and 'foo' member of (KeywordList01)", + postgres: "TemporalNamespaceDivision is null and KeywordList01 @> jsonb_build_array('foo')", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'KeywordList01 : ("foo")')`, + es: `{"bool":{"filter":{"term":{"KeywordList01":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "KeywordList not equal", + in: "AliasForKeywordList01 != 'foo'", + mysql: "TemporalNamespaceDivision is null and (not 'foo' member of (KeywordList01))", + postgres: "TemporalNamespaceDivision is null and (not KeywordList01 @> jsonb_build_array('foo'))", + sqlite: `TemporalNamespaceDivision is null and rowid not in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'KeywordList01 : ("foo")')`, + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"KeywordList01":"foo"}}]}}`, + }, + { + name: "KeywordList in", + in: "AliasForKeywordList01 IN ('foo', 'bar')", + mysql: `TemporalNamespaceDivision is null and json_overlaps(KeywordList01, cast('["foo","bar"]' as json))`, + postgres: "TemporalNamespaceDivision is null and (KeywordList01 @> jsonb_build_array('foo') or KeywordList01 @> jsonb_build_array('bar'))", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'KeywordList01 : ("foo" OR "bar")')`, + es: `{"bool":{"filter":{"terms":{"KeywordList01":["foo","bar"]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "KeywordList not in", + in: "AliasForKeywordList01 NOT IN ('foo', 'bar')", + mysql: `TemporalNamespaceDivision is null and (not json_overlaps(KeywordList01, cast('["foo","bar"]' as json)))`, + postgres: "TemporalNamespaceDivision is null and (not (KeywordList01 @> jsonb_build_array('foo') or KeywordList01 @> jsonb_build_array('bar')))", + sqlite: `TemporalNamespaceDivision is null and rowid not in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'KeywordList01 : ("foo" OR "bar")')`, + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"terms":{"KeywordList01":["foo","bar"]}}]}}`, + }, + { + name: "KeywordList predefined search attribute", + in: "TemporalChangeVersion = 'foo'", + mysql: "TemporalNamespaceDivision is null and 'foo' member of (TemporalChangeVersion)", + postgres: "TemporalNamespaceDivision is null and TemporalChangeVersion @> jsonb_build_array('foo')", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'TemporalChangeVersion : ("foo")')`, + es: `{"bool":{"filter":{"term":{"TemporalChangeVersion":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "KeywordList is null", + in: "AliasForKeywordList01 IS NULL", + sql: "TemporalNamespaceDivision is null and KeywordList01 is null", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"exists":{"field":"KeywordList01"}}]}}`, + }, + { + name: "KeywordList unsupported operator", + in: "AliasForKeywordList01 > 'foo'", + err: "operation is not supported: operator '>' not supported for KeywordList type search attribute 'AliasForKeywordList01'", + }, + { + name: "KeywordList starts with not supported", + in: "AliasForKeywordList01 STARTS_WITH 'foo'", + err: "operation is not supported: operator 'STARTS_WITH' not supported for KeywordList type search attribute 'AliasForKeywordList01'", + }, + { + name: "KeywordList invalid value type", + in: "AliasForKeywordList01 = 123", + err: "invalid expression: invalid value type for search attribute AliasForKeywordList01 of type KeywordList: 123 (type: int64)", + }, + { + name: "KeywordList with negative value", + in: "AliasForKeywordList01 = -'foo'", + err: `invalid expression: unary operator not supported in "-'foo'"`, + }, + { + name: "KeywordList with positive value", + in: "AliasForKeywordList01 = +'foo'", + err: `invalid expression: unary operator not supported in "+'foo'"`, + }, + { + name: "KeywordList in with negative value", + in: "AliasForKeywordList01 IN ('foo', -'bar')", + err: `invalid expression: unary operator not supported in "-'bar'"`, + }, + { + name: "KeywordList in with positive value", + in: "AliasForKeywordList01 IN ('foo', +'bar')", + err: `invalid expression: unary operator not supported in "+'bar'"`, + }, + { + name: "KeywordList not in with negative value", + in: "AliasForKeywordList01 NOT IN ('foo', -'bar')", + err: `invalid expression: unary operator not supported in "-'bar'"`, + }, + { + name: "KeywordList not in with positive value", + in: "AliasForKeywordList01 NOT IN ('foo', +'bar')", + err: `invalid expression: unary operator not supported in "+'bar'"`, + }, + + // Text type search attributes. + { + name: "Text equal", + in: "AliasForText01 = 'foo bar'", + mysql: "TemporalNamespaceDivision is null and match(Text01) against ('foo bar' in natural language mode)", + postgres: "TemporalNamespaceDivision is null and Text01 @@ 'foo | bar'::tsquery", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_text where executions_visibility_fts_text = 'Text01 : ("foo" OR "bar")')`, + es: `{"bool":{"filter":{"match":{"Text01":{"query":"foo bar"}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Text not equal", + in: "AliasForText01 != 'foo bar'", + mysql: "TemporalNamespaceDivision is null and (not match(Text01) against ('foo bar' in natural language mode))", + postgres: "TemporalNamespaceDivision is null and (not Text01 @@ 'foo | bar'::tsquery)", + sqlite: `TemporalNamespaceDivision is null and rowid not in (select rowid from executions_visibility_fts_text where executions_visibility_fts_text = 'Text01 : ("foo" OR "bar")')`, + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"match":{"Text01":{"query":"foo bar"}}}]}}`, + }, + { + name: "Text empty value", + in: "AliasForText01 = ''", + err: "invalid expression: no tokens found filtering on Text type search attribute AliasForText01", + }, + { + name: "Text blank value", + in: "AliasForText01 = ' '", + err: "invalid expression: no tokens found filtering on Text type search attribute AliasForText01", + }, + { + // Only blank values are rejected: the value is passed as is to the store converters, + // which tokenize it. + name: "Text value with surrounding spaces", + in: "AliasForText01 = ' foo bar '", + mysql: "TemporalNamespaceDivision is null and match(Text01) against (' foo bar ' in natural language mode)", + postgres: "TemporalNamespaceDivision is null and Text01 @@ 'foo | bar'::tsquery", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_text where executions_visibility_fts_text = 'Text01 : ("foo" OR "bar")')`, + es: `{"bool":{"filter":{"match":{"Text01":{"query":" foo bar "}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "Text unsupported operator", + in: "AliasForText01 > 'foo'", + err: "operation is not supported: operator '>' not supported for Text type search attribute 'AliasForText01'", + }, + { + name: "Text in not supported", + in: "AliasForText01 IN ('foo', 'bar')", + err: "operation is not supported: operator 'IN' not supported for Text type search attribute 'AliasForText01'", + }, + { + name: "Text range condition not supported", + in: "AliasForText01 BETWEEN 'a' AND 'b'", + err: "invalid expression: cannot do range condition on search attribute 'AliasForText01' of type Text", + }, + { + name: "Text invalid value type", + in: "AliasForText01 = 123", + err: "invalid expression: invalid value type for search attribute AliasForText01 of type Text: 123 (type: int64)", + }, + { + name: "Text with negative value", + in: "AliasForText01 = -'foo'", + err: `invalid expression: unary operator not supported in "-'foo'"`, + }, + { + name: "Text with positive value", + in: "AliasForText01 = +'foo'", + err: `invalid expression: unary operator not supported in "+'foo'"`, + }, + + // Search attribute name resolution. + { + name: "custom search attribute by field name", + in: "Keyword01 = 'foo'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "custom search attribute with backticks", + in: "`AliasForKeyword01` = 'foo'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "custom search attribute alias with hyphen", + in: "`AliasWithHyphenFor-Keyword01` = 'foo'", + sql: "TemporalNamespaceDivision is null and Keyword01 = 'foo'", + es: `{"bool":{"filter":{"term":{"Keyword01":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "predefined search attribute without Temporal prefix", + in: "PauseInfo = 'foo'", + mysql: "TemporalNamespaceDivision is null and 'foo' member of (TemporalPauseInfo)", + postgres: "TemporalNamespaceDivision is null and TemporalPauseInfo @> jsonb_build_array('foo')", + sqlite: `TemporalNamespaceDivision is null and rowid in (select rowid from executions_visibility_fts_keyword_list where executions_visibility_fts_keyword_list = 'TemporalPauseInfo : ("foo")')`, + es: `{"bool":{"filter":{"term":{"TemporalPauseInfo":"foo"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "system search attribute with Temporal prefix", + in: "TemporalHistoryLength > 10", + sql: "TemporalNamespaceDivision is null and history_length > 10", + es: `{"bool":{"filter":{"range":{"HistoryLength":{"from":10,"include_lower":false,"include_upper":true,"to":null}}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "ScheduleId is mapped to WorkflowId", + in: "ScheduleId = 'sched-id'", + sql: "TemporalNamespaceDivision is null and workflow_id = 'sched-id'", + es: `{"bool":{"filter":{"term":{"WorkflowId":"sched-id"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "TemporalScheduleId is mapped to WorkflowId", + in: "TemporalScheduleId = 'sched-id'", + sql: "TemporalNamespaceDivision is null and workflow_id = 'sched-id'", + es: `{"bool":{"filter":{"term":{"WorkflowId":"sched-id"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "unknown search attribute", + in: "Foo = 'bar'", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + { + name: "reserved field name is not a search attribute", + in: "NamespaceId = 'foo'", + err: "invalid expression: column name 'NamespaceId' is not a valid search attribute", + }, + + // Namespace division. + { + name: "explicit namespace division", + in: "TemporalNamespaceDivision = 'foo'", + sql: "TemporalNamespaceDivision = 'foo'", + es: `{"term":{"TemporalNamespaceDivision":"foo"}}`, + }, + { + name: "explicit namespace division is null", + in: "TemporalNamespaceDivision IS NULL", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "match any namespace division", + in: "TemporalNamespaceDivision IS NULL OR TemporalNamespaceDivision IS NOT NULL", + sql: "(TemporalNamespaceDivision is null or TemporalNamespaceDivision is not null)", + es: `{"bool":{"minimum_should_match":"1","should":[{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}},{"exists":{"field":"TemporalNamespaceDivision"}}]}}`, + }, + { + name: "namespace division in complex query", + in: "WorkflowId = 'wid' AND (TemporalNamespaceDivision = 'foo' OR TemporalNamespaceDivision = 'bar')", + sql: "(workflow_id = 'wid' and (TemporalNamespaceDivision = 'foo' or TemporalNamespaceDivision = 'bar'))", + es: `{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"bool":{"minimum_should_match":"1","should":[{"term":{"TemporalNamespaceDivision":"foo"}},{"term":{"TemporalNamespaceDivision":"bar"}}]}}]}}`, + }, + + // Logical operators. + { + name: "and expression", + in: "WorkflowId = 'wid' AND ExecutionStatus = 'Running'", + sql: "TemporalNamespaceDivision is null and (workflow_id = 'wid' and status = 1)", + es: `{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"term":{"ExecutionStatus":"Running"}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "or expression", + in: "WorkflowId = 'wid' OR RunId = 'rid'", + sql: "TemporalNamespaceDivision is null and (workflow_id = 'wid' or run_id = 'rid')", + es: `{"bool":{"filter":{"bool":{"minimum_should_match":"1","should":[{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}}]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "multiple and expressions", + in: "WorkflowId = 'wid' AND RunId = 'rid' AND WorkflowType = 'wtype'", + sql: "TemporalNamespaceDivision is null and ((workflow_id = 'wid' and run_id = 'rid') and workflow_type_name = 'wtype')", + es: `{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}},{"term":{"WorkflowType":"wtype"}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "multiple or expressions", + in: "WorkflowId = 'wid' OR RunId = 'rid' OR WorkflowType = 'wtype'", + sql: "TemporalNamespaceDivision is null and (workflow_id = 'wid' or run_id = 'rid' or workflow_type_name = 'wtype')", + es: `{"bool":{"filter":{"bool":{"minimum_should_match":"1","should":[{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}},{"term":{"WorkflowType":"wtype"}}]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "and or precedence", + in: "WorkflowId = 'wid' AND RunId = 'rid' OR WorkflowType = 'wtype'", + sql: "TemporalNamespaceDivision is null and (workflow_id = 'wid' and run_id = 'rid' or workflow_type_name = 'wtype')", + es: `{"bool":{"filter":{"bool":{"minimum_should_match":"1","should":[{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}}]}},{"term":{"WorkflowType":"wtype"}}]}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "parenthesized expression", + in: "WorkflowId = 'wid' AND (RunId = 'rid' OR WorkflowType = 'wtype')", + sql: "TemporalNamespaceDivision is null and (workflow_id = 'wid' and (run_id = 'rid' or workflow_type_name = 'wtype'))", + es: `{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"bool":{"minimum_should_match":"1","should":[{"term":{"RunId":"rid"}},{"term":{"WorkflowType":"wtype"}}]}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "nested parenthesized expression", + in: "((WorkflowId = 'wid'))", + sql: "TemporalNamespaceDivision is null and workflow_id = 'wid'", + es: `{"bool":{"filter":{"term":{"WorkflowId":"wid"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + }, + { + name: "not expression", + in: "NOT WorkflowId = 'wid'", + sql: "TemporalNamespaceDivision is null and (not workflow_id = 'wid')", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"WorkflowId":"wid"}}]}}`, + }, + { + name: "not parenthesized and expression", + in: "NOT (WorkflowId = 'wid' AND RunId = 'rid')", + sql: "TemporalNamespaceDivision is null and (not (workflow_id = 'wid' and run_id = 'rid'))", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"bool":{"filter":[{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}}]}}]}}`, + }, + { + name: "not parenthesized or expression", + in: "NOT (WorkflowId = 'wid' OR RunId = 'rid')", + sql: "TemporalNamespaceDivision is null and (not (workflow_id = 'wid' or run_id = 'rid'))", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"term":{"WorkflowId":"wid"}},{"term":{"RunId":"rid"}}]}}`, + }, + { + name: "not is null expression", + in: "NOT CloseTime IS NULL", + sql: "TemporalNamespaceDivision is null and (not close_time is null)", + es: `{"bool":{"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"bool":{"must_not":{"exists":{"field":"CloseTime"}}}}]}}`, + }, + { + name: "complex query", + in: "WorkflowType = 'wtype' AND ExecutionStatus IN ('Running', 'Failed') " + + "AND (StartTime > '2020-01-02T15:04:05Z' OR CloseTime IS NULL) " + + "AND NOT AliasForKeyword01 STARTS_WITH 'foo'", + mysql: "TemporalNamespaceDivision is null and " + + "(((workflow_type_name = 'wtype' and status in (1, 3)) and " + + "(start_time > '2020-01-02 15:04:05' or close_time is null)) and " + + "(not Keyword01 like 'foo%' escape '!'))", + postgres: "TemporalNamespaceDivision is null and " + + "(((workflow_type_name = 'wtype' and status in (1, 3)) and " + + "(start_time > '2020-01-02 15:04:05' or close_time is null)) and " + + "(not Keyword01 like 'foo%' escape '!'))", + sqlite: "TemporalNamespaceDivision is null and " + + "(((workflow_type_name = 'wtype' and status in (1, 3)) and " + + "(start_time > '2020-01-02 15:04:05+00:00' or close_time is null)) and " + + "(not Keyword01 like 'foo%' escape '!'))", + es: `{"bool":{"filter":[{"term":{"WorkflowType":"wtype"}},` + + `{"terms":{"ExecutionStatus":["Running","Failed"]}},` + + `{"bool":{"minimum_should_match":"1","should":[` + + `{"range":{"StartTime":{"from":"2020-01-02T15:04:05Z","include_lower":false,"include_upper":true,"to":null}}},` + + `{"bool":{"must_not":{"exists":{"field":"CloseTime"}}}}]}}],` + + `"must_not":[{"exists":{"field":"TemporalNamespaceDivision"}},{"prefix":{"Keyword01":"foo"}}]}}`, + }, + { + name: "error inside and expression", + in: "WorkflowId = 'wid' AND Foo = 'bar'", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + { + name: "error inside or expression", + in: "Foo = 'bar' OR WorkflowId = 'wid'", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + { + name: "error inside not expression", + in: "NOT Foo = 'bar'", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + + // GROUP BY clause. + { + name: "group by execution status", + in: "GROUP BY ExecutionStatus", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + groupBy: []string{"ExecutionStatus"}, + }, + { + name: "group by with where clause", + in: "WorkflowType = 'wtype' GROUP BY ExecutionStatus", + sql: "TemporalNamespaceDivision is null and workflow_type_name = 'wtype'", + es: `{"bool":{"filter":{"term":{"WorkflowType":"wtype"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + groupBy: []string{"ExecutionStatus"}, + }, + { + name: "group by namespace division", + in: "GROUP BY TemporalNamespaceDivision", + groupBy: []string{"TemporalNamespaceDivision"}, + }, + { + name: "group by unsupported search attribute", + in: "GROUP BY WorkflowId", + err: "operation is not supported: 'GROUP BY' clause is not supported for search attribute WorkflowId", + }, + { + name: "group by multiple search attributes", + in: "GROUP BY ExecutionStatus, TemporalNamespaceDivision", + err: "operation is not supported: 'GROUP BY' clause supports only a single field", + }, + { + name: "group by unknown search attribute", + in: "GROUP BY Foo", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + + // ORDER BY clause. + { + name: "order by", + in: "ORDER BY StartTime", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + orderBy: "order by start_time asc", + }, + { + name: "order by desc", + in: "ORDER BY CloseTime DESC", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + orderBy: "order by close_time desc", + }, + { + name: "order by multiple search attributes", + in: "ORDER BY StartTime DESC, RunId ASC", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + orderBy: "order by start_time desc, run_id asc", + }, + { + name: "order by with where clause", + in: "WorkflowId = 'wid' ORDER BY StartTime DESC", + sql: "TemporalNamespaceDivision is null and workflow_id = 'wid'", + es: `{"bool":{"filter":{"term":{"WorkflowId":"wid"}},"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + orderBy: "order by start_time desc", + }, + { + name: "order by custom search attribute", + in: "ORDER BY AliasForKeyword01", + sql: "TemporalNamespaceDivision is null", + es: `{"bool":{"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, + orderBy: "order by Keyword01 asc", + }, + { + name: "order by text search attribute", + in: "ORDER BY AliasForText01", + err: "operation is not supported: unable to sort by search attribute type Text", + }, + { + name: "order by unknown search attribute", + in: "ORDER BY Foo", + err: "invalid expression: column name 'Foo' is not a valid search attribute", + }, + + // Malformed and unsupported queries. + { + name: "malformed query", + in: "this is not a query", + err: "malformed SQL query: syntax error at position 41 near 'a'", + }, + { + name: "missing value", + in: "WorkflowId =", + err: "malformed SQL query: syntax error at position 40", + }, + { + name: "incomplete expression", + in: "WorkflowId", + err: "invalid expression: incomplete expression", + }, + { + name: "column name on right hand side", + in: "WorkflowId = RunId", + err: `operation is not supported: column name on the right side of comparison expression (did you forget to quote "RunId"?)`, + }, + { + name: "function expression", + in: "length(WorkflowId)", + err: "operation is not supported: function expression", + }, + { + name: "function expression on left hand side", + in: "length(WorkflowId) = 5", + err: "invalid expression: must be a column name but was *sqlparser.FuncExpr", + }, + { + name: "function expression on right hand side", + in: "WorkflowId = length('foo')", + err: "operation is not supported: nested func", + }, + { + name: "limit clause", + in: "WorkflowId = 'wid' LIMIT 10", + err: "operation is not supported: 'LIMIT' clause", + }, + { + name: "subquery", + in: "WorkflowId IN (SELECT * FROM table1)", + err: "invalid expression: unexpected value type *sqlparser.Subquery", + }, + { + name: "arithmetic expression", + in: "HistoryLength = 1 + 1", + err: "invalid expression: unexpected value type *sqlparser.BinaryExpr", + }, + { + name: "unsupported unary operator", + in: "AliasForInt01 = ~1", + err: `operation is not supported: unary operator "~"`, + }, +} + +func runQueryConverterTest[T any]( + t *testing.T, + store string, + newQueryConverter func() *query.QueryConverter[T], + serialize func(T) (string, error), +) { + for _, tc := range queryConverterTestCases { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + expectedOut, expectedErr := tc.expected(store) + // The query converter is stateful (e.g. it tracks if the query filtered by + // namespace division), thus each test case requires a new instance. + queryParams, err := newQueryConverter().Convert(tc.in) + if expectedErr != "" { + r.Error(err) + r.EqualError(err, expectedErr) + var converterErr *query.ConverterError + r.ErrorAs(err, &converterErr) + r.Nil(queryParams) + return + } + r.NoError(err) + out, err := serialize(queryParams.QueryExpr) + r.NoError(err) + r.Equal(expectedOut, out) + + var groupBy []string + for _, col := range queryParams.GroupBy { + groupBy = append(groupBy, col.FieldName) + } + r.Equal(tc.groupBy, groupBy) + r.Equal(tc.orderBy, strings.TrimSpace(sqlparser.String(queryParams.OrderBy))) + }) + } +} + +func serializeSQLQuery(expr sqlparser.Expr) (string, error) { + if expr == nil { + return "", nil + } + return sqlparser.String(expr), nil +} + +func serializeESQuery(q elastic.Query) (string, error) { + if q == nil { + return "", nil + } + source, err := q.Source() + if err != nil { + return "", err + } + data, err := json.Marshal(source) + if err != nil { + return "", err + } + return string(data), nil +} + +func TestSQLQueryConverter(t *testing.T) { + t.Parallel() + for _, plugin := range sqlPlugins { + t.Run(plugin, func(t *testing.T) { + t.Parallel() + pluginVisQC, err := sql.GetPluginVisibilityQueryConverter(plugin) + require.NoError(t, err) + runQueryConverterTest( + t, + plugin, + func() *query.QueryConverter[sqlparser.Expr] { + return query.NewQueryConverter( + &vissql.SQLQueryConverter{VisibilityQueryConverter: pluginVisQC}, + testNamespaceName, + searchattribute.TestNameTypeMap(), + &searchattribute.TestMapper{}, + nil, // metricsHandler + log.NewNoopLogger(), + ) + }, + serializeSQLQuery, + ) + }) + } +} + +func TestElasticsearchQueryConverter(t *testing.T) { + t.Parallel() + runQueryConverterTest( + t, + esStore, + func() *query.QueryConverter[elastic.Query] { + return elasticsearch.NewQueryConverter( + testNamespaceName, + searchattribute.TestNameTypeMap(), + &searchattribute.TestMapper{}, + nil, // metricsHandler + log.NewNoopLogger(), + ) + }, + serializeESQuery, + ) +} From 07a9dc313a7f6ee27fac7d9bdaec19ed15921814 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Wed, 2 Sep 2026 14:19:32 -0700 Subject: [PATCH 120/139] Extend Copilot code review guidelines (#11904) WISOTT --- .github/copilot-instructions.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f80f9ab2b41..ad7b68b0497 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -13,21 +13,25 @@ Apply these patterns when reviewing PRs or suggesting code changes. - Remove redundant nil checks after you just set a value - Do not export anything that doesn't need to be exported -## 2. Go Naming Conventions +## 2. Go Conventions - Don't use `Get` prefix for getters: `func (a *Activity) Store()` not `GetStore()` - Don't use `Impl` suffix for implementations - Don't put underscore after `Test` in test names: `TestRetry` not `Test_Retry` - Avoid stuttering: don't use `ActivityStatus` in package `activity`, just `Status` - Use `ok` boolean pattern instead of nil checks where idiomatic +- Prefer `cmp.Or` for defaults when zero means "unset"; it returns the first non-zero argument. Use an `if` statement for side-effecting or expensive fallbacks because all arguments are evaluated. -## 3. Testify Suite Correctness and Reliability +## 3. Testing Correctness and Reliability - Never use `s.T()` in subtests - use the subtest's `t` parameter - Never use suite assertion methods (`s.NoError`, `s.Equal`) from goroutines - causes panics - Use `EventuallyWithT` when you need assertions inside eventually blocks, and use that block's `t` - Use `require.ErrorAs(t, err, &specificErr)` for specific error type checks - Prefer `require` over `assert` - it's rarely useful to continue a test after a failed assertion +- Prefer table-driven tests over separate `Test*` methods when cases exercise the same behavior. Give each case a descriptive name and run it as a subtest. +- Run independent cases in parallel in plain `t.Run` tests. Testify suites do not support parallel subtests. +- Prefer comparing a function's full result with an expected value over separate `require.Equal` calls for each field. Use `protorequire.ProtoEqual` for proto results and proto fields within non-proto results. Use other field-level assertions only when part of the result is relevant. - Add comments explaining why `Eventually` is needed (e.g., eventual consistency) - Do not use single-value type assertions on errors (`err.(*T)`); this panics instead of failing the test when the type doesn't match. Use `errors.As` with a guarded return. - When launching a goroutine to maintain a precondition for later assertions (e.g., keeping pollers active so a deployment version gets registered), loop until context cancellation rather than running once. A single attempt that times out exits silently, leaving downstream Eventually/propagation waits to hang until their own deadline. @@ -69,6 +73,7 @@ Apply these patterns when reviewing PRs or suggesting code changes. ## 7. Code comments +- Write comments that read like sentences as full sentences, starting with a capital letter and ending with punctuation. Short labels, end-of-line fragments, directives, and `TODO` markers are exempt. - A comment should be removed if the behavior of the code without the comment should be apparent to a reader familiar with the codebase. - If the benefit of a comment can be achieved by improving variable/function names then suggest that. - A comment must not give unnecessary or verbose explanation. From 5618f0541682bec03a92030f01acbd6ef9887d23 Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Wed, 2 Sep 2026 14:26:12 -0700 Subject: [PATCH 121/139] Prevent Terminating an already Canceled SANO (#11897) ## What changed? Previously you could Terminate a standalone Nexus operation that was already in the Canceled state. This PR closes that gap. ## Why? That behavior was never intended, and is a bug. Canceled is already a terminal state. Not only does terminating an already canceled operation not make sense, it also opens up weird issues where we would mutate a resource that should be in a terminal state. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks This is a behavior change, and could break a test or something that was relying on this behavior earlier. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- chasm/lib/nexusoperation/operation.go | 4 + .../nexusoperation/operation_statemachine.go | 1 - chasm/lib/nexusoperation/operation_test.go | 56 ++++++++++++++ tests/nexus_standalone_test.go | 77 +++++++++++++++++-- 4 files changed, 132 insertions(+), 6 deletions(-) diff --git a/chasm/lib/nexusoperation/operation.go b/chasm/lib/nexusoperation/operation.go index 4ce985df2fb..4c9eecae8c0 100644 --- a/chasm/lib/nexusoperation/operation.go +++ b/chasm/lib/nexusoperation/operation.go @@ -417,6 +417,10 @@ func (o *Operation) Terminate( return chasm.TerminateComponentResponse{}, nil } + if !TransitionTerminated.Possible(o) { + return chasm.TerminateComponentResponse{}, ErrOperationAlreadyCompleted + } + return chasm.TerminateComponentResponse{}, TransitionTerminated.Apply(o, ctx, EventTerminated{ TerminateComponentRequest: req, }) diff --git a/chasm/lib/nexusoperation/operation_statemachine.go b/chasm/lib/nexusoperation/operation_statemachine.go index ba6f2498992..d7647246c26 100644 --- a/chasm/lib/nexusoperation/operation_statemachine.go +++ b/chasm/lib/nexusoperation/operation_statemachine.go @@ -259,7 +259,6 @@ var TransitionTerminated = chasm.NewTransition( nexusoperationpb.OPERATION_STATUS_SCHEDULED, nexusoperationpb.OPERATION_STATUS_STARTED, nexusoperationpb.OPERATION_STATUS_BACKING_OFF, - nexusoperationpb.OPERATION_STATUS_CANCELED, }, nexusoperationpb.OPERATION_STATUS_TERMINATED, func(o *Operation, ctx chasm.MutableContext, event EventTerminated) error { diff --git a/chasm/lib/nexusoperation/operation_test.go b/chasm/lib/nexusoperation/operation_test.go index e2ac5ea4919..ef1cfd304eb 100644 --- a/chasm/lib/nexusoperation/operation_test.go +++ b/chasm/lib/nexusoperation/operation_test.go @@ -67,6 +67,62 @@ func TestRequestCancelDeduplicationAfterTerminalState(t *testing.T) { }) } +func TestTerminate(t *testing.T) { + t.Parallel() + + newOpWithStatus := func(status nexusoperationpb.OperationStatus) (*Operation, *chasm.MockMutableContext) { + ctx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleNow: func(chasm.Component) time.Time { return defaultTime }, + HandleNamespaceEntry: func() *namespace.Namespace { + return namespace.NewNamespaceForTest( + &persistencespb.NamespaceInfo{Name: "ns-name"}, nil, false, nil, 0, + ) + }, + GoCtx: context.WithValue(context.Background(), OperationContextKey, &OperationContext{ + MetricTagConfig: dynamicconfig.GetTypedPropertyFn(NexusMetricTagConfig{}), + }), + }, + } + op := newTestOperation() + op.Status = status + return op, ctx + } + + t.Run("TerminatesARunningOperation", func(t *testing.T) { + op, ctx := newOpWithStatus(nexusoperationpb.OPERATION_STATUS_STARTED) + + _, err := op.Terminate(ctx, chasm.TerminateComponentRequest{RequestID: "req-id"}) + require.NoError(t, err) + require.Equal(t, nexusoperationpb.OPERATION_STATUS_TERMINATED, op.Status) + }) + + // Terminating an operation that already reached a terminal outcome is a FailedPrecondition. + for _, status := range []nexusoperationpb.OperationStatus{ + nexusoperationpb.OPERATION_STATUS_SUCCEEDED, + nexusoperationpb.OPERATION_STATUS_CANCELED, + nexusoperationpb.OPERATION_STATUS_FAILED, + nexusoperationpb.OPERATION_STATUS_TIMED_OUT, + } { + t.Run("Rejects"+status.String(), func(t *testing.T) { + op, ctx := newOpWithStatus(status) + + _, err := op.Terminate(ctx, chasm.TerminateComponentRequest{RequestID: "req-id"}) + require.ErrorIs(t, err, ErrOperationAlreadyCompleted) + require.Equal(t, status, op.Status) + }) + } + + t.Run("IsIdempotentForTheSameRequestID", func(t *testing.T) { + op, ctx := newOpWithStatus(nexusoperationpb.OPERATION_STATUS_STARTED) + + _, err := op.Terminate(ctx, chasm.TerminateComponentRequest{RequestID: "req-id"}) + require.NoError(t, err) + _, err = op.Terminate(ctx, chasm.TerminateComponentRequest{RequestID: "req-id"}) + require.NoError(t, err) + }) +} + func TestIsWaitStageReached(t *testing.T) { t.Parallel() diff --git a/tests/nexus_standalone_test.go b/tests/nexus_standalone_test.go index ab44d3efc47..c81d042c152 100644 --- a/tests/nexus_standalone_test.go +++ b/tests/nexus_standalone_test.go @@ -1194,7 +1194,9 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { s.Contains(err.Error(), "already terminated") }) - s.Run("AlreadyCanceled", func(s *NexusStandaloneTestSuite) { + // A cancel request on a running operation only records the cancellation; the operation stays + // running, so terminate is still accepted and closes it as TERMINATED. + s.Run("CancelRequestedThenTerminated", func(s *NexusStandaloneTestSuite) { env := s.newTestEnv() endpointName := env.createRandomNexusEndpoint(s.Context(), s.T()).GetSpec().GetName() @@ -1204,7 +1206,7 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { }) s.NoError(err) - // Cancel the operation first. + // Request cancellation first. _, err = env.FrontendClient().RequestCancelNexusOperationExecution(s.Context(), &workflowservice.RequestCancelNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), OperationId: "test-op", @@ -1212,7 +1214,16 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { }) s.NoError(err) - // Terminate a canceled operation — should succeed. + // No worker services this endpoint, so the operation is still running. + descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: "test-op", + RunId: startResp.RunId, + }) + s.NoError(err) + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_RUNNING, descResp.GetInfo().GetStatus()) + s.NotNil(descResp.GetInfo().GetCancellationInfo()) + _, err = env.FrontendClient().TerminateNexusOperationExecution(s.Context(), &workflowservice.TerminateNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), OperationId: "test-op", @@ -1222,8 +1233,7 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { }) s.NoError(err) - // Verify state changed to terminated (terminate overrides cancel request). - descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ + descResp, err = env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ Namespace: env.Namespace().String(), OperationId: "test-op", RunId: startResp.RunId, @@ -1232,6 +1242,63 @@ func (s *NexusStandaloneTestSuite) TestTerminateStandaloneNexusOperation() { s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_TERMINATED, descResp.GetInfo().GetStatus()) }) + s.Run("AlreadyCanceled", func(s *NexusStandaloneTestSuite) { + env := s.newTestEnv() + // The handler cancels on start, closing the operation as CANCELED. + endpointName := env.createRandomExternalNexusServer(s.Context(), s.T(), nexustest.Handler{ + OnStartOperation: func(ctx context.Context, service, operation string, input *nexus.LazyValue, options nexus.StartOperationOptions) (nexus.HandlerStartOperationResult[any], error) { + return nil, &nexus.OperationError{ + State: nexus.OperationStateCanceled, + Cause: &nexus.FailureError{Failure: nexus.Failure{Message: "canceled by handler"}}, + } + }, + }) + + startResp, err := s.startNexusOperation(env, &workflowservice.StartNexusOperationExecutionRequest{ + OperationId: "test-op", + Endpoint: endpointName, + }) + s.NoError(err) + + // Ensure the operation actually reached CANCELED before terminating. + _, err = env.FrontendClient().PollNexusOperationExecution(s.Context(), &workflowservice.PollNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: "test-op", + RunId: startResp.RunId, + WaitStage: enumspb.NEXUS_OPERATION_WAIT_STAGE_CLOSED, + }) + s.NoError(err) + + descResp, err := env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: "test-op", + RunId: startResp.RunId, + }) + s.NoError(err) + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_CANCELED, descResp.GetInfo().GetStatus()) + + // Terminating a canceled operation is rejected. + _, err = env.FrontendClient().TerminateNexusOperationExecution(s.Context(), &workflowservice.TerminateNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: "test-op", + RunId: startResp.RunId, + RequestId: "terminate-request-id", + Reason: "test termination", + }) + var failedPrecondition *serviceerror.FailedPrecondition + s.ErrorAs(err, &failedPrecondition) + s.ErrorContains(err, "operation already completed") + + // The rejected terminate left the outcome untouched. + descResp, err = env.FrontendClient().DescribeNexusOperationExecution(s.Context(), &workflowservice.DescribeNexusOperationExecutionRequest{ + Namespace: env.Namespace().String(), + OperationId: "test-op", + RunId: startResp.RunId, + }) + s.NoError(err) + s.Equal(enumspb.NEXUS_OPERATION_EXECUTION_STATUS_CANCELED, descResp.GetInfo().GetStatus()) + }) + s.Run("NotFound", func(s *NexusStandaloneTestSuite) { env := s.newTestEnv() From a07a006819510926ea418dfdbf06a8a5ac96ecd0 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Wed, 2 Sep 2026 15:22:19 -0700 Subject: [PATCH 122/139] Replace orphaned child workflows with no worker progress after force failover (#11815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? - Added request-only `OrphanedChildReplacementInfo`, carrying the parent’s current Version History as branch evidence. - When enabled, a retried child start may atomically terminate an orphaned conflicting run and create a replacement. - Replacement is allowed only when the conflicting child: - belongs to the same parent run; - was initiated on a losing parent branch, while the incoming initiation is on the current branch; - is the first run in its execution chain; - contains only `WorkflowExecutionStarted`; - has no pending in-memory Update. - Existing request-ID deduplication remains ahead of replacement handling. - Added outcome metrics, a post-commit success log, and the disabled-by-default `history.enableOrphanedChildWorkflowReplacement` setting. ## Why? After force failover, a child created from a losing parent branch may conflict with the same child start reissued by the winning branch. Normal workflow-ID conflict handling records `WORKFLOW_ALREADY_EXISTS`, leaving the parent unable to make progress and the original child orphaned. Re-linking the existing child is unsafe because its parent coordinates are stored in immutable history. This change instead replaces it only when the new active cluster sees no progress beyond `WorkflowExecutionStarted`. Ownership and progress are rechecked while holding the child lock, and termination plus replacement creation are committed atomically. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) Validated with: - `go test -tags test_dep ./service/history/api ./service/history -count=1` - Focused parent-child XDC integration test - `make proto` - `make lint-code` ## Potential risks - The decision uses the new active cluster’s locally visible state. During a network partition, it cannot know whether the previous active advanced the child beyond the replicated state. - Replacement terminates an existing run. The feature is disabled by default and fails closed unless every ownership, branch, and progress check passes under the child lock. - The request carries the parent’s full current `VersionHistoryItems` list. An unusually long version history may increase RPC size and could exceed internal gRPC limits; oversized requests fail instead of recovering the child. - All history hosts processing child starts in an enabled cluster must understand the request field. An older host may ignore it and record `WORKFLOW_ALREADY_EXISTS`. --- .../v1/request_response.go-helpers.pb.go | 37 + api/historyservice/v1/request_response.pb.go | 897 ++++++++++-------- common/dynamicconfig/constants.go | 7 + common/metrics/metric_defs.go | 1 + .../historyservice/v1/request_response.proto | 12 + service/history/api/startworkflow/api.go | 83 +- service/history/api/workflow_id_dedup.go | 120 +++ service/history/api/workflow_id_dedup_test.go | 249 +++++ service/history/configs/config.go | 37 +- service/history/history_engine2_test.go | 222 ++++- .../transfer_queue_active_task_executor.go | 207 ++-- ...ransfer_queue_active_task_executor_test.go | 196 +++- tests/xdc/parent_child_test.go | 228 ++--- 13 files changed, 1643 insertions(+), 653 deletions(-) diff --git a/api/historyservice/v1/request_response.go-helpers.pb.go b/api/historyservice/v1/request_response.go-helpers.pb.go index 68cf73e2c50..635ae4eeaaa 100644 --- a/api/historyservice/v1/request_response.go-helpers.pb.go +++ b/api/historyservice/v1/request_response.go-helpers.pb.go @@ -6072,3 +6072,40 @@ func (this *PollWorkflowExecutionTimeSkippingResponse) Equal(that interface{}) b return proto.Equal(this, that1) } + +// Marshal an object of type OrphanedChildReplacementInfo to the protobuf v3 wire format +func (val *OrphanedChildReplacementInfo) Marshal() ([]byte, error) { + return proto.Marshal(val) +} + +// Unmarshal an object of type OrphanedChildReplacementInfo from the protobuf v3 wire format +func (val *OrphanedChildReplacementInfo) Unmarshal(buf []byte) error { + return proto.Unmarshal(buf, val) +} + +// Size returns the size of the object, in bytes, once serialized +func (val *OrphanedChildReplacementInfo) Size() int { + return proto.Size(val) +} + +// Equal returns whether two OrphanedChildReplacementInfo values are equivalent by recursively +// comparing the message's fields. +// For more information see the documentation for +// https://pkg.go.dev/google.golang.org/protobuf/proto#Equal +func (this *OrphanedChildReplacementInfo) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + var that1 *OrphanedChildReplacementInfo + switch t := that.(type) { + case *OrphanedChildReplacementInfo: + that1 = t + case OrphanedChildReplacementInfo: + that1 = &t + default: + return false + } + + return proto.Equal(this, that1) +} diff --git a/api/historyservice/v1/request_response.pb.go b/api/historyservice/v1/request_response.pb.go index a7f13f19b10..571fe8cfba5 100644 --- a/api/historyservice/v1/request_response.pb.go +++ b/api/historyservice/v1/request_response.pb.go @@ -202,6 +202,10 @@ type StartWorkflowExecutionRequest struct { // WorkflowExecutionStartedEvent. DeclinedTargetVersionUpgrade *v17.DeclinedTargetVersionUpgrade `protobuf:"bytes,17,opt,name=declined_target_version_upgrade,json=declinedTargetVersionUpgrade,proto3" json:"declined_target_version_upgrade,omitempty"` TimeSkippingStatePropagation *v14.TimeSkippingStatePropagation `protobuf:"bytes,19,opt,name=time_skipping_state_propagation,json=timeSkippingStatePropagation,proto3" json:"time_skipping_state_propagation,omitempty"` + // When present, StartWorkflowExecution may replace a conflicting child only after verifying under + // the child's lock that its parent initiation is absent from the supplied current branch and that + // its first run has only WorkflowExecutionStarted. + OrphanedChildReplacementInfo *OrphanedChildReplacementInfo `protobuf:"bytes,20,opt,name=orphaned_child_replacement_info,json=orphanedChildReplacementInfo,proto3" json:"orphaned_child_replacement_info,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -362,6 +366,13 @@ func (x *StartWorkflowExecutionRequest) GetTimeSkippingStatePropagation() *v14.T return nil } +func (x *StartWorkflowExecutionRequest) GetOrphanedChildReplacementInfo() *OrphanedChildReplacementInfo { + if x != nil { + return x.OrphanedChildReplacementInfo + } + return nil +} + type StartWorkflowExecutionResponse struct { state protoimpl.MessageState `protogen:"open.v1"` RunId string `protobuf:"bytes,1,opt,name=run_id,json=runId,proto3" json:"run_id,omitempty"` @@ -10404,6 +10415,54 @@ func (x *PollWorkflowExecutionTimeSkippingResponse) GetResponse() *v1.PollWorkfl return nil } +// Carries the parent branch evidence needed to distinguish an orphaned child from another +// accepted initiation of the same child workflow ID. This is request-only replacement context and +// is not persisted into the child workflow. Re-linking requires a separate recovery mechanism +// because the existing parent coordinates are also immutable WorkflowExecutionStarted history. +type OrphanedChildReplacementInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + ParentCurrentVersionHistoryItems []*v19.VersionHistoryItem `protobuf:"bytes,1,rep,name=parent_current_version_history_items,json=parentCurrentVersionHistoryItems,proto3" json:"parent_current_version_history_items,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrphanedChildReplacementInfo) Reset() { + *x = OrphanedChildReplacementInfo{} + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[164] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrphanedChildReplacementInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrphanedChildReplacementInfo) ProtoMessage() {} + +func (x *OrphanedChildReplacementInfo) ProtoReflect() protoreflect.Message { + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[164] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrphanedChildReplacementInfo.ProtoReflect.Descriptor instead. +func (*OrphanedChildReplacementInfo) Descriptor() ([]byte, []int) { + return file_temporal_server_api_historyservice_v1_request_response_proto_rawDescGZIP(), []int{164} +} + +func (x *OrphanedChildReplacementInfo) GetParentCurrentVersionHistoryItems() []*v19.VersionHistoryItem { + if x != nil { + return x.ParentCurrentVersionHistoryItems + } + return nil +} + type ExecuteMultiOperationRequest_Operation struct { state protoimpl.MessageState `protogen:"open.v1"` // Types that are valid to be assigned to Operation: @@ -10417,7 +10476,7 @@ type ExecuteMultiOperationRequest_Operation struct { func (x *ExecuteMultiOperationRequest_Operation) Reset() { *x = ExecuteMultiOperationRequest_Operation{} - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[164] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10429,7 +10488,7 @@ func (x *ExecuteMultiOperationRequest_Operation) String() string { func (*ExecuteMultiOperationRequest_Operation) ProtoMessage() {} func (x *ExecuteMultiOperationRequest_Operation) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[164] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10501,7 +10560,7 @@ type ExecuteMultiOperationResponse_Response struct { func (x *ExecuteMultiOperationResponse_Response) Reset() { *x = ExecuteMultiOperationResponse_Response{} - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[166] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10513,7 +10572,7 @@ func (x *ExecuteMultiOperationResponse_Response) String() string { func (*ExecuteMultiOperationResponse_Response) ProtoMessage() {} func (x *ExecuteMultiOperationResponse_Response) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[166] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10583,7 +10642,7 @@ type ListQueuesResponse_QueueInfo struct { func (x *ListQueuesResponse_QueueInfo) Reset() { *x = ListQueuesResponse_QueueInfo{} - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[171] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[172] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10595,7 +10654,7 @@ func (x *ListQueuesResponse_QueueInfo) String() string { func (*ListQueuesResponse_QueueInfo) ProtoMessage() {} func (x *ListQueuesResponse_QueueInfo) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[171] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[172] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10646,7 +10705,7 @@ type AddTasksRequest_Task struct { func (x *AddTasksRequest_Task) Reset() { *x = AddTasksRequest_Task{} - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[172] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[173] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -10658,7 +10717,7 @@ func (x *AddTasksRequest_Task) String() string { func (*AddTasksRequest_Task) ProtoMessage() {} func (x *AddTasksRequest_Task) ProtoReflect() protoreflect.Message { - mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[172] + mi := &file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[173] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -10721,7 +10780,7 @@ const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc "task_token\x18\x06 \x01(\tR\ttaskToken\x12\x1d\n" + "\n" + "task_infos\x18\a \x01(\tR\ttaskInfos\x12.\n" + - "\x13chasm_component_ref\x18\b \x01(\tR\x11chasmComponentRef\"\x84\r\n" + + "\x13chasm_component_ref\x18\b \x01(\tR\x11chasmComponentRef\"\x91\x0e\n" + "\x1dStartWorkflowExecutionRequest\x12!\n" + "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12c\n" + "\rstart_request\x18\x02 \x01(\v2>.temporal.api.workflowservice.v1.StartWorkflowExecutionRequestR\fstartRequest\x12h\n" + @@ -10741,7 +10800,8 @@ const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc "\x18inherited_pinned_version\x18\x0f \x01(\v23.temporal.api.deployment.v1.WorkerDeploymentVersionR\x16inheritedPinnedVersion\x12s\n" + "\x1binherited_auto_upgrade_info\x18\x10 \x01(\v24.temporal.api.deployment.v1.InheritedAutoUpgradeInfoR\x18inheritedAutoUpgradeInfo\x12|\n" + "\x1fdeclined_target_version_upgrade\x18\x11 \x01(\v25.temporal.api.history.v1.DeclinedTargetVersionUpgradeR\x1cdeclinedTargetVersionUpgrade\x12{\n" + - "\x1ftime_skipping_state_propagation\x18\x13 \x01(\v24.temporal.api.common.v1.TimeSkippingStatePropagationR\x1ctimeSkippingStatePropagation:\x1f\x92\xc4\x03\x1b*\x19start_request.workflow_idJ\x04\b\x12\x10\x13\"\xb1\x03\n" + + "\x1ftime_skipping_state_propagation\x18\x13 \x01(\v24.temporal.api.common.v1.TimeSkippingStatePropagationR\x1ctimeSkippingStatePropagation\x12\x8a\x01\n" + + "\x1forphaned_child_replacement_info\x18\x14 \x01(\v2C.temporal.server.api.historyservice.v1.OrphanedChildReplacementInfoR\x1corphanedChildReplacementInfo:\x1f\x92\xc4\x03\x1b*\x19start_request.workflow_idJ\x04\b\x12\x10\x13\"\xb1\x03\n" + "\x1eStartWorkflowExecutionResponse\x12\x15\n" + "\x06run_id\x18\x01 \x01(\tR\x05runId\x12?\n" + "\x05clock\x18\x02 \x01(\v2).temporal.server.api.clock.v1.VectorClockR\x05clock\x12n\n" + @@ -11485,7 +11545,9 @@ const file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc "\fnamespace_id\x18\x01 \x01(\tR\vnamespaceId\x12c\n" + "\arequest\x18\x02 \x01(\v2I.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequestR\arequest:,\x92\xc4\x03(*&request.workflow_execution.workflow_id\"\x93\x01\n" + ")PollWorkflowExecutionTimeSkippingResponse\x12f\n" + - "\bresponse\x18\x01 \x01(\v2J.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponseR\bresponse:t\n" + + "\bresponse\x18\x01 \x01(\v2J.temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponseR\bresponse\"\xa3\x01\n" + + "\x1cOrphanedChildReplacementInfo\x12\x82\x01\n" + + "$parent_current_version_history_items\x18\x01 \x03(\v22.temporal.server.api.history.v1.VersionHistoryItemR parentCurrentVersionHistoryItems:t\n" + "\arouting\x12\x1f.google.protobuf.MessageOptions\x18\xc28 \x01(\v25.temporal.server.api.historyservice.v1.RoutingOptionsR\arouting\x88\x01\x01B temporal.api.workflowservice.v1.StartWorkflowExecutionRequest - 174, // 1: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.parent_execution_info:type_name -> temporal.server.api.workflow.v1.ParentExecutionInfo - 175, // 2: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.workflow_execution_expiration_time:type_name -> google.protobuf.Timestamp - 176, // 3: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.continue_as_new_initiator:type_name -> temporal.api.enums.v1.ContinueAsNewInitiator - 177, // 4: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.continued_failure:type_name -> temporal.api.failure.v1.Failure - 178, // 5: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.last_completion_result:type_name -> temporal.api.common.v1.Payloads - 179, // 6: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.first_workflow_task_backoff:type_name -> google.protobuf.Duration - 180, // 7: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.source_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp - 181, // 8: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.root_execution_info:type_name -> temporal.server.api.workflow.v1.RootExecutionInfo - 182, // 9: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.versioning_override:type_name -> temporal.api.workflow.v1.VersioningOverride - 183, // 10: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.inherited_pinned_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion - 184, // 11: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.inherited_auto_upgrade_info:type_name -> temporal.api.deployment.v1.InheritedAutoUpgradeInfo - 185, // 12: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.declined_target_version_upgrade:type_name -> temporal.api.history.v1.DeclinedTargetVersionUpgrade - 186, // 13: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.time_skipping_state_propagation:type_name -> temporal.api.common.v1.TimeSkippingStatePropagation - 187, // 14: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 188, // 15: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.eager_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse - 189, // 16: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 190, // 17: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link - 191, // 18: temporal.server.api.historyservice.v1.GetMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 192, // 19: temporal.server.api.historyservice.v1.GetMutableStateRequest.version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 193, // 20: temporal.server.api.historyservice.v1.GetMutableStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 191, // 21: temporal.server.api.historyservice.v1.GetMutableStateResponse.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 194, // 22: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 195, // 23: temporal.server.api.historyservice.v1.GetMutableStateResponse.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 195, // 24: temporal.server.api.historyservice.v1.GetMutableStateResponse.sticky_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 179, // 25: temporal.server.api.historyservice.v1.GetMutableStateResponse.sticky_task_queue_schedule_to_start_timeout:type_name -> google.protobuf.Duration - 196, // 26: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState - 189, // 27: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 197, // 28: temporal.server.api.historyservice.v1.GetMutableStateResponse.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories - 180, // 29: temporal.server.api.historyservice.v1.GetMutableStateResponse.most_recent_worker_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp - 193, // 30: temporal.server.api.historyservice.v1.GetMutableStateResponse.transition_history:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 198, // 31: temporal.server.api.historyservice.v1.GetMutableStateResponse.versioning_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionVersioningInfo - 199, // 32: temporal.server.api.historyservice.v1.GetMutableStateResponse.transient_or_speculative_tasks:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo - 191, // 33: temporal.server.api.historyservice.v1.PollMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 192, // 34: temporal.server.api.historyservice.v1.PollMutableStateRequest.version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 191, // 35: temporal.server.api.historyservice.v1.PollMutableStateResponse.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 194, // 36: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 195, // 37: temporal.server.api.historyservice.v1.PollMutableStateResponse.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 195, // 38: temporal.server.api.historyservice.v1.PollMutableStateResponse.sticky_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 179, // 39: temporal.server.api.historyservice.v1.PollMutableStateResponse.sticky_task_queue_schedule_to_start_timeout:type_name -> google.protobuf.Duration - 197, // 40: temporal.server.api.historyservice.v1.PollMutableStateResponse.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories - 196, // 41: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState - 189, // 42: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 191, // 43: temporal.server.api.historyservice.v1.ResetStickyTaskQueueRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 164, // 44: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.operations:type_name -> temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation - 165, // 45: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.responses:type_name -> temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response - 191, // 46: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 200, // 47: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest - 187, // 48: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 201, // 49: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo - 202, // 50: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment - 203, // 51: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 183, // 52: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.target_deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion - 194, // 53: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 199, // 54: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo - 195, // 55: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 175, // 56: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 57: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp - 166, // 58: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry - 187, // 59: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 204, // 60: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.messages:type_name -> temporal.api.protocol.v1.Message - 205, // 61: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.history:type_name -> temporal.api.history.v1.History - 205, // 62: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.raw_history:type_name -> temporal.api.history.v1.History - 194, // 63: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 199, // 64: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo - 195, // 65: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue - 175, // 66: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 67: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp - 167, // 68: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry - 187, // 69: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 204, // 70: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message - 205, // 71: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History - 191, // 72: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 206, // 73: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest - 187, // 74: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 201, // 75: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo - 202, // 76: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment - 203, // 77: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective - 207, // 78: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.scheduled_event:type_name -> temporal.api.history.v1.HistoryEvent - 175, // 79: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp - 175, // 80: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp - 178, // 81: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads - 194, // 82: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType - 187, // 83: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 208, // 84: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.priority:type_name -> temporal.api.common.v1.Priority - 209, // 85: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy - 210, // 86: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest - 12, // 87: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.started_response:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse - 211, // 88: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.activity_tasks:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueResponse - 188, // 89: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.new_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse - 212, // 90: temporal.server.api.historyservice.v1.RespondWorkflowTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest - 191, // 91: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 92: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 213, // 93: temporal.server.api.historyservice.v1.RecordActivityTaskHeartbeatRequest.heartbeat_request:type_name -> temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest - 214, // 94: temporal.server.api.historyservice.v1.RespondActivityTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest - 215, // 95: temporal.server.api.historyservice.v1.RespondActivityTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest - 216, // 96: temporal.server.api.historyservice.v1.RespondActivityTaskCanceledRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCanceledRequest - 191, // 97: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 98: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 217, // 99: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.signal_request:type_name -> temporal.api.workflowservice.v1.SignalWorkflowExecutionRequest - 191, // 100: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 190, // 101: temporal.server.api.historyservice.v1.SignalWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link - 218, // 102: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionRequest.signal_with_start_request:type_name -> temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest - 190, // 103: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionResponse.signal_link:type_name -> temporal.api.common.v1.Link - 191, // 104: temporal.server.api.historyservice.v1.RemoveSignalMutableStateRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 219, // 105: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.terminate_request:type_name -> temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest - 191, // 106: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 107: temporal.server.api.historyservice.v1.DeleteWorkflowExecutionRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 220, // 108: temporal.server.api.historyservice.v1.ResetWorkflowExecutionRequest.reset_request:type_name -> temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest - 221, // 109: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RequestCancelWorkflowExecutionRequest - 191, // 110: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 111: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 112: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.child_clock:type_name -> temporal.server.api.clock.v1.VectorClock - 187, // 113: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.parent_clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 114: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 115: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 116: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 117: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 207, // 118: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.completion_event:type_name -> temporal.api.history.v1.HistoryEvent - 187, // 119: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 191, // 120: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 121: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 187, // 122: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock - 222, // 123: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest - 223, // 124: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.execution_config:type_name -> temporal.api.workflow.v1.WorkflowExecutionConfig - 224, // 125: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_execution_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionInfo - 225, // 126: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_activities:type_name -> temporal.api.workflow.v1.PendingActivityInfo - 226, // 127: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_children:type_name -> temporal.api.workflow.v1.PendingChildExecutionInfo - 227, // 128: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_workflow_task:type_name -> temporal.api.workflow.v1.PendingWorkflowTaskInfo - 228, // 129: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.callbacks:type_name -> temporal.api.workflow.v1.CallbackInfo - 229, // 130: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_nexus_operations:type_name -> temporal.api.workflow.v1.PendingNexusOperationInfo - 230, // 131: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_extended_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionExtendedInfo - 191, // 132: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 192, // 133: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem - 231, // 134: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.events:type_name -> temporal.api.common.v1.DataBlob - 231, // 135: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.new_run_events:type_name -> temporal.api.common.v1.DataBlob - 232, // 136: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 233, // 137: temporal.server.api.historyservice.v1.ReplicateWorkflowStateRequest.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 175, // 138: temporal.server.api.historyservice.v1.SyncShardStatusRequest.status_time:type_name -> google.protobuf.Timestamp - 175, // 139: temporal.server.api.historyservice.v1.SyncActivityRequest.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 140: temporal.server.api.historyservice.v1.SyncActivityRequest.started_time:type_name -> google.protobuf.Timestamp - 175, // 141: temporal.server.api.historyservice.v1.SyncActivityRequest.last_heartbeat_time:type_name -> google.protobuf.Timestamp - 178, // 142: temporal.server.api.historyservice.v1.SyncActivityRequest.details:type_name -> temporal.api.common.v1.Payloads - 177, // 143: temporal.server.api.historyservice.v1.SyncActivityRequest.last_failure:type_name -> temporal.api.failure.v1.Failure - 234, // 144: temporal.server.api.historyservice.v1.SyncActivityRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 232, // 145: temporal.server.api.historyservice.v1.SyncActivityRequest.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo - 175, // 146: temporal.server.api.historyservice.v1.SyncActivityRequest.first_scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 147: temporal.server.api.historyservice.v1.SyncActivityRequest.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 179, // 148: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_initial_interval:type_name -> google.protobuf.Duration - 179, // 149: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_maximum_interval:type_name -> google.protobuf.Duration - 64, // 150: temporal.server.api.historyservice.v1.SyncActivitiesRequest.activities_info:type_name -> temporal.server.api.historyservice.v1.ActivitySyncInfo - 175, // 151: temporal.server.api.historyservice.v1.ActivitySyncInfo.scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 152: temporal.server.api.historyservice.v1.ActivitySyncInfo.started_time:type_name -> google.protobuf.Timestamp - 175, // 153: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_heartbeat_time:type_name -> google.protobuf.Timestamp - 178, // 154: temporal.server.api.historyservice.v1.ActivitySyncInfo.details:type_name -> temporal.api.common.v1.Payloads - 177, // 155: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_failure:type_name -> temporal.api.failure.v1.Failure - 234, // 156: temporal.server.api.historyservice.v1.ActivitySyncInfo.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 175, // 157: temporal.server.api.historyservice.v1.ActivitySyncInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp - 175, // 158: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp - 179, // 159: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_initial_interval:type_name -> google.protobuf.Duration - 179, // 160: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_maximum_interval:type_name -> google.protobuf.Duration - 191, // 161: temporal.server.api.historyservice.v1.DescribeMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 233, // 162: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.cache_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 233, // 163: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.database_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState - 191, // 164: temporal.server.api.historyservice.v1.DescribeHistoryHostRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution - 235, // 165: temporal.server.api.historyservice.v1.DescribeHistoryHostResponse.namespace_cache:type_name -> temporal.server.api.namespace.v1.NamespaceCacheInfo - 236, // 166: temporal.server.api.historyservice.v1.GetShardResponse.shard_info:type_name -> temporal.server.api.persistence.v1.ShardInfo - 175, // 167: temporal.server.api.historyservice.v1.RemoveTaskRequest.visibility_time:type_name -> google.protobuf.Timestamp - 237, // 168: temporal.server.api.historyservice.v1.GetReplicationMessagesRequest.tokens:type_name -> temporal.server.api.replication.v1.ReplicationToken - 168, // 169: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.shard_messages:type_name -> temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry - 238, // 170: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesRequest.task_infos:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo - 239, // 171: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 240, // 172: temporal.server.api.historyservice.v1.QueryWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest - 241, // 173: temporal.server.api.historyservice.v1.QueryWorkflowResponse.response:type_name -> temporal.api.workflowservice.v1.QueryWorkflowResponse - 242, // 174: temporal.server.api.historyservice.v1.ReapplyEventsRequest.request:type_name -> temporal.server.api.adminservice.v1.ReapplyEventsRequest - 243, // 175: temporal.server.api.historyservice.v1.GetDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 243, // 176: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 239, // 177: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask - 238, // 178: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks_info:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo - 243, // 179: temporal.server.api.historyservice.v1.PurgeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 243, // 180: temporal.server.api.historyservice.v1.MergeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType - 244, // 181: temporal.server.api.historyservice.v1.RefreshWorkflowTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.RefreshWorkflowTasksRequest - 191, // 182: temporal.server.api.historyservice.v1.GenerateLastHistoryReplicationTasksRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 96, // 183: temporal.server.api.historyservice.v1.GetReplicationStatusResponse.shards:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus - 175, // 184: temporal.server.api.historyservice.v1.ShardReplicationStatus.shard_local_time:type_name -> google.protobuf.Timestamp - 169, // 185: temporal.server.api.historyservice.v1.ShardReplicationStatus.remote_clusters:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry - 170, // 186: temporal.server.api.historyservice.v1.ShardReplicationStatus.handover_namespaces:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry - 175, // 187: temporal.server.api.historyservice.v1.ShardReplicationStatus.max_replication_task_visibility_time:type_name -> google.protobuf.Timestamp - 175, // 188: temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster.acked_task_visibility_time:type_name -> google.protobuf.Timestamp - 191, // 189: temporal.server.api.historyservice.v1.RebuildMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 191, // 190: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 231, // 191: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.history_batches:type_name -> temporal.api.common.v1.DataBlob - 234, // 192: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory - 191, // 193: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 175, // 194: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_start_time:type_name -> google.protobuf.Timestamp - 175, // 195: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_close_time:type_name -> google.protobuf.Timestamp - 245, // 196: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequest - 246, // 197: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponse - 247, // 198: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesRequest.sync_replication_state:type_name -> temporal.server.api.replication.v1.SyncReplicationState - 248, // 199: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesResponse.messages:type_name -> temporal.server.api.replication.v1.WorkflowReplicationMessages - 249, // 200: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateRequest - 250, // 201: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateResponse - 251, // 202: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest - 252, // 203: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse - 205, // 204: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.history:type_name -> temporal.api.history.v1.History - 252, // 205: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponseWithRaw.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse - 253, // 206: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest - 254, // 207: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse - 255, // 208: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Request.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Request - 256, // 209: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Response.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Response - 257, // 210: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryRequest.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryRequest - 258, // 211: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryResponse.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryResponse - 259, // 212: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionRequest.request:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionRequest - 260, // 213: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionResponse.response:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionResponse - 191, // 214: temporal.server.api.historyservice.v1.DeleteExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 261, // 215: temporal.server.api.historyservice.v1.GetDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey - 262, // 216: temporal.server.api.historyservice.v1.GetDLQTasksResponse.dlq_tasks:type_name -> temporal.server.api.common.v1.HistoryDLQTask - 261, // 217: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey - 263, // 218: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.inclusive_max_task_metadata:type_name -> temporal.server.api.common.v1.HistoryDLQTaskMetadata - 171, // 219: temporal.server.api.historyservice.v1.ListQueuesResponse.queues:type_name -> temporal.server.api.historyservice.v1.ListQueuesResponse.QueueInfo - 172, // 220: temporal.server.api.historyservice.v1.AddTasksRequest.tasks:type_name -> temporal.server.api.historyservice.v1.AddTasksRequest.Task - 264, // 221: temporal.server.api.historyservice.v1.ListTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksRequest - 265, // 222: temporal.server.api.historyservice.v1.ListTasksResponse.response:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksResponse - 266, // 223: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion - 267, // 224: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.success:type_name -> temporal.api.common.v1.Payload - 177, // 225: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.failure:type_name -> temporal.api.failure.v1.Failure - 175, // 226: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.close_time:type_name -> google.protobuf.Timestamp - 190, // 227: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.links:type_name -> temporal.api.common.v1.Link - 175, // 228: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.start_time:type_name -> google.protobuf.Timestamp - 266, // 229: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion - 267, // 230: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.success:type_name -> temporal.api.common.v1.Payload - 268, // 231: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.failure:type_name -> temporal.api.nexus.v1.Failure - 175, // 232: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.start_time:type_name -> google.protobuf.Timestamp - 190, // 233: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.links:type_name -> temporal.api.common.v1.Link - 269, // 234: temporal.server.api.historyservice.v1.InvokeStateMachineMethodRequest.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef - 270, // 235: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.state:type_name -> temporal.server.api.enums.v1.HealthState - 271, // 236: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.checks:type_name -> temporal.server.api.health.v1.HealthCheck - 191, // 237: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 193, // 238: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition - 197, // 239: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories - 272, // 240: temporal.server.api.historyservice.v1.SyncWorkflowStateResponse.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact - 273, // 241: temporal.server.api.historyservice.v1.UpdateActivityOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateActivityOptionsRequest - 274, // 242: temporal.server.api.historyservice.v1.UpdateActivityOptionsResponse.activity_options:type_name -> temporal.api.activity.v1.ActivityOptions - 275, // 243: temporal.server.api.historyservice.v1.PauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.PauseActivityRequest - 276, // 244: temporal.server.api.historyservice.v1.UnpauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.UnpauseActivityRequest - 277, // 245: temporal.server.api.historyservice.v1.ResetActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.ResetActivityRequest - 278, // 246: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionOptionsRequest - 279, // 247: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.workflow_execution_options:type_name -> temporal.api.workflow.v1.WorkflowExecutionOptions - 175, // 248: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.update_time:type_name -> google.protobuf.Timestamp - 280, // 249: temporal.server.api.historyservice.v1.PauseWorkflowExecutionRequest.pause_request:type_name -> temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest - 281, // 250: temporal.server.api.historyservice.v1.UnpauseWorkflowExecutionRequest.unpause_request:type_name -> temporal.api.workflowservice.v1.UnpauseWorkflowExecutionRequest - 282, // 251: temporal.server.api.historyservice.v1.StartNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.StartOperationRequest - 283, // 252: temporal.server.api.historyservice.v1.StartNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.StartOperationResponse - 284, // 253: temporal.server.api.historyservice.v1.CancelNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.CancelOperationRequest - 285, // 254: temporal.server.api.historyservice.v1.CancelNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.CancelOperationResponse - 286, // 255: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest - 287, // 256: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse - 1, // 257: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest - 105, // 258: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest - 2, // 259: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse - 106, // 260: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse - 288, // 261: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 288, // 262: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery - 289, // 263: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry.value:type_name -> temporal.server.api.replication.v1.ReplicationMessages - 98, // 264: temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry.value:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster - 97, // 265: temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry.value:type_name -> temporal.server.api.historyservice.v1.HandoverNamespaceInfo - 231, // 266: temporal.server.api.historyservice.v1.AddTasksRequest.Task.blob:type_name -> temporal.api.common.v1.DataBlob - 290, // 267: temporal.server.api.historyservice.v1.routing:extendee -> google.protobuf.MessageOptions - 0, // 268: temporal.server.api.historyservice.v1.routing:type_name -> temporal.server.api.historyservice.v1.RoutingOptions - 269, // [269:269] is the sub-list for method output_type - 269, // [269:269] is the sub-list for method input_type - 268, // [268:269] is the sub-list for extension type_name - 267, // [267:268] is the sub-list for extension extendee - 0, // [0:267] is the sub-list for field type_name + 174, // 0: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.start_request:type_name -> temporal.api.workflowservice.v1.StartWorkflowExecutionRequest + 175, // 1: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.parent_execution_info:type_name -> temporal.server.api.workflow.v1.ParentExecutionInfo + 176, // 2: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.workflow_execution_expiration_time:type_name -> google.protobuf.Timestamp + 177, // 3: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.continue_as_new_initiator:type_name -> temporal.api.enums.v1.ContinueAsNewInitiator + 178, // 4: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.continued_failure:type_name -> temporal.api.failure.v1.Failure + 179, // 5: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.last_completion_result:type_name -> temporal.api.common.v1.Payloads + 180, // 6: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.first_workflow_task_backoff:type_name -> google.protobuf.Duration + 181, // 7: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.source_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp + 182, // 8: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.root_execution_info:type_name -> temporal.server.api.workflow.v1.RootExecutionInfo + 183, // 9: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.versioning_override:type_name -> temporal.api.workflow.v1.VersioningOverride + 184, // 10: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.inherited_pinned_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion + 185, // 11: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.inherited_auto_upgrade_info:type_name -> temporal.api.deployment.v1.InheritedAutoUpgradeInfo + 186, // 12: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.declined_target_version_upgrade:type_name -> temporal.api.history.v1.DeclinedTargetVersionUpgrade + 187, // 13: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.time_skipping_state_propagation:type_name -> temporal.api.common.v1.TimeSkippingStatePropagation + 164, // 14: temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest.orphaned_child_replacement_info:type_name -> temporal.server.api.historyservice.v1.OrphanedChildReplacementInfo + 188, // 15: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 189, // 16: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.eager_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse + 190, // 17: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 191, // 18: temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link + 192, // 19: temporal.server.api.historyservice.v1.GetMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 193, // 20: temporal.server.api.historyservice.v1.GetMutableStateRequest.version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 194, // 21: temporal.server.api.historyservice.v1.GetMutableStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 192, // 22: temporal.server.api.historyservice.v1.GetMutableStateResponse.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 195, // 23: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 196, // 24: temporal.server.api.historyservice.v1.GetMutableStateResponse.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 196, // 25: temporal.server.api.historyservice.v1.GetMutableStateResponse.sticky_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 180, // 26: temporal.server.api.historyservice.v1.GetMutableStateResponse.sticky_task_queue_schedule_to_start_timeout:type_name -> google.protobuf.Duration + 197, // 27: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState + 190, // 28: temporal.server.api.historyservice.v1.GetMutableStateResponse.workflow_status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 198, // 29: temporal.server.api.historyservice.v1.GetMutableStateResponse.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories + 181, // 30: temporal.server.api.historyservice.v1.GetMutableStateResponse.most_recent_worker_version_stamp:type_name -> temporal.api.common.v1.WorkerVersionStamp + 194, // 31: temporal.server.api.historyservice.v1.GetMutableStateResponse.transition_history:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 199, // 32: temporal.server.api.historyservice.v1.GetMutableStateResponse.versioning_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionVersioningInfo + 200, // 33: temporal.server.api.historyservice.v1.GetMutableStateResponse.transient_or_speculative_tasks:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo + 192, // 34: temporal.server.api.historyservice.v1.PollMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 193, // 35: temporal.server.api.historyservice.v1.PollMutableStateRequest.version_history_item:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 192, // 36: temporal.server.api.historyservice.v1.PollMutableStateResponse.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 195, // 37: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 196, // 38: temporal.server.api.historyservice.v1.PollMutableStateResponse.task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 196, // 39: temporal.server.api.historyservice.v1.PollMutableStateResponse.sticky_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 180, // 40: temporal.server.api.historyservice.v1.PollMutableStateResponse.sticky_task_queue_schedule_to_start_timeout:type_name -> google.protobuf.Duration + 198, // 41: temporal.server.api.historyservice.v1.PollMutableStateResponse.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories + 197, // 42: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_state:type_name -> temporal.server.api.enums.v1.WorkflowExecutionState + 190, // 43: temporal.server.api.historyservice.v1.PollMutableStateResponse.workflow_status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 192, // 44: temporal.server.api.historyservice.v1.ResetStickyTaskQueueRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 165, // 45: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.operations:type_name -> temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation + 166, // 46: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.responses:type_name -> temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response + 192, // 47: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 201, // 48: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueRequest + 188, // 49: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 202, // 50: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo + 203, // 51: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment + 204, // 52: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 184, // 53: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedRequest.target_deployment_version:type_name -> temporal.api.deployment.v1.WorkerDeploymentVersion + 195, // 54: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 200, // 55: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo + 196, // 56: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 176, // 57: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 58: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp + 167, // 59: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry + 188, // 60: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 205, // 61: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.messages:type_name -> temporal.api.protocol.v1.Message + 206, // 62: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.history:type_name -> temporal.api.history.v1.History + 206, // 63: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.raw_history:type_name -> temporal.api.history.v1.History + 195, // 64: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 200, // 65: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.transient_workflow_task:type_name -> temporal.server.api.history.v1.TransientWorkflowTaskInfo + 196, // 66: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.workflow_execution_task_queue:type_name -> temporal.api.taskqueue.v1.TaskQueue + 176, // 67: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 68: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.started_time:type_name -> google.protobuf.Timestamp + 168, // 69: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.queries:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry + 188, // 70: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 205, // 71: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.messages:type_name -> temporal.api.protocol.v1.Message + 206, // 72: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.history:type_name -> temporal.api.history.v1.History + 192, // 73: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 207, // 74: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.poll_request:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueRequest + 188, // 75: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 202, // 76: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.build_id_redirect_info:type_name -> temporal.server.api.taskqueue.v1.BuildIdRedirectInfo + 203, // 77: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.scheduled_deployment:type_name -> temporal.api.deployment.v1.Deployment + 204, // 78: temporal.server.api.historyservice.v1.RecordActivityTaskStartedRequest.version_directive:type_name -> temporal.server.api.taskqueue.v1.TaskVersionDirective + 208, // 79: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.scheduled_event:type_name -> temporal.api.history.v1.HistoryEvent + 176, // 80: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.started_time:type_name -> google.protobuf.Timestamp + 176, // 81: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.current_attempt_scheduled_time:type_name -> google.protobuf.Timestamp + 179, // 82: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.heartbeat_details:type_name -> temporal.api.common.v1.Payloads + 195, // 83: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.workflow_type:type_name -> temporal.api.common.v1.WorkflowType + 188, // 84: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 209, // 85: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.priority:type_name -> temporal.api.common.v1.Priority + 210, // 86: temporal.server.api.historyservice.v1.RecordActivityTaskStartedResponse.retry_policy:type_name -> temporal.api.common.v1.RetryPolicy + 211, // 87: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskCompletedRequest + 12, // 88: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.started_response:type_name -> temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse + 212, // 89: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.activity_tasks:type_name -> temporal.api.workflowservice.v1.PollActivityTaskQueueResponse + 189, // 90: temporal.server.api.historyservice.v1.RespondWorkflowTaskCompletedResponse.new_workflow_task:type_name -> temporal.api.workflowservice.v1.PollWorkflowTaskQueueResponse + 213, // 91: temporal.server.api.historyservice.v1.RespondWorkflowTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondWorkflowTaskFailedRequest + 192, // 92: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 188, // 93: temporal.server.api.historyservice.v1.IsWorkflowTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 214, // 94: temporal.server.api.historyservice.v1.RecordActivityTaskHeartbeatRequest.heartbeat_request:type_name -> temporal.api.workflowservice.v1.RecordActivityTaskHeartbeatRequest + 215, // 95: temporal.server.api.historyservice.v1.RespondActivityTaskCompletedRequest.complete_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCompletedRequest + 216, // 96: temporal.server.api.historyservice.v1.RespondActivityTaskFailedRequest.failed_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskFailedRequest + 217, // 97: temporal.server.api.historyservice.v1.RespondActivityTaskCanceledRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RespondActivityTaskCanceledRequest + 192, // 98: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 188, // 99: temporal.server.api.historyservice.v1.IsActivityTaskValidRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 218, // 100: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.signal_request:type_name -> temporal.api.workflowservice.v1.SignalWorkflowExecutionRequest + 192, // 101: temporal.server.api.historyservice.v1.SignalWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 191, // 102: temporal.server.api.historyservice.v1.SignalWorkflowExecutionResponse.link:type_name -> temporal.api.common.v1.Link + 219, // 103: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionRequest.signal_with_start_request:type_name -> temporal.api.workflowservice.v1.SignalWithStartWorkflowExecutionRequest + 191, // 104: temporal.server.api.historyservice.v1.SignalWithStartWorkflowExecutionResponse.signal_link:type_name -> temporal.api.common.v1.Link + 192, // 105: temporal.server.api.historyservice.v1.RemoveSignalMutableStateRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 220, // 106: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.terminate_request:type_name -> temporal.api.workflowservice.v1.TerminateWorkflowExecutionRequest + 192, // 107: temporal.server.api.historyservice.v1.TerminateWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 108: temporal.server.api.historyservice.v1.DeleteWorkflowExecutionRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 221, // 109: temporal.server.api.historyservice.v1.ResetWorkflowExecutionRequest.reset_request:type_name -> temporal.api.workflowservice.v1.ResetWorkflowExecutionRequest + 222, // 110: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.cancel_request:type_name -> temporal.api.workflowservice.v1.RequestCancelWorkflowExecutionRequest + 192, // 111: temporal.server.api.historyservice.v1.RequestCancelWorkflowExecutionRequest.external_workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 112: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 188, // 113: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.child_clock:type_name -> temporal.server.api.clock.v1.VectorClock + 188, // 114: temporal.server.api.historyservice.v1.ScheduleWorkflowTaskRequest.parent_clock:type_name -> temporal.server.api.clock.v1.VectorClock + 192, // 115: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 188, // 116: temporal.server.api.historyservice.v1.VerifyFirstWorkflowTaskScheduledRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 192, // 117: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 118: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 208, // 119: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.completion_event:type_name -> temporal.api.history.v1.HistoryEvent + 188, // 120: temporal.server.api.historyservice.v1.RecordChildExecutionCompletedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 192, // 121: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.parent_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 122: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.child_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 188, // 123: temporal.server.api.historyservice.v1.VerifyChildExecutionCompletionRecordedRequest.clock:type_name -> temporal.server.api.clock.v1.VectorClock + 223, // 124: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.DescribeWorkflowExecutionRequest + 224, // 125: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.execution_config:type_name -> temporal.api.workflow.v1.WorkflowExecutionConfig + 225, // 126: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_execution_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionInfo + 226, // 127: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_activities:type_name -> temporal.api.workflow.v1.PendingActivityInfo + 227, // 128: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_children:type_name -> temporal.api.workflow.v1.PendingChildExecutionInfo + 228, // 129: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_workflow_task:type_name -> temporal.api.workflow.v1.PendingWorkflowTaskInfo + 229, // 130: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.callbacks:type_name -> temporal.api.workflow.v1.CallbackInfo + 230, // 131: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.pending_nexus_operations:type_name -> temporal.api.workflow.v1.PendingNexusOperationInfo + 231, // 132: temporal.server.api.historyservice.v1.DescribeWorkflowExecutionResponse.workflow_extended_info:type_name -> temporal.api.workflow.v1.WorkflowExecutionExtendedInfo + 192, // 133: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 193, // 134: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 232, // 135: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.events:type_name -> temporal.api.common.v1.DataBlob + 232, // 136: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.new_run_events:type_name -> temporal.api.common.v1.DataBlob + 233, // 137: temporal.server.api.historyservice.v1.ReplicateEventsV2Request.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 234, // 138: temporal.server.api.historyservice.v1.ReplicateWorkflowStateRequest.workflow_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 176, // 139: temporal.server.api.historyservice.v1.SyncShardStatusRequest.status_time:type_name -> google.protobuf.Timestamp + 176, // 140: temporal.server.api.historyservice.v1.SyncActivityRequest.scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 141: temporal.server.api.historyservice.v1.SyncActivityRequest.started_time:type_name -> google.protobuf.Timestamp + 176, // 142: temporal.server.api.historyservice.v1.SyncActivityRequest.last_heartbeat_time:type_name -> google.protobuf.Timestamp + 179, // 143: temporal.server.api.historyservice.v1.SyncActivityRequest.details:type_name -> temporal.api.common.v1.Payloads + 178, // 144: temporal.server.api.historyservice.v1.SyncActivityRequest.last_failure:type_name -> temporal.api.failure.v1.Failure + 235, // 145: temporal.server.api.historyservice.v1.SyncActivityRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 233, // 146: temporal.server.api.historyservice.v1.SyncActivityRequest.base_execution_info:type_name -> temporal.server.api.workflow.v1.BaseExecutionInfo + 176, // 147: temporal.server.api.historyservice.v1.SyncActivityRequest.first_scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 148: temporal.server.api.historyservice.v1.SyncActivityRequest.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 180, // 149: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_initial_interval:type_name -> google.protobuf.Duration + 180, // 150: temporal.server.api.historyservice.v1.SyncActivityRequest.retry_maximum_interval:type_name -> google.protobuf.Duration + 64, // 151: temporal.server.api.historyservice.v1.SyncActivitiesRequest.activities_info:type_name -> temporal.server.api.historyservice.v1.ActivitySyncInfo + 176, // 152: temporal.server.api.historyservice.v1.ActivitySyncInfo.scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 153: temporal.server.api.historyservice.v1.ActivitySyncInfo.started_time:type_name -> google.protobuf.Timestamp + 176, // 154: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_heartbeat_time:type_name -> google.protobuf.Timestamp + 179, // 155: temporal.server.api.historyservice.v1.ActivitySyncInfo.details:type_name -> temporal.api.common.v1.Payloads + 178, // 156: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_failure:type_name -> temporal.api.failure.v1.Failure + 235, // 157: temporal.server.api.historyservice.v1.ActivitySyncInfo.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 176, // 158: temporal.server.api.historyservice.v1.ActivitySyncInfo.first_scheduled_time:type_name -> google.protobuf.Timestamp + 176, // 159: temporal.server.api.historyservice.v1.ActivitySyncInfo.last_attempt_complete_time:type_name -> google.protobuf.Timestamp + 180, // 160: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_initial_interval:type_name -> google.protobuf.Duration + 180, // 161: temporal.server.api.historyservice.v1.ActivitySyncInfo.retry_maximum_interval:type_name -> google.protobuf.Duration + 192, // 162: temporal.server.api.historyservice.v1.DescribeMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 234, // 163: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.cache_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 234, // 164: temporal.server.api.historyservice.v1.DescribeMutableStateResponse.database_mutable_state:type_name -> temporal.server.api.persistence.v1.WorkflowMutableState + 192, // 165: temporal.server.api.historyservice.v1.DescribeHistoryHostRequest.workflow_execution:type_name -> temporal.api.common.v1.WorkflowExecution + 236, // 166: temporal.server.api.historyservice.v1.DescribeHistoryHostResponse.namespace_cache:type_name -> temporal.server.api.namespace.v1.NamespaceCacheInfo + 237, // 167: temporal.server.api.historyservice.v1.GetShardResponse.shard_info:type_name -> temporal.server.api.persistence.v1.ShardInfo + 176, // 168: temporal.server.api.historyservice.v1.RemoveTaskRequest.visibility_time:type_name -> google.protobuf.Timestamp + 238, // 169: temporal.server.api.historyservice.v1.GetReplicationMessagesRequest.tokens:type_name -> temporal.server.api.replication.v1.ReplicationToken + 169, // 170: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.shard_messages:type_name -> temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry + 239, // 171: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesRequest.task_infos:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo + 240, // 172: temporal.server.api.historyservice.v1.GetDLQReplicationMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 241, // 173: temporal.server.api.historyservice.v1.QueryWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.QueryWorkflowRequest + 242, // 174: temporal.server.api.historyservice.v1.QueryWorkflowResponse.response:type_name -> temporal.api.workflowservice.v1.QueryWorkflowResponse + 243, // 175: temporal.server.api.historyservice.v1.ReapplyEventsRequest.request:type_name -> temporal.server.api.adminservice.v1.ReapplyEventsRequest + 244, // 176: temporal.server.api.historyservice.v1.GetDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 244, // 177: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 240, // 178: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks:type_name -> temporal.server.api.replication.v1.ReplicationTask + 239, // 179: temporal.server.api.historyservice.v1.GetDLQMessagesResponse.replication_tasks_info:type_name -> temporal.server.api.replication.v1.ReplicationTaskInfo + 244, // 180: temporal.server.api.historyservice.v1.PurgeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 244, // 181: temporal.server.api.historyservice.v1.MergeDLQMessagesRequest.type:type_name -> temporal.server.api.enums.v1.DeadLetterQueueType + 245, // 182: temporal.server.api.historyservice.v1.RefreshWorkflowTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.RefreshWorkflowTasksRequest + 192, // 183: temporal.server.api.historyservice.v1.GenerateLastHistoryReplicationTasksRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 96, // 184: temporal.server.api.historyservice.v1.GetReplicationStatusResponse.shards:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus + 176, // 185: temporal.server.api.historyservice.v1.ShardReplicationStatus.shard_local_time:type_name -> google.protobuf.Timestamp + 170, // 186: temporal.server.api.historyservice.v1.ShardReplicationStatus.remote_clusters:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry + 171, // 187: temporal.server.api.historyservice.v1.ShardReplicationStatus.handover_namespaces:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry + 176, // 188: temporal.server.api.historyservice.v1.ShardReplicationStatus.max_replication_task_visibility_time:type_name -> google.protobuf.Timestamp + 176, // 189: temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster.acked_task_visibility_time:type_name -> google.protobuf.Timestamp + 192, // 190: temporal.server.api.historyservice.v1.RebuildMutableStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 192, // 191: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 232, // 192: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.history_batches:type_name -> temporal.api.common.v1.DataBlob + 235, // 193: temporal.server.api.historyservice.v1.ImportWorkflowExecutionRequest.version_history:type_name -> temporal.server.api.history.v1.VersionHistory + 192, // 194: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 176, // 195: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_start_time:type_name -> google.protobuf.Timestamp + 176, // 196: temporal.server.api.historyservice.v1.DeleteWorkflowVisibilityRecordRequest.workflow_close_time:type_name -> google.protobuf.Timestamp + 246, // 197: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest.request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionRequest + 247, // 198: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse.response:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionResponse + 248, // 199: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesRequest.sync_replication_state:type_name -> temporal.server.api.replication.v1.SyncReplicationState + 249, // 200: temporal.server.api.historyservice.v1.StreamWorkflowReplicationMessagesResponse.messages:type_name -> temporal.server.api.replication.v1.WorkflowReplicationMessages + 250, // 201: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateRequest + 251, // 202: temporal.server.api.historyservice.v1.PollWorkflowExecutionUpdateResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionUpdateResponse + 252, // 203: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryRequest + 253, // 204: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse + 206, // 205: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponse.history:type_name -> temporal.api.history.v1.History + 253, // 206: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryResponseWithRaw.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryResponse + 254, // 207: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseRequest.request:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseRequest + 255, // 208: temporal.server.api.historyservice.v1.GetWorkflowExecutionHistoryReverseResponse.response:type_name -> temporal.api.workflowservice.v1.GetWorkflowExecutionHistoryReverseResponse + 256, // 209: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Request.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Request + 257, // 210: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryV2Response.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryV2Response + 258, // 211: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryRequest.request:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryRequest + 259, // 212: temporal.server.api.historyservice.v1.GetWorkflowExecutionRawHistoryResponse.response:type_name -> temporal.server.api.adminservice.v1.GetWorkflowExecutionRawHistoryResponse + 260, // 213: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionRequest.request:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionRequest + 261, // 214: temporal.server.api.historyservice.v1.ForceDeleteWorkflowExecutionResponse.response:type_name -> temporal.server.api.adminservice.v1.DeleteWorkflowExecutionResponse + 192, // 215: temporal.server.api.historyservice.v1.DeleteExecutionRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 262, // 216: temporal.server.api.historyservice.v1.GetDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey + 263, // 217: temporal.server.api.historyservice.v1.GetDLQTasksResponse.dlq_tasks:type_name -> temporal.server.api.common.v1.HistoryDLQTask + 262, // 218: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.dlq_key:type_name -> temporal.server.api.common.v1.HistoryDLQKey + 264, // 219: temporal.server.api.historyservice.v1.DeleteDLQTasksRequest.inclusive_max_task_metadata:type_name -> temporal.server.api.common.v1.HistoryDLQTaskMetadata + 172, // 220: temporal.server.api.historyservice.v1.ListQueuesResponse.queues:type_name -> temporal.server.api.historyservice.v1.ListQueuesResponse.QueueInfo + 173, // 221: temporal.server.api.historyservice.v1.AddTasksRequest.tasks:type_name -> temporal.server.api.historyservice.v1.AddTasksRequest.Task + 265, // 222: temporal.server.api.historyservice.v1.ListTasksRequest.request:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksRequest + 266, // 223: temporal.server.api.historyservice.v1.ListTasksResponse.response:type_name -> temporal.server.api.adminservice.v1.ListHistoryTasksResponse + 267, // 224: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion + 268, // 225: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.success:type_name -> temporal.api.common.v1.Payload + 178, // 226: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.failure:type_name -> temporal.api.failure.v1.Failure + 176, // 227: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.close_time:type_name -> google.protobuf.Timestamp + 191, // 228: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.links:type_name -> temporal.api.common.v1.Link + 176, // 229: temporal.server.api.historyservice.v1.CompleteNexusOperationChasmRequest.start_time:type_name -> google.protobuf.Timestamp + 267, // 230: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.completion:type_name -> temporal.server.api.token.v1.NexusOperationCompletion + 268, // 231: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.success:type_name -> temporal.api.common.v1.Payload + 269, // 232: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.failure:type_name -> temporal.api.nexus.v1.Failure + 176, // 233: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.start_time:type_name -> google.protobuf.Timestamp + 191, // 234: temporal.server.api.historyservice.v1.CompleteNexusOperationRequest.links:type_name -> temporal.api.common.v1.Link + 270, // 235: temporal.server.api.historyservice.v1.InvokeStateMachineMethodRequest.ref:type_name -> temporal.server.api.persistence.v1.StateMachineRef + 271, // 236: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.state:type_name -> temporal.server.api.enums.v1.HealthState + 272, // 237: temporal.server.api.historyservice.v1.DeepHealthCheckResponse.checks:type_name -> temporal.server.api.health.v1.HealthCheck + 192, // 238: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 194, // 239: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.versioned_transition:type_name -> temporal.server.api.persistence.v1.VersionedTransition + 198, // 240: temporal.server.api.historyservice.v1.SyncWorkflowStateRequest.version_histories:type_name -> temporal.server.api.history.v1.VersionHistories + 273, // 241: temporal.server.api.historyservice.v1.SyncWorkflowStateResponse.versioned_transition_artifact:type_name -> temporal.server.api.replication.v1.VersionedTransitionArtifact + 274, // 242: temporal.server.api.historyservice.v1.UpdateActivityOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateActivityOptionsRequest + 275, // 243: temporal.server.api.historyservice.v1.UpdateActivityOptionsResponse.activity_options:type_name -> temporal.api.activity.v1.ActivityOptions + 276, // 244: temporal.server.api.historyservice.v1.PauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.PauseActivityRequest + 277, // 245: temporal.server.api.historyservice.v1.UnpauseActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.UnpauseActivityRequest + 278, // 246: temporal.server.api.historyservice.v1.ResetActivityRequest.frontend_request:type_name -> temporal.api.workflowservice.v1.ResetActivityRequest + 279, // 247: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsRequest.update_request:type_name -> temporal.api.workflowservice.v1.UpdateWorkflowExecutionOptionsRequest + 280, // 248: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.workflow_execution_options:type_name -> temporal.api.workflow.v1.WorkflowExecutionOptions + 176, // 249: temporal.server.api.historyservice.v1.UpdateWorkflowExecutionOptionsResponse.update_time:type_name -> google.protobuf.Timestamp + 281, // 250: temporal.server.api.historyservice.v1.PauseWorkflowExecutionRequest.pause_request:type_name -> temporal.api.workflowservice.v1.PauseWorkflowExecutionRequest + 282, // 251: temporal.server.api.historyservice.v1.UnpauseWorkflowExecutionRequest.unpause_request:type_name -> temporal.api.workflowservice.v1.UnpauseWorkflowExecutionRequest + 283, // 252: temporal.server.api.historyservice.v1.StartNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.StartOperationRequest + 284, // 253: temporal.server.api.historyservice.v1.StartNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.StartOperationResponse + 285, // 254: temporal.server.api.historyservice.v1.CancelNexusOperationRequest.request:type_name -> temporal.api.nexus.v1.CancelOperationRequest + 286, // 255: temporal.server.api.historyservice.v1.CancelNexusOperationResponse.response:type_name -> temporal.api.nexus.v1.CancelOperationResponse + 287, // 256: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingRequest.request:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingRequest + 288, // 257: temporal.server.api.historyservice.v1.PollWorkflowExecutionTimeSkippingResponse.response:type_name -> temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse + 193, // 258: temporal.server.api.historyservice.v1.OrphanedChildReplacementInfo.parent_current_version_history_items:type_name -> temporal.server.api.history.v1.VersionHistoryItem + 1, // 259: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionRequest + 105, // 260: temporal.server.api.historyservice.v1.ExecuteMultiOperationRequest.Operation.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionRequest + 2, // 261: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.start_workflow:type_name -> temporal.server.api.historyservice.v1.StartWorkflowExecutionResponse + 106, // 262: temporal.server.api.historyservice.v1.ExecuteMultiOperationResponse.Response.update_workflow:type_name -> temporal.server.api.historyservice.v1.UpdateWorkflowExecutionResponse + 289, // 263: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponse.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 289, // 264: temporal.server.api.historyservice.v1.RecordWorkflowTaskStartedResponseWithRawHistory.QueriesEntry.value:type_name -> temporal.api.query.v1.WorkflowQuery + 290, // 265: temporal.server.api.historyservice.v1.GetReplicationMessagesResponse.ShardMessagesEntry.value:type_name -> temporal.server.api.replication.v1.ReplicationMessages + 98, // 266: temporal.server.api.historyservice.v1.ShardReplicationStatus.RemoteClustersEntry.value:type_name -> temporal.server.api.historyservice.v1.ShardReplicationStatusPerCluster + 97, // 267: temporal.server.api.historyservice.v1.ShardReplicationStatus.HandoverNamespacesEntry.value:type_name -> temporal.server.api.historyservice.v1.HandoverNamespaceInfo + 232, // 268: temporal.server.api.historyservice.v1.AddTasksRequest.Task.blob:type_name -> temporal.api.common.v1.DataBlob + 291, // 269: temporal.server.api.historyservice.v1.routing:extendee -> google.protobuf.MessageOptions + 0, // 270: temporal.server.api.historyservice.v1.routing:type_name -> temporal.server.api.historyservice.v1.RoutingOptions + 271, // [271:271] is the sub-list for method output_type + 271, // [271:271] is the sub-list for method input_type + 270, // [270:271] is the sub-list for extension type_name + 269, // [269:270] is the sub-list for extension extendee + 0, // [0:269] is the sub-list for field type_name } func init() { file_temporal_server_api_historyservice_v1_request_response_proto_init() } @@ -12090,11 +12155,11 @@ func file_temporal_server_api_historyservice_v1_request_response_proto_init() { (*CompleteNexusOperationRequest_Success)(nil), (*CompleteNexusOperationRequest_Failure)(nil), } - file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[164].OneofWrappers = []any{ + file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165].OneofWrappers = []any{ (*ExecuteMultiOperationRequest_Operation_StartWorkflow)(nil), (*ExecuteMultiOperationRequest_Operation_UpdateWorkflow)(nil), } - file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[165].OneofWrappers = []any{ + file_temporal_server_api_historyservice_v1_request_response_proto_msgTypes[166].OneofWrappers = []any{ (*ExecuteMultiOperationResponse_Response_StartWorkflow)(nil), (*ExecuteMultiOperationResponse_Response_UpdateWorkflow)(nil), } @@ -12104,7 +12169,7 @@ func file_temporal_server_api_historyservice_v1_request_response_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc), len(file_temporal_server_api_historyservice_v1_request_response_proto_rawDesc)), NumEnums: 0, - NumMessages: 173, + NumMessages: 174, NumExtensions: 1, NumServices: 0, }, diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 38dce2f0142..f090903c077 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -2706,6 +2706,13 @@ where the user has set an explicit RetryPolicy, but not specified all the fields true, `Allows resetting of workflows with pending children when set to true`, ) + EnableOrphanedChildWorkflowReplacement = NewNamespaceBoolSetting( + "history.enableOrphanedChildWorkflowReplacement", + false, + `Allows a parent to replace an orphaned child only while the current cluster sees its first run with no history after WorkflowExecutionStarted. +The setting is evaluated against the parent namespace. +Enable only after all history hosts that may process child starts in this cluster support orphaned child replacement info; an older host ignores the request field and may permanently record WORKFLOW_ALREADY_EXISTS in the parent history`, + ) HistoryMaxAutoResetPoints = NewNamespaceIntSetting( "history.historyMaxAutoResetPoints", primitives.DefaultHistoryMaxAutoResetPoints, diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index dd4d74ce96b..4c1f3fc3098 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1021,6 +1021,7 @@ var ( // This metric has a "reason" tag attached to it to understand why eager start was denied. WorkflowEagerExecutionDeniedCounter = NewCounterDef("workflow_eager_execution_denied") StartWorkflowRequestDeduped = NewCounterDef("start_workflow_request_deduped") + OrphanedChildWorkflowReplacement = NewCounterDef("orphaned_child_workflow_replacement") EmptyCompletionCommandsCounter = NewCounterDef("empty_completion_commands") MultipleCompletionCommandsCounter = NewCounterDef("multiple_completion_commands") FailedWorkflowTasksCounter = NewCounterDef("failed_workflow_tasks") diff --git a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto index de813adcf43..283ed71ca9b 100644 --- a/proto/internal/temporal/server/api/historyservice/v1/request_response.proto +++ b/proto/internal/temporal/server/api/historyservice/v1/request_response.proto @@ -106,6 +106,10 @@ message StartWorkflowExecutionRequest { temporal.api.history.v1.DeclinedTargetVersionUpgrade declined_target_version_upgrade = 17; reserved 18; temporal.api.common.v1.TimeSkippingStatePropagation time_skipping_state_propagation = 19; + // When present, StartWorkflowExecution may replace a conflicting child only after verifying under + // the child's lock that its parent initiation is absent from the supplied current branch and that + // its first run has only WorkflowExecutionStarted. + OrphanedChildReplacementInfo orphaned_child_replacement_info = 20; } message StartWorkflowExecutionResponse { @@ -1451,3 +1455,11 @@ message PollWorkflowExecutionTimeSkippingRequest { message PollWorkflowExecutionTimeSkippingResponse { temporal.api.workflowservice.v1.PollWorkflowExecutionTimeSkippingResponse response = 1; } + +// Carries the parent branch evidence needed to distinguish an orphaned child from another +// accepted initiation of the same child workflow ID. This is request-only replacement context and +// is not persisted into the child workflow. Re-linking requires a separate recovery mechanism +// because the existing parent coordinates are also immutable WorkflowExecutionStarted history. +message OrphanedChildReplacementInfo { + repeated temporal.server.api.history.v1.VersionHistoryItem parent_current_version_history_items = 1; +} diff --git a/service/history/api/startworkflow/api.go b/service/history/api/startworkflow/api.go index 8197acc8c36..6a3886ce151 100644 --- a/service/history/api/startworkflow/api.go +++ b/service/history/api/startworkflow/api.go @@ -10,11 +10,13 @@ import ( historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" + enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/locks" + "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence" @@ -35,9 +37,11 @@ type ( ) const ( - eagerStartDeniedReasonDynamicConfigDisabled metrics.ReasonString = "dynamic_config_disabled" - eagerStartDeniedReasonFirstWorkflowTaskBackoff metrics.ReasonString = "first_workflow_task_backoff" - eagerStartDeniedReasonTaskAlreadyDispatched metrics.ReasonString = "task_already_dispatched" + eagerStartDeniedReasonDynamicConfigDisabled metrics.ReasonString = "dynamic_config_disabled" + eagerStartDeniedReasonFirstWorkflowTaskBackoff metrics.ReasonString = "first_workflow_task_backoff" + eagerStartDeniedReasonTaskAlreadyDispatched metrics.ReasonString = "task_already_dispatched" + orphanedChildReplacementReplaced = "replaced" + orphanedChildReplacementRejectedUnsupportedState = "rejected_unsupported_state" ) const ( @@ -463,21 +467,45 @@ func (s *Starter) resolveDuplicateWorkflowID( // previously created workflow context. newRunID := primitives.NewUUID().String() - currentExecutionUpdateAction, err := api.ResolveDuplicateWorkflowID( - s.shardContext, - workflowKey, - s.namespace, - newRunID, - currentWorkflowConditionFailed.State, - currentWorkflowConditionFailed.Status, - currentWorkflowConditionFailed.RequestIDs, - currentWorkflowConditionFailed.FirstExecutionRunID, - s.request.StartRequest.GetWorkflowIdReusePolicy(), - s.request.StartRequest.GetWorkflowIdConflictPolicy(), - currentWorkflowStartTime, - s.request.ParentExecutionInfo, - s.request.ChildWorkflowOnly, - ) + var currentExecutionUpdateAction api.UpdateWorkflowActionFunc + var err error + orphanedChildReplacementInfo := s.request.GetOrphanedChildReplacementInfo() + orphanedChildReplacementRequested := orphanedChildReplacementInfo != nil && + s.request.StartRequest.GetWorkflowIdConflictPolicy() == enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL + replaceOrphanedChild := orphanedChildReplacementRequested && + currentWorkflowConditionFailed.State == enumsspb.WORKFLOW_EXECUTION_STATE_CREATED + if orphanedChildReplacementRequested && !replaceOrphanedChild { + metrics.OrphanedChildWorkflowReplacement.With(s.getMetricsHandler()).Record( + 1, + metrics.OutcomeTag(orphanedChildReplacementRejectedUnsupportedState), + metrics.StringTag("workflow_state", currentWorkflowConditionFailed.State.String()), + ) + } + if replaceOrphanedChild { + currentExecutionUpdateAction = api.ReplaceOrphanedChildAction( + ctx, + s.request.ParentExecutionInfo, + orphanedChildReplacementInfo, + newRunID, + s.getMetricsHandler(), + ) + } else { + currentExecutionUpdateAction, err = api.ResolveDuplicateWorkflowID( + s.shardContext, + workflowKey, + s.namespace, + newRunID, + currentWorkflowConditionFailed.State, + currentWorkflowConditionFailed.Status, + currentWorkflowConditionFailed.RequestIDs, + currentWorkflowConditionFailed.FirstExecutionRunID, + s.request.StartRequest.GetWorkflowIdReusePolicy(), + s.request.StartRequest.GetWorkflowIdConflictPolicy(), + currentWorkflowStartTime, + s.request.ParentExecutionInfo, + s.request.ChildWorkflowOnly, + ) + } switch { case errors.Is(err, api.ErrUseCurrentExecution): @@ -539,6 +567,25 @@ func (s *Starter) resolveDuplicateWorkflowID( switch err { case nil: + if replaceOrphanedChild { + metrics.OrphanedChildWorkflowReplacement.With(s.getMetricsHandler()).Record( + 1, + metrics.OutcomeTag(orphanedChildReplacementReplaced), + ) + parentExecutionInfo := s.request.GetParentExecutionInfo() + parentExecution := parentExecutionInfo.GetExecution() + s.shardContext.GetLogger().Info( + "Replaced orphaned child workflow", + tag.WorkflowNamespaceID(s.namespace.ID().String()), + tag.WorkflowID(workflowID), + tag.NewStringTag("orphaned-child-run-id", currentWorkflowConditionFailed.RunID), + tag.WorkflowNewRunID(newRunID), + tag.NewStringTag("parent-workflow-id", parentExecution.GetWorkflowId()), + tag.NewStringTag("parent-run-id", parentExecution.GetRunId()), + tag.NewInt64("parent-initiated-event-id", parentExecutionInfo.GetInitiatedId()), + tag.NewInt64("parent-initiated-event-version", parentExecutionInfo.GetInitiatedVersion()), + ) + } if !s.requestEagerStart() { return &historyservice.StartWorkflowExecutionResponse{ RunId: newRunID, diff --git a/service/history/api/workflow_id_dedup.go b/service/history/api/workflow_id_dedup.go index 8586000de7d..b24239fffa0 100644 --- a/service/history/api/workflow_id_dedup.go +++ b/service/history/api/workflow_id_dedup.go @@ -1,6 +1,7 @@ package api import ( + "context" "errors" "fmt" "time" @@ -8,11 +9,16 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" enumsspb "go.temporal.io/server/api/enums/v1" + historyspb "go.temporal.io/server/api/history/v1" + "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" workflowspb "go.temporal.io/server/api/workflow/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/definition" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payloads" + "go.temporal.io/server/common/persistence/versionhistory" "go.temporal.io/server/service/history/consts" historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/workflow" @@ -22,6 +28,13 @@ import ( // use the current workflow execution instead of creating a new one var ErrUseCurrentExecution = errors.New("ErrUseCurrentExecution") +const ( + orphanedChildRaceClosed = "rejected_race_closed" + orphanedChildLocalProgress = "rejected_local_progress" + orphanedChildNotLosingBranch = "rejected_not_losing_branch" + terminateOrphanedChildReason = "Orphaned child workflow replaced by its parent" +) + // ResolveDuplicateWorkflowID determines how to resolve a workflow ID duplication upon workflow start according // to the WorkflowIdReusePolicy (for *completed* workflow) or WorkflowIdConflictPolicy (for *running* workflow). // @@ -83,6 +96,113 @@ func ResolveDuplicateWorkflowID( } } +// ReplaceOrphanedChildAction verifies and terminates the conflicting child while holding its lock. +func ReplaceOrphanedChildAction( + ctx context.Context, + parentExecutionInfo *workflowspb.ParentExecutionInfo, + orphanedChildReplacementInfo *historyservice.OrphanedChildReplacementInfo, + newRunID string, + metricsHandler metrics.Handler, +) UpdateWorkflowActionFunc { + return func(workflowLease WorkflowLease) (*UpdateWorkflowAction, error) { + mutableState := workflowLease.GetMutableState() + if !mutableState.IsWorkflowExecutionRunning() { + recordOrphanedChildReplacementRejection(metricsHandler, orphanedChildRaceClosed) + return nil, consts.ErrWorkflowCompleted + } + + executionState := mutableState.GetExecutionState() + reject := func(outcome string) (*UpdateWorkflowAction, error) { + recordOrphanedChildReplacementRejection(metricsHandler, outcome) + return nil, generateWorkflowAlreadyStartedError( + "Workflow execution is already running. WorkflowId: %v, RunId: %v.", + executionState.GetRequestIds(), + mutableState.GetWorkflowKey(), + executionState.GetFirstExecutionRunId(), + ) + } + + // The current-row snapshot only selects this path. Recheck under the child lock and accept only + // the first run with WorkflowExecutionStarted as its sole event. UpdateRegistry catches an + // admitted in-memory Update not yet present in history. + if executionState.GetFirstExecutionRunId() != mutableState.GetWorkflowKey().RunID || + executionState.GetState() != enumsspb.WORKFLOW_EXECUTION_STATE_CREATED || + mutableState.GetNextEventID() != common.FirstEventID+1 || + workflowLease.GetContext().UpdateRegistry(ctx).Len() != 0 { + return reject(orphanedChildLocalProgress) + } + if !isOrphanedChildOnLosingBranch( + mutableState.GetExecutionInfo(), + parentExecutionInfo, + orphanedChildReplacementInfo, + ) { + return reject(orphanedChildNotLosingBranch) + } + + if err := workflow.TerminateWorkflow( + mutableState, + terminateOrphanedChildReason, + payloads.EncodeString(fmt.Sprintf("terminated by new runID: %s", newRunID)), + consts.IdentityHistoryService, + false, + nil, // No links necessary. + ); err != nil { + return nil, err + } + return UpdateWorkflowTerminate, nil + } +} + +func isOrphanedChildOnLosingBranch( + executionInfo *persistencespb.WorkflowExecutionInfo, + parentExecutionInfo *workflowspb.ParentExecutionInfo, + orphanedChildReplacementInfo *historyservice.OrphanedChildReplacementInfo, +) bool { + parentExecution := parentExecutionInfo.GetExecution() + if parentExecutionInfo.GetNamespaceId() == "" || + parentExecution.GetWorkflowId() == "" || + parentExecution.GetRunId() == "" || + parentExecutionInfo.GetInitiatedId() < common.FirstEventID || + executionInfo.GetParentInitiatedId() < common.FirstEventID { + return false + } + // Failover versions are required to distinguish branches; local namespaces use EmptyVersion. + if parentExecutionInfo.GetInitiatedVersion() <= common.EmptyVersion || + executionInfo.GetParentInitiatedVersion() <= common.EmptyVersion { + return false + } + // Match the exact parent run. A shared workflow lineage is insufficient because a predecessor's + // ABANDON child may legitimately still be running. + if executionInfo.GetParentNamespaceId() != parentExecutionInfo.GetNamespaceId() || + executionInfo.GetParentWorkflowId() != parentExecution.GetWorkflowId() || + executionInfo.GetParentRunId() != parentExecution.GetRunId() { + return false + } + + parentCurrentVersionHistory := &historyspb.VersionHistory{ + Items: orphanedChildReplacementInfo.GetParentCurrentVersionHistoryItems(), + } + incomingInitiation := versionhistory.NewVersionHistoryItem( + parentExecutionInfo.GetInitiatedId(), + parentExecutionInfo.GetInitiatedVersion(), + ) + existingInitiation := versionhistory.NewVersionHistoryItem( + executionInfo.GetParentInitiatedId(), + executionInfo.GetParentInitiatedVersion(), + ) + // The new initiation must be on the parent's current branch while the conflicting child's + // initiation must not be; otherwise this is an ordinary duplicate or the evidence is ambiguous. + return versionhistory.ContainsVersionHistoryItem(parentCurrentVersionHistory, incomingInitiation) && + !versionhistory.ContainsVersionHistoryItem(parentCurrentVersionHistory, existingInitiation) +} + +func recordOrphanedChildReplacementRejection(metricsHandler metrics.Handler, outcome string) { + metrics.OrphanedChildWorkflowReplacement.With(metricsHandler).Record( + 1, + metrics.OutcomeTag(outcome), + ) +} + func ResolveWorkflowIDConflictPolicy( shardContext historyi.ShardContext, workflowKey definition.WorkflowKey, diff --git a/service/history/api/workflow_id_dedup_test.go b/service/history/api/workflow_id_dedup_test.go index 85f20e6cb6b..e45ff9f0253 100644 --- a/service/history/api/workflow_id_dedup_test.go +++ b/service/history/api/workflow_id_dedup_test.go @@ -1,21 +1,33 @@ package api import ( + "context" "testing" "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" + enumsspb "go.temporal.io/server/api/enums/v1" + historyspb "go.temporal.io/server/api/history/v1" + "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + workflowspb "go.temporal.io/server/api/workflow/v1" + "go.temporal.io/server/common" "go.temporal.io/server/common/clock" "go.temporal.io/server/common/definition" "go.temporal.io/server/common/dynamicconfig" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" + "go.temporal.io/server/service/history/consts" + historyi "go.temporal.io/server/service/history/interfaces" "go.temporal.io/server/service/history/shard" "go.temporal.io/server/service/history/tests" + "go.temporal.io/server/service/history/workflow/update" "go.uber.org/mock/gomock" ) @@ -80,6 +92,243 @@ func TestResolveDuplicateWorkflowStart(t *testing.T) { } } +func TestOrphanedChildReplacementDoesNotReplaceUnrelatedWorkflow(t *testing.T) { + parentExecutionInfo := &workflowspb.ParentExecutionInfo{ + NamespaceId: "parent-namespace", + Execution: &commonpb.WorkflowExecution{ + WorkflowId: "parent-workflow", + RunId: "parent-run", + }, + InitiatedId: 75, + InitiatedVersion: 925, + } + replacementInfo := &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{ + {EventId: 100, Version: 925}, + }, + } + + // A standalone workflow has no parent initiation to prove belongs to the requesting parent. + require.False(t, isOrphanedChildOnLosingBranch( + &persistencespb.WorkflowExecutionInfo{}, + parentExecutionInfo, + replacementInfo, + )) +} + +func TestReplaceOrphanedChildAction(t *testing.T) { + const ( + parentNamespaceID = "parent-namespace" + parentWorkflowID = "parent-workflow" + parentRunID = "parent-run" + childRunID = "child-run" + newRunID = "replacement-run" + ) + parentExecutionInfo := &workflowspb.ParentExecutionInfo{ + NamespaceId: parentNamespaceID, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: parentWorkflowID, + RunId: parentRunID, + }, + InitiatedId: 75, + InitiatedVersion: 925, + } + validRecoveryInfo := &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{ + {EventId: 100, Version: 925}, + }, + } + + testCases := []struct { + name string + closed bool + state enumsspb.WorkflowExecutionState + withoutParent bool + recoveryInfo *historyservice.OrphanedChildReplacementInfo + mutateInfo func(*persistencespb.WorkflowExecutionInfo) + firstRunID string + nextEventID int64 + admittedUpdate bool + expectedOutcome string + }{ + {name: "start-only child on losing branch"}, + { + name: "running child", + state: enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING, + expectedOutcome: orphanedChildLocalProgress, + }, + { + name: "created child has additional history", + nextEventID: common.FirstEventID + 2, + expectedOutcome: orphanedChildLocalProgress, + }, + { + name: "successor child run", + firstRunID: "first-child-run", + expectedOutcome: orphanedChildLocalProgress, + }, + { + name: "created child has admitted update", + admittedUpdate: true, + expectedOutcome: orphanedChildLocalProgress, + }, + { + name: "existing initiation is on current branch", + recoveryInfo: &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{ + {EventId: 74, Version: 892}, + {EventId: 100, Version: 925}, + }, + }, + expectedOutcome: orphanedChildNotLosingBranch, + }, + { + name: "empty recovery info", + recoveryInfo: &historyservice.OrphanedChildReplacementInfo{}, + expectedOutcome: orphanedChildNotLosingBranch, + }, + { + name: "different parent", + mutateInfo: func(info *persistencespb.WorkflowExecutionInfo) { + info.ParentRunId = "other-parent-run" + }, + expectedOutcome: orphanedChildNotLosingBranch, + }, + { + name: "missing request parent", + withoutParent: true, + expectedOutcome: orphanedChildNotLosingBranch, + }, + { + name: "same parent initiation", + mutateInfo: func(info *persistencespb.WorkflowExecutionInfo) { + info.ParentInitiatedId = parentExecutionInfo.GetInitiatedId() + info.ParentInitiatedVersion = parentExecutionInfo.GetInitiatedVersion() + }, + expectedOutcome: orphanedChildNotLosingBranch, + }, + { + name: "closed after snapshot", + closed: true, + expectedOutcome: orphanedChildRaceClosed, + }, + { + name: "zero-version branch prefix", + recoveryInfo: &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{ + {EventId: 50, Version: common.EmptyVersion}, + {EventId: 100, Version: 925}, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + executionInfo := &persistencespb.WorkflowExecutionInfo{ + ParentNamespaceId: parentNamespaceID, + ParentWorkflowId: parentWorkflowID, + ParentRunId: parentRunID, + ParentInitiatedId: 72, + ParentInitiatedVersion: 892, + } + if tc.mutateInfo != nil { + tc.mutateInfo(executionInfo) + } + firstRunID := tc.firstRunID + if firstRunID == "" { + firstRunID = childRunID + } + state := tc.state + if state == enumsspb.WORKFLOW_EXECUTION_STATE_UNSPECIFIED { + state = enumsspb.WORKFLOW_EXECUTION_STATE_CREATED + } + executionState := &persistencespb.WorkflowExecutionState{ + RunId: childRunID, + FirstExecutionRunId: firstRunID, + State: state, + } + workflowKey := definition.NewWorkflowKey("child-namespace", "child-workflow", childRunID) + nextEventID := tc.nextEventID + if nextEventID == 0 { + nextEventID = common.FirstEventID + 1 + } + controller := gomock.NewController(t) + mutableState := historyi.NewMockMutableState(controller) + mutableState.EXPECT().IsWorkflowExecutionRunning().Return(!tc.closed) + mutableState.EXPECT().GetExecutionInfo().Return(executionInfo).AnyTimes() + mutableState.EXPECT().GetExecutionState().Return(executionState).AnyTimes() + mutableState.EXPECT().GetWorkflowKey().Return(workflowKey).AnyTimes() + mutableState.EXPECT().GetNextEventID().Return(nextEventID).AnyTimes() + mutableState.EXPECT().GetCurrentVersion().Return(int64(1)) + mutableState.EXPECT().VisitUpdates(gomock.Any()) + updateRegistry := update.NewRegistry(mutableState) + if tc.admittedUpdate { + mutableState.EXPECT().GetUpdateOutcome(gomock.Any(), "update-id").Return( + nil, + serviceerror.NewNotFound("update not found"), + ) + _, _, err := updateRegistry.FindOrCreate(ctx, "update-id") + require.NoError(t, err) + } + workflowContext := historyi.NewMockWorkflowContext(controller) + workflowContext.EXPECT().UpdateRegistry(gomock.Any()).Return(updateRegistry).AnyTimes() + + expectReplacement := tc.expectedOutcome == "" + if expectReplacement { + // An untouched child has no started workflow task to fail first. + mutableState.EXPECT().GetStartedWorkflowTask().Return(nil) + mutableState.EXPECT().AddWorkflowExecutionTerminatedEvent( + terminateOrphanedChildReason, + gomock.Any(), + consts.IdentityHistoryService, + false, + nil, + ).Return(nil, nil) + } + + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + recoveryInfo := tc.recoveryInfo + if recoveryInfo == nil { + recoveryInfo = validRecoveryInfo + } + parent := parentExecutionInfo + if tc.withoutParent { + parent = nil + } + action, err := ReplaceOrphanedChildAction( + ctx, + parent, recoveryInfo, newRunID, + metricsHandler, + )( + NewWorkflowLease(workflowContext, nil, mutableState), + ) + switch tc.expectedOutcome { + case orphanedChildRaceClosed: + require.ErrorIs(t, err, consts.ErrWorkflowCompleted) + case "": + require.NoError(t, err) + require.Same(t, UpdateWorkflowTerminate, action) + default: + var alreadyStarted *serviceerror.WorkflowExecutionAlreadyStarted + require.ErrorAs(t, err, &alreadyStarted) + } + + recordings := capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()] + if expectReplacement { + require.Empty(t, recordings, "success must not be recorded before persistence commits") + } else { + require.Len(t, recordings, 1) + require.Equal(t, int64(1), recordings[0].Value) + require.Equal(t, tc.expectedOutcome, recordings[0].Tags["outcome"]) + } + }) + } +} + func TestMigrateWorkflowIDReusePolicyForRunningWorkflow(t *testing.T) { //nolint:staticcheck // SA1019: intentional migration coverage for deprecated policy reusePolicy := enumspb.WORKFLOW_ID_REUSE_POLICY_TERMINATE_IF_RUNNING diff --git a/service/history/configs/config.go b/service/history/configs/config.go index 44220a7aa1c..ccf0695ecd1 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -63,6 +63,8 @@ type Config struct { AllowResetWithPendingChildren dynamicconfig.BoolPropertyFnWithNamespaceFilter MaxAutoResetPoints dynamicconfig.IntPropertyFnWithNamespaceFilter + EnableOrphanedChildWorkflowReplacement dynamicconfig.BoolPropertyFnWithNamespaceFilter + // HistoryCache settings // Change of these configs require shard restart HistoryCacheLimitSizeBased bool @@ -480,23 +482,24 @@ func NewConfig( EnableSeparateReplicationEnableFlag: dynamicconfig.EnableSeparateReplicationEnableFlag.Get(dc), HistoryReplicationDLQV2: dynamicconfig.EnableHistoryReplicationDLQV2.Get(dc), - RPS: dynamicconfig.HistoryRPS.Get(dc), - NamespaceRPS: dynamicconfig.HistoryNamespaceRPS.Get(dc), - OperatorRPSRatio: dynamicconfig.OperatorRPSRatio.Get(dc), - MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), - PersistenceMaxQPS: dynamicconfig.HistoryPersistenceMaxQPS.Get(dc), - PersistenceGlobalMaxQPS: dynamicconfig.HistoryPersistenceGlobalMaxQPS.Get(dc), - PersistenceNamespaceMaxQPS: dynamicconfig.HistoryPersistenceNamespaceMaxQPS.Get(dc), - PersistenceGlobalNamespaceMaxQPS: dynamicconfig.HistoryPersistenceGlobalNamespaceMaxQPS.Get(dc), - PersistencePerShardNamespaceMaxQPS: dynamicconfig.HistoryPersistencePerShardNamespaceMaxQPS.Get(dc), - PersistenceDynamicRateLimitingParams: dynamicconfig.HistoryPersistenceDynamicRateLimitingParams.Get(dc), - PersistenceQPSBurstRatio: dynamicconfig.PersistenceQPSBurstRatio.Get(dc), - AlignMembershipChange: dynamicconfig.HistoryAlignMembershipChange.Get(dc), - ShutdownDrainDuration: dynamicconfig.HistoryShutdownDrainDuration.Get(dc), - StartupMembershipJoinDelay: dynamicconfig.HistoryStartupMembershipJoinDelay.Get(dc), - AllowResetWithPendingChildren: dynamicconfig.AllowResetWithPendingChildren.Get(dc), - MaxAutoResetPoints: dynamicconfig.HistoryMaxAutoResetPoints.Get(dc), - DefaultWorkflowTaskTimeout: dynamicconfig.DefaultWorkflowTaskTimeout.Get(dc), + RPS: dynamicconfig.HistoryRPS.Get(dc), + NamespaceRPS: dynamicconfig.HistoryNamespaceRPS.Get(dc), + OperatorRPSRatio: dynamicconfig.OperatorRPSRatio.Get(dc), + MaxIDLengthLimit: dynamicconfig.MaxIDLengthLimit.Get(dc), + PersistenceMaxQPS: dynamicconfig.HistoryPersistenceMaxQPS.Get(dc), + PersistenceGlobalMaxQPS: dynamicconfig.HistoryPersistenceGlobalMaxQPS.Get(dc), + PersistenceNamespaceMaxQPS: dynamicconfig.HistoryPersistenceNamespaceMaxQPS.Get(dc), + PersistenceGlobalNamespaceMaxQPS: dynamicconfig.HistoryPersistenceGlobalNamespaceMaxQPS.Get(dc), + PersistencePerShardNamespaceMaxQPS: dynamicconfig.HistoryPersistencePerShardNamespaceMaxQPS.Get(dc), + PersistenceDynamicRateLimitingParams: dynamicconfig.HistoryPersistenceDynamicRateLimitingParams.Get(dc), + PersistenceQPSBurstRatio: dynamicconfig.PersistenceQPSBurstRatio.Get(dc), + AlignMembershipChange: dynamicconfig.HistoryAlignMembershipChange.Get(dc), + ShutdownDrainDuration: dynamicconfig.HistoryShutdownDrainDuration.Get(dc), + StartupMembershipJoinDelay: dynamicconfig.HistoryStartupMembershipJoinDelay.Get(dc), + AllowResetWithPendingChildren: dynamicconfig.AllowResetWithPendingChildren.Get(dc), + EnableOrphanedChildWorkflowReplacement: dynamicconfig.EnableOrphanedChildWorkflowReplacement.Get(dc), + MaxAutoResetPoints: dynamicconfig.HistoryMaxAutoResetPoints.Get(dc), + DefaultWorkflowTaskTimeout: dynamicconfig.DefaultWorkflowTaskTimeout.Get(dc), MaxLocalParentWorkflowVerificationDuration: dynamicconfig.MaxLocalParentWorkflowVerificationDuration.Get(dc), diff --git a/service/history/history_engine2_test.go b/service/history/history_engine2_test.go index 5d2f889ff74..328d5d45acb 100644 --- a/service/history/history_engine2_test.go +++ b/service/history/history_engine2_test.go @@ -1582,7 +1582,7 @@ func makeMockStartRequest( func makeCurrentWorkflowConditionFailedError( tv *testvars.TestVars, startTime *timestamppb.Timestamp, -) error { +) *persistence.CurrentWorkflowConditionFailedError { lastWriteVersion := common.EmptyVersion tv1 := tv.WithRequestID("AttachedRequestID1") tv2 := tv.WithRequestID("AttachedRequestID2") @@ -1611,11 +1611,17 @@ func makeCurrentWorkflowConditionFailedError( } } -func (s *engine2Suite) setupStartWorkflowExecutionDedup(startTime *timestamppb.Timestamp) *workflow.MutableStateImpl { +func (s *engine2Suite) setupStartWorkflowExecutionDedup( + startTime *timestamppb.Timestamp, + currentState ...enumsspb.WorkflowExecutionState, +) *workflow.MutableStateImpl { brandNewExecutionRequest := mock.MatchedBy(func(request *persistence.CreateWorkflowExecutionRequest) bool { return request.Mode == persistence.CreateWorkflowModeBrandNew }) currentWorkflowConditionFailedError := makeCurrentWorkflowConditionFailedError(s.tv, startTime) + if len(currentState) != 0 { + currentWorkflowConditionFailedError.State = currentState[0] + } s.mockExecutionMgr.EXPECT().CreateWorkflowExecution(gomock.Any(), brandNewExecutionRequest). Return(nil, currentWorkflowConditionFailedError).AnyTimes() @@ -1684,6 +1690,168 @@ func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_SameRequestID() s.Equal(s.tv.RunID(), resp.GetRunId()) } +func (s *engine2Suite) TestStartWorkflowExecution_ReplaceOrphanedChild_RequestDedup() { + s.setupStartWorkflowExecutionForRunning() + startRequest := makeMockStartRequest( + s.tv, + enumspb.WORKFLOW_ID_REUSE_POLICY_UNSPECIFIED, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + ) + startRequest.StartRequest.RequestId = s.tv.RequestID() + startRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{} + + resp, err := s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + + s.NoError(err) + s.True(resp.Started) + s.Equal(s.tv.RunID(), resp.GetRunId()) +} + +func (s *engine2Suite) TestStartWorkflowExecution_ReplaceOrphanedChild_PostCommitObservability() { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + now := s.historyEngine.shardContext.GetTimeSource().Now() + ms := s.setupStartWorkflowExecutionDedup( + timestamppb.New(now.Add(-100*time.Millisecond)), + enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + ) + parentExecution := &commonpb.WorkflowExecution{ + WorkflowId: "parent-workflow-id", + RunId: uuid.NewString(), + } + addWorkflowExecutionStartedEventWithParent( + ms, + s.tv.WorkflowExecution(), + s.tv.WorkflowType().GetName(), + s.tv.TaskQueue().GetName(), + nil, + time.Second, + time.Second, + time.Second, + &workflowspb.ParentExecutionInfo{ + NamespaceId: tests.ParentNamespaceID.String(), + Execution: parentExecution, + InitiatedId: 72, + InitiatedVersion: 892, + }, + s.tv.WorkerIdentity(), + ) + persistedState := workflow.TestCloneToProto(context.Background(), ms) + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).DoAndReturn( + func(context.Context, *persistence.GetWorkflowExecutionRequest) (*persistence.GetWorkflowExecutionResponse, error) { + return &persistence.GetWorkflowExecutionResponse{State: common.CloneProto(persistedState)}, nil + }, + ).Times(2) + + replacementUpdate := mock.MatchedBy(func(request *persistence.UpdateWorkflowExecutionRequest) bool { + return request.UpdateWorkflowMutation.ExecutionState.GetStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED && + request.NewWorkflowSnapshot != nil && + request.NewWorkflowSnapshot.ExecutionState.GetStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + }) + persistenceErr := &persistence.WorkflowConditionFailedError{Msg: "replace orphaned child persistence failed"} + s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), replacementUpdate).Return(nil, persistenceErr) + s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), replacementUpdate).Return(tests.UpdateWorkflowExecutionResponse, nil) + + startRequest := makeMockStartRequest( + s.tv, + enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + ) + startRequest.ParentExecutionInfo = &workflowspb.ParentExecutionInfo{ + NamespaceId: tests.ParentNamespaceID.String(), + Execution: parentExecution, + InitiatedId: 75, + InitiatedVersion: 925, + } + startRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{{EventId: 100, Version: 925}}, + } + + resp, err := s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + s.ErrorIs(err, persistenceErr) + s.Nil(resp) + s.Empty(capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()]) + + resp, err = s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + s.NoError(err) + s.True(resp.GetStarted()) + s.NotEqual(s.tv.RunID(), resp.GetRunId()) + recordings := capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()] + s.Len(recordings, 1) + s.Equal(int64(1), recordings[0].Value) + s.Equal("replaced", recordings[0].Tags["outcome"]) +} + +func (s *engine2Suite) TestStartWorkflowExecution_ReplaceOrphanedChild_ClosedAfterSnapshot() { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + now := s.historyEngine.shardContext.GetTimeSource().Now() + ms := s.setupStartWorkflowExecutionDedup( + timestamppb.New(now.Add(-100*time.Millisecond)), + enumsspb.WORKFLOW_EXECUTION_STATE_CREATED, + ) + parentExecution := &commonpb.WorkflowExecution{ + WorkflowId: "parent-workflow-id", + RunId: uuid.NewString(), + } + addWorkflowExecutionStartedEventWithParent( + ms, + s.tv.WorkflowExecution(), + s.tv.WorkflowType().GetName(), + s.tv.TaskQueue().GetName(), + nil, + time.Second, + time.Second, + time.Second, + &workflowspb.ParentExecutionInfo{ + NamespaceId: tests.ParentNamespaceID.String(), + Execution: parentExecution, + InitiatedId: 72, + InitiatedVersion: 892, + }, + s.tv.WorkerIdentity(), + ) + persistedState := workflow.TestCloneToProto(context.Background(), ms) + persistedState.ExecutionState.State = enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED + persistedState.ExecutionState.Status = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return( + &persistence.GetWorkflowExecutionResponse{State: persistedState}, + nil, + ) + s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), gomock.Any()).Times(0) + + startRequest := makeMockStartRequest( + s.tv, + enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE, + enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, + ) + startRequest.ParentExecutionInfo = &workflowspb.ParentExecutionInfo{ + NamespaceId: tests.ParentNamespaceID.String(), + Execution: parentExecution, + InitiatedId: 75, + InitiatedVersion: 925, + } + startRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: []*historyspb.VersionHistoryItem{{EventId: 100, Version: 925}}, + } + + resp, err := s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + var unavailable *serviceerror.Unavailable + s.ErrorAs(err, &unavailable) + s.Contains(err.Error(), "Termination failed") + s.Nil(resp) + recordings := capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()] + s.Len(recordings, 1) + s.Equal(int64(1), recordings[0].Value) + s.Equal("rejected_race_closed", recordings[0].Tags["outcome"]) +} + func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_SameAttachedRequestID() { // no error when request ID is the same s.setupStartWorkflowExecutionForRunning() @@ -1709,7 +1877,6 @@ func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_SameAttachedRequ } func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_PolicyFail() { - // error when id conflict policy is POLICY_FAIL s.setupStartWorkflowExecutionForRunning() startRequest := makeMockStartRequest(s.tv, enumspb.WORKFLOW_ID_REUSE_POLICY_UNSPECIFIED, enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL) @@ -1721,6 +1888,52 @@ func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_PolicyFail() { s.Nil(resp) } +func (s *engine2Suite) TestStartWorkflowExecution_ReplaceOrphanedChild_RejectsCompletedSnapshot() { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + now := s.historyEngine.shardContext.GetTimeSource().Now() + s.setupStartWorkflowExecutionDedup( + timestamppb.New(now.Add(-100*time.Millisecond)), + enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED, + ) + startRequest := makeMockStartRequest(s.tv, enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL) + startRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{} + + resp, err := s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + + var expectedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(err, &expectedErr) + s.Nil(resp) + recordings := capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()] + s.Len(recordings, 1) + s.Equal("rejected_unsupported_state", recordings[0].Tags["outcome"]) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_COMPLETED.String(), recordings[0].Tags["workflow_state"]) +} + +func (s *engine2Suite) TestStartWorkflowExecution_ReplaceOrphanedChild_RejectsRunningSnapshot() { + metricsHandler := metricstest.NewCaptureHandler() + capture := metricsHandler.StartCapture() + defer metricsHandler.StopCapture(capture) + s.mockShard.SetMetricsHandler(metricsHandler) + + s.setupStartWorkflowExecutionForRunning() + startRequest := makeMockStartRequest(s.tv, enumspb.WORKFLOW_ID_REUSE_POLICY_UNSPECIFIED, enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL) + startRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{} + + resp, err := s.historyEngine.StartWorkflowExecution(metrics.AddMetricsContext(context.Background()), startRequest) + + var expectedErr *serviceerror.WorkflowExecutionAlreadyStarted + s.ErrorAs(err, &expectedErr) + s.Nil(resp) + recordings := capture.Snapshot()[metrics.OrphanedChildWorkflowReplacement.Name()] + s.Len(recordings, 1) + s.Equal("rejected_unsupported_state", recordings[0].Tags["outcome"]) + s.Equal(enumsspb.WORKFLOW_EXECUTION_STATE_RUNNING.String(), recordings[0].Tags["workflow_state"]) +} + func (s *engine2Suite) TestStartWorkflowExecution_Dedup_Running_UseExisting() { // ignore error when id conflict policy is USE_EXISTING s.setupStartWorkflowExecutionForRunning() @@ -1874,7 +2087,8 @@ func (s *engine2Suite) TestStartWorkflowExecution_Dedup() { resp, err := s.historyEngine.StartWorkflowExecution( metrics.AddMetricsContext(context.Background()), - makeStartRequest(enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE, enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL)) + makeStartRequest(enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE, enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL), + ) s.NoError(err) s.True(resp.Started) diff --git a/service/history/transfer_queue_active_task_executor.go b/service/history/transfer_queue_active_task_executor.go index 0b39f89e193..a3d64aef11d 100644 --- a/service/history/transfer_queue_active_task_executor.go +++ b/service/history/transfer_queue_active_task_executor.go @@ -1092,24 +1092,33 @@ func (t *transferQueueActiveTaskExecutor) processStartChildExecution( }, } - childRunID, childClock, err := t.startChildWorkflow( - ctx, - task, - parentNamespaceName, - targetNamespaceName, - namespace.ID(targetNamespaceID), - childInfo.CreateRequestId, - attributes, - sourceVersionStamp, - rootExecutionInfo, - inheritedBuildID, - initiatedEvent.GetUserMetadata(), - shouldTerminateAndStartChild, - inheritedVersioningOverride, - inheritedPinnedVersion, - priorities.Merge(mutableState.GetExecutionInfo().Priority, attributes.Priority), - inheritedAutoUpgradeInfo, - ) + childPriority := priorities.Merge(mutableState.GetExecutionInfo().Priority, attributes.Priority) + startParams := &startChildWorkflowParams{ + task: task, + parentNamespaceName: parentNamespaceName, + targetNamespaceName: targetNamespaceName, + targetNamespaceID: namespace.ID(targetNamespaceID), + requestID: childInfo.CreateRequestId, + attributes: attributes, + sourceVersionStamp: sourceVersionStamp, + rootExecutionInfo: rootExecutionInfo, + inheritedBuildID: inheritedBuildID, + userMetadata: initiatedEvent.GetUserMetadata(), + inheritedVersioningOverride: inheritedVersioningOverride, + inheritedPinnedVersion: inheritedPinnedVersion, + priority: childPriority, + inheritedAutoUpgradeInfo: inheritedAutoUpgradeInfo, + terminateExisting: shouldTerminateAndStartChild, + } + if !shouldTerminateAndStartChild { + startParams.orphanedChildReplacementInfo = t.getOrphanedChildReplacementInfo( + mutableState, + childInfo, + parentNamespaceName, + ) + } + + childRunID, childClock, err := t.startChildWorkflow(ctx, startParams) if err != nil { t.logger.Debug("Failed to start child workflow execution", tag.Error(err)) var failedCause enumspb.StartChildWorkflowExecutionFailedCause @@ -1174,6 +1183,53 @@ func (t *transferQueueActiveTaskExecutor) processStartChildExecution( }, parentClock, childClock) } +// getOrphanedChildReplacementInfo returns the parent branch evidence needed to validate a +// conflicting child. Nil keeps normal workflow-ID conflict handling. +func (t *transferQueueActiveTaskExecutor) getOrphanedChildReplacementInfo( + mutableState historyi.MutableState, + childInfo *persistencespb.ChildExecutionInfo, + parentNamespaceName namespace.Name, +) *historyservice.OrphanedChildReplacementInfo { + if !t.config.EnableOrphanedChildWorkflowReplacement(parentNamespaceName.String()) { + return nil + } + executionInfo := mutableState.GetExecutionInfo() + childNamespaceMatchesParent := childInfo.GetNamespaceId() == executionInfo.GetNamespaceId() + if childInfo.GetNamespaceId() == "" { + childNamespaceMatchesParent = childInfo.GetNamespace() == parentNamespaceName.String() + } + // The parent lock remains held across the child start RPC. Never send replacement intent when the + // conflicting workflow could be this parent, or the child-side transaction could wait on that lock. + if childNamespaceMatchesParent && childInfo.GetStartedWorkflowId() == executionInfo.GetWorkflowId() { + return nil + } + // Multiple pending initiations for the same target are ambiguous: the child cannot determine + // which initiation should own the replacement, so keep ordinary conflict handling. + for initiatedEventID, pendingChild := range mutableState.GetPendingChildExecutionInfos() { + if initiatedEventID != childInfo.GetInitiatedEventId() && sameChildWorkflowTarget(pendingChild, childInfo) { + return nil + } + } + + currentVersionHistory, err := versionhistory.GetCurrentVersionHistory(executionInfo.GetVersionHistories()) + if err != nil || len(currentVersionHistory.GetItems()) == 0 { + return nil + } + return &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: versionhistory.CopyVersionHistoryItems(currentVersionHistory.GetItems()), + } +} + +func sameChildWorkflowTarget(first, second *persistencespb.ChildExecutionInfo) bool { + if first.GetStartedWorkflowId() != second.GetStartedWorkflowId() { + return false + } + if first.GetNamespaceId() != "" && second.GetNamespaceId() != "" { + return first.GetNamespaceId() == second.GetNamespaceId() + } + return first.GetNamespace() == second.GetNamespace() +} + // verifyChildWorkflow describes the childWorkflowID and identifies its parent. It then checks if the current run was derived from that parent by comparing the OriginalRunID value. // It returns the child's runID if the checks pass. Empty runID is returned if the check doesn't pass. func (t *transferQueueActiveTaskExecutor) verifyChildWorkflow( @@ -1210,7 +1266,10 @@ func (t *transferQueueActiveTaskExecutor) verifyChildWorkflow( } childsParentRunID := response.WorkflowExecutionInfo.ParentExecution.RunId - // Check if the child's parent was the base run for the current run. + // A reset run may be missing ChildWorkflowExecutionStarted for a child that its original run + // already started before the reset point. If the child points directly to that original run, + // return the existing child's run IDs so the caller reconnects it to this reset run instead of + // starting a duplicate child. if childsParentRunID == mutableState.GetExecutionInfo().OriginalExecutionRunId { return response.WorkflowExecutionInfo.Execution.RunId, response.WorkflowExecutionInfo.FirstRunId, nil } @@ -1671,92 +1730,98 @@ func (t *transferQueueActiveTaskExecutor) signalExternalExecution( return err } +type startChildWorkflowParams struct { + task *tasks.StartChildExecutionTask + parentNamespaceName namespace.Name + targetNamespaceName namespace.Name + targetNamespaceID namespace.ID + requestID string + attributes *historypb.StartChildWorkflowExecutionInitiatedEventAttributes + sourceVersionStamp *commonpb.WorkerVersionStamp + rootExecutionInfo *workflowspb.RootExecutionInfo + inheritedBuildID string + userMetadata *sdkpb.UserMetadata + inheritedVersioningOverride *workflowpb.VersioningOverride + inheritedPinnedVersion *deploymentpb.WorkerDeploymentVersion + priority *commonpb.Priority + inheritedAutoUpgradeInfo *deploymentpb.InheritedAutoUpgradeInfo + terminateExisting bool + orphanedChildReplacementInfo *historyservice.OrphanedChildReplacementInfo +} + func (t *transferQueueActiveTaskExecutor) startChildWorkflow( ctx context.Context, - task *tasks.StartChildExecutionTask, - namespace namespace.Name, - targetNamespace namespace.Name, - targetNamespaceID namespace.ID, - childRequestID string, - attributes *historypb.StartChildWorkflowExecutionInitiatedEventAttributes, - sourceVersionStamp *commonpb.WorkerVersionStamp, - rootExecutionInfo *workflowspb.RootExecutionInfo, - inheritedBuildId string, - userMetadata *sdkpb.UserMetadata, - shouldTerminateAndStartChild bool, - inheritedVersioningOverride *workflowpb.VersioningOverride, - inheritedPinnedVersion *deploymentpb.WorkerDeploymentVersion, - priority *commonpb.Priority, - inheritedAutoUpgradeInfo *deploymentpb.InheritedAutoUpgradeInfo, + params *startChildWorkflowParams, ) (string, *clockspb.VectorClock, error) { - versioningOverride := inheritedVersioningOverride - if attributes.GetVersioningOverride() != nil { - versioningOverride = attributes.GetVersioningOverride() + versioningOverride := params.inheritedVersioningOverride + if params.attributes.GetVersioningOverride() != nil { + versioningOverride = params.attributes.GetVersioningOverride() } startRequest := &workflowservice.StartWorkflowExecutionRequest{ - Namespace: targetNamespace.String(), - WorkflowId: attributes.WorkflowId, - WorkflowType: attributes.WorkflowType, - TaskQueue: attributes.TaskQueue, - Input: attributes.Input, - Header: attributes.Header, - WorkflowExecutionTimeout: attributes.WorkflowExecutionTimeout, - WorkflowRunTimeout: attributes.WorkflowRunTimeout, - WorkflowTaskTimeout: attributes.WorkflowTaskTimeout, + Namespace: params.targetNamespaceName.String(), + WorkflowId: params.attributes.WorkflowId, + WorkflowType: params.attributes.WorkflowType, + TaskQueue: params.attributes.TaskQueue, + Input: params.attributes.Input, + Header: params.attributes.Header, + WorkflowExecutionTimeout: params.attributes.WorkflowExecutionTimeout, + WorkflowRunTimeout: params.attributes.WorkflowRunTimeout, + WorkflowTaskTimeout: params.attributes.WorkflowTaskTimeout, // Use the same request ID to dedupe StartWorkflowExecution calls - RequestId: childRequestID, - WorkflowIdReusePolicy: attributes.WorkflowIdReusePolicy, + RequestId: params.requestID, + WorkflowIdReusePolicy: params.attributes.WorkflowIdReusePolicy, WorkflowIdConflictPolicy: enumspb.WORKFLOW_ID_CONFLICT_POLICY_FAIL, - RetryPolicy: attributes.RetryPolicy, - CronSchedule: attributes.CronSchedule, - Memo: attributes.Memo, - SearchAttributes: attributes.SearchAttributes, - UserMetadata: userMetadata, + RetryPolicy: params.attributes.RetryPolicy, + CronSchedule: params.attributes.CronSchedule, + Memo: params.attributes.Memo, + SearchAttributes: params.attributes.SearchAttributes, + UserMetadata: params.userMetadata, VersioningOverride: versioningOverride, - Priority: priority, - TimeSkippingConfig: attributes.GetTimeSkippingConfig(), + Priority: params.priority, + TimeSkippingConfig: params.attributes.GetTimeSkippingConfig(), } - - statePropagation := attributes.GetTimeSkippingStatePropagation() + statePropagation := params.attributes.GetTimeSkippingStatePropagation() nowForExpirationAndBackoff := workflow.AdjustNowWithTimeSkipping( t.shardContext.GetTimeSource().Now(), statePropagation, ) request := common.CreateHistoryStartWorkflowRequest( - targetNamespaceID.String(), + params.targetNamespaceID.String(), startRequest, &workflowspb.ParentExecutionInfo{ - NamespaceId: task.NamespaceID, - Namespace: namespace.String(), + NamespaceId: params.task.NamespaceID, + Namespace: params.parentNamespaceName.String(), Execution: &commonpb.WorkflowExecution{ - WorkflowId: task.WorkflowID, - RunId: task.RunID, + WorkflowId: params.task.WorkflowID, + RunId: params.task.RunID, }, - InitiatedId: task.InitiatedEventID, - InitiatedVersion: task.Version, - Clock: vclock.NewVectorClock(t.shardContext.GetClusterMetadata().GetClusterID(), t.shardContext.GetShardID(), task.TaskID), + InitiatedId: params.task.InitiatedEventID, + InitiatedVersion: params.task.Version, + Clock: vclock.NewVectorClock(t.shardContext.GetClusterMetadata().GetClusterID(), t.shardContext.GetShardID(), params.task.TaskID), }, - rootExecutionInfo, + params.rootExecutionInfo, nowForExpirationAndBackoff, ) - request.SourceVersionStamp = sourceVersionStamp - request.InheritedBuildId = inheritedBuildId - request.InheritedPinnedVersion = inheritedPinnedVersion + request.SourceVersionStamp = params.sourceVersionStamp + request.InheritedBuildId = params.inheritedBuildID + request.InheritedPinnedVersion = params.inheritedPinnedVersion request.TimeSkippingStatePropagation = statePropagation // Only set the AutoUpgrade info if the Pinned version is not set. if request.InheritedPinnedVersion == nil { - request.InheritedAutoUpgradeInfo = inheritedAutoUpgradeInfo + request.InheritedAutoUpgradeInfo = params.inheritedAutoUpgradeInfo } - if shouldTerminateAndStartChild { + if params.terminateExisting { request.StartRequest.WorkflowIdReusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE request.StartRequest.WorkflowIdConflictPolicy = enumspb.WORKFLOW_ID_CONFLICT_POLICY_TERMINATE_EXISTING request.ChildWorkflowOnly = true + } else { + request.OrphanedChildReplacementInfo = params.orphanedChildReplacementInfo } response, err := t.historyRawClient.StartWorkflowExecution(ctx, request) diff --git a/service/history/transfer_queue_active_task_executor_test.go b/service/history/transfer_queue_active_task_executor_test.go index 7d5c11e95f9..cf2775da2b4 100644 --- a/service/history/transfer_queue_active_task_executor_test.go +++ b/service/history/transfer_queue_active_task_executor_test.go @@ -19,6 +19,7 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" + historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/historyservicemock/v1" "go.temporal.io/server/api/matchingservice/v1" @@ -2070,6 +2071,18 @@ func (s *transferQueueActiveTaskExecutorSuite) validateUpdateExecutionRequestWit } func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Success() { + s.testProcessStartChildExecutionSuccess(false) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Success_OrphanedChildReplacementEnabled() { + s.testProcessStartChildExecutionSuccess(true) +} + +func (s *transferQueueActiveTaskExecutorSuite) testProcessStartChildExecutionSuccess(replacementEnabled bool) { + if replacementEnabled { + s.mockShard.GetConfig().EnableOrphanedChildWorkflowReplacement = func(string) bool { return true } + } + execution := &commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", RunId: uuid.NewString(), @@ -2153,15 +2166,28 @@ func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Su childClock := vclock.NewVectorClock(rand.Int63(), rand.Int31(), rand.Int63()) persistenceMutableState := s.createPersistenceMutableState(mutableState, event.GetEventId(), event.GetVersion()) - s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) - s.mockHistoryClient.EXPECT().StartWorkflowExecution(gomock.Any(), s.createChildWorkflowExecutionRequest( + expectedStartRequest := s.createChildWorkflowExecutionRequest( s.childNamespace, transferTask, mutableState, ci, rootExecutionInfo, userMetadata, - )).Return(&historyservice.StartWorkflowExecutionResponse{RunId: childRunID, Clock: childClock}, nil) + ) + if replacementEnabled { + parentCurrentVersionHistory, err := versionhistory.GetCurrentVersionHistory( + persistenceMutableState.GetExecutionInfo().GetVersionHistories(), + ) + s.NoError(err) + expectedStartRequest.OrphanedChildReplacementInfo = &historyservice.OrphanedChildReplacementInfo{ + ParentCurrentVersionHistoryItems: versionhistory.CopyVersionHistoryItems(parentCurrentVersionHistory.GetItems()), + } + } + s.mockExecutionMgr.EXPECT().GetWorkflowExecution(gomock.Any(), gomock.Any()).Return(&persistence.GetWorkflowExecutionResponse{State: persistenceMutableState}, nil) + s.mockHistoryClient.EXPECT().StartWorkflowExecution(gomock.Any(), protomock.Eq(expectedStartRequest)).Return( + &historyservice.StartWorkflowExecutionResponse{RunId: childRunID, Clock: childClock}, + nil, + ) s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), gomock.Any()).Return(tests.UpdateWorkflowExecutionResponse, nil) s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(s.namespaceEntry.IsGlobalNamespace(), s.version).Return(cluster.TestCurrentClusterName).AnyTimes() currentShardClock := s.mockShard.CurrentVectorClock() @@ -2620,6 +2646,8 @@ func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Fa rootExecutionInfo, nil, )).Return(nil, serviceerror.NewWorkflowExecutionAlreadyStarted("msg", "", "")) + // Orphaned-child replacement is opt-in, so this request carries no replacement intent and records + // StartChildExecutionFailed(WORKFLOW_ALREADY_EXISTS) exactly as it did before that feature. s.mockExecutionMgr.EXPECT().UpdateWorkflowExecution(gomock.Any(), gomock.Any()).Return(tests.UpdateWorkflowExecutionResponse, nil) s.mockClusterMetadata.EXPECT().ClusterNameForFailoverVersion(s.namespaceEntry.IsGlobalNamespace(), s.version).Return(cluster.TestCurrentClusterName).AnyTimes() @@ -2627,6 +2655,168 @@ func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Fa s.NoError(resp.ExecutionErr) } +func (s *transferQueueActiveTaskExecutorSuite) TestGetOrphanedChildReplacementInfo() { + const childWorkflowID = "child-workflow" + parentVersionHistoryItems := []*historyspb.VersionHistoryItem{{EventId: 100, Version: 925}} + childInfo := &persistencespb.ChildExecutionInfo{ + InitiatedEventId: 75, + StartedWorkflowId: childWorkflowID, + NamespaceId: tests.ChildNamespaceID.String(), + Namespace: tests.ChildNamespace.String(), + } + + // The parent supplies only branch evidence. ReplaceOrphanedChildAction applies child-local policy + // while holding the conflicting run's lock. + testCases := []struct { + name string + enabled bool + pendingChildren map[int64]*persistencespb.ChildExecutionInfo + expected bool + }{ + { + name: "disabled", + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{75: childInfo}, + }, + { + name: "only this initiation holds the workflow ID", + enabled: true, + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{75: childInfo}, + expected: true, + }, + { + name: "another accepted child uses the workflow ID", + enabled: true, + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{ + 72: { + InitiatedEventId: 72, + StartedWorkflowId: childWorkflowID, + NamespaceId: tests.ChildNamespaceID.String(), + }, + 75: childInfo, + }, + }, + { + name: "three accepted initiations use the workflow ID", + enabled: true, + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{ + 70: { + InitiatedEventId: 70, + StartedWorkflowId: childWorkflowID, + NamespaceId: tests.ChildNamespaceID.String(), + }, + 72: { + InitiatedEventId: 72, + StartedWorkflowId: childWorkflowID, + NamespaceId: tests.ChildNamespaceID.String(), + }, + 75: childInfo, + }, + }, + { + name: "another namespace uses the workflow ID", + enabled: true, + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{ + 72: { + InitiatedEventId: 72, + StartedWorkflowId: childWorkflowID, + NamespaceId: tests.NamespaceID.String(), + }, + 75: childInfo, + }, + expected: true, + }, + { + name: "other accepted children use different workflow IDs", + enabled: true, + pendingChildren: map[int64]*persistencespb.ChildExecutionInfo{ + 72: {InitiatedEventId: 72, StartedWorkflowId: "other-child"}, + 75: childInfo, + }, + expected: true, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + s.mockShard.GetConfig().EnableOrphanedChildWorkflowReplacement = func(string) bool { + return tc.enabled + } + mutableState := historyi.NewMockMutableState(gomock.NewController(s.T())) + mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: "parent-namespace", + WorkflowId: "parent-workflow", + VersionHistories: versionhistory.NewVersionHistories(versionhistory.NewVersionHistory( + nil, + parentVersionHistoryItems, + )), + }).AnyTimes() + mutableState.EXPECT().GetPendingChildExecutionInfos().Return(tc.pendingChildren).AnyTimes() + + replacementInfo := s.transferQueueActiveTaskExecutor.getOrphanedChildReplacementInfo( + mutableState, + childInfo, + tests.Namespace, + ) + if tc.expected { + s.NotNil(replacementInfo) + s.Equal(parentVersionHistoryItems, replacementInfo.GetParentCurrentVersionHistoryItems()) + s.NotSame(parentVersionHistoryItems[0], replacementInfo.GetParentCurrentVersionHistoryItems()[0]) + } else { + s.Nil(replacementInfo) + } + }) + } + + s.Run("same workflow ID in another namespace", func() { + s.mockShard.GetConfig().EnableOrphanedChildWorkflowReplacement = func(string) bool { return true } + mutableState := historyi.NewMockMutableState(gomock.NewController(s.T())) + mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: tests.NamespaceID.String(), + WorkflowId: "same-workflow", + VersionHistories: versionhistory.NewVersionHistories(versionhistory.NewVersionHistory( + nil, + parentVersionHistoryItems, + )), + }) + mutableState.EXPECT().GetPendingChildExecutionInfos().Return(map[int64]*persistencespb.ChildExecutionInfo{ + 75: childInfo, + }) + crossNamespaceChild := &persistencespb.ChildExecutionInfo{ + InitiatedEventId: 75, + StartedWorkflowId: "same-workflow", + NamespaceId: tests.ChildNamespaceID.String(), + } + + s.NotNil(s.transferQueueActiveTaskExecutor.getOrphanedChildReplacementInfo( + mutableState, + crossNamespaceChild, + tests.Namespace, + )) + }) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestOrphanedChildReplacementDoesNotDeadlockOnSelfWorkflowID() { + s.mockShard.GetConfig().EnableOrphanedChildWorkflowReplacement = func(string) bool { return true } + mutableState := historyi.NewMockMutableState(gomock.NewController(s.T())) + mutableState.EXPECT().GetExecutionInfo().Return(&persistencespb.WorkflowExecutionInfo{ + NamespaceId: "same-namespace", + WorkflowId: "same-workflow", + }) + selfChild := &persistencespb.ChildExecutionInfo{ + InitiatedEventId: 75, + Namespace: tests.Namespace.String(), + StartedWorkflowId: "same-workflow", + } + + // The parent lock spans startChildWorkflow, so targeting the parent itself would recursively + // acquire that lock. Omitting replacement info keeps the request on ordinary conflict handling. + s.Nil(s.transferQueueActiveTaskExecutor.getOrphanedChildReplacementInfo( + mutableState, + selfChild, + tests.Namespace, + )) +} + func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Failure_InvalidVersioningOverride() { execution := &commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", diff --git a/tests/xdc/parent_child_test.go b/tests/xdc/parent_child_test.go index 03f11e5e3dc..29cb034ecc7 100644 --- a/tests/xdc/parent_child_test.go +++ b/tests/xdc/parent_child_test.go @@ -8,10 +8,13 @@ import ( "time" "github.com/stretchr/testify/suite" + commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" historypb "go.temporal.io/api/history/v1" "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" enumsspb "go.temporal.io/server/api/enums/v1" + "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/tests/testcore" ) @@ -34,161 +37,138 @@ func (s *parentChildXDCTestSuite) TearDownSuite() { s.tearDownSuite() } -// TestReproOrphanedChildAfterForceFailover covers the cross-shard ordering where the child start -// reaches the passive while parent replication from WorkflowTaskStarted onward, including -// StartChildWorkflowExecutionInitiated, remains delayed, followed by force failover. +// TestRecoversOrphanedChildAfterForceFailover covers the cross-shard ordering where the child start +// reaches the passive without its initial workflow task. Parent replication from WorkflowTaskStarted +// onward, including StartChildWorkflowExecutionInitiated, remains delayed before force failover. // -// | parent on target | child on target | active | outcome -// -----------------------+------------------------------------------+---------------------------------+-----------------+------------------------------ -// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix -// child start arrives | parent update remains delayed | RUNNING, points to old version | initial active | cross-shard partial state -// force failover | incomplete branch becomes current | unchanged | initial standby | target begins active recovery -// retry StartChild | same initiated event ID at a new version | unchanged | initial standby | WORKFLOW_ALREADY_EXISTS -// assert | no ChildWorkflowExecutionStarted | still points to losing version | initial standby | child is orphaned +// | parent on target | child on target | active | outcome +// -----------------------+------------------------------------------+----------------------------------+-----------------+------------------------------ +// parent prefix arrives | WFT scheduled | does not exist | initial active | common parent prefix +// child start arrives | parent update remains delayed | pristine, points to old version | initial active | cross-shard partial state +// force failover | incomplete branch becomes current | unchanged | initial standby | target begins active recovery +// retry StartChild | reissued initiation on the new branch | old run terminated, new run open | initial standby | orphan is replaced atomically +// assert | ChildWorkflowExecutionStarted | replacement is current | initial standby | parent can make progress // -// Event checkpoints select an entire replication task, not an individual event. The delayed parent -// task intentionally remains unapplied through the assertions. -func (s *parentChildXDCTestSuite) TestReproOrphanedChildAfterForceFailover() { +// This scenario uses the default transition-history replication. Event checkpoints select an entire +// replication task, and the delayed parent task remains unapplied through the assertions. +func (s *parentChildXDCTestSuite) TestRecoversOrphanedChildAfterForceFailover() { s.runParentChildScenario(parentChildScenario{ steps: []parentChildScenarioStep{ - // Create the parent and its first workflow task on the initial active cluster. + enableOrphanedChildWorkflowReplacement(), startParentWorkflow(), - // Give the passive a common parent prefix before introducing the cross-shard gap. applyReplicationThroughTaskContainingEvent( initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, ), - // Complete the parent task with StartChild, creating the child on the source. completeParentWorkflowTaskWithStartChildCommand(), - // Stop parent replication before its StartChild branch is complete on the passive. delayReplicationAtTaskContainingEvent( initialStandbyCluster, parentWorkflow, enumspb.EVENT_TYPE_WORKFLOW_TASK_STARTED, ), - // Let the child arrive independently, still pointing to the old parent branch. applyReplicationThroughTaskContainingEvent( initialStandbyCluster, childWorkflow, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, ), - // Promote the incomplete passive state without draining the delayed parent task. forceFailoverNamespaceTo(initialStandbyCluster), - // Retry StartChild on the new active branch, where the child ID already exists. completeParentWorkflowTaskWithStartChildCommand(), }, expectations: []parentChildExpectation{ - { - name: "parent StartChild fails because the child already exists", - check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { - events, err := runtime.activeWorkflowHistory(ctx, parentWorkflow) - if err != nil { - return err - } - failedEvent := findHistoryEvent( - events, - enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED, - func(event *historypb.HistoryEvent) bool { - attrs := event.GetStartChildWorkflowExecutionFailedEventAttributes() - return attrs.GetWorkflowId() == runtime.childID && - attrs.GetCause() == enumspb.START_CHILD_WORKFLOW_EXECUTION_FAILED_CAUSE_WORKFLOW_ALREADY_EXISTS - }, - ) - if failedEvent == nil { - return fmt.Errorf("parent has no WORKFLOW_ALREADY_EXISTS failure for child %q", runtime.childID) - } - return nil - }, - }, - currentWorkflowHasStatusOnCluster( + workflowHasEventOnCluster( initialStandbyCluster, childWorkflow, - enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_TERMINATED, ), - { - name: "child remains attached to the losing parent branch", - check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { - if runtime.childRunID == "" { - return errors.New("child WorkflowExecutionStarted was not applied") - } - childEvents, err := runtime.activeWorkflowHistory(ctx, childWorkflow) - if err != nil { - return err - } - childStartedEvent := findHistoryEvent( - childEvents, - enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED, - nil, - ) - if childStartedEvent == nil { - return fmt.Errorf("child run %q has no persisted WorkflowExecutionStarted event", runtime.childRunID) - } - childStartedAttrs := childStartedEvent.GetWorkflowExecutionStartedEventAttributes() - if childStartedAttrs == nil { - return errors.New("persisted child start event has no attributes") - } - parentExecution := childStartedAttrs.GetParentWorkflowExecution() - if parentExecution.GetWorkflowId() != runtime.parentID || parentExecution.GetRunId() != runtime.parentRunID { - return fmt.Errorf( - "child parent is %s/%s, want %s/%s", - parentExecution.GetWorkflowId(), - parentExecution.GetRunId(), - runtime.parentID, - runtime.parentRunID, - ) - } + parentOwnsReplacementChild(), + orphanedChildReplacementWasRecorded(), + }, + }) +} - parentEvents, err := runtime.activeWorkflowHistory(ctx, parentWorkflow) - if err != nil { - return err - } - currentInitiatedEvent := findHistoryEvent( - parentEvents, - enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED, - func(event *historypb.HistoryEvent) bool { - attrs := event.GetStartChildWorkflowExecutionInitiatedEventAttributes() - return attrs.GetWorkflowId() == runtime.childID && - event.GetEventId() == childStartedAttrs.GetParentInitiatedEventId() - }, - ) - currentChildStartedEvent := findHistoryEvent( - parentEvents, - enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, - func(event *historypb.HistoryEvent) bool { - attrs := event.GetChildWorkflowExecutionStartedEventAttributes() - return attrs.GetWorkflowExecution().GetWorkflowId() == runtime.childID && - attrs.GetWorkflowExecution().GetRunId() == runtime.childRunID - }, - ) - if currentChildStartedEvent != nil { - return errors.New("current parent branch owns the applied child") - } - if currentInitiatedEvent == nil { - return fmt.Errorf( - "current parent branch has no StartChild event %d for child %q", - childStartedAttrs.GetParentInitiatedEventId(), - runtime.childID, - ) - } - if currentInitiatedEvent.GetVersion() == childStartedAttrs.GetParentInitiatedEventVersion() { - return fmt.Errorf( - "child and current parent branch have the same initiation version %d", - currentInitiatedEvent.GetVersion(), - ) - } - runtime.tracef( - "orphan confirmed: child parent pointer=(%d, v%d), current parent initiation=(%d, v%d)", - childStartedAttrs.GetParentInitiatedEventId(), - childStartedAttrs.GetParentInitiatedEventVersion(), - currentInitiatedEvent.GetEventId(), - currentInitiatedEvent.GetVersion(), - ) - return nil +func enableOrphanedChildWorkflowReplacement() parentChildScenarioStep { + return parentChildScenarioStep{ + name: "enable orphaned child workflow replacement", + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + for _, cluster := range runtime.suite.clusters { + runtime.cleanups = append(runtime.cleanups, cluster.OverrideDynamicConfig( + runtime.suite.T(), + dynamicconfig.EnableOrphanedChildWorkflowReplacement, + true, + )) + } + return nil + }, + } +} + +func parentOwnsReplacementChild() parentChildExpectation { + return parentChildExpectation{ + name: "parent owns a running replacement child", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + if runtime.childRunID == "" { + return errors.New("orphaned child WorkflowExecutionStarted was not applied") + } + currentChild, err := runtime.activeCluster().FrontendClient().DescribeWorkflowExecution( + ctx, + &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: runtime.namespace, + Execution: &commonpb.WorkflowExecution{WorkflowId: runtime.childID}, }, - }, + ) + if err != nil { + return err + } + currentChildInfo := currentChild.GetWorkflowExecutionInfo() + replacementRunID := currentChildInfo.GetExecution().GetRunId() + if replacementRunID == "" || replacementRunID == runtime.childRunID { + return fmt.Errorf("current child run is %q, want a replacement for %q", replacementRunID, runtime.childRunID) + } + if currentChildInfo.GetStatus() != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING { + return fmt.Errorf("replacement child status is %s, want RUNNING", currentChildInfo.GetStatus()) + } + + parentEvents, err := runtime.activeWorkflowHistory(ctx, parentWorkflow) + if err != nil { + return err + } + if findHistoryEvent( + parentEvents, + enumspb.EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_FAILED, + func(event *historypb.HistoryEvent) bool { + return event.GetStartChildWorkflowExecutionFailedEventAttributes().GetWorkflowId() == runtime.childID + }, + ) != nil { + return fmt.Errorf("parent recorded StartChildWorkflowExecutionFailed for child %q", runtime.childID) + } + if findHistoryEvent( + parentEvents, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + func(event *historypb.HistoryEvent) bool { + startedExecution := event.GetChildWorkflowExecutionStartedEventAttributes().GetWorkflowExecution() + return startedExecution.GetWorkflowId() == runtime.childID && + startedExecution.GetRunId() == replacementRunID + }, + ) == nil { + return fmt.Errorf("parent has no ChildWorkflowExecutionStarted for replacement run %q", replacementRunID) + } + return nil }, - }) + } +} + +func orphanedChildReplacementWasRecorded() parentChildExpectation { + return parentChildExpectation{ + name: "orphaned child replacement metric is recorded", + check: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + return runtime.requireCapturedMetric( + initialStandbyCluster, + metrics.OrphanedChildWorkflowReplacement.Name(), + map[string]string{"outcome": "replaced"}, + ) + }, + } } // TestStandbyResendsMissingChild covers the cross-shard ordering where the parent update From adb4a6ce8060a6a4a7e0ae3d5ca19624a02d05d6 Mon Sep 17 00:00:00 2001 From: Jiechen Zhong Date: Wed, 2 Sep 2026 16:12:54 -0700 Subject: [PATCH 123/139] Fix child completion recovery after late parent replication (#11868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? When an active Parent `StartChildExecution` task regenerated from late replication finds that the Child is already closed: - Resolve the Child’s current run by Workflow ID using `GetMutableState`. - Verify that the current run belongs to the expected execution chain using `FirstExecutionRunId`. - Refresh only a final current run (`Completed`, `Failed`, `Canceled`, `Terminated`, or `TimedOut`) to regenerate its `CloseExecution` task. - Allow the regenerated task to record Child completion in the Parent. Recovery is restricted to the path where the Parent still has a pending, started `ChildExecutionInfo`. If completion has already been recorded, that entry has been removed and recovery is not triggered. The change also adds: - A namespace-level `history.enableChildWorkflowCompletionRecovery` kill switch. - A `child_workflow_completion_recovery_attempts` metric. - Logging for terminal recovery attempts and Workflow ID reuse mismatches. - Unit coverage for terminal recovery, Workflow ID reuse, disabled recovery, deleted current runs, skipped running/paused successors, and retryable `GetMutableState`/Refresh failures. - XDC coverage for: - A normally completed Child. - A Child that Continued-As-New before its final run completed. - A reset Child whose current reset run completed. - A namespace-scoped, test-only source replication conversion interceptor. The XDC harness uses it to pause Parent raw-task conversion and deterministically reproduce source-side replication backlog without disabling replication, changing cluster metadata, or acknowledging/dropping tasks. ## Why? With cross-shard replication ordering, the Child can arrive and close on the new active cluster before the Parent arrives. The Child’s original `CloseExecution` task cannot notify the missing Parent, and the resulting `NotFound` is acknowledged. When Parent replication arrives later, task refresh regenerates its `StartChildExecution` task. That task finds that the Child has already closed, but previously did not recreate the lost completion notification. Consequently, the Parent could permanently retain the pending Child without recording `ChildWorkflowExecutionCompleted`. The regenerated Parent task now resolves the Child’s current execution chain and refreshes its final current run. This recreates the Child’s `CloseExecution` task, allowing completion recording to converge. `GetMutableState` is used instead of `DescribeWorkflowExecution` because recovery only needs the current execution, workflow status, and first execution run ID. It avoids constructing unrelated Describe data such as pending operations, memo, search attributes, callbacks, and Nexus state. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [x] added new functional test(s) Commands run: - Targeted `TransferQueueActiveTaskExecutorSuite` recovery unit tests with `-tags test_dep`. - Normal, Continue-As-New, and reset XDC recovery tests with `-tags test_dep`. - Full `TestParentChildXDCTestSuite`. - `make lint-code-fast` with zero issues. - CI-equivalent `govulncheck v1.7.0 -json ./...` with exit code 0. Not run: - Full `saas-temporal` dependency-injection and service-startup compatibility validation. ## Potential risks - `RefreshWorkflowTasks` performs a full task refresh rather than regenerating only `CloseExecution`. It invalidates previously generated tasks and may also regenerate visibility, retention, archival, pending Child, and HSM/CHASM tasks. - In the narrow case where an existing `CloseExecution` task has `DeleteAfterClose=true`, full refresh regenerates it with `DeleteAfterClose=false`. If the original task is invalidated before processing, deletion may not complete. - If the current Child run has already been deleted before Parent replication arrives, there is no execution to refresh and completion still cannot be recovered. A TODO records this remaining gap. --- common/dynamicconfig/constants.go | 6 + common/metrics/metric_defs.go | 4 + common/testing/testhooks/hooks.go | 27 +- service/history/configs/config.go | 2 + service/history/history_engine.go | 9 + .../transfer_queue_active_task_executor.go | 113 +++++++- ...ransfer_queue_active_task_executor_test.go | 172 ++++++++++++ tests/xdc/base.go | 4 +- tests/xdc/parent_child_harness.go | 193 ++++++++++---- tests/xdc/parent_child_test.go | 245 ++++++++++++++++++ 10 files changed, 712 insertions(+), 63 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index f090903c077..21bdf0e412e 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -2830,6 +2830,12 @@ is not held for the duration of the cross-cluster sync.`, verification may resend a missing child workflow in the background from the active cluster. When disabled, verification remains local-only. StandbyTaskMissingEventsResendDelay plus ReplicationTaskApplyTimeout should remain below StandbyTaskMissingEventsDiscardDelay.`, + ) + EnableChildWorkflowCompletionRecovery = NewNamespaceBoolSetting( + "history.enableChildWorkflowCompletionRecovery", + true, + `EnableChildWorkflowCompletionRecovery controls whether an active StartChildExecution task may +refresh a terminal child workflow to recover a completion notification lost while the parent was missing.`, ) WorkflowResendHostMaxInFlight = NewGlobalIntSetting( "history.workflowResendHostMaxInFlight", diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 4c1f3fc3098..78433aa0e50 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1158,6 +1158,10 @@ var ( ChildWorkflowResendLimited = NewCounterDef("child_workflow_resend_limited") // ChildWorkflowResendLatency measures a child resend and subsequent verification. ChildWorkflowResendLatency = NewTimerDef("child_workflow_resend_latency") + // ChildWorkflowCompletionRecoveryAttempts counts terminal child refreshes triggered by a late parent. + ChildWorkflowCompletionRecoveryAttempts = NewCounterDef("child_workflow_completion_recovery_attempts") + // ChildWorkflowCompletionRecoveryChainMismatch counts recoveries skipped after Workflow ID reuse. + ChildWorkflowCompletionRecoveryChainMismatch = NewCounterDef("child_workflow_completion_recovery_chain_mismatch") // WorkflowResendSchedulerAtCapacity counts host-level workflow resends rejected at the concurrency limit. WorkflowResendSchedulerAtCapacity = NewCounterDef("workflow_resend_scheduler_at_capacity") // ReplicationOrphanedHistoryBranch tracks cases where history branch cleanup was skipped on error diff --git a/common/testing/testhooks/hooks.go b/common/testing/testhooks/hooks.go index 72cd5f8cac6..37d8979bea8 100644 --- a/common/testing/testhooks/hooks.go +++ b/common/testing/testhooks/hooks.go @@ -26,19 +26,20 @@ type HistoryPassiveReplicationTestHook interface { // Test hook keys with their return type and scope. // Try to avoid global scope as it requires a dedicated test cluster. var ( - MatchingDisableSyncMatch = newKey[bool, namespace.ID]() - MatchingLBForceReadPartition = newKey[int, namespace.ID]() - MatchingLBForceWritePartition = newKey[int, namespace.ID]() - UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() - UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() - TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() - MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() - MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() - MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() - HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() - HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() - HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() - HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() + MatchingDisableSyncMatch = newKey[bool, namespace.ID]() + MatchingLBForceReadPartition = newKey[int, namespace.ID]() + MatchingLBForceWritePartition = newKey[int, namespace.ID]() + UpdateWithStartInBetweenLockAndStart = newKey[func(), namespace.ID]() + UpdateWithStartOnClosingWorkflowRetry = newKey[func(), namespace.ID]() + TaskQueuesInDeploymentSyncBatchSize = newKey[int, global]() + MatchingIgnoreRoutingConfigRevisionCheck = newKey[bool, namespace.ID]() + MatchingDeploymentRegisterErrorBackoff = newKey[time.Duration, namespace.ID]() + MatchingForwardTaskDelay = newKey[time.Duration, namespace.ID]() + HistoryReplicationTaskInterceptor = newKey[func(*replicationspb.ReplicationTask, func() error) error, global]() + HistoryReplicationTaskConversionInterceptor = newKey[func(historytasks.Task, func() (*replicationspb.ReplicationTask, error)) (*replicationspb.ReplicationTask, error), namespace.ID]() + HistoryReplicationDLQWriteInterceptor = newKey[func(*persistencespb.ReplicationTaskInfo, func() error) error, global]() + HistoryChasmRuntimeProvider = newKey[func(chasm.Engine, chasm.VisibilityManager, *chasm.Registry), global]() + HistoryTransferTaskInterceptor = newKey[func(historytasks.Task, func()), namespace.ID]() // HistoryPassiveReplicationTest enables the single-cluster passive replication // test stack for one namespace. Production builds always report it as unset. HistoryPassiveReplicationTest = newKey[HistoryPassiveReplicationTestHook, namespace.ID]() diff --git a/service/history/configs/config.go b/service/history/configs/config.go index ccf0695ecd1..1c9d0e94cd8 100644 --- a/service/history/configs/config.go +++ b/service/history/configs/config.go @@ -291,6 +291,7 @@ type Config struct { ReplicationTaskApplyTimeout dynamicconfig.DurationPropertyFn EnableAsyncParentWorkflowResend dynamicconfig.BoolPropertyFn EnableChildWorkflowResend dynamicconfig.BoolPropertyFn + EnableChildWorkflowCompletionRecovery dynamicconfig.BoolPropertyFnWithNamespaceFilter WorkflowResendHostMaxInFlight dynamicconfig.IntPropertyFn ReplicationTaskFetcherParallelism dynamicconfig.IntPropertyFn ReplicationTaskFetcherAggregationInterval dynamicconfig.DurationPropertyFn @@ -738,6 +739,7 @@ func NewConfig( ReplicationTaskApplyTimeout: dynamicconfig.ReplicationTaskApplyTimeout.Get(dc), EnableAsyncParentWorkflowResend: dynamicconfig.EnableAsyncParentWorkflowResend.Get(dc), EnableChildWorkflowResend: dynamicconfig.EnableChildWorkflowResend.Get(dc), + EnableChildWorkflowCompletionRecovery: dynamicconfig.EnableChildWorkflowCompletionRecovery.Get(dc), WorkflowResendHostMaxInFlight: dynamicconfig.WorkflowResendHostMaxInFlight.Get(dc), ReplicationTaskFetcherParallelism: dynamicconfig.ReplicationTaskFetcherParallelism.Get(dc), ReplicationTaskFetcherAggregationInterval: dynamicconfig.ReplicationTaskFetcherAggregationInterval.Get(dc), diff --git a/service/history/history_engine.go b/service/history/history_engine.go index 019638b02b9..f1d4774efa7 100644 --- a/service/history/history_engine.go +++ b/service/history/history_engine.go @@ -1004,6 +1004,15 @@ func (e *historyEngineImpl) ConvertReplicationTask( task tasks.Task, clusterID int32, ) (*replicationspb.ReplicationTask, error) { + if hook, ok := testhooks.Get( + e.testHooks, + testhooks.HistoryReplicationTaskConversionInterceptor, + namespace.ID(task.GetNamespaceID()), + ); ok { + return hook(task, func() (*replicationspb.ReplicationTask, error) { + return e.replicationAckMgr.ConvertTaskByCluster(ctx, task, clusterID) + }) + } return e.replicationAckMgr.ConvertTaskByCluster(ctx, task, clusterID) } diff --git a/service/history/transfer_queue_active_task_executor.go b/service/history/transfer_queue_active_task_executor.go index a3d64aef11d..47f102c0ede 100644 --- a/service/history/transfer_queue_active_task_executor.go +++ b/service/history/transfer_queue_active_task_executor.go @@ -2,6 +2,7 @@ package history import ( "context" + "errors" "fmt" "github.com/google/uuid" @@ -14,6 +15,7 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/adminservice/v1" clockspb "go.temporal.io/server/api/clock/v1" "go.temporal.io/server/api/historyservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" @@ -913,7 +915,25 @@ func (t *transferQueueActiveTaskExecutor) processStartChildExecution( return serviceerror.NewNamespaceNotFound(childInfo.Namespace) } - return t.createFirstWorkflowTask(ctx, targetNamespaceID.String(), childExecution, parentClock, childClock) + // Recovery depends on this separate scheduling request returning ErrWorkflowCompleted. If + // child start and first Workflow Task scheduling are combined into one transaction, this + // path cannot detect a completion that happened before parent replication arrived. + scheduleErr := t.createFirstWorkflowTask(ctx, targetNamespaceID.String(), childExecution, parentClock, childClock) + if scheduleErr == nil || !isWorkflowCompletedError(scheduleErr) { + return scheduleErr + } + + // Child completion removes ChildExecutionInfo, so reaching this point means the parent still + // needs a completion event for this child execution chain. + return t.recoverClosedChildCompletion( + ctx, + task, + targetNamespaceName, + targetNamespaceID.String(), + childExecution.GetWorkflowId(), + childExecution.GetRunId(), + scheduleErr, + ) } // remaining 2 cases: @@ -1510,6 +1530,97 @@ func (t *transferQueueActiveTaskExecutor) createFirstWorkflowTask( return err } +func isWorkflowCompletedError(err error) bool { + var notFoundErr *serviceerror.NotFound + // ScheduleWorkflowTask is a gRPC call, so errors.Is cannot identify the local sentinel after + // it has been serialized. Match both the service error type and the sentinel message instead. + return errors.As(err, ¬FoundErr) && notFoundErr.Error() == consts.ErrWorkflowCompleted.Error() +} + +func (t *transferQueueActiveTaskExecutor) recoverClosedChildCompletion( + ctx context.Context, + parentTask *tasks.StartChildExecutionTask, + childNamespaceName namespace.Name, + childNamespaceID string, + childWorkflowID string, + childFirstRunID string, + scheduleErr error, +) error { + if !t.config.EnableChildWorkflowCompletionRecovery(childNamespaceName.String()) { + return scheduleErr + } + + // Parent replication can regenerate this task after the child's original close task was + // acknowledged while the parent was missing. + mutableStateResponse, mutableStateErr := t.historyRawClient.GetMutableState(ctx, &historyservice.GetMutableStateRequest{ + NamespaceId: childNamespaceID, + Execution: &commonpb.WorkflowExecution{ + WorkflowId: childWorkflowID, + }, + }) + if mutableStateErr != nil { + if common.IsNotFoundError(mutableStateErr) { + // TODO: Revisit this recovery gap. If the current run is deleted before parent + // replication arrives, there is no execution to refresh and completion cannot be recovered. + return nil + } + return mutableStateErr + } + + if mutableStateResponse.GetFirstExecutionRunId() != childFirstRunID { + metrics.ChildWorkflowCompletionRecoveryChainMismatch.With( + t.metricHandler.WithTags(metrics.NamespaceTag(childNamespaceName.String())), + ).Record(1) + t.logger.Warn( + "Unable to recover child completion because the workflow ID points to a different execution chain", + tag.NewStringTag("parent-namespace-id", parentTask.GetNamespaceID()), + tag.NewStringTag("parent-workflow-id", parentTask.GetWorkflowID()), + tag.NewStringTag("parent-run-id", parentTask.GetRunID()), + tag.NewStringTag("child-namespace-id", childNamespaceID), + tag.NewStringTag("child-workflow-id", childWorkflowID), + tag.NewStringTag("expected-child-first-run-id", childFirstRunID), + tag.NewStringTag("current-child-first-run-id", mutableStateResponse.GetFirstExecutionRunId()), + tag.NewStringTag("current-child-run-id", mutableStateResponse.GetExecution().GetRunId()), + ) + return nil + } + // A running successor will generate its own CloseExecution task after the parent is present. + // Only a terminal current run needs recovery for the completion notification that was lost. + switch mutableStateResponse.GetWorkflowStatus() { + case enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + enumspb.WORKFLOW_EXECUTION_STATUS_FAILED, + enumspb.WORKFLOW_EXECUTION_STATUS_CANCELED, + enumspb.WORKFLOW_EXECUTION_STATUS_TERMINATED, + enumspb.WORKFLOW_EXECUTION_STATUS_TIMED_OUT: + default: + return nil + } + + currentExecution := mutableStateResponse.GetExecution() + metrics.ChildWorkflowCompletionRecoveryAttempts.With( + t.metricHandler.WithTags(metrics.NamespaceTag(childNamespaceName.String())), + ).Record(1) + t.logger.Info( + "Refreshing terminal child workflow to recover completion notification", + tag.NewStringTag("parent-namespace-id", parentTask.GetNamespaceID()), + tag.NewStringTag("parent-workflow-id", parentTask.GetWorkflowID()), + tag.NewStringTag("parent-run-id", parentTask.GetRunID()), + tag.NewStringTag("child-namespace-id", childNamespaceID), + tag.NewStringTag("child-workflow-id", childWorkflowID), + tag.NewStringTag("child-first-run-id", childFirstRunID), + tag.NewStringTag("child-terminal-run-id", currentExecution.GetRunId()), + ) + + _, err := t.historyRawClient.RefreshWorkflowTasks(ctx, &historyservice.RefreshWorkflowTasksRequest{ + NamespaceId: childNamespaceID, + ArchetypeId: chasm.WorkflowArchetypeID, + Request: &adminservice.RefreshWorkflowTasksRequest{ + Execution: currentExecution, + }, + }) + return err +} + func (t *transferQueueActiveTaskExecutor) requestCancelExternalExecutionCompleted( ctx context.Context, task *tasks.CancelExecutionTask, diff --git a/service/history/transfer_queue_active_task_executor_test.go b/service/history/transfer_queue_active_task_executor_test.go index cf2775da2b4..258818e5ddc 100644 --- a/service/history/transfer_queue_active_task_executor_test.go +++ b/service/history/transfer_queue_active_task_executor_test.go @@ -19,6 +19,7 @@ import ( taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/adminservice/v1" historyspb "go.temporal.io/server/api/history/v1" "go.temporal.io/server/api/historyservice/v1" "go.temporal.io/server/api/historyservicemock/v1" @@ -36,6 +37,7 @@ import ( "go.temporal.io/server/common/definition" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" @@ -3102,6 +3104,176 @@ func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Su s.NoError(resp.ExecutionErr) } +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_RefreshesTerminalCurrentRun() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + terminalRunID := uuid.NewString() + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), protomock.Eq(&historyservice.GetMutableStateRequest{ + NamespaceId: s.childNamespaceID.String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId}, + })).Return(&historyservice.GetMutableStateResponse{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: terminalRunID}, + WorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + FirstExecutionRunId: childExecution.RunId, + }, nil) + s.mockHistoryClient.EXPECT().RefreshWorkflowTasks(gomock.Any(), protomock.Eq(&historyservice.RefreshWorkflowTasksRequest{ + NamespaceId: s.childNamespaceID.String(), + ArchetypeId: chasm.WorkflowArchetypeID, + Request: &adminservice.RefreshWorkflowTasksRequest{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: terminalRunID}, + }, + })).Return(&historyservice.RefreshWorkflowTasksResponse{}, nil) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().NoError(err) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestIsWorkflowCompletedError() { + s.True(isWorkflowCompletedError(consts.ErrWorkflowCompleted)) + s.True(isWorkflowCompletedError(serviceerror.NewNotFound(consts.ErrWorkflowCompleted.Error()))) + s.False(isWorkflowCompletedError(serviceerror.NewNotFound("workflow execution not found"))) + s.False(isWorkflowCompletedError(serviceerror.NewUnavailable("history unavailable"))) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_RefreshFailureIsRetried() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + terminalRunID := uuid.NewString() + refreshErr := serviceerror.NewUnavailable("refresh failed") + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(&historyservice.GetMutableStateResponse{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: terminalRunID}, + WorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + FirstExecutionRunId: childExecution.RunId, + }, nil) + s.mockHistoryClient.EXPECT().RefreshWorkflowTasks(gomock.Any(), gomock.Any()). + Return(nil, refreshErr) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().ErrorIs(err, refreshErr) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_RunningSuccessorIsNotRefreshed() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(&historyservice.GetMutableStateResponse{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: uuid.NewString()}, + WorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING, + FirstExecutionRunId: childExecution.RunId, + }, nil) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().NoError(err) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_PausedSuccessorIsNotRefreshed() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(&historyservice.GetMutableStateResponse{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: uuid.NewString()}, + WorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_PAUSED, + FirstExecutionRunId: childExecution.RunId, + }, nil) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().NoError(err) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_WorkflowIDReused() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + metricHandler := metricstest.NewCaptureHandler() + metricCapture := metricHandler.StartCapture() + defer metricHandler.StopCapture(metricCapture) + s.transferQueueActiveTaskExecutor.metricHandler = metricHandler + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(&historyservice.GetMutableStateResponse{ + Execution: &commonpb.WorkflowExecution{WorkflowId: childExecution.WorkflowId, RunId: uuid.NewString()}, + WorkflowStatus: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + FirstExecutionRunId: uuid.NewString(), + }, nil) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().NoError(err) + s.Require().Len(metricCapture.Snapshot()[metrics.ChildWorkflowCompletionRecoveryChainMismatch.Name()], 1) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_CurrentRunNotFound() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(nil, serviceerror.NewNotFound("current run deleted")) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().NoError(err) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_GetMutableStateFailureIsRetried() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + mutableStateErr := serviceerror.NewUnavailable("mutable state unavailable") + + s.mockHistoryClient.EXPECT().GetMutableState(gomock.Any(), gomock.Any()). + Return(nil, mutableStateErr) + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().ErrorIs(err, mutableStateErr) +} + +func (s *transferQueueActiveTaskExecutorSuite) TestRecoverClosedChildCompletion_Disabled() { + childExecution := &commonpb.WorkflowExecution{ + WorkflowId: "child-workflow-id", + RunId: uuid.NewString(), + } + s.transferQueueActiveTaskExecutor.config.EnableChildWorkflowCompletionRecovery = func(string) bool { return false } + + err := s.recoverClosedChildCompletion(childExecution, consts.ErrWorkflowCompleted) + s.Require().ErrorIs(err, consts.ErrWorkflowCompleted) +} + +func (s *transferQueueActiveTaskExecutorSuite) recoverClosedChildCompletion( + childExecution *commonpb.WorkflowExecution, + scheduleErr error, +) error { + return s.transferQueueActiveTaskExecutor.recoverClosedChildCompletion( + context.Background(), + &tasks.StartChildExecutionTask{ + WorkflowKey: definition.NewWorkflowKey( + s.namespaceID.String(), + "parent-workflow-id", + uuid.NewString(), + ), + }, + s.childNamespace, + s.childNamespaceID.String(), + childExecution.GetWorkflowId(), + childExecution.GetRunId(), + scheduleErr, + ) +} + func (s *transferQueueActiveTaskExecutorSuite) TestProcessStartChildExecution_Duplication() { execution := &commonpb.WorkflowExecution{ WorkflowId: "some random workflow ID", diff --git a/tests/xdc/base.go b/tests/xdc/base.go index ee5d891e378..ea4b267e5ef 100644 --- a/tests/xdc/base.go +++ b/tests/xdc/base.go @@ -92,7 +92,9 @@ func (s *xdcBaseSuite) setupSuite(opts ...testcore.TestClusterOption) { if s.dynamicConfigOverrides == nil { s.dynamicConfigOverrides = make(map[dynamicconfig.Key]any) } - s.dynamicConfigOverrides[dynamicconfig.ClusterMetadataRefreshInterval.Key()] = time.Second * 5 + if _, ok := s.dynamicConfigOverrides[dynamicconfig.ClusterMetadataRefreshInterval.Key()]; !ok { + s.dynamicConfigOverrides[dynamicconfig.ClusterMetadataRefreshInterval.Key()] = time.Second * 5 + } s.dynamicConfigOverrides[dynamicconfig.NamespaceCacheRefreshInterval.Key()] = testcore.NamespaceCacheRefreshInterval s.dynamicConfigOverrides[dynamicconfig.EnableTransitionHistory.Key()] = s.enableTransitionHistory // TODO (prathyush): remove this after setting it to true by default. diff --git a/tests/xdc/parent_child_harness.go b/tests/xdc/parent_child_harness.go index bc1a930036b..1bdcaeb69a8 100644 --- a/tests/xdc/parent_child_harness.go +++ b/tests/xdc/parent_child_harness.go @@ -26,11 +26,13 @@ import ( "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" + "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/persistence/serialization" "go.temporal.io/server/common/testing/await" "go.temporal.io/server/common/testing/taskpoller" "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/testing/testvars" + historytasks "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/tests/testcore" "google.golang.org/protobuf/types/known/durationpb" ) @@ -84,6 +86,7 @@ type ( delayedTasks map[parentChildReplicationLane]*parentChildReplicationTask metricCaptures [2]parentChildMetricCapture trace []string + resumeParentConversion func() } parentChildMetricCapture struct { @@ -226,6 +229,9 @@ func (r *parentChildScenarioRuntime) initialize(ctx context.Context) error { } func (r *parentChildScenarioRuntime) close() { + if r.resumeParentConversion != nil { + r.resumeParentConversion() + } for _, gate := range r.gates { if gate != nil { gate.close() @@ -317,6 +323,42 @@ func setStandbyClusterDelay( } } +func delayParentReplicationTaskConversion(cluster parentChildCluster) parentChildScenarioStep { + return parentChildScenarioStep{ + name: fmt.Sprintf("delay parent replication task conversion on %s", cluster), + run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { + clusterIndex := int(cluster) + if clusterIndex < 0 || clusterIndex >= len(runtime.suite.clusters) { + return fmt.Errorf("unknown parent-child cluster %d", cluster) + } + if runtime.resumeParentConversion != nil { + return errors.New("parent replication task conversion is already delayed") + } + + release := make(chan struct{}) + var releaseOnce sync.Once + runtime.resumeParentConversion = func() { + releaseOnce.Do(func() { close(release) }) + } + runtime.removeHooks = append(runtime.removeHooks, runtime.suite.clusters[clusterIndex].InjectHook( + runtime.suite.T(), + testhooks.NewHook( + testhooks.HistoryReplicationTaskConversionInterceptor, + func(task historytasks.Task, convert func() (*replicationspb.ReplicationTask, error)) (*replicationspb.ReplicationTask, error) { + if task.GetWorkflowID() != runtime.parentID { + return convert() + } + <-release + return convert() + }, + ), + namespace.ID(runtime.namespaceID), + )) + return nil + }, + } +} + func setStandbyTaskDiscardDelay( cluster parentChildCluster, taskType enumsspb.TaskType, @@ -418,7 +460,7 @@ func applyDelayedReplication( return parentChildScenarioStep{ name: fmt.Sprintf("apply delayed %s replication to %s", workflow, targetCluster), run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { - return runtime.applyDelayedReplication(targetCluster, workflow) + return runtime.resolveDelayedReplication(targetCluster, workflow, true) }, } } @@ -430,7 +472,7 @@ func acknowledgeDelayedReplicationWithoutApplying( return parentChildScenarioStep{ name: fmt.Sprintf("acknowledge delayed %s replication to %s without applying", workflow, targetCluster), run: func(_ context.Context, runtime *parentChildScenarioRuntime) error { - return runtime.acknowledgeDelayedReplicationWithoutApplying(targetCluster, workflow) + return runtime.resolveDelayedReplication(targetCluster, workflow, false) }, } } @@ -482,7 +524,8 @@ func completeChildWorkflowTask() parentChildScenarioStep { return parentChildScenarioStep{ name: "complete the child workflow task", run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { - return runtime.completeChildWorkflowTask(ctx) + _, err := runtime.closeChildWorkflowTask(ctx, enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION) + return err }, } } @@ -762,9 +805,10 @@ func (r *parentChildScenarioRuntime) acknowledgeNextReplicationTaskWithoutApplyi return task.acknowledgeWithoutApplying() } -func (r *parentChildScenarioRuntime) applyDelayedReplication( +func (r *parentChildScenarioRuntime) resolveDelayedReplication( targetCluster parentChildCluster, workflow parentChildWorkflow, + shouldApply bool, ) error { targetClusterIndex := int(targetCluster) if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { @@ -784,43 +828,21 @@ func (r *parentChildScenarioRuntime) applyDelayedReplication( return err } - r.tracef( - " apply delayed task %d to cluster %d for %s [%s]", - task.task.GetSourceTaskId(), - targetClusterIndex, - workflow, - formatParentChildReplicationTask(task.task, events), - ) delete(r.delayedTasks, lane) - if err := task.apply(); err != nil { - return err - } - r.recordChildRunIDFromAppliedTask(workflow, task, events) - return nil -} - -func (r *parentChildScenarioRuntime) acknowledgeDelayedReplicationWithoutApplying( - targetCluster parentChildCluster, - workflow parentChildWorkflow, -) error { - targetClusterIndex := int(targetCluster) - if targetClusterIndex < 0 || targetClusterIndex >= len(r.gates) { - return fmt.Errorf("unknown parent-child cluster %d", targetCluster) - } - if _, err := r.workflowID(workflow); err != nil { - return err - } - - lane := parentChildReplicationLane{targetClusterIndex: targetClusterIndex, workflow: workflow} - task, delayed := r.delayedTasks[lane] - if !delayed { - return fmt.Errorf("no delayed %s replication task to %s", workflow, targetCluster) - } - events, err := decodeParentChildReplicationEvents(task.task) - if err != nil { - return err + if shouldApply { + r.tracef( + " apply delayed task %d to cluster %d for %s [%s]", + task.task.GetSourceTaskId(), + targetClusterIndex, + workflow, + formatParentChildReplicationTask(task.task, events), + ) + if err := task.apply(); err != nil { + return err + } + r.recordChildRunIDFromAppliedTask(workflow, task, events) + return nil } - r.tracef( " acknowledge delayed task %d to cluster %d for %s without applying [%s]", task.task.GetSourceTaskId(), @@ -828,7 +850,6 @@ func (r *parentChildScenarioRuntime) acknowledgeDelayedReplicationWithoutApplyin workflow, formatParentChildReplicationTask(task.task, events), ) - delete(r.delayedTasks, lane) return task.acknowledgeWithoutApplying() } @@ -868,7 +889,36 @@ func (r *parentChildScenarioRuntime) completeParentWorkflowTaskWithStartChildCom return err } -func (r *parentChildScenarioRuntime) completeChildWorkflowTask(ctx context.Context) error { +func (r *parentChildScenarioRuntime) closeChildWorkflowTask( + ctx context.Context, + commandType enumspb.CommandType, +) (string, error) { + var command *commandpb.Command + switch commandType { + case enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION: + command = &commandpb.Command{ + CommandType: commandType, + Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ + CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, + }, + } + case enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION: + command = &commandpb.Command{ + CommandType: commandType, + Attributes: &commandpb.Command_ContinueAsNewWorkflowExecutionCommandAttributes{ + ContinueAsNewWorkflowExecutionCommandAttributes: &commandpb.ContinueAsNewWorkflowExecutionCommandAttributes{ + WorkflowType: &commonpb.WorkflowType{Name: "child-workflow"}, + TaskQueue: r.childTestVars.TaskQueue(), + WorkflowRunTimeout: durationpb.New(time.Minute), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + }, + }, + } + default: + return "", fmt.Errorf("unsupported child close command %s", commandType) + } + + closedRunID := r.childRunID poller := taskpoller.New(r.suite.T(), r.activeCluster().FrontendClient(), r.namespace) _, err := poller.PollAndHandleWorkflowTask(r.childTestVars, func( task *workflowservice.PollWorkflowTaskQueueResponse, @@ -879,20 +929,67 @@ func (r *parentChildScenarioRuntime) completeChildWorkflowTask(ctx context.Conte } if r.childRunID == "" { r.childRunID = execution.GetRunId() + closedRunID = r.childRunID } if execution.GetRunId() != r.childRunID { return nil, fmt.Errorf("polled child run %s, want %s", execution.GetRunId(), r.childRunID) } return &workflowservice.RespondWorkflowTaskCompletedRequest{ - Commands: []*commandpb.Command{{ - CommandType: enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION, - Attributes: &commandpb.Command_CompleteWorkflowExecutionCommandAttributes{ - CompleteWorkflowExecutionCommandAttributes: &commandpb.CompleteWorkflowExecutionCommandAttributes{}, - }, - }}, + Commands: []*commandpb.Command{command}, }, nil }, taskpoller.WithTimeout(testTimeout)) - return err + if err != nil || commandType != enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION { + return closedRunID, err + } + + events, err := r.activeWorkflowHistory(ctx, childWorkflow) + if err != nil { + return "", err + } + continuedAsNewEvent := findHistoryEvent(events, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_CONTINUED_AS_NEW, nil) + if continuedAsNewEvent == nil { + return "", fmt.Errorf("child run %s has no WorkflowExecutionContinuedAsNew event", closedRunID) + } + newRunID := continuedAsNewEvent.GetWorkflowExecutionContinuedAsNewEventAttributes().GetNewExecutionRunId() + if newRunID == "" { + return "", fmt.Errorf("child run %s continued as new without a new run ID", closedRunID) + } + r.childRunID = newRunID + r.tracef(" child continued as new from run %s to %s", closedRunID, newRunID) + return closedRunID, nil +} + +func (r *parentChildScenarioRuntime) resetChildWorkflow(ctx context.Context) error { + events, err := r.activeWorkflowHistory(ctx, childWorkflow) + if err != nil { + return err + } + workflowTaskCompletedEvent := findHistoryEvent(events, enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED, nil) + if workflowTaskCompletedEvent == nil { + return fmt.Errorf("child run %s has no WorkflowTaskCompleted event", r.childRunID) + } + + baseRunID := r.childRunID + response, err := r.activeCluster().FrontendClient().ResetWorkflowExecution(ctx, &workflowservice.ResetWorkflowExecutionRequest{ + Namespace: r.namespace, + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: r.childID, + RunId: baseRunID, + }, + Reason: "parent-child XDC test", + WorkflowTaskFinishEventId: workflowTaskCompletedEvent.GetEventId(), + RequestId: uuid.NewString(), + Identity: r.childTestVars.WorkerIdentity(), + }) + if err != nil { + return err + } + if response.GetRunId() == "" { + return fmt.Errorf("reset child run %s without a new run ID", baseRunID) + } + r.childRunID = response.GetRunId() + r.tracef(" reset child from run %s to %s", baseRunID, r.childRunID) + return nil } func (r *parentChildScenarioRuntime) signalChildWorkflow(ctx context.Context) error { diff --git a/tests/xdc/parent_child_test.go b/tests/xdc/parent_child_test.go index 29cb034ecc7..5cab27bda0c 100644 --- a/tests/xdc/parent_child_test.go +++ b/tests/xdc/parent_child_test.go @@ -16,9 +16,20 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/testing/testhooks" + historytasks "go.temporal.io/server/service/history/tasks" "go.temporal.io/server/tests/testcore" ) +type childRunAction int + +const ( + completeChildRun childRunAction = iota + continueAsNewChildRun + resetChildRun +) + func TestParentChildXDCTestSuite(t *testing.T) { t.Parallel() suite.Run(t, new(parentChildXDCTestSuite)) @@ -87,6 +98,240 @@ func (s *parentChildXDCTestSuite) TestRecoversOrphanedChildAfterForceFailover() }) } +// TestActiveRecoversChildCompletionWhenParentReplicationArrivesLate covers the cross-shard ordering +// where parent replication remains delayed until after the child closes on the new active cluster. +// +// | parent on target | child on target | active | outcome +// --------------------+---------------------------------+-------------------+-----------------+----------------------------------------------- +// conversion paused | does not exist | does not exist | initial active | source cannot emit the parent snapshot +// child start arrives | does not exist | RUNNING | initial active | child points to the missing parent +// force failover | does not exist | RUNNING | initial standby | target becomes active with partial state +// child closes | does not exist | COMPLETED | initial standby | CloseExecution cannot notify the missing parent +// parent tasks arrive | child relationship restored | COMPLETED | initial standby | regenerated StartChild finds the child closed +// refresh child tasks | ChildWorkflowExecutionCompleted | COMPLETED | initial standby | recreated CloseExecution records completion +// +// Parent task conversion is paused before the first task is emitted. After the child closes, the +// first raw task is converted against the latest source state, producing a complete initial snapshot +// that includes ChildWorkflowExecutionStarted. No task is acknowledged or dropped. Event checkpoints +// select an entire replication task, not an individual event. +func (s *parentChildXDCTestSuite) TestActiveRecoversChildCompletionWhenParentReplicationArrivesLate() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + setStandbyClusterDelay(initialStandbyCluster, 0), + delayParentReplicationTaskConversion(initialActiveCluster), + startParentWorkflow(), + completeParentWorkflowTaskWithStartChildCommand(), + waitForWorkflowEventOnCluster( + initialActiveCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + confirmWorkflowIsMissingOnCluster(initialStandbyCluster, parentWorkflow), + forceFailoverNamespaceTo(initialStandbyCluster), + closeChildRunsBeforeApplyingParent(completeChildRun), + }, + expectations: []parentChildExpectation{ + parentRecordsFinalChildRunCompletion(), + }, + }) +} + +// TestActiveRecoversContinuedAsNewChildCompletionWhenParentReplicationArrivesLate covers the same +// delayed-parent ordering when the child continues as new before its terminal run closes. +// +// | parent on target | child on target | active | outcome +// -----------------------+----------------------------------+-----------------------------------------+-----------------+----------------------------------------------- +// conversion paused | does not exist | does not exist | initial active | source cannot emit the parent snapshot +// child start arrives | does not exist | run 1 RUNNING | initial active | child points to the missing parent +// force failover | does not exist | run 1 RUNNING | initial standby | target becomes active with partial state +// child continues as new | does not exist | run 1 CONTINUED_AS_NEW; run 2 RUNNING | initial standby | run 1 CloseExecution processes without replying +// terminal run closes | does not exist | run 1 CONTINUED_AS_NEW; run 2 COMPLETED | initial standby | run 2 cannot notify the missing parent +// parent tasks arrive | child relationship restored | run 2 COMPLETED | initial standby | StartChild resolves the current chain member +// refresh current run | ChildWorkflowExecutionCompleted | run 2 COMPLETED | initial standby | recreated run 2 CloseExecution records completion +// +// Both child CloseExecution tasks are processed before the parent snapshot is applied. Scheduling +// the first task on run 1 reports that it is closed; loading current mutable state finds terminal run 2 +// with the same first-run ID, so recovery refreshes run 2. Parent replication is delayed, not lost. +func (s *parentChildXDCTestSuite) TestActiveRecoversContinuedAsNewChildCompletionWhenParentReplicationArrivesLate() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + setStandbyClusterDelay(initialStandbyCluster, 0), + delayParentReplicationTaskConversion(initialActiveCluster), + startParentWorkflow(), + completeParentWorkflowTaskWithStartChildCommand(), + waitForWorkflowEventOnCluster( + initialActiveCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + confirmWorkflowIsMissingOnCluster(initialStandbyCluster, parentWorkflow), + forceFailoverNamespaceTo(initialStandbyCluster), + closeChildRunsBeforeApplyingParent( + continueAsNewChildRun, + completeChildRun, + ), + }, + expectations: []parentChildExpectation{ + parentRecordsFinalChildRunCompletion(), + }, + }) +} + +// TestActiveRecoversResetChildCompletionWhenParentReplicationArrivesLate covers the same +// delayed-parent ordering when a completed child is reset and its reset run also closes. +// +// | parent on target | child on target | active | outcome +// --------------------+----------------------------------+-----------------------------------------+-----------------+----------------------------------------------- +// conversion paused | does not exist | does not exist | initial active | source cannot emit the parent snapshot +// child start arrives | does not exist | original run RUNNING | initial active | child points to the missing parent +// force failover | does not exist | original run RUNNING | initial standby | target becomes active with partial state +// original run closes | does not exist | original run COMPLETED | initial standby | completion cannot reach the missing parent +// child is reset | does not exist | original COMPLETED; reset run RUNNING | initial standby | reset run preserves the original first-run ID +// reset run closes | does not exist | original COMPLETED; reset run COMPLETED | initial standby | reset completion cannot reach the missing parent +// parent tasks arrive | child relationship restored | reset run COMPLETED | initial standby | StartChild resolves the current reset run +// refresh reset run | ChildWorkflowExecutionCompleted | reset run COMPLETED | initial standby | recreated reset-run CloseExecution records completion +// +// The original and reset-run CloseExecution tasks are processed before the parent snapshot is +// applied. Because the reset run inherits the original first-run ID, recovery recognizes it as the +// same child chain and refreshes the terminal reset run. Parent replication is delayed, not lost. +func (s *parentChildXDCTestSuite) TestActiveRecoversResetChildCompletionWhenParentReplicationArrivesLate() { + s.runParentChildScenario(parentChildScenario{ + steps: []parentChildScenarioStep{ + setStandbyClusterDelay(initialStandbyCluster, 0), + delayParentReplicationTaskConversion(initialActiveCluster), + startParentWorkflow(), + completeParentWorkflowTaskWithStartChildCommand(), + waitForWorkflowEventOnCluster( + initialActiveCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + ), + applyReplicationThroughTaskContainingEvent( + initialStandbyCluster, + childWorkflow, + enumspb.EVENT_TYPE_WORKFLOW_TASK_SCHEDULED, + ), + confirmWorkflowIsMissingOnCluster(initialStandbyCluster, parentWorkflow), + forceFailoverNamespaceTo(initialStandbyCluster), + closeChildRunsBeforeApplyingParent( + completeChildRun, + resetChildRun, + completeChildRun, + ), + }, + expectations: []parentChildExpectation{ + parentRecordsFinalChildRunCompletion(), + }, + }) +} + +func parentRecordsFinalChildRunCompletion() parentChildExpectation { + return parentChildExpectation{ + name: "parent records completion of the final child run", + check: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + events, err := runtime.workflowHistoryOnCluster(ctx, initialStandbyCluster, parentWorkflow) + if err != nil { + return err + } + if findHistoryEvent( + events, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED, + func(event *historypb.HistoryEvent) bool { + execution := event.GetChildWorkflowExecutionCompletedEventAttributes().GetWorkflowExecution() + return execution.GetWorkflowId() == runtime.childID && execution.GetRunId() == runtime.childRunID + }, + ) == nil { + return fmt.Errorf("parent has no completion for final child run %s/%s", runtime.childID, runtime.childRunID) + } + return nil + }, + } +} + +func closeChildRunsBeforeApplyingParent(actions ...childRunAction) parentChildScenarioStep { + return parentChildScenarioStep{ + name: "close every child run before applying parent replication", + run: func(ctx context.Context, runtime *parentChildScenarioRuntime) error { + if len(actions) == 0 { + return errors.New("no child close commands") + } + processedCloseTasks := make(chan string, len(actions)) + cleanup := runtime.suite.clusters[initialStandbyCluster].InjectHook( + runtime.suite.T(), + testhooks.NewHook(testhooks.HistoryTransferTaskInterceptor, func(task historytasks.Task, execute func()) { + execute() + if task.GetWorkflowID() == runtime.childID && task.GetType() == enumsspb.TASK_TYPE_TRANSFER_CLOSE_EXECUTION { + select { + case processedCloseTasks <- task.GetRunID(): + default: + } + } + }), + namespace.ID(runtime.namespaceID), + ) + defer cleanup() + + closeRun := func(commandType enumspb.CommandType) error { + closedRunID, err := runtime.closeChildWorkflowTask(ctx, commandType) + if err != nil { + return err + } + for { + select { + case processedRunID := <-processedCloseTasks: + if processedRunID == closedRunID { + runtime.tracef(" processed child CloseExecution for run %s", closedRunID) + return nil + } + case <-ctx.Done(): + return ctx.Err() + } + } + } + + for _, action := range actions { + switch action { + case completeChildRun: + if err := closeRun(enumspb.COMMAND_TYPE_COMPLETE_WORKFLOW_EXECUTION); err != nil { + return err + } + case continueAsNewChildRun: + if err := closeRun(enumspb.COMMAND_TYPE_CONTINUE_AS_NEW_WORKFLOW_EXECUTION); err != nil { + return err + } + case resetChildRun: + if err := runtime.resetChildWorkflow(ctx); err != nil { + return err + } + default: + return fmt.Errorf("unknown child run action %d", action) + } + } + if runtime.resumeParentConversion == nil { + return errors.New("parent replication task conversion is not delayed") + } + runtime.resumeParentConversion() + return runtime.processReplicationThroughTaskContainingEvent( + ctx, + initialStandbyCluster, + parentWorkflow, + enumspb.EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED, + applyReplicationTask, + ) + }, + } +} + func enableOrphanedChildWorkflowReplacement() parentChildScenarioStep { return parentChildScenarioStep{ name: "enable orphaned child workflow replacement", From cd667daadb88d189df0302d8f473858ee7168ce5 Mon Sep 17 00:00:00 2001 From: David Porter Date: Wed, 2 Sep 2026 19:46:50 -0600 Subject: [PATCH 124/139] Sch v1 wf (#11821) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary We held back several fixes in the V1 schedule workflow and bundled them into a single atomic commit with the intention of avoiding having to perform multiple increments of the workflow versioning. The changes are broadly: Migration v1->v2 fixes + CGS version changes. ## Risk This touches V1 schedules workflow, and a mistake risks nondeterminitism. This is relatively high impact and has quite a bit of subtlety. This is therefore a commit we need to merge with care ## Testing and validation 1. Changes have been manually tested on individual PRs already 2. Changes have been manually run through CGS's validation suite (cc @liam-lowe) Our intention is to a) make the 163 release cut and b) continue to manually testing a few operational scenarios manually while this is being merged in. If we see problems we can hotfix this. Manually doing some scenario testing will take time so we intend to do this in parallel. Some of the scenarios we will test manually are: 1. Deploy and rollback to an earlier version (probably with LLM assist, but running locally) 2. Manually lifting the the version of a schedule via dynamic config (manually) ensuring there's no nondeterminitism risk 3. V2->v1 Rollback works as expected Some of the tests we've already run and manually validated(cc @liam-lowe) 1. Version floor works as expected 2. That this change fixes v1->v2 migration 3. That this change avoids the problematic history entries which broke a customer using the coinbase Ruby SDK ## LLM Summary This PR stacks a series of changes to the V1 (legacy) scheduler workflow: two migration-correctness fixes, and dynamic-config levers to safely roll the V1 workflow version forward/backward across a multi-cluster deployment without requiring continue-as-new. Summary of each PR in the stack, in order: ### #11462 — V1→V2 migration-eligibility fix and migrated-start ID (David Porter) Combines two previously separate fixes under one shared version bump (`v13`), avoiding two separate version-bump deploys for the same version number: - `RefreshBeforeMigrationCheck`: fixes a bug that was preventing V1→V2 migration from ever succeeding under default configuration. - `PreserveMigratedStartIDs`: preserves the request IDs workflows were originally started with, in case of a rollback. - Adds a guard against late migrations caused by a transient error bouncing a migration attempt and the schedule re-attempting it after waking again. - Follows the existing two-phase-rollout pattern: this PR only teaches the scheduler to *understand* v13 (gated behind `hasMinVersion(13)`); `CurrentTweakablePolicies.Version` itself stays at v12 pending a follow-up activation deploy. ### #11588 — Fix schedule action delay after refresh (Alex Stanfield) - `processWatcherResult` only recorded `DesiredTime` on the long-poll path; when a refresh instead discovered the prior action had completed, `DesiredTime` stayed unset, inflating the reported `ScheduleActionDelay` for back-to-back buffered actions. - Backdates `DesiredTime` on the refresh path too, gated behind a new version, `RefreshCompletionDesiredTime` (v14). - Handles two follow-on correctness gaps surfaced in review: `ALLOW_ALL` starts (never blocked by a running workflow, so shouldn't be backdated to an unrelated close time) and multiple tracked executions in one refresh pass (must only move the recorded close time forward, never backward). - Extracts the decision into a pure, unit-testable `shouldBackdateDesiredTime` function, and a shared `IgnoresRunningWorkflow` helper so `ProcessBuffer` and the new backdate logic can't drift apart on what "waits for a running workflow" means. ### #11827 — Sort BufferedStarts by due time on CHASM-to-V1 rollback (Alex Stanfield) - `CHASMToLegacyStartScheduleArgs` (the CHASM→V1 rollback conversion) appended trigger-derived `BufferedStarts` after the regular pending ones unconditionally, without sorting by due time. - V1's buffer-processing code assumes `BufferedStarts[0]` is always the earliest-due pending start — an invariant not guaranteed across a rollback, since manual triggers are built by iterating a Go map (randomized order) and appended regardless of their own due time. - Fix: sort the combined list by `ActualTime` after appending, mirroring the sort already applied to `RecentActions` a few lines above. ### #11831 — Re-evaluate V1 version ceiling per iteration (Alex Stanfield) - The base PR (#10817, by liam-lowe) added `worker.schedulerV1VersionCeiling` but only applied it on the first `tweakables` evaluation, so raising/removing a ceiling left an in-flight workflow stuck at the capped version until continue-as-new. - Re-reads the ceiling on every evaluation: `next version = max(recorded version, min(binary default, current ceiling))`. The recorded version stays monotonic — a newly-lowered ceiling never downgrades a version already recorded in the current run, but a raised/removed ceiling lets the workflow advance at its next wakeup instead of waiting for continue-as-new. - Recorded via the existing `MutableSideEffect`, so replay consumes history rather than re-evaluating live dynamic config. ### #11856 — Add worker.schedulerV1VersionOverride (Alex Stanfield) - #11831 makes the ceiling dynamic, but a ceiling can only restrict a binary default, never promote past it. Adds the namespace-level override needed to activate a newer version already supported by the binary. - `requested version = valid override, otherwise binary default`; `next version = max(recorded version, min(requested version, current ceiling))`. Default `-1` retains the binary default; values below the binary default or above `LatestSchedulerWorkflowVersion` are ignored. - The frontend uses the same override for initial schedule memo/list-info construction, so version-dependent metadata agrees with the first worker task. Base of the stack: #10817 (liam-lowe) introduced the original (static-per-run) `worker.schedulerV1VersionCeiling` dynamic config that #11831/#11856 build on. ## Why? To support safe, gradual rollout/rollback of V1 scheduler workflow version bumps (migration fixes, action-delay-after-refresh fix) in a cross-version multi-cluster topology, without requiring continue-as-new to pick up config changes. ## How did you test it? - [x] built - [x] covered by existing tests - [x] added new unit test(s) --------- Co-authored-by: liam-lowe <56076876+liam-lowe@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 Co-authored-by: alex.stanfield <13949480+chaptersix@users.noreply.github.com> Co-authored-by: Stephan Behnke Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: michaely520 Co-authored-by: Feiyang Xie Co-authored-by: Kannan Co-authored-by: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Co-authored-by: Lakshay <54310363+Lakshaymiddha@users.noreply.github.com> Co-authored-by: samm Co-authored-by: Quinn Klassen Co-authored-by: Will Duan Co-authored-by: Qian Chen Co-authored-by: Prathyush PV Co-authored-by: Sean Kane Co-authored-by: mavemuri <74267563+mavemuri@users.noreply.github.com> Co-authored-by: Rodrigo Zhou Co-authored-by: Brian VanLoo Co-authored-by: akbala Co-authored-by: Dan Davison Co-authored-by: Chris Smith --- chasm/lib/scheduler/migration/migration.go | 45 +- .../lib/scheduler/migration/migration_test.go | 141 +++ common/dynamicconfig/constants.go | 13 + service/frontend/service.go | 6 + service/frontend/workflow_handler.go | 7 +- service/worker/scheduler/activities.go | 10 + service/worker/scheduler/activities_test.go | 68 +- service/worker/scheduler/buffer.go | 14 +- service/worker/scheduler/export_test.go | 6 +- service/worker/scheduler/fx.go | 18 +- service/worker/scheduler/replay_test.go | 144 ++- .../replay_migration_v1_to_v2.json.gz | Bin 0 -> 3902 bytes ....23-pre.json.gz => replay_v1.23.0.json.gz} | Bin .../scheduler/testdata/replay_v1.24.0.json.gz | Bin 0 -> 3713 bytes .../scheduler/testdata/replay_v1.25.0.json.gz | Bin 0 -> 3704 bytes .../scheduler/testdata/replay_v1.27.0.json.gz | Bin 0 -> 3669 bytes .../scheduler/testdata/replay_v1.29.0.json.gz | Bin 0 -> 3685 bytes .../replay_version_override_v14.json.gz | Bin 0 -> 4061 bytes .../worker/scheduler/version_ceiling_test.go | 261 +++++ service/worker/scheduler/workflow.go | 334 +++++- service/worker/scheduler/workflow_test.go | 984 +++++++++++++++++- tests/schedule_migration_test.go | 225 ++++ tests/schedule_test.go | 150 +++ 23 files changed, 2333 insertions(+), 93 deletions(-) create mode 100644 service/worker/scheduler/testdata/replay_migration_v1_to_v2.json.gz rename service/worker/scheduler/testdata/{replay_v1.23-pre.json.gz => replay_v1.23.0.json.gz} (100%) create mode 100644 service/worker/scheduler/testdata/replay_v1.24.0.json.gz create mode 100644 service/worker/scheduler/testdata/replay_v1.25.0.json.gz create mode 100644 service/worker/scheduler/testdata/replay_v1.27.0.json.gz create mode 100644 service/worker/scheduler/testdata/replay_v1.29.0.json.gz create mode 100644 service/worker/scheduler/testdata/replay_version_override_v14.json.gz create mode 100644 service/worker/scheduler/version_ceiling_test.go diff --git a/chasm/lib/scheduler/migration/migration.go b/chasm/lib/scheduler/migration/migration.go index 2affc16234f..161eb41cb34 100644 --- a/chasm/lib/scheduler/migration/migration.go +++ b/chasm/lib/scheduler/migration/migration.go @@ -169,19 +169,38 @@ func CHASMToLegacyStartScheduleArgs( invokerBuffered = invoker.GetBufferedStarts() } bufferedStarts, running, recent := splitBufferedStartsForLegacy(invokerBuffered) - if len(info.GetRecentActions()) > 0 { + recentFromInfo := len(info.GetRecentActions()) > 0 + if recentFromInfo { storedRecent := make([]*schedulepb.ScheduleActionResult, 0, len(info.GetRecentActions())) for _, action := range info.GetRecentActions() { storedRecent = append(storedRecent, common.CloneProto(action)) } recent = append(storedRecent, recent...) + } + ongoingBackfills, triggerStarts := convertBackfillersCHASMToLegacy(backfillers, migrationTime) + + // recent is a concatenation of independently-ordered sources (stored info + invoker-derived), + // and RecentActions has no order-sensitive consumer -- it's just a display/history list -- so + // a plain re-sort by ActualTime is correct. + if recentFromInfo { slices.SortFunc(recent, func(a, b *schedulepb.ScheduleActionResult) int { return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) }) recent = util.SliceTail(recent, legacyRecentActionCount) } - ongoingBackfills, triggerStarts := convertBackfillersCHASMToLegacy(backfillers, migrationTime) - bufferedStarts = append(bufferedStarts, triggerStarts...) + + // bufferedStarts is different: it's already in invoker enqueue order, and that order is + // load-bearing -- it's fed into the same ProcessBuffer V1 uses, where BUFFER_ONE and + // "nothing running" both take whichever entry comes first in iteration order, never + // comparing ActualTime. A pending trigger Backfiller hasn't been enqueued yet -- + // BackfillerTaskHandler.processTrigger builds its single BufferedStart and only appends it + // via Invoker.EnqueueBufferedStarts once its task actually executes, regardless of the + // trigger's own ActualTime -- so simulating "if CHASM kept running" means every still-pending + // trigger belongs after whatever's already buffered, not repositioned into it by time. + // triggerStarts are sorted only among themselves (built from a randomized map iteration, so + // they have no defined relative order of their own) purely for a deterministic tie-break, + // then appended after bufferedStarts unchanged. + bufferedStarts = appendSortedTriggerStarts(bufferedStarts, triggerStarts) var generatorLastProcessed *timestamppb.Timestamp if generator != nil { @@ -217,6 +236,26 @@ func CHASMToLegacyStartScheduleArgs( } } +// appendSortedTriggerStarts appends triggerStarts after bufferedStarts, leaving bufferedStarts' +// own relative order untouched. See the comment at the call site for why: a pending trigger +// Backfiller only ever gets enqueued (appended) once its task actually executes, never +// repositioned earlier by its own ActualTime. triggerStarts are sorted only among themselves -- +// built from a randomized map iteration over pending backfillers, they have no defined relative +// order of their own -- purely so that multiple simultaneously-pending triggers land in the +// resulting list in a deterministic (not map-iteration-dependent) order. +func appendSortedTriggerStarts( + bufferedStarts []*schedulespb.BufferedStart, + triggerStarts []*schedulespb.BufferedStart, +) []*schedulespb.BufferedStart { + if len(triggerStarts) == 0 { + return bufferedStarts + } + slices.SortFunc(triggerStarts, func(a, b *schedulespb.BufferedStart) int { + return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) + }) + return append(bufferedStarts, triggerStarts...) +} + // convertBufferedStartsLegacyToCHASM transforms V1 buffered starts to V2 format. // V2 requires request_id, workflow_id, attempt, and backoff_time fields. func convertBufferedStartsLegacyToCHASM( diff --git a/chasm/lib/scheduler/migration/migration_test.go b/chasm/lib/scheduler/migration/migration_test.go index fc3ee84c784..76efdb48a81 100644 --- a/chasm/lib/scheduler/migration/migration_test.go +++ b/chasm/lib/scheduler/migration/migration_test.go @@ -327,6 +327,147 @@ func TestCHASMToLegacyStartScheduleArgs(t *testing.T) { require.True(t, triggerFound) } +// TestCHASMToLegacyStartScheduleArgs_PendingTriggerAppendsAfterExistingQueue verifies that a +// still-pending manual trigger Backfiller -- even though its own due time predates an +// already-pending regular buffered start -- ends up AFTER that start in the converted V1 +// BufferedStarts list, not before it. A pending trigger Backfiller hasn't been enqueued into the +// invoker's buffer yet: BackfillerTaskHandler.processTrigger only calls +// Invoker.EnqueueBufferedStarts (append-only) once its task actually executes, regardless of the +// trigger's own due time. So simulating "if CHASM kept running" means the trigger belongs after +// whatever's already buffered. V1's BUFFER_ONE (and "nothing running") always keeps whichever +// entry is first in the list; if the earlier-due-but-not-yet-enqueued trigger were placed first +// instead, a rolled-back scheduler with a running workflow would retain the trigger and +// permanently drop the already-deferred regular start. +func TestCHASMToLegacyStartScheduleArgs_PendingTriggerAppendsAfterExistingQueue(t *testing.T) { + now := time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + scheduler := &schedulerpb.SchedulerState{ + Namespace: "ns", + NamespaceId: "ns-id", + ScheduleId: "sched-id", + ConflictToken: 1, + Schedule: newTestSchedule(), + Info: &schedulepb.ScheduleInfo{}, + } + invoker := &schedulerpb.InvokerState{ + BufferedStarts: []*schedulespb.BufferedStart{ + { + // Due at -1m: later than the trigger below, but already enqueued. + NominalTime: timestamppb.New(now.Add(-time.Minute)), + ActualTime: timestamppb.New(now.Add(-time.Minute)), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + }, + } + backfillers := map[string]*schedulerpb.BackfillerState{ + "trigger-1": { + BackfillId: "trigger-1", + // Requested well before the pending buffered start's own due time, but not + // yet enqueued into the invoker's buffer. + LastProcessedTime: timestamppb.New(now.Add(-10 * time.Minute)), + Request: &schedulerpb.BackfillerState_TriggerRequest{ + TriggerRequest: &schedulepb.TriggerImmediatelyRequest{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + }, + }, + } + + args := CHASMToLegacyStartScheduleArgs(scheduler, nil, invoker, backfillers, nil, nil, nil, now) + + require.Len(t, args.State.BufferedStarts, 2) + require.False(t, args.State.BufferedStarts[0].GetManual(), "the already-enqueued regular start must stay first") + require.True(t, args.State.BufferedStarts[1].GetManual(), "the not-yet-enqueued trigger must land after it") +} + +// TestCHASMToLegacyStartScheduleArgs_MultipleTriggersDeterministicOrder verifies that when +// multiple trigger Backfillers are pending simultaneously (built from a randomized map +// iteration, so they have no defined relative order of their own), they still land in the +// converted V1 BufferedStarts list in a deterministic order -- sorted by ActualTime -- rather +// than depending on map iteration order. +func TestCHASMToLegacyStartScheduleArgs_MultipleTriggersDeterministicOrder(t *testing.T) { + now := time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + scheduler := &schedulerpb.SchedulerState{ + Namespace: "ns", + NamespaceId: "ns-id", + ScheduleId: "sched-id", + ConflictToken: 1, + Schedule: newTestSchedule(), + Info: &schedulepb.ScheduleInfo{}, + } + backfillers := map[string]*schedulerpb.BackfillerState{ + "trigger-later": { + BackfillId: "trigger-later", + LastProcessedTime: timestamppb.New(now.Add(-2 * time.Minute)), + Request: &schedulerpb.BackfillerState_TriggerRequest{ + TriggerRequest: &schedulepb.TriggerImmediatelyRequest{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + }, + }, + "trigger-earlier": { + BackfillId: "trigger-earlier", + LastProcessedTime: timestamppb.New(now.Add(-5 * time.Minute)), + Request: &schedulerpb.BackfillerState_TriggerRequest{ + TriggerRequest: &schedulepb.TriggerImmediatelyRequest{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + }, + }, + }, + } + + for range 20 { + args := CHASMToLegacyStartScheduleArgs(scheduler, nil, nil, backfillers, nil, nil, nil, now) + require.Len(t, args.State.BufferedStarts, 2) + require.True(t, + args.State.BufferedStarts[0].GetActualTime().AsTime().Before(args.State.BufferedStarts[1].GetActualTime().AsTime()), + "triggers must be in ActualTime order regardless of map iteration order") + } +} + +// TestCHASMToLegacyStartScheduleArgs_PreservesInvokerBufferedOrder verifies that two +// already-enqueued invoker BufferedStarts keep their original relative order even when the +// later-enqueued one has an OLDER ActualTime -- exactly what a backfill (processing a +// historical time range) can produce when it enqueues after a regular tick. A plain sort of +// the combined list by ActualTime would put the backfill-derived entry first, and since V1's +// BUFFER_ONE (and "nothing running") always keeps whichever entry is first in the list, that +// would make the rolled-back scheduler execute the backfill and permanently discard the +// already-deferred regular start instead. +func TestCHASMToLegacyStartScheduleArgs_PreservesInvokerBufferedOrder(t *testing.T) { + now := time.Date(2024, 6, 1, 12, 0, 0, 0, time.UTC) + scheduler := &schedulerpb.SchedulerState{ + Namespace: "ns", + NamespaceId: "ns-id", + ScheduleId: "sched-id", + ConflictToken: 1, + Schedule: newTestSchedule(), + Info: &schedulepb.ScheduleInfo{}, + } + invoker := &schedulerpb.InvokerState{ + BufferedStarts: []*schedulespb.BufferedStart{ + { + // Enqueued first: a regular deferred BUFFER_ONE start, due "now". + NominalTime: timestamppb.New(now), + ActualTime: timestamppb.New(now), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + { + // Enqueued second, e.g. by a backfill processing a historical window -- + // its own due time is OLDER than the entry already ahead of it. + NominalTime: timestamppb.New(now.Add(-time.Hour)), + ActualTime: timestamppb.New(now.Add(-time.Hour)), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + }, + } + + args := CHASMToLegacyStartScheduleArgs(scheduler, nil, invoker, nil, nil, nil, nil, now) + + require.Len(t, args.State.BufferedStarts, 2) + require.True(t, args.State.BufferedStarts[0].GetActualTime().AsTime().Equal(now), + "the first-enqueued entry must stay first even though its own ActualTime is later") + require.True(t, args.State.BufferedStarts[1].GetActualTime().AsTime().Equal(now.Add(-time.Hour))) +} + func TestCHASMToLegacyStartScheduleArgs_ExcludesAllowAllFromRunningWorkflows(t *testing.T) { // Regression test: workflows started under ALLOW_ALL are tracked in V2 as // BufferedStarts with a RunId (and no Completed) while they run. Modern V1 diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index 21bdf0e412e..b6252268ca3 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -3629,6 +3629,19 @@ error and stopping the schedule.`, `SchedulerSpecWarnIterations is how many excluded candidate times the scheduler evaluates while searching for a schedule's next action time before emitting a warning (metric + log). It is non-fatal: the search continues past this threshold.`, + ) + SchedulerV1VersionCeiling = NewNamespaceIntSetting( + "worker.schedulerV1VersionCeiling", + -1, + `SchedulerV1VersionCeiling caps the workflow version the V1 scheduler records into history, so histories written on this cluster stay replayable on peer clusters that do not support newer versions. Set it to the highest scheduler version supported by the lowest peer. Intended for multi-cluster failover and rollback. The supported floor is version 1 (OSS v1.20). A negative value (the default) disables the cap. +The ceiling is reread on every tweakables evaluation. The version never decreases within a run, but raising or removing a ceiling can advance it on the next evaluation. A lower ceiling is recorded immediately; if it is below the version already recorded for the run, that version is retained. +Operational notes: (1) A ceiling below 12 holds fresh or not-yet-advanced runs below CHASM migration support, so it pauses their V1->V2 CHASM migrations until the ceiling is lifted (deferred, not dropped). It cannot downgrade a version already recorded in an existing run. (2) A ceiling below 6 skips custom search-attribute updates on schedule edits in fresh or not-yet-advanced runs. (3) This caps V1 scheduler histories only; schedules already migrated to CHASM V2 are not made rollback-safe by it.`, + ) + SchedulerV1VersionOverride = NewNamespaceIntSetting( + "worker.schedulerV1VersionOverride", + -1, + `SchedulerV1VersionOverride selects a supported V1 scheduler workflow version without waiting for a server release to change the default. Set it to a version from the current default through the latest version supported by this binary. A negative value (the default), a value below the default, or a value above the latest supported version is ignored. +The override is reread during every scheduler tweakables evaluation through MutableSideEffect. It can advance the version in the current workflow run at the next evaluation, subject to the current SchedulerV1VersionCeiling. Neither a lower override nor a newly lower ceiling can reduce a version already recorded in that run.`, ) WorkerDeleteNamespaceActivityLimits = NewGlobalTypedSetting( "worker.deleteNamespaceActivityLimitsConfig", diff --git a/service/frontend/service.go b/service/frontend/service.go index f30a7b2c03e..7b7e2e7e386 100644 --- a/service/frontend/service.go +++ b/service/frontend/service.go @@ -176,6 +176,10 @@ type Config struct { // Enable schedule-related RPCs EnableSchedules dynamicconfig.BoolPropertyFnWithNamespaceFilter + // Ceiling on the V1 scheduler workflow's recorded version. + SchedulerV1VersionCeiling dynamicconfig.IntPropertyFnWithNamespaceFilter + // Requested V1 scheduler workflow version. + SchedulerV1VersionOverride dynamicconfig.IntPropertyFnWithNamespaceFilter // Enable CHASM tree infrastructure EnableChasm dynamicconfig.BoolPropertyFnWithNamespaceFilter @@ -385,6 +389,8 @@ func NewConfig( MaxFairnessWeightOverrideConfigLimit: dynamicconfig.MatchingMaxFairnessKeyWeightOverrides.Get(dc), EnableSchedules: dynamicconfig.FrontendEnableSchedules.Get(dc), + SchedulerV1VersionCeiling: dynamicconfig.SchedulerV1VersionCeiling.Get(dc), + SchedulerV1VersionOverride: dynamicconfig.SchedulerV1VersionOverride.Get(dc), EnableChasm: dynamicconfig.EnableChasm.Get(dc), EnableCHASMSchedulerCreation: dynamicconfig.EnableCHASMSchedulerCreation.Get(dc), CHASMSchedulerCreationRolloutPercent: dynamicconfig.CHASMSchedulerCreationRolloutPercent.Get(dc), diff --git a/service/frontend/workflow_handler.go b/service/frontend/workflow_handler.go index d410c60dbae..3d0159f5307 100644 --- a/service/frontend/workflow_handler.go +++ b/service/frontend/workflow_handler.go @@ -7217,7 +7217,12 @@ func (wh *WorkflowHandler) cleanScheduleMemo(memo *commonpb.Memo) *commonpb.Memo // This mutates request (but idempotent so safe for retries) func (wh *WorkflowHandler) addInitialScheduleMemo(request *workflowservice.CreateScheduleRequest, args *schedulespb.StartScheduleArgs) { - info := scheduler.GetListInfoFromStartArgs(args, time.Now().UTC(), wh.scheduleSpecBuilder) + versionCeiling := wh.config.SchedulerV1VersionCeiling(request.Namespace) + versionOverride := wh.config.SchedulerV1VersionOverride(request.Namespace) + info := scheduler.GetListInfoFromStartArgs(args, time.Now().UTC(), wh.scheduleSpecBuilder, scheduler.VersionSelection{ + Ceiling: versionCeiling, + Override: versionOverride, + }) infoBytes, err := info.Marshal() if err != nil { wh.logger.Error("encoding initial schedule memo failed", tag.Error(err)) diff --git a/service/worker/scheduler/activities.go b/service/worker/scheduler/activities.go index 1f9c7a7d01f..ec0b0739b09 100644 --- a/service/worker/scheduler/activities.go +++ b/service/worker/scheduler/activities.go @@ -38,6 +38,7 @@ type ( startWorkflowRateLimiter quotas.RateLimiter maxBlobSize dynamicconfig.IntPropertyFn localActivitySleepLimit dynamicconfig.DurationPropertyFn + migrationEnabled dynamicconfig.BoolPropertyFn } errFollow string @@ -394,6 +395,15 @@ func (a *activities) MigrateScheduleToChasm(ctx context.Context, req *schedulerp nil, ) } + if a.migrationEnabled != nil && !a.migrationEnabled() { + // A live (uncached) check, deliberately re-read here rather than + // trusting the caller's possibly-stale view: this is what stops a + // pending migration from completing after EnableCHASMSchedulerMigration + // has been rolled back, even if the workflow retrying it has been + // asleep since well before the rollback. The caller just logs this + // like any other failure and keeps retrying at its normal cadence. + return errors.New("MigrateScheduleToChasm: migration is currently disabled") + } _, err := a.SchedulerClient.CreateFromMigrationState(ctx, req) if err != nil { // Treat "already exists" as success (idempotency). diff --git a/service/worker/scheduler/activities_test.go b/service/worker/scheduler/activities_test.go index c667033b366..5594f85187c 100644 --- a/service/worker/scheduler/activities_test.go +++ b/service/worker/scheduler/activities_test.go @@ -16,7 +16,8 @@ import ( type mockSchedulerClient struct { schedulerpb.SchedulerServiceClient - migrateErr error + migrateErr error + createCalls int } func (m *mockSchedulerClient) CreateFromMigrationState( @@ -24,6 +25,7 @@ func (m *mockSchedulerClient) CreateFromMigrationState( _ *schedulerpb.CreateFromMigrationStateRequest, _ ...grpc.CallOption, ) (*schedulerpb.CreateFromMigrationStateResponse, error) { + m.createCalls++ return &schedulerpb.CreateFromMigrationStateResponse{}, m.migrateErr } @@ -34,7 +36,8 @@ func newTestActivities(client schedulerpb.SchedulerServiceClient, nsID namespace SchedulerClient: client, MetricsHandler: metrics.NoopMetricsHandler, }, - namespaceID: nsID, + namespaceID: nsID, + migrationEnabled: func() bool { return true }, } } @@ -100,3 +103,64 @@ func TestMigrateScheduleToChasm_NamespaceMismatch(t *testing.T) { require.Contains(t, err.Error(), "different-namespace-id") require.Contains(t, err.Error(), testNamespaceID) } + +// TestMigrateScheduleToChasm_MigrationDisabled verifies the activity performs +// its own live check of EnableCHASMSchedulerMigration -- independent of +// whatever the calling workflow last cached -- and refuses when it's off. +// This is what stops a pending migration from completing after a rollback, +// even if the workflow retrying it has been asleep since before the rollback. +func TestMigrateScheduleToChasm_MigrationDisabled(t *testing.T) { + client := &mockSchedulerClient{} + a := newTestActivities(client, testNamespaceID) + a.migrationEnabled = func() bool { return false } + + err := a.MigrateScheduleToChasm(context.Background(), &schedulerpb.CreateFromMigrationStateRequest{ + NamespaceId: testNamespaceID, + }) + require.Error(t, err) + require.Contains(t, err.Error(), "migration is currently disabled") +} + +// The guard must refuse before touching the scheduler service, not merely translate its +// error -- otherwise it creates the V2 target and then reports failure, leaving both a V1 +// workflow and a V2 schedule behind. +func TestMigrateScheduleToChasm_MigrationDisabledShortCircuits(t *testing.T) { + client := &mockSchedulerClient{} + a := newTestActivities(client, testNamespaceID) + a.migrationEnabled = func() bool { return false } + + err := a.MigrateScheduleToChasm(context.Background(), &schedulerpb.CreateFromMigrationStateRequest{ + NamespaceId: testNamespaceID, + }) + require.Error(t, err) + require.Zero(t, client.createCalls, "the V2 schedule must not be created while migration is disabled") +} + +// The non-retryable namespace mismatch must not be masked by the retryable disabled +// error, which would turn a permanent misroute into an endless retry loop. +func TestMigrateScheduleToChasm_NamespaceMismatchBeatsDisabledCheck(t *testing.T) { + client := &mockSchedulerClient{} + a := newTestActivities(client, testNamespaceID) + a.migrationEnabled = func() bool { return false } + + err := a.MigrateScheduleToChasm(context.Background(), &schedulerpb.CreateFromMigrationStateRequest{ + NamespaceId: "different-namespace-id", + }) + require.Error(t, err) + require.Contains(t, err.Error(), "does not match activity namespace ID") + require.Zero(t, client.createCalls) +} + +// Documents the fail-open default of the `migrationEnabled != nil` guard: an unwired +// dependency migrates as if enabled, which is the wrong polarity for a rollback guard. +func TestMigrateScheduleToChasm_MigrationEnabledUnwired(t *testing.T) { + client := &mockSchedulerClient{} + a := newTestActivities(client, testNamespaceID) + a.migrationEnabled = nil + + err := a.MigrateScheduleToChasm(context.Background(), &schedulerpb.CreateFromMigrationStateRequest{ + NamespaceId: testNamespaceID, + }) + require.NoError(t, err) + require.Equal(t, 1, client.createCalls, "unwired migrationEnabled currently fails open") +} diff --git a/service/worker/scheduler/buffer.go b/service/worker/scheduler/buffer.go index d2a20a2ecdb..f29c00d6748 100644 --- a/service/worker/scheduler/buffer.go +++ b/service/worker/scheduler/buffer.go @@ -27,6 +27,18 @@ type ( } ) +// IgnoresRunningWorkflow reports whether a start with the given (already +// resolved) overlap policy is started regardless of whether a workflow is +// currently running for the schedule. This is only true for ALLOW_ALL: every +// other policy either waits for the running workflow to finish, skips, or +// cancels/terminates it first. Callers that need to know whether a buffered +// start was actually blocked on a prior run finishing (as opposed to merely +// following it in time) should use this instead of re-deriving the same +// condition. +func IgnoresRunningWorkflow(overlapPolicy enumspb.ScheduleOverlapPolicy) bool { + return overlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL +} + func ProcessBuffer[T Overlappable]( buffer []T, isRunning bool, @@ -48,7 +60,7 @@ func ProcessBuffer[T Overlappable]( overlapPolicy := resolve(start.GetOverlapPolicy()) // For allow-all, just collect and start all at once - if overlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL { + if IgnoresRunningWorkflow(overlapPolicy) { action.OverlappingStarts = append(action.OverlappingStarts, start) continue } diff --git a/service/worker/scheduler/export_test.go b/service/worker/scheduler/export_test.go index 817a6157a6b..fda0b26526b 100644 --- a/service/worker/scheduler/export_test.go +++ b/service/worker/scheduler/export_test.go @@ -10,10 +10,6 @@ import ( // a SpecBuilder with specific Max/WarnIterations accessors. Compiled only under test. func SchedulerWorkflowWithSpecBuilder(b *SpecBuilder) func(workflow.Context, *schedulespb.StartScheduleArgs) error { return func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { - return schedulerWorkflowWithSpecBuilder( - ctx, args, b, - func() bool { return false }, // enableCHASMMigration - func() bool { return false }, // migrateWithRunningWorkflows - ) + return schedulerWorkflowWithSpecBuilder(ctx, args, b, schedulerDynamicConfig{}) } } diff --git a/service/worker/scheduler/fx.go b/service/worker/scheduler/fx.go index fd8d755342a..c7b61e674a6 100644 --- a/service/worker/scheduler/fx.go +++ b/service/worker/scheduler/fx.go @@ -55,6 +55,8 @@ type ( enableCHASMMigration dynamicconfig.BoolPropertyFnWithNamespaceFilter chasmMigrationRolloutPercent dynamicconfig.IntPropertyFnWithNamespaceFilter migrateWithRunningWorkflows dynamicconfig.BoolPropertyFnWithNamespaceFilter + schedulerV1VersionCeiling dynamicconfig.IntPropertyFnWithNamespaceFilter + schedulerV1VersionOverride dynamicconfig.IntPropertyFnWithNamespaceFilter globalNSStartWorkflowRPS dynamicconfig.TypedSubscribableWithNamespaceFilter[float64] maxBlobSize dynamicconfig.IntPropertyFnWithNamespaceFilter localActivitySleepLimit dynamicconfig.DurationPropertyFnWithNamespaceFilter @@ -93,6 +95,8 @@ func NewResult( enableCHASMMigration: dynamicconfig.EnableCHASMSchedulerMigration.Get(dc), chasmMigrationRolloutPercent: dynamicconfig.CHASMSchedulerMigrationRolloutPercent.Get(dc), migrateWithRunningWorkflows: dynamicconfig.EnableCHASMSchedulerMigrationWithRunningWorkflows.Get(dc), + schedulerV1VersionCeiling: dynamicconfig.SchedulerV1VersionCeiling.Get(dc), + schedulerV1VersionOverride: dynamicconfig.SchedulerV1VersionOverride.Get(dc), globalNSStartWorkflowRPS: dynamicconfig.SchedulerNamespaceStartWorkflowRPS.Subscribe(dc), maxBlobSize: dynamicconfig.BlobSizeLimitError.Get(dc), localActivitySleepLimit: dynamicconfig.SchedulerLocalActivitySleepLimit.Get(dc), @@ -117,7 +121,18 @@ func (s *workerComponent) Register(registry sdkworker.Registry, ns *namespace.Na migrateWithRunningWorkflows := func() bool { return s.migrateWithRunningWorkflows(nsName) } - return schedulerWorkflowWithSpecBuilder(ctx, args, s.specBuilder, enableMigration, migrateWithRunningWorkflows) + versionCeiling := func() int { + return s.schedulerV1VersionCeiling(nsName) + } + versionOverride := func() int { + return s.schedulerV1VersionOverride(nsName) + } + return schedulerWorkflowWithSpecBuilder(ctx, args, s.specBuilder, schedulerDynamicConfig{ + enableCHASMMigration: enableMigration, + migrateWithRunningWorkflows: migrateWithRunningWorkflows, + versionCeiling: versionCeiling, + versionOverride: versionOverride, + }) } registry.RegisterWorkflowWithOptions(wfFunc, workflow.RegisterOptions{Name: WorkflowType}) @@ -145,5 +160,6 @@ func (s *workerComponent) newActivities(name namespace.Name, id namespace.ID, de startWorkflowRateLimiter: lim, maxBlobSize: func() int { return s.maxBlobSize(name.String()) }, localActivitySleepLimit: func() time.Duration { return s.localActivitySleepLimit(name.String()) }, + migrationEnabled: func() bool { return s.enableCHASMMigration(name.String()) }, }, cancel } diff --git a/service/worker/scheduler/replay_test.go b/service/worker/scheduler/replay_test.go index 52352144566..1a8e3aa397e 100644 --- a/service/worker/scheduler/replay_test.go +++ b/service/worker/scheduler/replay_test.go @@ -8,38 +8,68 @@ import ( "testing" "github.com/stretchr/testify/require" + commonpb "go.temporal.io/api/common/v1" historypb "go.temporal.io/api/history/v1" "go.temporal.io/sdk/client" "go.temporal.io/sdk/worker" "go.temporal.io/sdk/workflow" "go.temporal.io/server/common/log" + "go.temporal.io/server/common/payload" "go.temporal.io/server/service/worker/scheduler" + "google.golang.org/protobuf/proto" ) -// TestReplays tests workflow logic backwards compatibility from previous versions. -// Whenever there's a change in logic, consider capturing a new history with the -// testdata/generate_history.sh script and checking it in. -func TestReplays(t *testing.T) { - replayer := worker.NewWorkflowReplayer() - replayer.RegisterWorkflowWithOptions(scheduler.SchedulerWorkflow, workflow.RegisterOptions{Name: scheduler.WorkflowType}) +// versionRelease maps a SchedulerWorkflowVersion to its respective initial server release. +// The snapshot is testdata/replay_.json.gz. +type versionRelease struct { + version scheduler.SchedulerWorkflowVersion + server string +} - files, err := filepath.Glob("testdata/replay_*.json.gz") - require.NoError(t, err) +// Versions sharing a release share a snapshot, but are kept here for completeness. +var versionReleases = []versionRelease{ + // v0. https://github.com/temporalio/temporal/blob/v1.19.1/service/worker/scheduler/workflow.go + {scheduler.InitialVersion, "v1.19.1"}, + // v1. https://github.com/temporalio/temporal/blob/v1.20.4/service/worker/scheduler/workflow.go + {scheduler.BatchAndCacheTimeQueries, "v1.20.4"}, + // v2. https://github.com/temporalio/temporal/blob/v1.22.0/service/worker/scheduler/workflow.go + {scheduler.NewCacheAndJitter, "v1.22.0"}, + // v3. https://github.com/temporalio/temporal/blob/v1.23.0/service/worker/scheduler/workflow.go + {scheduler.DontTrackOverlapping, "v1.23.0"}, + // v4, v5, v6: https://github.com/temporalio/temporal/blob/v1.24.0/service/worker/scheduler/workflow.go + {scheduler.InclusiveBackfillStartTime, "v1.24.0"}, + {scheduler.IncrementalBackfill, "v1.24.0"}, + {scheduler.UpdateFromPrevious, "v1.24.0"}, + // v7, v8: https://github.com/temporalio/temporal/blob/v1.25.0/service/worker/scheduler/workflow.go + {scheduler.CANAfterSignals, "v1.25.0"}, + {scheduler.UseLastAction, "v1.25.0"}, + // v9, v10: https://github.com/temporalio/temporal/blob/v1.27.0/service/worker/scheduler/workflow.go + {scheduler.AccurateFutureActionTimes, "v1.27.0"}, + {scheduler.ActionResultIncludesStatus, "v1.27.0"}, + // v11, v12: https://github.com/temporalio/temporal/blob/v1.29.0/service/worker/scheduler/workflow.go + {scheduler.LimitMemoSpecSize, "v1.29.0"}, + {scheduler.TriggerImmediatelyTimestamp, "v1.29.0"}, +} - logger := log.NewSdkLogger(log.NewTestLogger()) +// versionFixtures cover versions that are newer than the latest released scheduler history. +// TestEveryVersionIsMapped verifies that each fixture records the version it represents. +var versionFixtures = map[scheduler.SchedulerWorkflowVersion]string{ + scheduler.MigrationHandoffFixes: "replay_migration_v1_to_v2.json.gz", + scheduler.RefreshCompletionDesiredTime: "replay_version_override_v14.json.gz", +} - for _, filename := range files { - logger.Info("Replaying", "file", filename) - f, err := os.Open(filename) - require.NoError(t, err) - r, err := gzip.NewReader(f) - require.NoError(t, err) - history, err := client.HistoryFromJSON(r, client.HistoryJSONOptions{}) - require.NoError(t, err) - err = replayer.ReplayWorkflowHistory(logger, history) - require.NoError(t, err) - _ = r.Close() - _ = f.Close() +// TestReplays replays every recorded history under the current binary, so a history an older +// server wrote still replays after an upgrade. It covers both the per-release version snapshots +// (versionReleases) and the targeted behavior fixtures (replay_with_*, replay_recent_*) that +// exercise specific version logic. +func TestReplays(t *testing.T) { + files, err := filepath.Glob(filepath.Join("testdata", "replay_*.json.gz")) + require.NoError(t, err) + require.NotEmpty(t, files) + for _, f := range files { + t.Run(filepath.Base(f), func(t *testing.T) { + replay(t, loadHistory(t, f)) + }) } } @@ -105,3 +135,75 @@ func TestReplaysWithDynamicConfigChange(t *testing.T) { "replay after a dynamic-config change should still be deterministic: %s", filename) } } + +// TestEveryVersionIsMapped checks every version from InitialVersion through +// LatestSchedulerWorkflowVersion maps to a history that records that version. +func TestEveryVersionIsMapped(t *testing.T) { + mapped := map[scheduler.SchedulerWorkflowVersion]bool{} + for _, vr := range versionReleases { + mapped[vr.version] = true + _, err := os.Stat(filepath.Join("testdata", "replay_"+vr.server+".json.gz")) + require.NoErrorf(t, err, "missing snapshot replay_%s.json.gz", vr.server) + } + for version, fixture := range versionFixtures { + mapped[version] = true + path := filepath.Join("testdata", fixture) + _, err := os.Stat(path) + require.NoErrorf(t, err, "missing snapshot %s", fixture) + require.Containsf(t, fixtureTweakablesVersions(t, path), version, + "fixture %s does not record a v%d tweakables marker", fixture, int(version)) + } + for v := scheduler.InitialVersion; v <= scheduler.LatestSchedulerWorkflowVersion; v++ { + require.Truef(t, mapped[v], "scheduler version v%d is not mapped to a replay history", int(v)) + } +} + +func fixtureTweakablesVersions(t *testing.T, path string) []scheduler.SchedulerWorkflowVersion { + t.Helper() + var versions []scheduler.SchedulerWorkflowVersion + for _, event := range loadHistory(t, path).GetEvents() { + attrs := event.GetMarkerRecordedEventAttributes() + if attrs.GetMarkerName() != "MutableSideEffect" { + continue + } + data := attrs.GetDetails()["data"].GetPayloads() + if len(data) != 2 { + continue + } + var id string + if payload.Decode(data[0], &id) != nil || id != "tweakables" { + continue + } + var payloads commonpb.Payloads + if proto.Unmarshal(data[1].GetData(), &payloads) != nil || len(payloads.GetPayloads()) != 1 { + continue + } + var tweakables scheduler.TweakablePolicies + if payload.Decode(payloads.GetPayloads()[0], &tweakables) == nil { + versions = append(versions, tweakables.Version) + } + } + return versions +} + +// replay asserts the history replays under the current scheduler workflow binary. +func replay(t *testing.T, h *historypb.History) { + t.Helper() + replayer := worker.NewWorkflowReplayer() + replayer.RegisterWorkflowWithOptions(scheduler.SchedulerWorkflow, workflow.RegisterOptions{Name: scheduler.WorkflowType}) + require.NoError(t, replayer.ReplayWorkflowHistory(log.NewSdkLogger(log.NewTestLogger()), h)) +} + +// loadHistory reads a gzipped JSON history fixture. +func loadHistory(t *testing.T, path string) *historypb.History { + t.Helper() + f, err := os.Open(path) + require.NoError(t, err, "missing history %s", path) + defer func() { _ = f.Close() }() + r, err := gzip.NewReader(f) + require.NoError(t, err) + defer func() { _ = r.Close() }() + h, err := client.HistoryFromJSON(r, client.HistoryJSONOptions{}) + require.NoError(t, err) + return h +} diff --git a/service/worker/scheduler/testdata/replay_migration_v1_to_v2.json.gz b/service/worker/scheduler/testdata/replay_migration_v1_to_v2.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..7cc141ea49afa0fa4f9a5ea531b77a0ae2e5e7e0 GIT binary patch literal 3902 zcmV-E55e#siwFP!00002|Lk0AkDEB#{wu3|-KDb4DO9TTB!o~hu~QC#;8&}PQxXUy znLviXs{j2JWKQiI%5-L*-R`p=DowGC@3Zgg8Z&?NqN`|Ib>tFfCL7J4D4F~y=T#Tz`YZYz;Te?O|ACLRK-?^_1SViDW9F0aWS6GoX^Ta zk>{2+BcDxcU|7niSyXuW=DZu=`n)JK&&u6lvrKoTa*Bdu5%0EU;T?)7RGJM2MRADz zV-e-O_kY|@J#YI*Z|k*^7E!j}9lVsutDMP$brc@c=)l~}?tQu5SwBX{C)e$x1GCMU z;~!H8yx`KjDV0U|r<=!Ng3 z+aMbQYQoH0tU)WQ`W1qB%yo0$pM zI8oTCwT`hxGGe(ksg=q-X&dYqYm(3u*PDVQLG4xm;93JyYr~S-EoLfBU1weqYnT$G zv9rNgo^7A?S4=Feq#161rs^bae1(xx%p4=ZGK{G+PDqVes@e%w^$)!-zkj)xR+g7< znvC8x@!W$TZz6Mdh*$B}OK)?|IBwg#1W;OvEQAd3A!0NPtqP%DxE#pI8{O}9CqRGah_a5H7%DJb&&V-KDqgwTKEFDEm0Jy0@wWQ)${Z%m zcyc>_8MNacdSAXYQs$9&2-aU0#CRJUi(3K742k^|@x5$mT;NRdn!m?a@*r^%ujdARFiN)=UQURlCxf5PIr0qAlL6Q>suqS-!~HfPJkkZ&HqM&b z6zx;58efN-6o?`xhLB+q5E+gOCB{QgWcog2m%PA=5CzTNhv@Pcj!pBF~J`qke9ySmrfWF`{612p@IudBU}t_*i&;8kunzDa&OPw7ozW z6<(aSlW(lmrICJc65T&?wgyX`@xgT_IbGnm*V^dKI&yb0+)p!dcNb~y zqw(fJ=bgKXyml{S)jqmN+ISAXRaX1U07=K2#zMKwm8hC1)S>IpIRPDwfs58K7>9d5 zGp@o)6L76ST)QP64etNTwGG_+8?p{nvI`f(h3D`apAF(!qI@-XDo;$hFj9ZpcCd5L zfvJ}45FY8=!*`OkpOMs8?TxM4mG3kbGE%CLtI*+rz96dWqe9_K}W_z3T5l-GoPKLI+Bi|njS;P9tMCppJV&RLHa1q%oZ7>6xF9LL zrPZ}>F&Xte&+(YQee4o=BF%I1wd!Bn1Dfodm4 zmZrXHVCwMOP+eosf%{vVrZf$jF_mc15DthDHH6D+afx=MFqh6 zhZ}Hnf15ab`|?7T#MIi&zC3&IU7yC^)@z|_z2?E|>UIC+_4-W;^_w#uGrCxVHV@`x z-Hd%zZwJ5}4dS!~No%=0`ers*?^Qr9QL*}{ijj&R<#sRT>Ei50*d^#b3>-zB~HPq>vHt@g=~Q{f#W@?B%lfQ-kg1fAf>&~~PGBD(ip9hv@ zXv4Xsm=8TGfjdXsP%h>-RKpK?_eG4wjl(~A$2B|3xs<=Mb9<{pE)#?KG zoYgH2E5_5nS8A#CBF>`2(}=enz7y&i(U0T9JwOlMv3Mt421AvjF zP-K<}d3_KrB?P5s7$4Kp3&^bK* zF4Pc;yd?+Ofbo4EFr4QB#tZrp<8#3>h*rPp8u0q}njR(P%vb}Eg>NH0l}rdS3gwZ=))X3Z?O za4pQ!7-p`cA?zqT&2)~`n|LZuHpFtvnWLYG3D<6dPgBd-;0zb8H7{ulAaO{H>&b?U z=5S^y@l@70nygb|xhJ=R@k~|fsioIgofor_&blTju{A%@WCiJHP|>7aQ&JV5+{B1+ z1xh>`rc+C=u(c{?R{M;n*}TNoWKFY4GPAU1&54eN$xL;5Dua?(t7>W)iEH6vW|7Q| zL5bCIMlB{rtv5O0X>l3r z3j`Mq^FDAHhxB9HRV~{^eIT?nttEX4io#9naX~9x9FX5?g6TYJ9>b zIN?#4POSi7M=4xO<7YBl(`1;DBw%Nb4%}pA%4M9L$&b;Xn&uy05zRu^XlBuyu?GSW z-dvIeIQ?21ATW(At|fTi{VpU~4_cBTc}y zCg2hEloto*dNr^#RlZo>(H!`e2HD@yAcwp0{?jxF6uX8e4}Tny3!wiZM1Gd)2}myW z1%!A3@m)~)V;R|j`Vkwf#rD}Bh2_GzneGV7@2UHh?BYY&rH-<5Kzs{eE}YZ;e;1e| z(9Pg1RM%$c@W&>;?_hLTYso%yol{L0w~+V0a_OS{FJ8L9i57Rn zwg^7xMiJY!VoUIK6`TEun40)0rY3Al-48Q06<1P3Q?qavM&fm0Y6i-O zS41P%IU1$6V_FdR)=bN}jr^Bs`P(op4?Ym7hO7AUj%m5N9Y5fwnHI!%3{M`4uV7jb z|01U4JpD^ABYgyUH%0H(^>> z@ITJ9^t&0Hmqg(Wm=>0M-N!NF--&4vB0lt&UdSNOtm#LP=}SVu^jSU(r9RJ#;!{^3 zM8LoL3Ivimxejq0dK;#N?Z5W(n8z8{a}56DnHCmw^_TFf>(T-%{X~6x_2=~MGr8&8 zS2ecuayBCW%)VWV^58!n}2PkxU>jz zaRg&vYrgxxz$BBIm{~K)KD+xoFQ%M8H|Ku({7#=gdi0j&RoT1vzx)5{Veg{H_J00w z{L_vz+PmmM07V9f4CHJ8E?9WM@gK!LfGl*sEUmZgQf;AO`^F~M+Kn?auSV0EbE7Ra zWp3*;d}Gjt2WGE^@2#b`=e^mVjJUyV&1X zwCr8{@woIz_K#lfb(~f-+ZH=7Wy-o_%3w{yeM)!C!|LAWd&a_5}qqZ_yd7)Q%?F*U*MHuH%@1jStbWfHOVag3_pEGYszQ|h1GFp~i zMDLXpju%xhmTKZ0_5obfVYaBJ+34U|LPOv>buSmCYY7lpf*p=WbvVL_H(qQ)=Post zlRP;5`fJDAk%OL+2UogUP$e>q*_{eFw@rBl?ubukJ-l0Muz~dZ$pLU!eOyf;t&KDxFfc^!75(6 z3TUv3v#D+H1ZovY%ote!kR;upWW6Ryn4)H5$kqfZiI9bkkc8v(*3U-ebZMpmXHlH~xDbCE8I2uoq4VcLzMIC=5pWN#6_s>sU(Duqr>-|M* z`yLC*){4b0j^f-)AN|wZ=baCWf-iU?&oki)4j7)}AmdX(WJJ$fgRqr8pK_k?KlFl4v0C*mdPnv%niZ|Tu40;oT@+dO z5u)d-?#-WjbxE9b|CR5>vUnL4nx>XyXzGflmY%$pmwIIAvMH~wj=yLY>Bexan@S|* zuESRW6xP8dE_}|YLe8j%QPSl5l{rlGv2q<#LDI%3BKj|GKL-Y>hf%ELKi$9A>8gF- z`~11(q@><1Sbv=#;yk2xG#ovOYTrjV_A**-y@0CmZCu7h{vEl!9N+q}bnwTctsnjR zYnQi4bF=(VQuB zZ(FX5s@Lv%nxEl$`I!}2{;d3L%huJ6rCe(2e0nlD_j}(&#)IFS%FM4wuRnk2ZQ{J8 z)3IR8wYhe4rDOF5&rBpwJH?jm8DJMmT9ouzM1Bc?ZFX?6 zKC%55XZ+jj`zDLD;)EOYBtfQL9?eS{j#4jbiy`j!;~M-;tVc!f_~MarkNpV6kH?Q@ zw4%E>kDh$_ne~M;tuI0Ld)7y?d8L^*rl!nH^`zB1$IZM}O(qSBU3k*m#|8IdqJ8E0 zDYfE|YO56us`e8ht-Ltx7C&g~oT~BJIrH>D$xY66Mmyq6N_UCkUZm$qJ|uhDnI>K>Kk_JZ}2)C<04!Rm!2bRd@hJ*iT2gn zsWP$LrJ4GGx~SHriVBLHKG$HYm? za5+hXS{L1HR0eQf>YGs+!jTYw(IFTw_U_2s`p%*W$D=AxbTPb~u&#B8&3pmcy@rE< z=SX`4O01t79nT$~4sQ7z60YM_^|{49RCE#9EpILxUM^~PK021gnhsAk+?0dU(qnfJ~u($5(7Oc*r_s^gyTg`mK~kO$QGH;O;Q6&`*57@LuWAzfRJ|j zR}S$U-lcA<<8fNLi{Uz4;?#X2m+g#rBVS~0wy5VY6Q!So45J|Q)S|~%Sm%j2d<3AvOPOSBU8p9WoQ?VOyH#{6BPTrqPzEqMD&H+ zQ!;V*{$Y++*wQ;nc{%{#(pG4e5m(effGUVmUh_7HZIB0*J z(>-|f4>?)?TqonC?>X80+R{w>ONC}>vZ-9&$hK|j!?~?lC)`eIpY69L-96%6>l+OC zy_V5S622BnKA@m$Z(q{z$8e;PNw@wt81nuM>hR*XS|6ONb&>m?)@>bW=F4zK0^PGX zqq~=JO!xW=Q};M+6z`tEH1LswLLc&F??+rr`g3807Q?Y3_;X=Lx_MGm{!!GH^>YP% z%J|?J<19FvbMajux+J_6qQ3w?E__cErC=p6VnA06&kY95z~@&C4Ej7}IZ>1b;5$Ix zvUTN(vOU8< zv)sAX8whWyJ;kHa|AyN0dj*wKPcob#h#I+`DWFC+fm7RnqzNiwM?6vhOYAv|>_viX zgLM-vo+^_KYVdmIjIx;p?p#|3Q`@L9Bo)cf!Hq#PQ#TD(!`O8eYXpsWs_262AOSbB zfGhxKs@_a(qnX)8HM4adDQt!l;|`7I2iGxUtfF=gi^fKc61+h*5L`Q4x4dE7s7Rtg zS7B3PM-6V=4I0lmH__WYfIuB!XmB_N99A_rwapD_qKYJ>!8KqB)N5j=8=Pr8YNB>e zLo&k6)OCz4z+uNsTHdf4RWmi>T#MbHOg|7uJCF?VIGMM5YTV$IIQjuwY~tGMmNypJ zK{a#c5L@g9tLYl6ix{cl8rw%YGaGSkq6s7hHBJ_3%Nxq(70C#|77&OXz}Z-X$R;&z zIvqe{FJf1f*_OAN8r`+JMu}dL%z&L_Mlt(To50-Ps5Td z(}&Jej1sxVF*x8D)QB|>NG*?_!HkKxE9rn<5fvksaij=vr zE}~2D;5R#3?$f{h-Z{JtvMTe(5;xBlO)!@BM_}^w{?FiCn*cmUHCBLt9K=m+17HpX*(*rMoJoB`U&r6@Pzb;YHCNVJ!{OKlrltlPecv<$Bg#(rVE`9|+wgtr6?q zr^8|5*>bTtHC!;Y^#e*sg;Y|Z1kaHwi^$eR)GR#h^^Q~zqlEu*X$&7Pjic=^50}Q5 zUoMU7$4ld9E+n4mLIcqMaab)Li=(q)^^vQSxSIdSK@JGKFgOcZpNg9PIU)}REY}xa zJLRK~v3v@qim-or%qNa=?=(~`yI1$e!;Y?Yv|N8vR>yfoce$5-xPSd&wJWj@AuWSl z?9&hTSm(I{2IneZ0Nw+mi*WEikI|*`R1Vqx;N5(VgKzS=Rj~^w6BKzB=X=U{C6m*` ziCLa!LmJ>rStU_=dp|_3O)MY~IIpLUe&^cmhGe7lY^)1-SfEE+%#=7mx^i;T zSc`_=Od6Z}{K}009606#Zz6doln3d;3`y literal 0 HcmV?d00001 diff --git a/service/worker/scheduler/testdata/replay_v1.25.0.json.gz b/service/worker/scheduler/testdata/replay_v1.25.0.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..19bb14e53620d6216e09316d76d8933d4678a5c6 GIT binary patch literal 3704 zcmV-;4u|m{iwFP!00002|KwVGbDK!E|0+}byC&N-4+&RI)gnLyS=|Z<2{pI2Ts&G* zBOZ!Cmc*35`z^4YIDTa0ICCd^_m3%0)D7qLJEza@(0_3Bh8A_jUH;Gg|Jn(6$qC%W z-M>7RQ|^+304h9Ccqn>;aw!Oxviw056$uEzmz}3=yVIPT*t_<~C-d5$xL3pR#J@Hd zratrR3BI0sn(LVccfr+RwYd`oq`ur!xC`!3?l#M;JeWrs?`u{T(>mPMG%?%PYjwS2 z(Y~e?clpPct#?BI;EJ$Qv!?mB+=UrmohrVH*EHE@bjLsJ=5FugI(xc*^|3A8@kPbg zpWEH6*zW6g=QccLWtiOgDW`Rqgmu_1&>}7qR;;*7j^x>%EJpHJ9GoK2y&=Us=PtOM zR#mv7_x?$&c^zBoiTHgFJ318p@h2mR08U+ti1Yt1q< zbTxCzPXV0o#|ykCi$N1wDi|-4++3Si228Su2~Cs_4>4TdB)T9bgL0>{1jJJzM~!yG z4QT=$G$co0?756IaZNk|CR1TUnqh{#fK5E@fW4WcO-E2q$kn*}`yI6KBe-7q~_}Tt3=9s2@T$QG2RAk`Vx@lCQw$>K*%CR+9TVCD$MU$274A!=* zuhb&&r8PVBFJ4kIy$BUIxn-*FT8}5zIz#;AW^2_?f>(;s3 z+uIASqTw!He_Ik*ki6H< zLUot7Q#h9ssGb7toBXpgH;Wd7n>Zg6}vRY16Cgss7P4X5*LYaF6>o6CTj=QW1VbDZSt}tZ1cl z@|~GEGq0~~Y+U1si?ng;(5fx=Ua4ngMW5#1SHwSYj`*j}Q9cN2zbA|5JLi=ns!P3Z zT=VMbRDI*LFHw9{1uD9`xj&fb?q8~`^pt;%S(sNm3gbapZun2T@#axdukF(eUUCDS(TJf>C)kG@T4<>2Z1a}@a*8}6ZC0pz8F5`N%-_K z-)8hp0jwTH{0sd4uBg5%5*J*RY|x_`n1#h^R?%dbg{!v75&zw7!B4V!RdU_$U&M04 z2iW@n+BfF3raM-w9=-Ny#O3oMF8Ac`L>y`EmFZr)rap0vmm=NbG!I+vxHK+z$%`Sr z+i-6m+Sg%}(J4!)xm?n?<}SE|)*;Kfy$?=0rwM#^f;>7a3RevL;g0yDDp(-Kwa)oj zy(C!-w&NV%M80qnv95n~$RevEINUt#8TX~3ugR5e&G!VVo6wgFbG7{(Lv{af+?!S3 zpr?@;_`y2ykHCRLK;T-4hsid|-J9fOE2uFA)Q|%*w8#Hd_D0x78@x^|TqcXbBJ|}= zB*tu#n&0dl>!WF~aI>gzw-JmN4@Onj60 zGH@2yB7RmSi%}MzY$eEtRSaj9y%|;s9Lg~m9%5_04~FhG^5;!r4eMC9mE_|{2&M<- z7IV-(>-ZRizPfjyDny0T>D=k*<5tfB5j$OtNSyBz-Bys->J|ve$N4Fk4exr~&7Dqe zuAV$AYubHydSJ7Bo(8S1-^g&SNF1SXfq|Wt!dM@T5^H`Ui%w4C3DRNK5CT2hZ>Fkgr%mdD>+nABS1S$K}4L z>Fy^aA^IZhu@IL0^!&z4JheMfx8FN>?oYxeg;F`UP^#}Kl>Ijs%AZjoe>P2?1dBE3 zFgU~OcI|Pg(^gackfM%8tc_BoBWi6RfmJ>;w(K7gbyoS+Sgwr;S#PbOyG=iT#&eRR zp0@S(q^eV488E+(d`AmXWi5xP7NoTJM<{4Ps@J{9)&iC{QWR+cyOPzY)`E2Z&(EQz zt^P8kJ%IQJkkva6S*r9sIx+h+HC^v(?=>|wSO0jed7f(zW}Z2Hjy09L;P%^!?!K_q z=@Sz7y)n;g#=VgWKCqcj;l85D5Ai@JS8t+kIpFF#wx(k*{c=$>f}+3pe15_I5> zLAdNC((B}V>e>1g;V!tM+~s$JlJ8DKe~6+B!a1H60POeP&Cyl$JsiEm#IHcoV<8SD zQBirhCog#^iez0z4~zu&>DPr|Ak1-$&EC7(Pq? z4uk(Mi28XvTM*BTS+Dow{i6b;zomctPG97oY>2FYVMA-MF0hF|XJgOJv1Qg1FUye* zPQ=GK_VI>5`$V9_#=3pPkQ}iU|#7)b5Z2?=e18L&+nFhm|IPn~a84w6`z}RzSlDBf~22F(w8=8o7oM8j! zfz|3}%c#lnz! z(shQK*#UWX_5?ZJkW`b<2}nWS`r2jKpVh>hHNjM9V+U}vl3GvWW^?BQKVSqBG0=q! z`{dqlPCfC48DT&|mU)`^GZ|aMtc~g7uOoSAb#Ch|z-LW|bZ%Oz?@3Gf&1pGqE1eeC z?53@ce??}})2N%BBzXqtEHmx2WN2zv_NJSez;NNp^N-mfnA(b+VL%omFo)_Xgh3UF zEpkTFU@@r3B72wa7u(1(**2w@JpZwU(b1Hi? zS`%2`1UX@obfDfLWk;)h_IH2lO>W}6&Lc||XZgH|Ep>lKEFOJ(bB;JSPuR#d0RtlP z*6X(NU})FGLjq>tNT8b%&lHItlqTrc{W$uW97;>aM)%RvTvaxh zbRR_>@JVb2^DWYu`}{db>Q|FuSWZWV35jQeiQgUx@o{3AH8xfvVZs_UGdT8!X<*?s z@{)Cs6N$Xx!)&{*&~E)DA}lCpVn-E!lnE=K_ZBAHY4jo$mOiNczAOV7fHTCG5$(v`ivig^|QBMGC?g zfJCO>faR~eM|#HPr(ge`Y3U5!*4mCdJ$v~+Y4Cd68(tmCBKc_6L5dE@iy?wejo>K7 zIsha`;@CGKHa2x&p=02gkffUfGG>CQnW65i2IgvJuO`$qPPS%N_CVK6_KURAV;wbj zI!sbQ%H%6gD>pCQ!yS9}0i~oyMyms!A&)4?vlV0vLi6qXkF7NG>Qf#FQGR#tfcS?x zvF1&p3wz=VSu76L1kP$?IO4=IPbA&c6OTxU0Wk4r!r05!fnnDqHxI*2K3@WGZa1M-B)&02}BOq+0m(&gW|2td39o?%JC3^ggb; zFTOTK&l>Wl{nDQ_>)m;@reV_ECAj^i@5lbfQ^5b6d%73@hhK=>V!x(4Ru3Ld@_M(Y Wx3~Yk{oeop0RR8oP0u}yF#rGx4|D7Q literal 0 HcmV?d00001 diff --git a/service/worker/scheduler/testdata/replay_v1.27.0.json.gz b/service/worker/scheduler/testdata/replay_v1.27.0.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..b83b9515c58f462bff02c92e5c3cd4268f25dc02 GIT binary patch literal 3669 zcmV-b4yy4ViwFP!00002|KwU}Z==Z8{wpKCPG}XyVjQWZivbfHR}nU2cr}XHY=c?S z1_#3^|NRwY>2%uZaXOjh&i&AGhg!~l>O80FcOt%xi@GAt|L@_yYDAn96mfR{mzCr( zaZW%0ITCOrq%Dd&r$*;2_h~$)SPpnkGuxYfro51`b#1v<^0lKGm(#iCT+2&Yv@KP` z*QO;HmMjrxMD17Gds2Ym*%3#a5ly+>te^?YokMMc(M`q@cw zv#S%^&5ij-*^O>ycgi*4i_GJQEK z!shemp0j)Ugjj@^s#M0}TGzG1dMd9~T|1PbJPDJDu%5^h{>$(yG@P&z6gfkewq9O& zaIv2+@!nldI$s%s`7+AojeIG=C<_v*3-W0b!p&_YQgk#acVZHO(4-Pi3Ol6CQr83! zIWnAEI;B}^?K*0!nSe&A%b?tDy)a&;_mM3dCG++rCiAH{3$8d63okM zQ@oHRdwwk6j)-p~;n7=;CEGWo%rjz}6g`!JzHZC5e0`~6=^ASW66U=^t3Ih)rI?jf z{4)N&di)FJ(0{0$(I=i7jVShX<-Bqf@$jW;UOk1XZ? z15FqLn&%=Cjz%kz;b=he9L^BfotuK(On%qfO~6g{F(Hk7ELtGRsC zRNMgP^M1mEd}v?$K^B`y6w9ks9M;4c5yiEiWW(G?C7n{leRncEI|B-X_MGX?b!L^f zLOb9GW>o;E+m|)pzLLa^dnqc5y$i=%-(d@Ree)%R zyuyiMdBfDWg?g&QiIMmIrj+(xa+p-!ey=Qcc6|(3e(j=^Qh=z(R%hY zjg;&**xvJG7p)C5&NBy| zn|`gHPg5UyH-U2KY-Q1!OCt;O>A;(smWN@EbEpD3^1bEk_MVoIXob;6j$NtQEbHkl zl&toa>eugEHj6Ke!Jayt_VB|&8NP{gyVoa}i!;Yt&blW$7vS@KZJpo8Az39+OvXbb zlA)hvWLWNtI^KOi5F-8a1F`2slIX+Z8?Ugb4y5jWbjUoMdM^?scWRuHtP!&Mv6%3 z7wT1?r^Y{)l0OIF z696MfjsH-}scIaPk$IG{XZl15pR>%rWSIgd@(k$@fXid{opAXT)Nsy@MghaXkYxRs zB^hSKkpa&$WCZ$<3#mBvSFeL=(^ADtmpCJea+lvXKz`WG{3+;7Q737d2N1p+dymKO z!QO*U{MF!l6$JqB0ZY>Ch$9&gab&D{%Tk=%$y`(!Z0QF#H8M{GwA<;-+%j@6m220MJ8Xj1 zO;Lv|V~e3nYYyty01RgvY)$$#joayPZs`zrYOP5!gGFkC(k+Wq7rLDbG)e8(P)J$i zNU{%Z0%(+?!`#wY*XikODV%@xAYFDYVBI6K?&&GPHi-4%eblAc*;Nrpl-VH6tzVnXt1PrgWfE; z)*Kt47IfH}B(CKGH?=#Q-s@VE?1N^lJ1+sWW(_)^JxQ+J+&*X~0A9N8^#EN5Z;%uY zYfw5#P&e7QCY536vUAJWcoM2L1$CYY4qj>l*U=%iq7++pvi$SF8(N=QQd97|VuCp} zEV=iDCHv;EocEbc2k|1$6HPGt*m6DtDPZ_J0zT3N7xvH32Q6^aD=IgC>;?5}5``+fI`Z zgS&;VCQYI*4d6LcGxTE2$DYPG_Y#%++E>`c)js>X?_s0cFt77K8Pj&Y=t5<@zXuJ^ z?#uKBSy|S5Jx)M}bHvWADDj3YCv<%#@hqD`5@OwM-C##H#pdB|Y$G=Ol_3AL3$rJ> zFniLC{gt6z$%duM<~7nJb*dio=jYw`t6g~$@9HG3P#Gnw_W#|V7LMRYj-rk}Z%NjL ze;h+r!_dPiu!~o_xY~SM7AHj=?+QQr$n*-e7 zyFcF?;Qxl^Kol}T7zBWf=}|y3Jm-_+5nxD~@hO^GMZqYHk2ePxh8lfva{y8&*@_vD zxDRL!K={t)z|lS+1pnUV0E8#mhova^wu~Wa*ix}AOV@&Vjb-P1ThYfdhxm{SBB8=@ zbHr*z+Ix$NFTz&>g2>{sIwsU8ipmbZ~PKBB@%TVsp zC|%^TAp4@ep2%xkU2Cx{wW=Uj_Cyrq<2pE-J2rIF@Zeg;W^Ua_lo@1n_TWmGOrZve+ zm9_h*>!a^rP2urk^D(fDeet!h0WXW{mo?8H6x_o(vx)s^xbMIFN54(|S3tS{o_Dfm nf8kATpWHX`E~zJvCwINu$9H$X-Ti+6009608{;F2^e_MbcxX)J literal 0 HcmV?d00001 diff --git a/service/worker/scheduler/testdata/replay_v1.29.0.json.gz b/service/worker/scheduler/testdata/replay_v1.29.0.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..5c84767d35b7bae71e68ba3f778537cdb662400e GIT binary patch literal 3685 zcmV-r4w~^FiwFP!00002|KwVEbDPSt|0*f{doQ^#EP_;>s*3<&WX)hUA$WDl#WIpW zED9iFIF;}I3S>J@Z0E?a&-UIQDJ#s`e}6dr z*G_Ug`7{9};0Tf<0Buv;Cl-97rXM+B%J7W;b!vAQrYb81+1IvrrChtZc{yL|?zOU3 zB*)ftbZyz9X)E%us0r%*i5ki9?35e6wPm&6WM!+|;^^2UWnncz)!Bb z_2koU4^vNMf14D+$Z3P~Ls_M>`xJGh7k(vnOBfC>uw7d~w|YS42@0&F67? zQXG!WF!d00SsBD9Y2~;H;-CqJ0bE37oD|#1r-_$mM{m7gm-MU)Y4gS_R{3zU9M^TQ z#dkuA)m0OzLg%^dF#@YD&R5+spSOX<_8!<}{>7^HEfzqFwPSVO#dDMf>S`amx0$hC z6jA%jFC%X!4kl6&U21ZPrLAG;x7)e0)eQZ%l$2SN%*5?Xnem^;zhcXYJ5g0Kc*6WR1ua82=D?zfGX!p;K*fc>`hT!T~+qtji7)m?}7Eh!!B*3^X+ZpYxlmkRS2`mxOBu8;%K=LdeHb`PzG)dD&UwoMiO zUzIsaHC2K|&ozn)x1QGDTzw`DZy2{9#!)(sPriH^IH_?^MZ0g~KvKl`7RU34quO_# zjDj53hakdoa+B0aS^VJKL4j_tKOdMrZE-==MR2HhWiwDcR7qKlhV$FF$>XpL&I`*t zL(yl2Wm~i^uPy09k)7qUF?l-q&Qb2o=(&LWim3YU50iaT40J{o3`cR4>q`yE*GM;^ zxE$qN524>KrL3&+69@h358n%k{(T|wAGs+%6{fEc(o5GC51xqV#T%>sMtEN}`Y2XB zmRE!G|1OgtuL&4Lv$EV1S5^5jE2AL$Z*gzigb*{@vIvU zG2v;CI zNGkI~hF_dimg-Qc%B8@A#%`H5oLGZo7-fH~=b9Tm;vLztU4ai?*xS18zfEtJK_|7vYd9l|}@#VRV}^=uuu z>^`KUL?3d#M{}3b!t&Q<78d3q1gsOdRHmgRfU?0n-4pG?9A%-p+OO1A7rKLl%*eYF7KJ*)CqA%ztz)+nqB;vC92?)j1&Jd~4*n9o8E8 zJPm+<2-Vx(QCFR{HnS+7kGyGVc{|Q=Z|jhb1Ao1^Infdmy)e7jb!shJWCOjGnl=7f z`}%#)X86J!^=W|lfKObQp##pHL7wDNTG)}gO3ggWLe+%6%NDV^gMmvO9%P#NSyZ4y z2$*3_w=o7A9`p13yk87CO-g%6;JEX;lj<=5 z`$z){kaRuy850wJ_ppHcNRqt3o^jZ@W*pTyG2{Fb#wqe#&N=>Fz_-A@ z6@0&dKt2WWM&KiwBVs0q2xfx?f)6kzf`AXgIGRqQAo>ABw`@(i^o9vVS>>lX%a04F zKZ)ci>MW5{6gvg)0ObYYO;G-1Ab&cRpOPWwH{6DZBaSB+YO^82Fq{&Y-!Lo%HW4+A zU%>K!6~J4;JO$3v5yvs_g627rej}Pc>FJ5bpN#GAkLS+?;}m_aVy66CGDePM{-TWW zM`@7TX|!iU8ulLTOJs*^tzFs%ryGrj{Wb3641=IQ7SN%P-y<{nA9dZ|Y-_8=$ zQ5mw$Y-t;`*GG+)MlF=c9;puSRgczPt%dSFSxUqFEd%X6izF)b?Pz?H&|R(Z+(qU~ zB}(&;P?)rk+7roo;};^GRh&C@AQ$5Mx$Xr zdru*YWKn7ibXX|E#DEoQ`T@Im5jGE+T2!?-*kK%4I)>+~_Be9byT3RO1(rRr#)>lBV zHrdrh*7}ylX&EBD^@3ahp$mXthjf&j>PoZPTW9w*r39v8hZ3?y3E4r)xJ7A6!L;n4 z#I(>hZc(pMx@-no(k8~*Bz>1^$6ADSUndcAKhZdNLEhUmRB1fdqrHv&R(Dr9 zvOQ|)DwOAX4eCWO#A82=)1~e8C`XB(u2SFLrM@c8;MUW+&BNQR{`Kwl>fvn{Ts@4( zJ&NQPb$^ASS=E&)*!0?HyXVEb8QQ%n~Oc{zh2M|B!In#5FOmQR!z-yar zG!==@flKa`?LO_cNw!7sJix5Sp}Xs$!kY@$yYDjQq-bze1lfnX;SZat%s<4qj;bWY zAMQ}gQvvX23qW`eWX)0F|2?u6&J#Js@zeLSIqpq1w<)U#6H!?-NpZwPP!k1ipJ}yu z(lkMycUHhr@6Zsyoviuis|fsGQbk~SI-bUuBLsd*5zKVT69G??gfJD@h{79+;kXy7 z2n;>t-&#cg)Ok8$_$m7?RRjRuT19xiZV1SKXjcIH>$!ym0M64DOOxy?V~3<8TSJZ_ zUyIf?Qru_sMn4-vq`RmPi#5XSGa4*#`=s@51E?qh0Yiivo+6kHPZ9w=Wr+|lbj*=7 zPt*7tEBQkkP0wnH`X_Xvmd5OL8ScWCvgbQa{g<=f;Ib`?_(B=7S<QpmiG1*s&MQwg)XF_>fJ(F>lIN6Bx5^MwvM*du_AYxsMO9(5|!ldlTo)?d+LTX zx+U$l?c&lF%l&iRx=UMYVd^zd9{4~Gv^@^l8VA!^pu8RuCXrYk!+hTOYt(u!%9eJP zBH3Vc+utqiEb&s)p%x*^XUHe0VXvcQ5l;e9d>rC#m<7TZ4{|9kwe7PmgOVC&}i zlbz;C>tl-rtjGX~0jv!>9|is+-~J#%PLf#vys}x*m5QTa`@#0^l?PWh?*=2?eNZNf zJhn9*KUj9xw3UAArd0>k;&>F`J3?D*-L$qvxtNk-tL&0!U8lv;s)MplVufzst?M!k z*L700KK^xHdOY;6)-pJVStt3bCWyZOOB$pV%4c5RcbtUyN+0=iF2nJ<%4Zt344#76aUt!d`n=FCD^e#XyO6R0@Yc> z?v5Cf;WFC(=RZ`kMD0=>8#@kCcX^k9%rsXmala? z+N1th@N%X0?fD$Z0C={>j_d{J_q#R9Jm}|`=zWrVdu^x8mAgK7-iHghVd>|)O}tyh zG9e(c;MkvFV6x8IpA6@*>e%NiOopyM8SVpDTHBmYqUCVqtB&PQdX+aJS}?&Ndn8kgAL#~7MSJ`_h*!#Ahjcjv)>{;#E5kL)0F^eE( z&4u1y@O1y~SnC{QV96#-;pQxl1)HNj7Bx~G;%7ebY@_z}gZyZl?sjA^c-)U_WRC@8 zJ994|L%pxS*5^MzKShikR0|rZ1$EXI1Xb&%Rg~#0T?XVFP3C%e2si)(AaZSq;oDry zw5M>&NHH%mFis)~gwr?_tod=1lvFUL*d%3{#tBm< zD-tveNRp|T1UwVLDPZ^j07mNYZ3YSw6oN1jI#6icv{q$Wln4L$iynDdUD&It3(pGh zn(AVAt-A+HzE%3;(MzSJeNkFypqzZ^rMmKt>TuIqq)U3r;e2DPjFpEw4fh|gZldmp znxm?NvbGDE6jky(JfGs>nR2){mDBkk0#O9*>y`6L)$OnsRrjEpR}1dREn?*hD}IqA zp43z)`F{hF?5a9g9)p-cLDVaX1#@2(A4m}e)9H^sXC7Sc zHm&NSY?yYt6N-V*W}*ag3=ajCks^U*!dMEXVaP}AfZnJ~KCP4L=xTz*Ps6DcGc3=u z3=iWD6C^NU1U3u&G~CA2|1zDx!RJlj_yawi?+P%pr|e7B43f@UDm8F%zyU9-_peIvC2)W ziWlf2wUS^t8&^p@Ai<0_blgq*&%+}8EZ}DaRZ6@#pVNXLc>V*Rd*(@t_EYyVLZR2UHzq5*20qqQQ>~9`A?Nbr6!oO5;SCPLqh% z_Bg47lpNNc)O1bD`&DVWC_+n<^W8z|xx>nzptSY*=GAT9-8p-c-fEQNP3Xc+m>#-i z!<&#QWCx4q9r6Y#^4zJyZDrXwj+6H`4f|b zOWKVKV2Fjk$$n;g#B8@ZE*+>jWow2oD>j$Nl00f8L!FK^Q^yCXi; zu}Ui6>CKPG}jwAia=aM@G8m2 zRG(lZ5A8^GGBYP6R88c&!ZKFpC~)EGtkW}E;ZN_J(UZYCt579tztO~ztz4`X`o2pl z>d#}=0a~^K(b6F!c{m93K^m$~-%f z;wRs7q$A?UkGX@XXhWDBvIi3dgfLODUe-zZ6A}k9=#TWU%R|!mr{^R*#g=wB>hrfs z=VL8@mfXebl6wdKP;&3QJGuXitp2l!4Bem1!69zOcupCD3zNuQ-s*QV{(z?2a$0v54DP>`jYKnGMuGQgwBw zO~v7-`+qm1qTH0>aGl9QIQt~x&>&X35X(*T3q9<#^>=tw`$~`Ez%}jSy{X?TrmZQ~ z1Je82Kzqr(q>G;MzARQrS*J;bia4F7FS^>Bce-ht{9W{fkxQHKOKRe=`n;rV={jxm;9H!tts%vH74S-w*RIn%DPO0S z50B3RJwo_FT3#S{@PU&!PU5Q8&D&~!c1sA}yr%R4K+x%MZ(ehtnJ-;ZI9-xTedQG| zWtN_2CiwQ}fP>5u-}woIjr1JJo;>kd!sabl`VjySa6-T_Vw;Z{zLRi_6z~Bfuv{C2 zp%lZm@H)R^*_wRkwQgF=qRa#G95jB(5`8ZV1mEV!sLBNqir1_B<){z=c&_mG&_EFW zt|Y-F4-~QO5JbGlOs7%E@KP)?5`YoY5qQ8QVkq%YdYK2}Aj|z24+P()^rR1f{@7WstE|u#lg1)^-NAhrivcbZ$=LDsY=(eAaY?FTDt95%J zpauxY9posR0bMm0eij|x@v`3B&w3f%Q@7*A;hS2&TYI*$$0^|SzQ>~d0wP!Ak&4oh ztU=!$63i+)-2yyvhv~>R1YI51o~v=X3^TtU2}m^rWP5wh_2@eQ?H4;gGjrSk&dUe$ zk!@^sSE+SZYy1>0uxjRr)wbTgH^;6<-(mHBw8yyuycA}h-4#Z*nfvrUE@Hg~usHK} zxKA>a4dysk=zWNn`*Ykk0G8q0??>B_ZLq%T)w(@T>9;+9psLwu6Nji zsoytKyw@7M?>4@D(1D=4#vb(%@KZ=|j;Y>uenxHT(fdH8dQk6=S(L@O-e07XUkI0G z*TN;pebtq`&95HAQs93iEX8++<%lM{Zd9imnk#=rWG2nw0@kl6H+kUr4qu+hiHA&4)@^VKwz>tjh!{J)UqIY1 zfNt?yly&!Lzi3c@9%$({H6|wTUAR5)SJsJJlX5@RM05)ty`@a5HTk=zC!T=eICu7u zDy@(3s&{A8XpRgsiYJE@Kyv@u=A0PxQ?7VY6O;d6A1yCx&Q>*#qfLc0R2INIGREI~8 z6=7PGQNl#UvQC%lgb6BUnQXsos85}E`LE-c#lNQMu3nePzh=j#xhTI(bnmpc4Ak>7JeUc+cX$g*wzYFoYz$yxY=ko-lOjs?G~m47`XXRo2h zAY^&=odrPFv8~~;(tqe$4_I-(w&jb3Kz>Z^aks+B_64ghlkHcNlYD174FMZ6d=QEZ zAI2;r#c;}Wq)svw*>=Fj(ZylEbk_NbX|~^g7P>gu)Iqpi-JY9i_i>sd1p?lqEw)MOS&-S%+X52#7`dkxh{$;&rCuDb}Ll=3)a?&rf4tBCV#%^LQ- z9bz5g*_6M*1m~#6s^BEy>D= int(wantVersion) && override <= int(LatestSchedulerWorkflowVersion) { + wantVersion = SchedulerWorkflowVersion(override) + } + if ceiling >= 0 && ceiling < int(wantVersion) { + wantVersion = SchedulerWorkflowVersion(ceiling) + } + if recordedVersion > wantVersion { + wantVersion = recordedVersion + } + + version, capturedCeiling := determineVersionTransition(defaultVersion, recordedVersion, ceiling, override) + require.Equalf(t, wantVersion, version, "default=%d recorded=%d ceiling=%d override=%d", defaultVersion, recordedVersion, ceiling, override) + require.Equalf(t, ceiling, capturedCeiling, "default=%d recorded=%d ceiling=%d override=%d", defaultVersion, recordedVersion, ceiling, override) + } + } + } + } +} + +func TestDetermineVersionDiagnostics(t *testing.T) { + t.Run("reports invalid override warnings", func(t *testing.T) { + logger := &warningLogger{} + s := &scheduler{ + logger: logger, + versionCeiling: func() int { return -1 }, + versionOverride: func() int { return int(LatestSchedulerWorkflowVersion) + 1 }, + } + + s.determineVersion(TriggerImmediatelyTimestamp) + s.determineVersion(TriggerImmediatelyTimestamp) + + require.Equal(t, []string{ + "worker.schedulerV1VersionOverride is outside the supported range; ignored", + "worker.schedulerV1VersionOverride is outside the supported range; ignored", + }, logger.warnings) + }) + + t.Run("does not report a ceiling that caps an override as ineffective", func(t *testing.T) { + logger := &warningLogger{} + s := &scheduler{ + logger: logger, + versionCeiling: func() int { return int(MigrationHandoffFixes) }, + versionOverride: func() int { return int(LatestSchedulerWorkflowVersion) }, + } + + version, _ := s.determineVersion(TriggerImmediatelyTimestamp) + + require.Equal(t, SchedulerWorkflowVersion(MigrationHandoffFixes), version) + require.Empty(t, logger.warnings) + }) +} + +// TestVersionCeilingDefersCHASMMigration verifies that a clamp below the CHASM gate keeps +// migration markers out of history. The pending migration is retained for a later fresh run. +func (s *workflowSuite) TestVersionCeilingDefersCHASMMigration() { + migrateCalls := 0 + s.expectMigrate(&migrateCalls) + + signalMigrate := func(after time.Duration) { + s.env.RegisterDelayedCallback(func() { s.env.SignalWorkflow(SignalNameMigrateToChasm, nil) }, after) + } + // The schedule is paused (no timers), so the two signals advance its iterations to + // continue-as-new. + signalMigrate(30 * time.Minute) // sets PendingMigration (deferred under the clamp) + signalMigrate(60 * time.Minute) // runs the iteration budget out so it continues-as-new + + s.runWithCeiling( + func() bool { return true }, + func() bool { return true }, + func() int { return oldPeerCeiling }, + pausedHourlySchedule(), 2) + + s.True(s.env.IsWorkflowCompleted()) + s.Equal(0, migrateCalls, "neither DC- nor signal-driven migration may run while the ceiling holds") + + // PendingMigration is in continue-as-new payload on the next run. + canArgs := s.continueAsNewArgs() + s.True(canArgs.GetState().GetPendingMigration(), "deferred migration must survive continue-as-new") + + // After ceiling removed, the deferred migration executes immediately. + endOfRun1 := s.now() + s.env = s.NewTestWorkflowEnvironment() + s.expectMigrate(&migrateCalls) + s.env.SetStartTime(endOfRun1) + // ceiling unset (clamp lifted): the default workflow entrypoint disables the clamp. + s.env.ExecuteWorkflow(SchedulerWorkflow, canArgs) + + s.True(s.env.IsWorkflowCompleted()) + s.Require().NoError(s.env.GetWorkflowError(), "second run completes via the deferred migration after the lift") + s.Equal(1, migrateCalls) +} + +func (s *workflowSuite) TestVersionCeilingLiftAdvancesWithinRun() { + migrateCalls := 0 + s.expectMigrate(&migrateCalls) + + ceiling := int(oldPeerCeiling) + s.env.RegisterDelayedCallback(func() { + ceiling = -1 + s.env.SignalWorkflow(SignalNameMigrateToChasm, nil) + }, 30*time.Minute) + + s.runWithCeiling( + func() bool { return true }, + func() bool { return true }, + func() int { return ceiling }, + pausedHourlySchedule(), 0) + + s.True(s.env.IsWorkflowCompleted()) + s.Require().NoError(s.env.GetWorkflowError()) + s.Equal(1, migrateCalls) +} + +func (s *workflowSuite) TestVersionOverrideAdvancesWithinRunAfterCeilingLift() { + migrateCalls := 0 + s.expectMigrate(&migrateCalls) + + ceiling := int(oldPeerCeiling) + s.env.RegisterDelayedCallback(func() { + ceiling = -1 + s.env.SignalWorkflow(SignalNameMigrateToChasm, nil) + }, 30*time.Minute) + + s.runWorkflowFn(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: func() bool { return true }, + migrateWithRunningWorkflows: func() bool { return true }, + versionCeiling: func() int { return ceiling }, + versionOverride: func() int { return int(LatestSchedulerWorkflowVersion) }, + }) + }, pausedHourlySchedule(), 0) + + s.True(s.env.IsWorkflowCompleted()) + s.Require().NoError(s.env.GetWorkflowError()) + s.Equal(1, migrateCalls) +} + +// oldPeerCeiling is one below the CHASM migration gate, modeling an older rollback peer that has no CHASM scheduler. +const oldPeerCeiling = TriggerImmediatelyTimestamp - 1 + +func (s *workflowSuite) runWithCeiling(enableCHASMMigration, migrateWithRunningWorkflows func() bool, versionCeiling func() int, sched *schedulepb.Schedule, iterations int) { + s.runWorkflowFn(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), schedulerDynamicConfig{ + enableCHASMMigration: enableCHASMMigration, + migrateWithRunningWorkflows: migrateWithRunningWorkflows, + versionCeiling: versionCeiling, + }) + }, sched, iterations) +} + +// expectMigrate stubs the MigrateScheduleToChasm activity and counts invocations into calls. +func (s *workflowSuite) expectMigrate(calls *int) { + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Maybe().Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + *calls++ + return nil + }) +} + +// continueAsNewArgs asserts the workflow finished by continue-as-new and returns the +// StartScheduleArgs it carried into the next run. +func (s *workflowSuite) continueAsNewArgs() *schedulespb.StartScheduleArgs { + s.T().Helper() + var canErr *workflow.ContinueAsNewError + s.Require().ErrorAs(s.env.GetWorkflowError(), &canErr) + var args schedulespb.StartScheduleArgs + s.Require().NoError(payloads.Decode(canErr.Input, &args)) + return &args +} + +// pausedHourlySchedule is paused (so only signals advance iterations) on a coarse interval that +// never fires on its own. +func pausedHourlySchedule() *schedulepb.Schedule { + return &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}}, + State: &schedulepb.ScheduleState{Paused: true}, + } +} diff --git a/service/worker/scheduler/workflow.go b/service/worker/scheduler/workflow.go index 1d9b60728ec..014ba65b058 100644 --- a/service/worker/scheduler/workflow.go +++ b/service/worker/scheduler/workflow.go @@ -36,8 +36,8 @@ import ( type SchedulerWorkflowVersion int64 const ( - // Versions of workflow logic. When introducing a new version, consider generating a new - // history for TestReplays using generate_history.sh. + // Versions of workflow logic. When introducing a new version, generate a new history for + // TestReplays using generate_history.sh. // represents the state before Version is introduced InitialVersion SchedulerWorkflowVersion = 0 @@ -66,6 +66,20 @@ const ( LimitMemoSpecSize = 11 // trigger immediately timestamp is added to the PatchRequest TriggerImmediatelyTimestamp = 12 + // MigrationHandoffFixes bundles two V1<->V2 migration-handoff fixes under a + // single version so only one activation deploy is needed for both -- see + // the TODO on CurrentTweakablePolicies below. + // - Reconcile running-workflow status before evaluating CHASM migration + // eligibility, so an actively-firing schedule can observe an empty + // RunningWorkflows window and migrate instead of deferring forever. + // - Preserve workflow/request IDs already assigned to starts migrated + // from CHASM, preserving idempotency identity across the migration + // handoff. + MigrationHandoffFixes = 13 + // update the desired time for a buffered start when refresh discovers the prior action completed + RefreshCompletionDesiredTime = 14 + // LatestSchedulerWorkflowVersion is the newest workflow behavior this binary can activate. + LatestSchedulerWorkflowVersion SchedulerWorkflowVersion = RefreshCompletionDesiredTime ) const ( @@ -115,11 +129,24 @@ type ( // without modifying state). specBuilder *SpecBuilder cspec *CompiledSpec - // enableCHASMMigration and migrateWithRunningWorkflows are re-evaluated - // every iteration inside the "tweakables" MutableSideEffect. - enableCHASMMigration func() bool - migrateWithRunningWorkflows func() bool + // Dynamic-config readers are injected per namespace. updateTweakables samples them inside a + // MutableSideEffect and records the resulting values in tweakables for deterministic replay. + // enableCHASMMigration reads history.enableCHASMSchedulerMigration and + // history.chasmSchedulerMigrationRolloutPercent to select V1 schedules for CHASM migration. + enableCHASMMigration func() bool + // migrateWithRunningWorkflows reads history.enableCHASMSchedulerMigrationWithRunningWorkflows + // to allow CHASM migration while schedule actions are running. + migrateWithRunningWorkflows func() bool + // versionCeiling reads worker.schedulerV1VersionCeiling to cap the V1 scheduler version + // written to history for peer-cluster replay and rollback compatibility. + versionCeiling func() int + // versionOverride reads worker.schedulerV1VersionOverride to select a newer supported + // V1 scheduler version while preserving the recorded version as a monotonic floor. + versionOverride func() int + + // State produced by updateTweakables must remain in TweakablePolicies. MutableSideEffect + // callbacks do not run during replay, so scheduler fields are not restored from history. tweakables TweakablePolicies currentTimer workflow.Future @@ -143,31 +170,42 @@ type ( } TweakablePolicies struct { - DefaultCatchupWindow time.Duration // Default for catchup window - MinCatchupWindow time.Duration // Minimum for catchup window - RetentionTime time.Duration // How long to keep schedules after they're done - CanceledTerminatedCountAsFailures bool // Whether cancelled+terminated count for pause-on-failure - AlwaysAppendTimestamp bool // Whether to append timestamp for non-overlapping workflows too - FutureActionCount int // The number of future action times to include in Describe. - RecentActionCount int // The number of recent actual action results to include in Describe. - FutureActionCountForList int // The number of future action times to include in List (search attr). - RecentActionCountForList int // The number of recent actual action results to include in List (search attr). - IterationsBeforeContinueAsNew int // Number of iterations per run, or 0 to use server-suggested - SleepWhilePaused bool // If true, don't set timers while paused/out of actions - MaxBufferSize int // MaxBufferSize limits the number of buffered starts and backfills - BackfillsPerIteration int // How many backfilled actions to take per iteration (implies rate limit since min sleep is 1s) - AllowZeroSleep bool // Whether to allow a zero-length timer. Used for workflow compatibility. - ReuseTimer bool // Whether to reuse timer. Used for workflow compatibility. - NextTimeCacheV2Size int // Size of next time cache (v2) - SpecFieldLengthLimit int // item limit per spec field on the ScheduleInfo memo - Version SchedulerWorkflowVersion // Used to keep track of schedules version to release new features and for backward compatibility - // version 0 corresponds to the schedule version that comes before introducing the Version parameter - - EnableCHASMMigration bool // Whether to automatically migrate this schedule to CHASM (V2) - MigrateWithRunningWorkflows bool // Whether to migrate this schedule to CHASM (V2) while it has running workflows - - // When introducing a new field with new workflow logic, consider generating a new - // history for TestReplays using generate_history.sh. + DefaultCatchupWindow time.Duration // Default for catchup window + MinCatchupWindow time.Duration // Minimum for catchup window + RetentionTime time.Duration // How long to keep schedules after they're done + CanceledTerminatedCountAsFailures bool // Whether cancelled+terminated count for pause-on-failure + AlwaysAppendTimestamp bool // Whether to append timestamp for non-overlapping workflows too + FutureActionCount int // The number of future action times to include in Describe. + RecentActionCount int // The number of recent actual action results to include in Describe. + FutureActionCountForList int // The number of future action times to include in List (search attr). + RecentActionCountForList int // The number of recent actual action results to include in List (search attr). + IterationsBeforeContinueAsNew int // Number of iterations per run, or 0 to use server-suggested + SleepWhilePaused bool // If true, don't set timers while paused/out of actions + MaxBufferSize int // MaxBufferSize limits the number of buffered starts and backfills + BackfillsPerIteration int // How many backfilled actions to take per iteration (implies rate limit since min sleep is 1s) + AllowZeroSleep bool // Whether to allow a zero-length timer. Used for workflow compatibility. + ReuseTimer bool // Whether to reuse timer. Used for workflow compatibility. + NextTimeCacheV2Size int // Size of next time cache (v2) + SpecFieldLengthLimit int // item limit per spec field on the ScheduleInfo memo + + // Dynamic-config-derived values are captured in the "tweakables" MutableSideEffect. Replay + // reads the recorded values, rather than querying the current dynamic-config values. + // Version is the binary's default version, capped by worker.schedulerV1VersionCeiling and + // retained monotonically for the run to preserve backwards compatibility. + // Version 0 corresponds to the schedule version that came before introducing this field. + Version SchedulerWorkflowVersion + // VersionCeiling captures the raw worker.schedulerV1VersionCeiling value, even when Version + // is retained. This makes a ceiling change part of the recorded tweakables value. + VersionCeiling int + // VersionCeilingSet distinguishes a captured zero ceiling from history written before this + // flag was introduced. + VersionCeilingSet bool + // EnableCHASMMigration captures history.enableCHASMSchedulerMigration and + // history.chasmSchedulerMigrationRolloutPercent to enable automatic CHASM migration. + EnableCHASMMigration bool + // MigrateWithRunningWorkflows captures history.enableCHASMSchedulerMigrationWithRunningWorkflows + // to allow CHASM migration while schedule actions are running. + MigrateWithRunningWorkflows bool } // this is for backwards compatibility; current code serializes cache as proto @@ -215,7 +253,9 @@ var ( ReuseTimer: true, NextTimeCacheV2Size: 14, // see note below SpecFieldLengthLimit: 10, - Version: TriggerImmediatelyTimestamp, + // TODO: bump to MigrationHandoffFixes (13) in a follow-up deploy to + // activate both v13 fixes at once. + Version: TriggerImmediatelyTimestamp, } // Note on NextTimeCacheV2Size: This value must be > FutureActionCountForList. Each @@ -228,17 +268,35 @@ var ( errUpdateConflict = errors.New("conflicting concurrent update") ) +type schedulerDynamicConfig struct { + enableCHASMMigration func() bool + migrateWithRunningWorkflows func() bool + versionCeiling func() int + versionOverride func() int +} + func SchedulerWorkflow(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { - disabled := func() bool { return false } dc := dynamicconfig.NewNoopCollection() specBuilder := NewSpecBuilder( dynamicconfig.SchedulerSpecWarnIterations.Get(dc), dynamicconfig.SchedulerSpecMaxIterations.Get(dc), ) - return schedulerWorkflowWithSpecBuilder(ctx, args, specBuilder, disabled, disabled) + return schedulerWorkflowWithSpecBuilder(ctx, args, specBuilder, schedulerDynamicConfig{}) } -func schedulerWorkflowWithSpecBuilder(ctx workflow.Context, args *schedulespb.StartScheduleArgs, specBuilder *SpecBuilder, enableCHASMMigration func() bool, migrateWithRunningWorkflows func() bool) error { +func schedulerWorkflowWithSpecBuilder(ctx workflow.Context, args *schedulespb.StartScheduleArgs, specBuilder *SpecBuilder, config schedulerDynamicConfig) error { + if config.enableCHASMMigration == nil { + config.enableCHASMMigration = func() bool { return false } + } + if config.migrateWithRunningWorkflows == nil { + config.migrateWithRunningWorkflows = func() bool { return false } + } + if config.versionCeiling == nil { + config.versionCeiling = func() int { return -1 } + } + if config.versionOverride == nil { + config.versionOverride = func() int { return -1 } + } scheduler := &scheduler{ StartScheduleArgs: args, ctx: ctx, @@ -249,8 +307,10 @@ func schedulerWorkflowWithSpecBuilder(ctx workflow.Context, args *schedulespb.St metrics.ScheduleBackendTag: metrics.ScheduleBackendLegacy, }), specBuilder: specBuilder, - enableCHASMMigration: enableCHASMMigration, - migrateWithRunningWorkflows: migrateWithRunningWorkflows, + enableCHASMMigration: config.enableCHASMMigration, + migrateWithRunningWorkflows: config.migrateWithRunningWorkflows, + versionCeiling: config.versionCeiling, + versionOverride: config.versionOverride, } return scheduler.run() } @@ -334,11 +394,29 @@ func (s *scheduler) run() error { ) } + if s.hasMinVersion(MigrationHandoffFixes) && + s.tweakables.EnableCHASMMigration && !s.State.PendingMigration && + s.State.NeedRefresh { + s.refreshWorkflows(slices.Clone(s.Info.RunningWorkflows)) + s.State.NeedRefresh = false + } + + // if there's been a rollback, turn off the pending migration flag + if s.hasMinVersion(MigrationHandoffFixes) && s.State.PendingMigration { + if !s.tweakables.EnableCHASMMigration { + s.State.PendingMigration = false + } + } + if !s.State.PendingMigration && s.tweakables.EnableCHASMMigration && (s.tweakables.MigrateWithRunningWorkflows || len(s.Info.RunningWorkflows) == 0) { s.State.PendingMigration = true } - if s.State.PendingMigration { + // CHASM migration markers are only safe to write at or after version TriggerImmediatelyTimestamp. + // A clamped run defers migration without dropping it: PendingMigration stays set, and the migration runs once the ceiling is lifted. + if s.State.PendingMigration && !s.hasMinVersion(TriggerImmediatelyTimestamp) { + s.logger.Debug("deferring schedule migration to CHASM: recorded version is clamped below migration support") + } else if s.State.PendingMigration { err := s.executeMigration() if err == nil { s.logger.Info("schedule migration to CHASM succeeded", @@ -602,6 +680,31 @@ func (s *scheduler) fillNextTimeCacheV2(start time.Time) { // Run this logic in a SideEffect so that we can fix bugs there without breaking // existing schedule workflows. val := workflow.SideEffect(s.ctx, func(ctx workflow.Context) any { + // Versions before UpdateFromPrevious (6) recorded this cache as JSON. + // When clamped below 6, keep writing JSON to stay replayable by those versions. + if !s.hasMinVersion(UpdateFromPrevious) { + cache := jsonNextTimeCacheV2{Version: s.tweakables.Version, Start: start} + for t := start; len(cache.Results) < s.tweakables.NextTimeCacheV2Size; { + next, err := s.cspec.GetNextTime(s.jitterSeed(), t) + if errors.Is(err, ErrComputeLimitExceeded) { + s.metrics.Counter(metrics.ScheduleComputeLimitExceeded.Name()).Inc(1) + s.logger.Warn("schedule spec next-time search hit the compute limit; taking no further action until the spec is changed") + cache.Completed = true + break + } + if next.ComputeLimitWarning { + s.metrics.Counter(metrics.ScheduleComputeLimitWarning.Name()).Inc(1) + s.logger.Warn("schedule spec next-time search crossed the warn threshold; continuing (spec may be over-excluded)") + } + if next.Next.IsZero() { + cache.Completed = true + break + } + cache.Results = append(cache.Results, next) + t = next.Next + } + return cache + } cache := &schedulespb.NextTimeCache{ Version: int64(s.tweakables.Version), StartTime: timestamppb.New(start), @@ -960,8 +1063,22 @@ func (s *scheduler) processWatcherResult(id string, f workflow.Future, long bool // starting the next). The legacy path doesn't use deferred starts // (Attempt == -1) like CHASM, so BufferedStarts[0] is always the next // pending start. - if long && len(s.State.BufferedStarts) > 0 { - s.State.BufferedStarts[0].DesiredTime = res.CloseTime + // + // Branched by version explicitly (rather than folding hasMinVersion into + // shouldBackdateDesiredTime) so replay of histories recorded below + // RefreshCompletionDesiredTime keeps taking the old, unconditional-on-long-poll-only path + // byte-for-byte -- a mixed-fleet rolling deploy of this very change must not let a + // newer binary recompute a different DesiredTime, and therefore a different + // continue-as-new Input, than what an older binary already recorded for the same history. + if len(s.State.BufferedStarts) > 0 { + next := s.State.BufferedStarts[0] + if s.hasMinVersion(RefreshCompletionDesiredTime) { + if shouldBackdateDesiredTime(long, res.CloseTime, next.DesiredTime, next.ActualTime, s.resolveOverlapPolicy(next.OverlapPolicy)) { + next.DesiredTime = res.CloseTime + } + } else if long { + next.DesiredTime = res.CloseTime + } } s.logger.Debug("started workflow finished", "workflow", id, "status", res.Status, "pause-after-failure", pauseOnFailure, "long", long) @@ -1009,7 +1126,11 @@ func (s *scheduler) processUpdate(req *schedulespb.FullUpdateRequest) { s.ensureFields() s.compileSpec() - s.updateCustomSearchAttributes(req.SearchAttributes) + // Gated at UpdateFromPrevious (6): the search-attribute upsert shipped in v1.24.0 + // (https://github.com/temporalio/temporal/pull/5632). + if s.hasMinVersion(UpdateFromPrevious) { + s.updateCustomSearchAttributes(req.SearchAttributes) + } // Record customer start workflow memo payload size on each update. s.recordActionPayloadMetrics() @@ -1319,9 +1440,13 @@ func (s *scheduler) updateTweakables() { // Use MutableSideEffect so that we can change the defaults without breaking determinism. get := func(ctx workflow.Context) any { p := CurrentTweakablePolicies - // Re-evaluates migration dynamic config each iteration. - p.EnableCHASMMigration = s.enableCHASMMigration() - p.MigrateWithRunningWorkflows = s.migrateWithRunningWorkflows() + p.Version, p.VersionCeiling = s.determineVersion(p.Version) + p.VersionCeilingSet = true + // Only set migration config at/after the TriggerImmediatelyTimestamp version. + if p.Version >= TriggerImmediatelyTimestamp { + p.EnableCHASMMigration = s.enableCHASMMigration() + p.MigrateWithRunningWorkflows = s.migrateWithRunningWorkflows() + } return p } eq := func(a, b any) bool { return a.(TweakablePolicies) == b.(TweakablePolicies) } @@ -1351,6 +1476,45 @@ func (s *scheduler) resolveOverlapPolicy(overlapPolicy enumspb.ScheduleOverlapPo return overlapPolicy } +// shouldBackdateDesiredTime reports whether a watcher result discovering a workflow's completion +// should backdate the next buffered start's DesiredTime to that completion's CloseTime, for the +// purpose of computing ScheduleActionDelay. +// +// The long-poll path predates this check and stays unconditional: it always backdates, since a +// long-poll watcher is only ever installed to wait for the exact workflow the next buffered start +// is blocked on. +// +// The refresh path is conditional: a refresh can discover a completion long after the fact, once +// an unrelated, later buffered start already exists. Only backdate when the close genuinely came +// after the next start's own due time (i.e. it was actually blocked waiting on this workflow, +// e.g. under BUFFER_ONE/BUFFER_ALL/CANCEL_OTHER/TERMINATE_OTHER) and the next start's own resolved +// overlap policy actually waits for a running workflow to finish. A start whose resolved policy +// ignores running workflows (ALLOW_ALL, see IgnoresRunningWorkflow) is never blocked by this +// close, no matter the timing, since processBuffer starts it regardless of isRunning -- and +// backdating would understate its real delay. +// +// refreshWorkflows calls this once per tracked execution, so a run with multiple RunningWorkflows +// (e.g. ALLOW_ALL runs inherited from before a pre-DontTrackOverlapping version ceiling was +// lifted) can process several results against the same buffered start in one pass. On the refresh +// path, only move currentDesiredTime forward -- never let a later-processed but earlier-closing +// execution overwrite a genuinely later close already recorded this pass, or the start's real +// blocked duration would be understated. +func shouldBackdateDesiredTime( + long bool, + closeTime *timestamppb.Timestamp, + currentDesiredTime *timestamppb.Timestamp, + nextActualTime *timestamppb.Timestamp, + nextResolvedOverlapPolicy enumspb.ScheduleOverlapPolicy, +) bool { + if long { + return true + } + return closeTime != nil && + closeTime.AsTime().After(nextActualTime.AsTime()) && + !IgnoresRunningWorkflow(nextResolvedOverlapPolicy) && + (currentDesiredTime == nil || closeTime.AsTime().After(currentDesiredTime.AsTime())) +} + func (s *scheduler) addStart(nominalTime, actualTime time.Time, overlapPolicy enumspb.ScheduleOverlapPolicy, manual bool) { s.logger.Debug("addStart", "start-time", nominalTime, "actual-start-time", actualTime, "overlap-policy", overlapPolicy, "manual", manual) if s.tweakables.MaxBufferSize > 0 && len(s.State.BufferedStarts) >= s.tweakables.MaxBufferSize { @@ -1473,8 +1637,15 @@ func (s *scheduler) startWorkflow( newWorkflow *workflowpb.NewWorkflowExecutionInfo, ) (*schedulepb.ScheduleActionResult, error) { nominalTimeSec := start.NominalTime.AsTime().UTC().Truncate(time.Second) - workflowID := newWorkflow.WorkflowId - if start.OverlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL || s.tweakables.AlwaysAppendTimestamp { + workflowID := "" + if s.hasMinVersion(MigrationHandoffFixes) { + workflowID = start.WorkflowId + } + if workflowID == "" { + workflowID = newWorkflow.WorkflowId + } + if (!s.hasMinVersion(MigrationHandoffFixes) || start.WorkflowId == "") && + (start.OverlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL || s.tweakables.AlwaysAppendTimestamp) { // must match AppendedTimestampForValidation workflowID += "-" + nominalTimeSec.Format(time.RFC3339) } @@ -1510,6 +1681,13 @@ func (s *scheduler) startWorkflow( reusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE } + requestID := "" + if s.hasMinVersion(MigrationHandoffFixes) { + requestID = start.RequestId + } + if requestID == "" { + requestID = s.newUUIDString() + } req := &schedulespb.StartWorkflowRequest{ Request: &workflowservice.StartWorkflowExecutionRequest{ WorkflowId: workflowID, @@ -1520,7 +1698,7 @@ func (s *scheduler) startWorkflow( WorkflowRunTimeout: newWorkflow.WorkflowRunTimeout, WorkflowTaskTimeout: newWorkflow.WorkflowTaskTimeout, Identity: s.identity(), - RequestId: s.newUUIDString(), + RequestId: requestID, WorkflowIdReusePolicy: reusePolicy, RetryPolicy: newWorkflow.RetryPolicy, Memo: newWorkflow.Memo, @@ -1723,18 +1901,76 @@ func (s *scheduler) hasMinVersion(version SchedulerWorkflowVersion) bool { return s.tweakables.Version >= version } +// determineVersion returns this iteration's effective version and raw ceiling. The version cannot +// decrease, but the latest worker.schedulerV1VersionCeiling namespace dynamic-config value is +// applied on each iteration. The raw ceiling is captured in tweakables even when the effective +// version is retained. +func (s *scheduler) determineVersion(defaultVersion SchedulerWorkflowVersion) (SchedulerWorkflowVersion, int) { + ceiling := s.versionCeiling() + override := s.versionOverride() + validOverride := override >= int(defaultVersion) && override <= int(LatestSchedulerWorkflowVersion) + if shouldWarnForVersionCeiling(s.tweakables, resolveVersionBeforeCeiling(defaultVersion, override), ceiling) { + s.logger.Warn("worker.schedulerV1VersionCeiling above the version this binary records; no effect", + "ceiling", ceiling, "recordedVersion", defaultVersion) + } + if override >= 0 && !validOverride { + s.logger.Warn("worker.schedulerV1VersionOverride is outside the supported range; ignored", + "override", override, "defaultVersion", defaultVersion, "latestSupportedVersion", LatestSchedulerWorkflowVersion) + } + return determineVersionTransition(defaultVersion, s.tweakables.Version, ceiling, override) +} + +func shouldWarnForVersionCeiling(tweakables TweakablePolicies, defaultVersion SchedulerWorkflowVersion, ceiling int) bool { + return ceiling > int(defaultVersion) && (!tweakables.VersionCeilingSet || ceiling != tweakables.VersionCeiling) +} + +// determineVersionTransition applies ceiling to defaultVersion, then retains recordedVersion when it +// is higher. This allows a raised ceiling to advance a run without allowing a lowered ceiling to +// downgrade a version already recorded in history. It returns that effective version and the raw ceiling. +func determineVersionTransition(defaultVersion, recordedVersion SchedulerWorkflowVersion, ceiling, override int) (SchedulerWorkflowVersion, int) { + return max(resolveVersion(defaultVersion, ceiling, override), recordedVersion), ceiling +} + +// clampVersion lowers v to ceiling. A negative ceiling is unset (no clamp); a ceiling at or above +// the highest version leaves v unchanged. +func clampVersion(v SchedulerWorkflowVersion, ceiling int) SchedulerWorkflowVersion { + if ceiling < 0 { + return v + } + return min(v, SchedulerWorkflowVersion(ceiling)) +} + +// resolveVersion applies a valid override, then lowers the result to the ceiling. +func resolveVersion(v SchedulerWorkflowVersion, ceiling, override int) SchedulerWorkflowVersion { + return clampVersion(resolveVersionBeforeCeiling(v, override), ceiling) +} + +func resolveVersionBeforeCeiling(v SchedulerWorkflowVersion, override int) SchedulerWorkflowVersion { + if override >= int(v) && override <= int(LatestSchedulerWorkflowVersion) { + return SchedulerWorkflowVersion(override) + } + return v +} + func panicIfErr(err error) { if err != nil { panic(err) } } -func GetListInfoFromStartArgs(args *schedulespb.StartScheduleArgs, now time.Time, specBuilder *SpecBuilder) *schedulepb.ScheduleListInfo { +type VersionSelection struct { + Ceiling int + Override int +} + +func GetListInfoFromStartArgs(args *schedulespb.StartScheduleArgs, now time.Time, specBuilder *SpecBuilder, selection VersionSelection) *schedulepb.ScheduleListInfo { // Create a scheduler outside of workflow context with just the fields we need to call // getListInfo. Note that this does not take into account InitialPatch. + tweakables := CurrentTweakablePolicies + tweakables.Version = resolveVersion(tweakables.Version, selection.Ceiling, selection.Override) s := &scheduler{ StartScheduleArgs: args, - tweakables: CurrentTweakablePolicies, + tweakables: tweakables, specBuilder: specBuilder, } s.ensureFields() diff --git a/service/worker/scheduler/workflow_test.go b/service/worker/scheduler/workflow_test.go index 0207d827109..2dc8f04fad1 100644 --- a/service/worker/scheduler/workflow_test.go +++ b/service/worker/scheduler/workflow_test.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -17,10 +18,12 @@ import ( sdkpb "go.temporal.io/api/sdk/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" workflowpb "go.temporal.io/api/workflow/v1" + sdkclient "go.temporal.io/sdk/client" "go.temporal.io/sdk/testsuite" "go.temporal.io/sdk/workflow" schedulespb "go.temporal.io/server/api/schedule/v1" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/searchattribute/sadefs" @@ -33,7 +36,24 @@ type ( workflowSuite struct { suite.Suite testsuite.WorkflowTestSuite - env *testsuite.TestWorkflowEnvironment + env *testsuite.TestWorkflowEnvironment + metrics *capturingMetricsHandler + } + + capturingMetricsHandler struct { + timers []capturedTimer + } + + capturedTimer struct { + name string + value time.Duration + } + + noopMetricsCounter struct{} + noopMetricsGauge struct{} + capturingMetricsTimer struct { + handler *capturingMetricsHandler + name string } ) @@ -46,9 +66,45 @@ func TestWorkflow(t *testing.T) { } func (s *workflowSuite) SetupTest() { + s.metrics = &capturingMetricsHandler{} + s.SetMetricsHandler(s.metrics) s.env = s.NewTestWorkflowEnvironment() } +func (h *capturingMetricsHandler) WithTags(map[string]string) sdkclient.MetricsHandler { + return h +} + +func (h *capturingMetricsHandler) Counter(string) sdkclient.MetricsCounter { + return noopMetricsCounter{} +} + +func (h *capturingMetricsHandler) Gauge(string) sdkclient.MetricsGauge { + return noopMetricsGauge{} +} + +func (h *capturingMetricsHandler) Timer(name string) sdkclient.MetricsTimer { + return capturingMetricsTimer{handler: h, name: name} +} + +func (h *capturingMetricsHandler) timerValues(name string) []time.Duration { + var values []time.Duration + for _, timer := range h.timers { + if timer.name == name { + values = append(values, timer.value) + } + } + return values +} + +func (noopMetricsCounter) Inc(int64) {} + +func (noopMetricsGauge) Update(float64) {} + +func (t capturingMetricsTimer) Record(value time.Duration) { + t.handler.timers = append(t.handler.timers, capturedTimer{name: t.name, value: value}) +} + func (s *workflowSuite) AfterTest(suiteName, testName string) { s.env.AssertExpectations(s.T()) } @@ -84,6 +140,10 @@ func (s *workflowSuite) defaultAction(id string) *schedulepb.ScheduleAction { func (s *workflowSuite) run(sched *schedulepb.Schedule, iterations int) { // test workflows will run until "completion", in our case that means until // continue-as-new. we only need a small number of iterations to test, though. + s.runWorkflowFn(SchedulerWorkflow, sched, iterations) +} + +func (s *workflowSuite) runWorkflowFn(wf any, sched *schedulepb.Schedule, iterations int) { CurrentTweakablePolicies.IterationsBeforeContinueAsNew = iterations // fixed start time @@ -94,7 +154,7 @@ func (s *workflowSuite) run(sched *schedulepb.Schedule, iterations int) { sched.Action = s.defaultAction("myid") } - s.env.ExecuteWorkflow(SchedulerWorkflow, &schedulespb.StartScheduleArgs{ + s.env.ExecuteWorkflow(wf, &schedulespb.StartScheduleArgs{ Schedule: sched, State: &schedulespb.InternalState{ Namespace: "myns", @@ -332,6 +392,7 @@ func (s *workflowSuite) TestStart() { s.Nil(req.Request.LastCompletionResult) s.Nil(req.Request.ContinuedFailure) s.Equal("myid-2022-06-01T00:15:00Z", req.Request.WorkflowId) + s.NotEmpty(req.Request.RequestId) s.Equal("mywf", req.Request.WorkflowType.Name) s.Equal("mytq", req.Request.TaskQueue.Name) s.Equal(`"value"`, payload.ToString(req.Request.Memo.Fields["mymemo"])) @@ -356,6 +417,100 @@ func (s *workflowSuite) TestStart() { s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) } +func (s *workflowSuite) TestMigratedBufferedStartPreservesIdempotencyIDs() { + // MigrationHandoffFixes isn't yet the shipped CurrentTweakablePolicies.Version + // (see the TODO on CurrentTweakablePolicies in workflow.go), so force it here to + // exercise the branch regardless of the current rollout state. + prevVersion := CurrentTweakablePolicies.Version + CurrentTweakablePolicies.Version = MigrationHandoffFixes + defer func() { CurrentTweakablePolicies.Version = prevVersion }() + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("migrated-workflow-id", req.Request.WorkflowId) + s.Equal("migrated-request-id", req.Request.RequestId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, s.migratedStartScheduleArgs()) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +func (s *workflowSuite) TestMigratedBufferedStartUsesLegacyIDsAtOldVersion() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = TriggerImmediatelyTimestamp + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("configured-workflow-id-2022-06-01T00:00:00Z", req.Request.WorkflowId) + s.NotEmpty(req.Request.RequestId) + s.NotEqual("migrated-request-id", req.Request.RequestId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, s.migratedStartScheduleArgs()) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +func (s *workflowSuite) TestNativeBufferedStartFallsBackAtNewVersion() { + // Complement to TestMigratedBufferedStart*: a schedule that never went through + // CHASM has BufferedStart.WorkflowId/RequestId empty (the common case -- see + // the proto comment on those fields), even once MigrationHandoffFixes is + // active. Guards against the migrated-ID branch swallowing the native-start + // fallback (legacy generated workflow ID, including the AlwaysAppendTimestamp + // suffix, and a freshly generated request ID). + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("myid-2022-06-01T00:15:00Z", req.Request.WorkflowId) + s.NotEmpty(req.Request.RequestId) + return nil, nil + }) + + s.run(&schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(55 * time.Minute), + }}, + }, + }, 2) + // two iterations to start one workflow: first will sleep, second will start and then sleep again + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +func (s *workflowSuite) migratedStartScheduleArgs() *schedulespb.StartScheduleArgs { + return &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}, + }, + Action: s.defaultAction("configured-workflow-id"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + BufferedStarts: []*schedulespb.BufferedStart{{ + NominalTime: timestamppb.New(baseStartTime), + ActualTime: timestamppb.New(baseStartTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + Manual: true, + RequestId: "migrated-request-id", + WorkflowId: "migrated-workflow-id", + }}, + }, + } +} + func (s *workflowSuite) TestInitialPatch() { // written using low-level mocks so we can set initial patch @@ -368,7 +523,10 @@ func (s *workflowSuite) TestInitialPatch() { s.True(time.Date(2022, 6, 1, 0, 15, 0, 0, time.UTC).Equal(s.now())) s.Equal("myid-2022-06-01T00:00:00Z", req.Execution.WorkflowId) s.False(req.LongPoll) - return &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED}, nil + return &schedulespb.WatchWorkflowResponse{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(baseStartTime.Add(10 * time.Minute)), + }, nil }) s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { s.True(time.Date(2022, 6, 1, 0, 15, 0, 0, time.UTC).Equal(s.now())) @@ -399,6 +557,329 @@ func (s *workflowSuite) TestInitialPatch() { }) s.True(s.env.IsWorkflowCompleted()) s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) + // The prior action closed (00:10) before the next start was even due (00:15), so the next + // start ran exactly on time -- it was never blocked waiting on the prior action. DesiredTime + // must stay at ActualTime, reporting zero delay, not the idle gap since the prior close. + s.Equal([]time.Duration{0}, s.metrics.timerValues(metrics.ScheduleActionDelay.Name())) +} + +// TestRefreshCompletionDesiredTimeBacklog verifies the actual regression this fix targets: a +// refresh discovering that the prior action closed AFTER the next buffered start's own due time +// -- i.e. that start was genuinely blocked waiting on it -- backdates DesiredTime, so +// ScheduleActionDelay measures from the real close instead of the start's original scheduled +// time. +// +// A long-poll watcher gets installed as soon as a start is buffered behind a still-running +// action, so that watcher (not a later refresh) is what normally discovers a genuine backlog's +// completion. To exercise the refresh path specifically, this test never lets the long-poll +// watcher resolve; discovery instead comes from an explicit refresh signal, mirroring the +// watcher-interruption/worker-restart case described where refreshWorkflows finds a completion +// the long-poll watcher missed. +func (s *workflowSuite) TestRefreshCompletionDesiredTimeBacklog() { + prevVersion := CurrentTweakablePolicies.Version + CurrentTweakablePolicies.Version = RefreshCompletionDesiredTime + defer func() { CurrentTweakablePolicies.Version = prevVersion }() + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 0, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Request.WorkflowId) + return nil, nil + }) + // The first scheduled tick (:15) buffers behind the still-running manual action. + s.expectWatch(func(req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 15, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Execution.WorkflowId) + s.False(req.LongPoll) + return &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING}, nil + }) + // A long-poll watcher is installed right after. Resolve it well after the explicit refresh + // below is expected to have already discovered completion and started the buffered action, + // so discovery here is unambiguously via refresh; this resolution becomes a harmless, + // already-a-no-op duplicate (BufferedStarts is empty and the action is no longer running by + // then). + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.MatchedBy(func(req *schedulespb.WatchWorkflowRequest) bool { + return req.LongPoll + })).Once().AfterFn(func() time.Duration { + return 20 * time.Minute + }).Return(func(ctx context.Context, req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + return &schedulespb.WatchWorkflowResponse{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(baseStartTime.Add(20 * time.Minute)), + }, nil + }) + // An explicit refresh signal at :25 discovers the manual action actually closed at :20 -- + // after the buffered start's own due time of :15, i.e. genuinely blocked. + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 25*time.Minute) + s.expectWatch(func(req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 25, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Execution.WorkflowId) + s.False(req.LongPoll) + return &schedulespb.WatchWorkflowResponse{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(baseStartTime.Add(20 * time.Minute)), + }, nil + }) + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 25, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:15:00Z", req.Request.WorkflowId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 3 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(15 * time.Minute), + }}, + }, + Action: s.defaultAction("myid"), + Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + InitialPatch: &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{}, + }, + }) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) + // Blocked from :15 (its own due time) until :20 (when the prior action actually closed): + // zero delay here would hide the very bug this test exists to catch. + s.Equal([]time.Duration{5 * time.Minute}, s.metrics.timerValues(metrics.ScheduleActionDelay.Name())) +} + +// TestRefreshCompletionDesiredTimeBacklogAtOldVersion pins the pre-fix behavior for histories +// recorded below RefreshCompletionDesiredTime: it runs the exact same genuinely-blocked scenario +// as TestRefreshCompletionDesiredTimeBacklog above (refresh discovers a close after the buffered +// start's own due time), but at the old version, where the refresh path must stay a no-op even +// though the close time would otherwise qualify for backdating. This is the case the version gate +// exists to protect: without it, a mixed-fleet rolling deploy of this fix could have an old binary +// record a continue-as-new Input with DesiredTime unset, and a new binary later replay that same +// history and compute DesiredTime differently, breaking determinism. +func (s *workflowSuite) TestRefreshCompletionDesiredTimeBacklogAtOldVersion() { + prevVersion := CurrentTweakablePolicies.Version + CurrentTweakablePolicies.Version = TriggerImmediatelyTimestamp + defer func() { CurrentTweakablePolicies.Version = prevVersion }() + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 0, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Request.WorkflowId) + return nil, nil + }) + s.expectWatch(func(req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 15, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Execution.WorkflowId) + s.False(req.LongPoll) + return &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING}, nil + }) + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.MatchedBy(func(req *schedulespb.WatchWorkflowRequest) bool { + return req.LongPoll + })).Once().AfterFn(func() time.Duration { + return 20 * time.Minute + }).Return(func(ctx context.Context, req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + return &schedulespb.WatchWorkflowResponse{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(baseStartTime.Add(20 * time.Minute)), + }, nil + }) + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 25*time.Minute) + s.expectWatch(func(req *schedulespb.WatchWorkflowRequest) (*schedulespb.WatchWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 25, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:00:00Z", req.Execution.WorkflowId) + s.False(req.LongPoll) + return &schedulespb.WatchWorkflowResponse{ + Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED, + CloseTime: timestamppb.New(baseStartTime.Add(20 * time.Minute)), + }, nil + }) + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.True(time.Date(2022, 6, 1, 0, 25, 0, 0, time.UTC).Equal(s.now())) + s.Equal("myid-2022-06-01T00:15:00Z", req.Request.WorkflowId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 3 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(15 * time.Minute), + }}, + }, + Action: s.defaultAction("myid"), + Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + InitialPatch: &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{}, + }, + }) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) + // Falls back to the scheduled (actual) time, not the prior action's close time: due at :15, + // started at :25, a 10 minute delay -- the pre-fix baseline. If this ever comes back 5 minutes + // (the fixed value from the sibling test above), the version gate has stopped doing its job. + s.Equal([]time.Duration{10 * time.Minute}, s.metrics.timerValues(metrics.ScheduleActionDelay.Name())) +} + +func TestShouldBackdateDesiredTime(t *testing.T) { + base := baseStartTime + dueAt15 := timestamppb.New(base.Add(15 * time.Minute)) + + dueAt10 := timestamppb.New(base.Add(10 * time.Minute)) + dueAt20 := timestamppb.New(base.Add(20 * time.Minute)) + dueAt25 := timestamppb.New(base.Add(25 * time.Minute)) + + testCases := []struct { + name string + long bool + closeTime *timestamppb.Timestamp + currentDesiredTime *timestamppb.Timestamp + nextActualTime *timestamppb.Timestamp + nextResolvedOverlapPolicy enumspb.ScheduleOverlapPolicy + want bool + }{ + { + name: "long poll always backdates regardless of policy or timing", + long: true, + // close before due and ALLOW_ALL would both independently forbid backdating on the + // refresh path; the long-poll path predates and ignores both. + closeTime: timestamppb.New(base.Add(10 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + want: true, + }, + { + name: "refresh with nil close time never backdates", + long: false, + closeTime: nil, + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + want: false, + }, + { + name: "refresh with close before due time does not backdate: start ran on time", + long: false, + closeTime: timestamppb.New(base.Add(10 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + want: false, + }, + { + name: "refresh with close after due time backdates under BUFFER_ONE: genuinely blocked", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + want: true, + }, + { + name: "refresh with close after due time backdates under BUFFER_ALL", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: true, + }, + { + name: "refresh with close after due time backdates under CANCEL_OTHER", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, + want: true, + }, + { + name: "refresh with close after due time backdates under TERMINATE_OTHER", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, + want: true, + }, + { + name: "refresh with close after due time backdates under SKIP", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + want: true, + }, + { + // The regression this test guards against: a schedule policy change to ALLOW_ALL + // means processBuffer starts this entry regardless of isRunning, so it was never + // blocked on the workflow whose close the refresh just discovered. Backdating here + // would understate schedule_action_delay by attributing an unrelated close time. + name: "refresh with close after due time does NOT backdate under ALLOW_ALL", + long: false, + closeTime: timestamppb.New(base.Add(20 * time.Minute)), + nextActualTime: dueAt15, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + want: false, + }, + { + // refreshWorkflows calls this once per tracked RunningWorkflows execution. If a + // later-processed result closed earlier than one already recorded this pass, it must + // not overwrite the later (maximum) close time with an earlier one. + name: "refresh does NOT overwrite a later already-recorded close time with an earlier one", + long: false, + closeTime: dueAt20, + currentDesiredTime: dueAt25, + nextActualTime: dueAt10, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: false, + }, + { + name: "refresh DOES move the close time forward when the new one is later", + long: false, + closeTime: dueAt25, + currentDesiredTime: dueAt20, + nextActualTime: dueAt10, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: true, + }, + { + name: "refresh with no prior desired time set backdates normally", + long: false, + closeTime: dueAt20, + currentDesiredTime: nil, + nextActualTime: dueAt10, + nextResolvedOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, + want: true, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := shouldBackdateDesiredTime( + tc.long, + tc.closeTime, + tc.currentDesiredTime, + tc.nextActualTime, + tc.nextResolvedOverlapPolicy, + ) + require.Equal(t, tc.want, got) + }) + } } func (s *workflowSuite) TestCatchupWindow() { @@ -2283,7 +2764,7 @@ func (s *workflowSuite) TestMigrateSuccess() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), - func() bool { return enableMigration }, func() bool { return true }) + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2306,6 +2787,146 @@ func (s *workflowSuite) TestMigrateSuccess() { s.NoError(s.env.GetWorkflowError()) } +func (s *workflowSuite) TestAutoMigrateReconcilesRunningWorkflowBeforeCheck() { + // The early-refresh behavior is gated on MigrationHandoffFixes, which is + // intentionally NOT yet the shipped CurrentTweakablePolicies.Version (it is + // activated in a follow-up deploy for rollback safety -- see the TODO on + // CurrentTweakablePolicies in workflow.go). Force the version here so this + // guard exercises the branch regardless of the current rollout state. + prevVersion := CurrentTweakablePolicies.Version + CurrentTweakablePolicies.Version = MigrationHandoffFixes + defer func() { CurrentTweakablePolicies.Version = prevVersion }() + + staleWID := "myid-2022-06-01T00:00:00Z" + + // The refresh watcher reports the stale workflow as already completed. + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.Anything).Return( + &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED}, nil) + + // No action should ever be started: migration must happen during the idle + // window before the first action fires. + s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Times(0).Maybe().Return( + func(_ context.Context, req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Failf("unexpected start", "for %s at %s", req.Request.WorkflowId, s.now()) + return nil, nil + }) + + var migratedAt time.Time + migrated := false + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Once().Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrated = true + migratedAt = s.now() + return nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 100 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + // enableCHASMMigration=true, migrateWithRunningWorkflows=false (guard on). + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return false }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(1 * time.Hour), + }}, + }, + Action: s.defaultAction("myid"), + }, + Info: &schedulepb.ScheduleInfo{ + RunningWorkflows: []*commonpb.WorkflowExecution{{WorkflowId: staleWID}}, + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + LastProcessedTime: timestamppb.New(baseStartTime), + // Mimics processTimeRange having just buffered an action: production + // sets NeedRefresh in that path, and it is what previously ran only + // inside processBuffer, after the eligibility check. + NeedRefresh: true, + }, + }) + + s.True(s.env.IsWorkflowCompleted()) + s.Require().NoError(s.env.GetWorkflowError(), "schedule should migrate (complete), not defer/CAN") + s.True(migrated, "MigrateScheduleToChasm should have been called via auto-eligibility") + s.True(migratedAt.Before(baseStartTime.Add(time.Hour)), + "migration should occur in the idle window before the first action fires, at %s", migratedAt) +} + +// TestAutoMigrateStaysDeferredAtOldVersionWhileBusy is the old-version +// counterpart to TestAutoMigrateReconcilesRunningWorkflowBeforeCheck: identical +// setup (a stale RunningWorkflows entry plus NeedRefresh), but run at the +// version that predates MigrationHandoffFixes. It pins the bug the fix +// addresses: the eligibility check reads len(RunningWorkflows) before that same +// iteration's processBuffer() call reconciles it via NeedRefresh, so migration +// is deferred during what would otherwise be the idle window -- even though the +// "running" workflow has already completed. Without the fix, this schedule +// continues-as-new instead of migrating. +func (s *workflowSuite) TestAutoMigrateStaysDeferredAtOldVersionWhileBusy() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = TriggerImmediatelyTimestamp + + staleWID := "myid-2022-06-01T00:00:00Z" + + // The refresh watcher reports the stale workflow as already completed. + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.Anything).Return( + &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED}, nil) + + s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Times(0).Maybe().Return( + func(_ context.Context, req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Failf("unexpected start", "for %s at %s", req.Request.WorkflowId, s.now()) + return nil, nil + }) + + // At the pre-fix version, the eligibility check still sees the stale (not + // yet reconciled) RunningWorkflows entry, so migration must not fire in this + // iteration. + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Times(0).Maybe().Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + s.Fail("migration should not run at the pre-fix version while RunningWorkflows looks busy") + return nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + // enableCHASMMigration=true, migrateWithRunningWorkflows=false (guard on) -- + // same knobs as TestAutoMigrateReconcilesRunningWorkflowBeforeCheck. + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return false }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(1 * time.Hour), + }}, + }, + Action: s.defaultAction("myid"), + }, + Info: &schedulepb.ScheduleInfo{ + RunningWorkflows: []*commonpb.WorkflowExecution{{WorkflowId: staleWID}}, + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + LastProcessedTime: timestamppb.New(baseStartTime), + NeedRefresh: true, + }, + }) + + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError()), + "schedule should continue-as-new (deferring migration), not complete") +} + func (s *workflowSuite) TestMigrateFailure() { // Mock MigrateSchedule activity to always fail. Migration is retried // each iteration since PendingMigration is persisted in State. @@ -2334,7 +2955,7 @@ func (s *workflowSuite) TestMigrateFailure() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), - func() bool { return enableMigration }, func() bool { return true }) + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2388,7 +3009,7 @@ func (s *workflowSuite) TestMigrateFailureThenRetrySuccess() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), - func() bool { return enableMigration }, func() bool { return true }) + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2450,7 +3071,7 @@ func (s *workflowSuite) TestMigrateFailureThenSignal() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), - func() bool { return enableMigration }, func() bool { return true }) + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2482,6 +3103,83 @@ func (s *workflowSuite) TestMigrateFailureThenSignal() { s.True(canArgs.State.PendingMigration, "PendingMigration should be set in CAN state") } +// TestMigrateRollbackDoesNotBlockScheduleActions verifies the actual +// rollback-safety property: once EnableCHASMSchedulerMigration is rolled back +// mid-flight, the pending migration keeps failing (mirroring the real +// activity's own live disabled-check -- see +// TestMigrateScheduleToChasm_MigrationDisabled), but the V1 schedule itself +// is entirely unaffected -- it keeps firing its own actions on schedule. A +// stuck, perpetually-failing migration must never block the schedule's real +// work. +// +// Pinned pre-v13: from v13 on the reset clears PendingMigration, so migration stops +// retrying entirely (see TestMigrateRollbackClearsPendingMigrationAtNewVersion). +func (s *workflowSuite) TestMigrateRollbackDoesNotBlockScheduleActions() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = TriggerImmediatelyTimestamp + + enableMigration := true + migrateCalls := 0 + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrateCalls++ + if !enableMigration { + // What the real activity returns once its own live + // migrationEnabled() check goes false. + return errors.New("MigrateScheduleToChasm: migration is currently disabled") + } + return errors.New("migration failed") + }) + + startCalls := 0 + s.env.OnActivity(new(activities).StartWorkflow, mock.Anything, mock.Anything).Return( + func(_ context.Context, req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + startCalls++ + return &schedulespb.StartWorkflowResponse{ + RunId: uuid.NewString(), + RealStartTime: timestamppb.New(s.now()), + }, nil + }) + // Report every fired workflow as immediately completed so the default + // SKIP overlap policy never withholds the next scheduled action. + s.env.OnActivity(new(activities).WatchWorkflow, mock.Anything, mock.Anything).Return( + &schedulespb.WatchWorkflowResponse{Status: enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED}, nil) + + // Roll the flag back shortly after start -- well before the schedule's + // own hourly actions fire -- simulating an operator reverting the + // migration switch shortly after a bounced attempt. + s.env.RegisterDelayedCallback(func() { + enableMigration = false + }, 1*time.Minute) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 100 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(1 * time.Hour), + }}, + }, + Action: s.defaultAction("myid"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + }) + + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError()), "schedule should keep running (CAN), not fail or get stuck") + s.Greater(migrateCalls, 1, "migration should keep retrying (and failing) throughout") + s.GreaterOrEqual(startCalls, 3, "schedule should keep firing its own actions on schedule despite the stuck migration") +} + func (s *workflowSuite) TestMigrateDynamicConfig() { // Enable migration by threading enableCHASMMigration=true through the closure (race-safe). // Mock MigrateSchedule activity to succeed. @@ -2493,7 +3191,8 @@ func (s *workflowSuite) TestMigrateDynamicConfig() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { - return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), func() bool { return true }, func() bool { return true }) + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2542,7 +3241,8 @@ func (s *workflowSuite) TestMigrateDynamicConfigFlipsMidRun() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { - return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), func() bool { return enabled }, func() bool { return true }) + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enabled }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2581,7 +3281,8 @@ func (s *workflowSuite) TestMigrateDynamicConfigFailure() { s.env.SetStartTime(baseStartTime) s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { - return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), func() bool { return true }, func() bool { return true }) + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return true }, migrateWithRunningWorkflows: func() bool { return true }}) }, &schedulespb.StartScheduleArgs{ Schedule: &schedulepb.Schedule{ Spec: &schedulepb.ScheduleSpec{ @@ -2660,3 +3361,266 @@ func (s *workflowSuite) TestMigrateDynamicConfigDisabledNoMigration() { s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) s.Zero(migrateCalls) } + +// migratedStartArgs builds StartScheduleArgs with a single BufferedStart, so tests can +// vary just the migration-carried identity fields and the overlap policy. +func (s *workflowSuite) migratedStartArgs(start *schedulespb.BufferedStart) *schedulespb.StartScheduleArgs { + return &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(time.Hour)}}, + }, + Action: s.defaultAction("configured-workflow-id"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + BufferedStarts: []*schedulespb.BufferedStart{start}, + }, + } +} + +// The rollback conversion only fills fields that were empty, so a start can arrive back +// in V1 with a workflow ID but no request ID. The two fields are read independently. +func (s *workflowSuite) TestMigratedBufferedStartKeepsWorkflowIdWithGeneratedRequestId() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("migrated-workflow-id", req.Request.WorkflowId) + s.NotEmpty(req.Request.RequestId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, s.migratedStartArgs(&schedulespb.BufferedStart{ + NominalTime: timestamppb.New(baseStartTime), + ActualTime: timestamppb.New(baseStartTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + Manual: true, + WorkflowId: "migrated-workflow-id", + })) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +// Mirror of the above: request ID preserved, workflow ID falls back to V1's legacy +// derivation. +func (s *workflowSuite) TestMigratedBufferedStartKeepsRequestIdWithGeneratedWorkflowId() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("configured-workflow-id-2022-06-01T00:00:00Z", req.Request.WorkflowId) + s.Equal("migrated-request-id", req.Request.RequestId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, s.migratedStartArgs(&schedulespb.BufferedStart{ + NominalTime: timestamppb.New(baseStartTime), + ActualTime: timestamppb.New(baseStartTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL, + Manual: true, + RequestId: "migrated-request-id", + })) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +// A migrated workflow ID already carries the nominal-time suffix (V2 applies it +// unconditionally), so the suffix must be suppressed for any overlap policy, not just +// ALLOW_ALL, or the ID comes back doubled and no longer dedups against V2's run. +func (s *workflowSuite) TestMigratedBufferedStartSkipsTimestampSuffixForNonAllowAll() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + s.True(CurrentTweakablePolicies.AlwaysAppendTimestamp, + "this test is only meaningful while AlwaysAppendTimestamp is on") + + s.expectStart(func(req *schedulespb.StartWorkflowRequest) (*schedulespb.StartWorkflowResponse, error) { + s.Equal("configured-workflow-id-2022-06-01T00:00:00Z", req.Request.WorkflowId) + s.Equal("migrated-request-id", req.Request.RequestId) + return nil, nil + }) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 1 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(SchedulerWorkflow, s.migratedStartArgs(&schedulespb.BufferedStart{ + NominalTime: timestamppb.New(baseStartTime), + ActualTime: timestamppb.New(baseStartTime), + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, + // Precomputed as V2 would: base ID + truncated nominal time. + WorkflowId: "configured-workflow-id-2022-06-01T00:00:00Z", + RequestId: "migrated-request-id", + })) + s.True(s.env.IsWorkflowCompleted()) + s.True(workflow.IsContinueAsNewError(s.env.GetWorkflowError())) +} + +// Once migration is rolled back with a migration still pending, the pending flag must be +// dropped so the schedule stops retrying instead of migrating late. +func (s *workflowSuite) TestMigrateRollbackClearsPendingMigrationAtNewVersion() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + + enableMigration := true + migrateCalls := 0 + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrateCalls++ + return errors.New("migration failed") + }) + + // Roll the flag back, then wake the workflow so it re-reads tweakables. + s.env.RegisterDelayedCallback(func() { + enableMigration = false + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 1*time.Second) + // Wake it once more, to prove no further migration attempt is made. + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 5*time.Second) + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameForceCAN, nil) + }, 10*time.Second) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 100 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(1 * time.Hour)}}, + }, + Action: s.defaultAction("myid"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + }) + + s.Equal(1, migrateCalls, "migration must not be retried after the rollback") + + var canErr *workflow.ContinueAsNewError + s.Require().ErrorAs(s.env.GetWorkflowError(), &canErr) + var canArgs schedulespb.StartScheduleArgs + s.Require().NoError(payloads.Decode(canErr.Input, &canArgs)) + s.False(canArgs.State.PendingMigration, "PendingMigration should be cleared after rollback") +} + +// Pre-v13 counterpart: the reset is version-gated, so an older recorded version must +// keep retrying and keep the flag across continue-as-new. +func (s *workflowSuite) TestMigrateRollbackKeepsPendingMigrationAtOldVersion() { + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = TriggerImmediatelyTimestamp + + enableMigration := true + migrateCalls := 0 + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrateCalls++ + return errors.New("migration failed") + }) + + s.env.RegisterDelayedCallback(func() { + enableMigration = false + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 1*time.Second) + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameRefresh, nil) + }, 5*time.Second) + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameForceCAN, nil) + }, 10*time.Second) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 100 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return enableMigration }, migrateWithRunningWorkflows: func() bool { return true }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(1 * time.Hour)}}, + }, + Action: s.defaultAction("myid"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + }) + + s.Greater(migrateCalls, 1, "pre-v13 behavior is unchanged: migration keeps being retried") + + var canErr *workflow.ContinueAsNewError + s.Require().ErrorAs(s.env.GetWorkflowError(), &canErr) + var canArgs schedulespb.StartScheduleArgs + s.Require().NoError(payloads.Decode(canErr.Input, &canArgs)) + s.True(canArgs.State.PendingMigration, "pre-v13, PendingMigration survives continue-as-new") +} + +// The workflow's EnableCHASMMigration tweakable ANDs the namespace bool with the rollout +// percent (fx.go), which defaults to 0, while the activity guard reads only the bool. So +// an operator migrating one schedule on demand (bool on, percent 0, migrate signal) +// passes the activity guard but reads as rolled back to the reset, which drops +// PendingMigration before executeMigration runs. +func (s *workflowSuite) TestOperatorMigrateSignalSurvivesRolloutPercentZero() { + s.T().Skip("known gap: the v13 rollback reset discards operator-initiated migrations " + + "whenever CHASMSchedulerMigrationRolloutPercent is 0 (its default)") + + previousTweakables := CurrentTweakablePolicies + defer func() { CurrentTweakablePolicies = previousTweakables }() + CurrentTweakablePolicies.Version = MigrationHandoffFixes + + migrateCalls := 0 + s.env.OnActivity(new(activities).MigrateScheduleToChasm, mock.Anything, mock.Anything).Return( + func(context.Context, *schedulerpb.CreateFromMigrationStateRequest) error { + migrateCalls++ + return nil + }) + + s.env.RegisterDelayedCallback(func() { + s.env.SignalWorkflow(SignalNameMigrateToChasm, nil) + }, 1*time.Second) + + CurrentTweakablePolicies.IterationsBeforeContinueAsNew = 100 + s.env.SetStartTime(baseStartTime) + s.env.ExecuteWorkflow(func(ctx workflow.Context, args *schedulespb.StartScheduleArgs) error { + // enableCHASMMigration=false models rolloutPercent=0 with the namespace bool on. + return schedulerWorkflowWithSpecBuilder(ctx, args, newSpecBuilderForTest(0, 0), + schedulerDynamicConfig{enableCHASMMigration: func() bool { return false }, migrateWithRunningWorkflows: func() bool { return true }}) + }, &schedulespb.StartScheduleArgs{ + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(1 * time.Hour)}}, + }, + Action: s.defaultAction("myid"), + }, + State: &schedulespb.InternalState{ + Namespace: "myns", + NamespaceId: "mynsid", + ScheduleId: "myschedule", + ConflictToken: InitialConflictToken, + }, + }) + + s.Equal(1, migrateCalls, "an operator-requested migration must not be discarded by the rollback reset") + s.Require().NoError(s.env.GetWorkflowError(), "workflow should complete after a successful migration") +} diff --git a/tests/schedule_migration_test.go b/tests/schedule_migration_test.go index e7623ffc848..f433a4854f6 100644 --- a/tests/schedule_migration_test.go +++ b/tests/schedule_migration_test.go @@ -1,6 +1,7 @@ package tests import ( + "context" "encoding/binary" "errors" "strings" @@ -20,6 +21,7 @@ import ( "go.temporal.io/server/api/adminservice/v1" "go.temporal.io/server/api/historyservice/v1" schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm" chasmscheduler "go.temporal.io/server/chasm/lib/scheduler" schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common" @@ -35,6 +37,7 @@ import ( "go.temporal.io/server/tests/testcore" "google.golang.org/grpc" "google.golang.org/grpc/metadata" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -2306,3 +2309,225 @@ func TestScheduleMigration_NoRunningWorkflows_GeneratorStarts(t *testing.T) { require.ErrorAs(t, err, &closedErr) }, 6*time.Second, 100*time.Millisecond) } + +func TestScheduleMigrationV1ToV2_AdminMigratePreservesRunningWorkflowHistory(t *testing.T) { + + // TODO: This is the admin-API directly poking a migration to v2: + t.Skip("admin MigrateSchedule path still injects EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED " + + "into a running workflow's history; remove this skip when the retroactive callback-attach is fixed") + + env := newScheduleEnv(t, testcore.WithWorkerService("V1 scheduler")) + ctx := testcore.NewContext() + + sid := testcore.RandomizeStr("sched-admin-migrate") + wid := testcore.RandomizeStr("sched-admin-migrate-wf") + wt := testcore.RandomizeStr("sched-admin-migrate-wt") + + // A workflow that blocks until signaled, so it is guaranteed to still be + // running at the moment migration happens. + resumeSignal := "resume" + env.SdkWorker().RegisterWorkflowWithOptions(func(ctx workflow.Context) error { + workflow.GetSignalChannel(ctx, resumeSignal).Receive(ctx, nil) + return nil + }, workflow.RegisterOptions{Name: wt}) + + sched := &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{Interval: []*schedulepb.IntervalSpec{{Interval: durationpb.New(1 * time.Hour)}}}, + Action: startWorkflowAction(env, wid, wt), + } + createV1Schedule(ctx, t, env, sid, sched, &schedulepb.SchedulePatch{ + TriggerImmediately: &schedulepb.TriggerImmediatelyRequest{}, + }) + + runningWfID := awaitRunningAction(ctx, t, env, sid) + + // Ensure the workflow gets unblocked at the end regardless of outcome, so a + // failing assertion doesn't leak a permanently-running execution. + defer func() { + _, _ = env.FrontendClient().SignalWorkflowExecution(testcore.NewContext(), &workflowservice.SignalWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: runningWfID}, + SignalName: resumeSignal, + }) + }() + + // Sanity: V1 never emits this event type before migration. + requireNoOptionsUpdatedEvent(ctx, t, env, runningWfID) + + // Migrate via the admin RPC while the fired workflow is still running. + env.OverrideDynamicConfig(dynamicconfig.EnableChasm, true) + _, err := env.AdminClient().MigrateSchedule(ctx, &adminservice.MigrateScheduleRequest{ + Namespace: env.Namespace().String(), + ScheduleId: sid, + Target: adminservice.MigrateScheduleRequest_SCHEDULER_TARGET_CHASM, + Identity: "test", + RequestId: testcore.RandomizeStr("request-id"), + }) + require.NoError(t, err) + + // Migration is fully applied once the V1 scheduler workflow completes, so its + // side effects (including any retroactive callback attach) are already + // reflected in the fired workflow's history by this point. + awaitV1SchedulerCompleted(ctx, t, env, sid) + + requireNoOptionsUpdatedEvent(ctx, t, env, runningWfID) +} + +// requireNoChasmSentinel asserts that no CHASM entity -- sentinel or otherwise +// -- exists yet for scheduleID under the scheduler archetype. Used right after +// creating a V1 schedule to confirm the test's "no sentinel gets written" +// assumption directly (rather than only inferring it from the EnableChasm +// value passed to CreateSchedule), since a stray/unexpired sentinel would +// invisibly gate migration behind chasm/lib/scheduler/config.go's +// SentinelIdleTime (15 minutes) and make an otherwise-passing test hang or +// flake for the wrong reason. +func requireNoChasmSentinel(ctx context.Context, t *testing.T, env *testcore.TestEnv, scheduleID string) { + t.Helper() + + resp, err := env.AdminClient().DescribeMutableState(ctx, &adminservice.DescribeMutableStateRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: scheduleID}, + Archetype: string(chasm.SchedulerArchetype), + }) + if err != nil { + // NotFound means nothing at all was written to the CHASM key space for + // this schedule ID yet -- definitely no sentinel. Any other error is + // unexpected and should fail the test. + var notFoundErr *serviceerror.NotFound + require.ErrorAs(t, err, ¬FoundErr, "unexpected error checking for a CHASM sentinel") + return + } + + node := resp.GetDatabaseMutableState().GetChasmNodes()[""] + require.NotNil(t, node, "CHASM execution exists for %q but has no root node", scheduleID) + + var state schedulerpb.SchedulerState + require.NoError(t, proto.Unmarshal(node.GetData().GetData(), &state)) + require.False(t, state.GetSentinel(), + "a CHASM sentinel exists for schedule %q -- it would block migration for up to "+ + "SentinelIdleTime (chasm/lib/scheduler/config.go), invalidating this test's timing", scheduleID) +} + +// createV1Schedule creates a V1 (workflow-backed) schedule with CHASM disabled, +// then asserts no CHASM sentinel was written. initialPatch may be nil. +func createV1Schedule( + ctx context.Context, + t *testing.T, + env *testcore.TestEnv, + scheduleID string, + sched *schedulepb.Schedule, + initialPatch *schedulepb.SchedulePatch, +) { + t.Helper() + + // EnableChasm is unset at creation time, so no CHASM sentinel gets written (which would block migration). + env.OverrideDynamicConfig(dynamicconfig.EnableChasm, false) + + _, err := env.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: env.Namespace().String(), + ScheduleId: scheduleID, + Schedule: sched, + InitialPatch: initialPatch, + Identity: "test", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + requireNoChasmSentinel(ctx, t, env, scheduleID) +} + +// awaitRunningAction waits until the schedule has fired an action whose workflow +// is still RUNNING, and returns that workflow's ID. +func awaitRunningAction(ctx context.Context, t *testing.T, env *testcore.TestEnv, scheduleID string) string { + t.Helper() + + var runningWfID string + await.RequireTrue(t, func() bool { + descResp, err := env.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: env.Namespace().String(), + ScheduleId: scheduleID, + }) + if err != nil || len(descResp.GetInfo().GetRecentActions()) == 0 { + return false + } + a := descResp.Info.RecentActions[0] + if a.GetStartWorkflowStatus() != enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING { + return false + } + runningWfID = a.GetStartWorkflowResult().GetWorkflowId() + return true + }, 15*time.Second, 500*time.Millisecond) + require.NotEmpty(t, runningWfID) + return runningWfID +} + +// awaitAnyAction waits until the schedule has fired at least one action +// (regardless of the fired workflow's status) and returns that workflow's ID. +func awaitAnyAction(ctx context.Context, t *testing.T, env *testcore.TestEnv, scheduleID string) string { + t.Helper() + + var wfID string + await.RequireTrue(t, func() bool { + descResp, err := env.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: env.Namespace().String(), + ScheduleId: scheduleID, + }) + if err != nil || len(descResp.GetInfo().GetRecentActions()) == 0 { + return false + } + wfID = descResp.Info.RecentActions[0].GetStartWorkflowResult().GetWorkflowId() + return wfID != "" + }, 15*time.Second, 500*time.Millisecond) + require.NotEmpty(t, wfID) + return wfID +} + +// awaitV1SchedulerCompleted waits until the V1 scheduler workflow reaches +// COMPLETED. The V1 scheduler workflow only completes when executeMigration() +// succeeds, so its completion is a reliable "migration happened" signal. +func awaitV1SchedulerCompleted(ctx context.Context, t *testing.T, env *testcore.TestEnv, scheduleID string) { + t.Helper() + + v1WorkflowID := scheduler.WorkflowIDPrefix + scheduleID + await.RequireTruef(t, func() bool { + desc, err := env.GetTestCluster().HistoryClient().DescribeWorkflowExecution(ctx, &historyservice.DescribeWorkflowExecutionRequest{ + NamespaceId: env.NamespaceID().String(), + Request: &workflowservice.DescribeWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: v1WorkflowID}, + }, + }) + return err == nil && desc.GetWorkflowExecutionInfo().GetStatus() == enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED + }, 30*time.Second, 1*time.Second, "V1 scheduler workflow should complete once migration succeeds") +} + +// requireNoOptionsUpdatedEvent asserts the workflow's history does not contain +// EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED. V1 never emits this event, so +// older/community SDKs (whose vendored protobuf predates it) cannot decode it +// and crash -- permanently stalling the workflow (see +// repros/scheduler-migration-bug-evidence.md). +func requireNoOptionsUpdatedEvent(ctx context.Context, t *testing.T, env *testcore.TestEnv, workflowID string) { + t.Helper() + + history, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{WorkflowId: workflowID}, + }) + require.NoError(t, err) + for _, event := range history.GetHistory().GetEvents() { + require.NotEqual(t, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, event.GetEventType(), + "workflow %q gained %s (event id %d) during migration -- older SDKs cannot decode this "+ + "event type and will permanently stall on it (see repros/scheduler-migration-bug-evidence.md)", + workflowID, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_OPTIONS_UPDATED, event.GetEventId()) + } +} + +// requireV2ScheduleExists asserts a V2 (CHASM) schedule exists for scheduleID. +func requireV2ScheduleExists(ctx context.Context, t *testing.T, env *testcore.TestEnv, scheduleID string) { + t.Helper() + + _, err := env.GetTestCluster().SchedulerClient().DescribeSchedule(ctx, &schedulerpb.DescribeScheduleRequest{ + NamespaceId: env.NamespaceID().String(), + FrontendRequest: &workflowservice.DescribeScheduleRequest{Namespace: env.Namespace().String(), ScheduleId: scheduleID}, + }) + require.NoError(t, err) +} diff --git a/tests/schedule_test.go b/tests/schedule_test.go index 6abb995b528..59439e24829 100644 --- a/tests/schedule_test.go +++ b/tests/schedule_test.go @@ -31,6 +31,7 @@ import ( schedulerpb "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/headers" + "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/payload" "go.temporal.io/server/common/payloads" "go.temporal.io/server/common/primitives" @@ -507,6 +508,9 @@ func TestScheduleV1(t *testing.T) { newContext := v1ContextFactory t.Run("TestCreateScheduleDuplicateSdkError", func(t *testing.T) { t.Parallel(); testCreateScheduleDuplicateSdkError(t, false) }) t.Run("TestCHASMCanListV1Schedules", func(t *testing.T) { t.Parallel(); testCHASMCanListV1Schedules(t, newContext) }) + // Not parallel: testActionDelayMetrics temporarily overrides the package-level + // scheduler.CurrentTweakablePolicies.Version, which every parallel sibling schedule shares. + t.Run("TestActionDelayMetrics", func(t *testing.T) { testActionDelayMetrics(t, newContext) }) t.Run("TestRefresh", func(t *testing.T) { t.Parallel(); testRefresh(t, newContext) }) t.Run("TestListBeforeRun", func(t *testing.T) { t.Parallel(); testListBeforeRun(t, newContext) }) t.Run("TestRateLimit", func(t *testing.T) { t.Parallel(); testRateLimit(t, newContext) }) @@ -516,6 +520,152 @@ func TestScheduleV1(t *testing.T) { t.Run("TestUpdateScheduleMemoRejected", func(t *testing.T) { t.Parallel(); testUpdateScheduleMemoRejected(t, newContext) }) } +func testActionDelayMetrics(t *testing.T, newContext contextFactory) { + // The short-refresh DesiredTime path this test exercises is gated behind + // RefreshCompletionDesiredTime; the shipped default stays at TriggerImmediatelyTimestamp + // until a follow-up deploy activates it. Force it on for this run (see the caller: this + // subtest is intentionally not run with t.Parallel(), since this override is shared + // package state). + prevVersion := scheduler.CurrentTweakablePolicies.Version + scheduler.CurrentTweakablePolicies.Version = scheduler.RefreshCompletionDesiredTime + t.Cleanup(func() { scheduler.CurrentTweakablePolicies.Version = prevVersion }) + + s := newScheduleEnv(t, scheduleCommonOpts(t)...) + + sid := testcore.RandomizeStr("sched-action-delay") + wid := testcore.RandomizeStr("sched-action-delay-wf") + wt := testcore.RandomizeStr("sched-action-delay-wt") + + var activityCalls atomic.Int32 + releaseManualActivity := make(chan struct{}) + releaseFirstScheduledActivity := make(chan struct{}) + activityFn := func(ctx context.Context) error { + var release chan struct{} + switch activityCalls.Add(1) { + case 1: + release = releaseManualActivity + case 2: + release = releaseFirstScheduledActivity + default: + return nil + } + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + } + s.SdkWorker().RegisterActivity(activityFn) + s.SdkWorker().RegisterWorkflowWithOptions(func(ctx workflow.Context) error { + ctx = workflow.WithActivityOptions(ctx, workflow.ActivityOptions{ + StartToCloseTimeout: 30 * time.Second, + }) + return workflow.ExecuteActivity(ctx, activityFn).Get(ctx, nil) + }, workflow.RegisterOptions{Name: wt}) + + ctx := newContext(testcontext.For(t)) + metricCapture := s.StartNamespaceMetricCapture() + interval := 10 * time.Second + phaseOffset := 5 * time.Second + _, err := s.FrontendClient().CreateSchedule(ctx, &workflowservice.CreateScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: sid, + Schedule: &schedulepb.Schedule{ + Spec: &schedulepb.ScheduleSpec{ + Interval: []*schedulepb.IntervalSpec{{ + Interval: durationpb.New(interval), + Phase: durationpb.New(time.Duration((time.Now().Unix()+int64(phaseOffset/time.Second))%int64(interval/time.Second)) * time.Second), + }}, + }, + Action: startWorkflowAction(s, wid, wt), + Policies: &schedulepb.SchedulePolicies{ + OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, + }, + State: &schedulepb.ScheduleState{ + LimitedActions: true, + RemainingActions: 2, + }, + }, + InitialPatch: triggerPatch(enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED), + Identity: "test", + RequestId: uuid.NewString(), + }) + require.NoError(t, err) + + // Query the scheduler workflow directly (not DescribeSchedule, which signals a refresh and + // would race the completion-discovery path each stage below is intended to cover). + queryBufferAndRunning := func() (bufferSize int64, running int) { + encoded, queryErr := s.SdkClient().QueryWorkflow(ctx, scheduler.WorkflowIDPrefix+sid, "", scheduler.QueryNameDescribe) + if queryErr != nil { + return -1, -1 + } + var response schedulespb.DescribeResponse + if encoded.Get(&response) != nil { + return -1, -1 + } + return response.GetInfo().GetBufferSize(), len(response.GetInfo().GetRunningWorkflows()) + } + await.RequireTruef(t, func() bool { + bufferSize, running := queryBufferAndRunning() + return activityCalls.Load() == 1 && bufferSize == 1 && running == 1 + }, awaitTimeout, pollInterval, "manual action should be running with the first scheduled action buffered") + close(releaseManualActivity) + + // Hold the first scheduled action open past the second scheduled tick, so the second + // scheduled action is buffered behind a still-running (not yet closed) action -- i.e. + // genuinely blocked, not merely started after an unrelated, already-finished one. Note this + // exercises DesiredTime backdating end-to-end (via whichever path discovers the completion -- + // a long-poll watcher is installed as soon as a start is buffered behind a running action, so + // that watcher, not a later refresh, is what normally wins here). The specific + // refresh-discovers-a-real-backlog branch (refreshBackdate == true in processWatcherResult) is + // pinned deterministically by TestRefreshCompletionDesiredTimeBacklog in workflow_test.go, + // which controls discovery timing directly instead of racing a real watcher. + await.RequireTruef(t, func() bool { + bufferSize, running := queryBufferAndRunning() + return activityCalls.Load() == 2 && bufferSize == 1 && running == 1 + }, awaitTimeout, pollInterval, "first scheduled action should be running with the second scheduled action buffered") + close(releaseFirstScheduledActivity) + + // Do not describe the schedule while waiting. The second scheduled action must discover + // the prior completion through processBuffer's short refresh, not a DescribeSchedule signal. + await.RequireTruef(t, func() bool { + return activityCalls.Load() == 3 && len(metricCapture.Metric(metrics.ScheduleActionDelay.Name())) == 2 + }, awaitTimeout, pollInterval, "manual action and two scheduled actions should start") + desc, err := s.FrontendClient().DescribeSchedule(ctx, &workflowservice.DescribeScheduleRequest{ + Namespace: s.Namespace().String(), + ScheduleId: sid, + }) + require.NoError(t, err) + require.Equal(t, int64(3), desc.GetInfo().GetActionCount()) + actions := desc.GetInfo().GetRecentActions() + require.Len(t, actions, 3) + + closeTime := func(action *schedulepb.ScheduleActionResult) time.Time { + t.Helper() + events := s.GetHistory(s.Namespace().String(), action.GetStartWorkflowResult()) + require.NotEmpty(t, events) + lastEvent := events[len(events)-1] + require.Equal(t, enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED, lastEvent.GetEventType()) + return lastEvent.GetEventTime().AsTime() + } + // The prior action's close only backdates the desired time when it comes after this + // action's own scheduled time -- i.e. this action was actually blocked waiting on it. + // Otherwise the action ran on time and the desired time stays at its scheduled time. + expectedDesiredTime := func(priorClose time.Time, action *schedulepb.ScheduleActionResult) time.Time { + if scheduleTime := action.GetScheduleTime().AsTime(); priorClose.After(scheduleTime) { + return priorClose + } + return action.GetScheduleTime().AsTime() + } + recordings := metricCapture.Metric(metrics.ScheduleActionDelay.Name()) + for _, recording := range recordings { + require.Equal(t, metrics.ScheduleBackendLegacy, recording.Tags[metrics.ScheduleBackendTag]) + } + require.Equal(t, actions[1].GetActualTime().AsTime().Sub(expectedDesiredTime(closeTime(actions[0]), actions[1])), recordings[0].Value) + require.Equal(t, actions[2].GetActualTime().AsTime().Sub(expectedDesiredTime(closeTime(actions[1]), actions[2])), recordings[1].Value) +} + func runSharedScheduleTests(t *testing.T, newContext contextFactory) { t.Run("TestBasics", func(t *testing.T) { t.Parallel(); testBasics(t, newContext) }) t.Run("TestInput", func(t *testing.T) { t.Parallel(); testInput(t, newContext) }) From cb650ca63807222de7d9c31da1e1995417e3ac8f Mon Sep 17 00:00:00 2001 From: Alan Wu Date: Thu, 3 Sep 2026 11:13:37 -0400 Subject: [PATCH 125/139] Pin thrift dependency to 0.24 (#11907) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 42dea9ec419..55125f0e90f 100644 --- a/go.mod +++ b/go.mod @@ -125,7 +125,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.56.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver/v3 v3.4.0 // indirect - github.com/apache/thrift v0.23.0 // indirect + github.com/apache/thrift v0.24.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.22 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.22 // indirect diff --git a/go.sum b/go.sum index c16758ef876..1d241d3c204 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTN github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= -github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/aws/aws-sdk-go-v2 v1.41.6 h1:1AX0AthnBQzMx1vbmir3Y4WsnJgiydmnJjiLu+LvXOg= github.com/aws/aws-sdk-go-v2 v1.41.6/go.mod h1:dy0UzBIfwSeot4grGvY1AqFWN5zgziMmWGzysDnHFcQ= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 h1:adBsCIIpLbLmYnkQU+nAChU5yhVTvu5PerROm+/Kq2A= From 91b7d77a2113dd65e54ffa0da28857aa219722ed Mon Sep 17 00:00:00 2001 From: Fred Tzeng <41805201+fretz12@users.noreply.github.com> Date: Thu, 3 Sep 2026 10:15:05 -0700 Subject: [PATCH 126/139] Add standalone activity dispatch hook (#11798) ## What changed? - Corrected standalone activity dispatch-reason classification for resets and retries. - Added a dispatch hook that runs after CHASM validation and before sending the activity to Matching. - Added coverage for retry, reset, and pause/unpause transitions. ## Why? Consumers need a reliable way to classify and intercept standalone activity dispatches. Reset attempt 1 was previously classified as a retry, and processing before CHASM validation could include stale tasks replaced by reset, update, pause, or unpause operations. These changes ensure dispatch reasons reflect the actual attempt and allow integrations to act only on valid dispatches. ## How did you test it? - [X] built - [ ] run locally and tested manually - [X] covered by existing tests - [X] added new unit test(s) - [ ] added new functional test(s) ## Potential risks - An incorrectly implemented dispatch hook could block, drop, or duplicate an activity dispatch. - Hook errors now propagate through task processing and may cause the dispatch task to retry. - Consumers relying on the previous reset dispatch classification may observe reset attempt 1 changing from RETRY to IMMEDIATE. When no hook is configured, dispatch behavior remains unchanged. --- chasm/lib/activity/attempt.go | 7 +- chasm/lib/activity/attempt_test.go | 25 ++- chasm/lib/activity/statemachine_test.go | 94 +++++++++-- chasm/lib/activity/tasks.go | 64 +++++-- chasm/lib/activity/tasks_test.go | 214 ++++++++++++++++++++++++ 5 files changed, 369 insertions(+), 35 deletions(-) diff --git a/chasm/lib/activity/attempt.go b/chasm/lib/activity/attempt.go index 59646982c31..ffff6b985dc 100644 --- a/chasm/lib/activity/attempt.go +++ b/chasm/lib/activity/attempt.go @@ -293,15 +293,16 @@ func (a *Activity) firstDispatchTime() time.Time { } func (a *Activity) newActivityDispatchTask(ctx chasm.Context) *activitypb.ActivityDispatchTask { + attempt := a.LastAttempt.Get(ctx) dispatchReason := activitypb.DISPATCH_REASON_IMMEDIATE - if a.GetFirstAttemptStartedTime() != nil { + if attempt.GetCount() > 1 { dispatchReason = activitypb.DISPATCH_REASON_RETRY - } else if a.GetStartDelay().AsDuration() > 0 { + } else if a.GetFirstAttemptStartedTime() == nil && a.GetStartDelay().AsDuration() > 0 { dispatchReason = activitypb.DISPATCH_REASON_START_DELAY } return &activitypb.ActivityDispatchTask{ - Stamp: a.LastAttempt.Get(ctx).GetStamp(), + Stamp: attempt.GetStamp(), DispatchReason: dispatchReason, StartDelayBucket: startDelayBucket(a.GetStartDelay().AsDuration()), } diff --git a/chasm/lib/activity/attempt_test.go b/chasm/lib/activity/attempt_test.go index 38e5c506324..7aab0d92923 100644 --- a/chasm/lib/activity/attempt_test.go +++ b/chasm/lib/activity/attempt_test.go @@ -109,6 +109,7 @@ func TestStartDelayBucket(t *testing.T) { func TestNewActivityDispatchTask(t *testing.T) { testCases := []struct { name string + attemptCount int32 startDelay time.Duration firstAttemptStartedTime *timestamppb.Timestamp expectedReason activitypb.DispatchReason @@ -116,28 +117,39 @@ func TestNewActivityDispatchTask(t *testing.T) { }{ { name: "immediate", + attemptCount: 1, expectedReason: activitypb.DISPATCH_REASON_IMMEDIATE, expectedBucket: activitypb.START_DELAY_BUCKET_NONE, }, { name: "start delay", + attemptCount: 1, startDelay: time.Hour, expectedReason: activitypb.DISPATCH_REASON_START_DELAY, expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, }, { - name: "retry without configured start delay", - firstAttemptStartedTime: timestamppb.Now(), - expectedReason: activitypb.DISPATCH_REASON_RETRY, - expectedBucket: activitypb.START_DELAY_BUCKET_NONE, + name: "attempt number is authoritative for retry", + attemptCount: 2, + expectedReason: activitypb.DISPATCH_REASON_RETRY, + expectedBucket: activitypb.START_DELAY_BUCKET_NONE, }, { name: "retry preserves configured start delay bucket", + attemptCount: 2, startDelay: time.Hour, firstAttemptStartedTime: timestamppb.Now(), expectedReason: activitypb.DISPATCH_REASON_RETRY, expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, }, + { + name: "attempt 1 after a prior worker start with configured start delay", + attemptCount: 1, + startDelay: time.Hour, + firstAttemptStartedTime: timestamppb.Now(), + expectedReason: activitypb.DISPATCH_REASON_IMMEDIATE, + expectedBucket: activitypb.START_DELAY_BUCKET_1H_6H, + }, } for _, tc := range testCases { @@ -148,7 +160,10 @@ func TestNewActivityDispatchTask(t *testing.T) { StartDelay: durationpb.New(tc.startDelay), FirstAttemptStartedTime: tc.firstAttemptStartedTime, }, - LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{Stamp: 1}), + LastAttempt: chasm.NewDataField(ctx, &activitypb.ActivityAttemptState{ + Count: tc.attemptCount, + Stamp: 1, + }), } task := activity.newActivityDispatchTask(ctx) diff --git a/chasm/lib/activity/statemachine_test.go b/chasm/lib/activity/statemachine_test.go index ded2b6272b7..1583c7b7ff5 100644 --- a/chasm/lib/activity/statemachine_test.go +++ b/chasm/lib/activity/statemachine_test.go @@ -369,6 +369,53 @@ func TestTransitionRescheduled(t *testing.T) { } } +func TestPauseUnpauseReplacesDispatch(t *testing.T) { + testCases := []struct { + attemptCount int32 + expectedReason activitypb.DispatchReason + }{ + {attemptCount: 1, expectedReason: activitypb.DISPATCH_REASON_IMMEDIATE}, + {attemptCount: 3, expectedReason: activitypb.DISPATCH_REASON_RETRY}, + } + + for _, tc := range testCases { + t.Run(fmt.Sprintf("attempt=%d", tc.attemptCount), func(t *testing.T) { + ctx := &chasm.MockMutableContext{} + ctx.HandleNow = func(chasm.Component) time.Time { return defaultTime } + attemptState := &activitypb.ActivityAttemptState{Count: tc.attemptCount, Stamp: 7} + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + ScheduleTime: timestamppb.New(defaultTime), + }, + LastAttempt: chasm.NewDataField(ctx, attemptState), + } + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{}) + oldTask := &activitypb.ActivityDispatchTask{Stamp: 7} + + err := TransitionPaused.Apply(activity, ctx, pauseEvent{metricsHandler: metrics.NoopMetricsHandler}) + require.NoError(t, err) + require.Equal(t, activitypb.ACTIVITY_EXECUTION_STATUS_PAUSED, activity.GetStatus()) + valid, err := handler.Validate(ctx, activity, chasm.TaskInvocation{}, oldTask) + require.NoError(t, err) + require.False(t, valid) + + err = TransitionUnpaused.Apply(activity, ctx, unpauseEvent{ + req: &workflowservice.UnpauseActivityExecutionRequest{}, + }) + require.NoError(t, err) + require.Equal(t, tc.attemptCount, attemptState.GetCount()) + require.Len(t, ctx.Tasks, 1) + dispatchTask := ctx.Tasks[0].Payload.(*activitypb.ActivityDispatchTask) + require.Equal(t, int32(9), dispatchTask.GetStamp()) + require.Equal(t, tc.expectedReason, dispatchTask.GetDispatchReason()) + valid, err = handler.Validate(ctx, activity, chasm.TaskInvocation{}, dispatchTask) + require.NoError(t, err) + require.True(t, valid) + }) + } +} + func TestTransitionStarted(t *testing.T) { ctx := &chasm.MockMutableContext{} ctx.HandleNow = func(chasm.Component) time.Time { return defaultTime } @@ -1100,6 +1147,8 @@ func TestTransitionResetHeartbeat(t *testing.T) { metricsHandler: metrics.NoopMetricsHandler, }) require.NoError(t, err) + dispatchTask := ctx.Tasks[len(ctx.Tasks)-1].Payload.(*activitypb.ActivityDispatchTask) + require.Equal(t, activitypb.DISPATCH_REASON_IMMEDIATE, dispatchTask.GetDispatchReason()) if resetHeartbeat { require.Nil(t, act.LastHeartbeat.Get(ctx).GetDetails()) require.Nil(t, act.LastHeartbeat.Get(ctx).GetRecordedTime()) @@ -1186,6 +1235,10 @@ func TestDeferredResetHeartbeat(t *testing.T) { require.Equal(t, timestamppb.New(defaultTime), act.LastHeartbeat.Get(ctx).GetRecordedTime()) } require.Len(t, ctx.Tasks, tc.expectedTaskCount) + if tc.expectedTaskCount > 0 { + dispatchTask := ctx.Tasks[tc.expectedTaskCount-1].Payload.(*activitypb.ActivityDispatchTask) + require.Equal(t, activitypb.DISPATCH_REASON_IMMEDIATE, dispatchTask.GetDispatchReason()) + } }) } } @@ -1246,38 +1299,53 @@ func TestTransitionResetFromPaused(t *testing.T) { require.Len(t, ctx.Tasks, tc.expectedTaskCount) // Last task is always the dispatch task - _, ok := ctx.Tasks[tc.expectedTaskCount-1].Payload.(*activitypb.ActivityDispatchTask) + dispatchTask, ok := ctx.Tasks[tc.expectedTaskCount-1].Payload.(*activitypb.ActivityDispatchTask) require.True(t, ok, "expected ActivityDispatchTask as last task") + require.Equal(t, activitypb.DISPATCH_REASON_IMMEDIATE, dispatchTask.GetDispatchReason()) }) } } -// TestTransitionResetClearsCurrentRetryInterval verifies that TransitionReset clears the retry -// interval so a reset activity is not delayed by a previous backoff period. -func TestTransitionResetClearsCurrentRetryInterval(t *testing.T) { +// TestResetDuringRetryBackoff verifies that a reset during retry backoff clears the retry interval, +// restarts at attempt 1, and replaces the pending retry dispatch with an immediate one. +func TestResetDuringRetryBackoff(t *testing.T) { ctx := &chasm.MockMutableContext{} ctx.HandleNow = func(chasm.Component) time.Time { return defaultTime } attemptState := &activitypb.ActivityAttemptState{ Count: 2, + Stamp: 7, CurrentRetryInterval: durationpb.New(30 * time.Second), } act := &Activity{ ActivityState: &activitypb.ActivityState{ - ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, - RetryPolicy: defaultRetryPolicy, - ScheduleToCloseTimeout: durationpb.New(defaultScheduleToCloseTimeout), - ScheduleToStartTimeout: durationpb.New(defaultScheduleToStartTimeout), - StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), - Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, - TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + ActivityType: &commonpb.ActivityType{Name: "test-activity-type"}, + RetryPolicy: defaultRetryPolicy, + ScheduleToCloseTimeout: durationpb.New(defaultScheduleToCloseTimeout), + ScheduleToStartTimeout: durationpb.New(defaultScheduleToStartTimeout), + StartToCloseTimeout: durationpb.New(defaultStartToCloseTimeout), + Status: activitypb.ACTIVITY_EXECUTION_STATUS_SCHEDULED, + TaskQueue: &taskqueuepb.TaskQueue{Name: "test-task-queue"}, + FirstAttemptStartedTime: timestamppb.New(defaultTime.Add(-time.Minute)), }, LastAttempt: chasm.NewDataField(ctx, attemptState), Outcome: chasm.NewDataField(ctx, &activitypb.ActivityOutcome{}), } + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{}) + oldTask := &activitypb.ActivityDispatchTask{Stamp: 7} err := TransitionReset.Apply(act, ctx, resetEvent{resetTime: defaultTime, metricsHandler: metrics.NoopMetricsHandler}) require.NoError(t, err) - require.Nil(t, attemptState.GetCurrentRetryInterval(), "TransitionReset must clear CurrentRetryInterval") - require.Equal(t, int32(1), attemptState.Count, "TransitionReset must reset Count to 1") + require.Nil(t, attemptState.GetCurrentRetryInterval()) + require.Equal(t, int32(1), attemptState.GetCount()) + + dispatchTask := ctx.Tasks[len(ctx.Tasks)-1].Payload.(*activitypb.ActivityDispatchTask) + require.Equal(t, int32(8), dispatchTask.GetStamp()) + require.Equal(t, activitypb.DISPATCH_REASON_IMMEDIATE, dispatchTask.GetDispatchReason()) + valid, err := handler.Validate(ctx, act, chasm.TaskInvocation{}, oldTask) + require.NoError(t, err) + require.False(t, valid) + valid, err = handler.Validate(ctx, act, chasm.TaskInvocation{}, dispatchTask) + require.NoError(t, err) + require.True(t, valid) } diff --git a/chasm/lib/activity/tasks.go b/chasm/lib/activity/tasks.go index b7e87887b02..90ceb4a598d 100644 --- a/chasm/lib/activity/tasks.go +++ b/chasm/lib/activity/tasks.go @@ -4,6 +4,7 @@ import ( "context" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/server/api/matchingservice/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common/metrics" @@ -12,10 +13,23 @@ import ( "go.uber.org/fx" ) +// DispatchTaskHook wraps a validated activity dispatch on the active cluster before it is sent to +// Matching. The hook must call dispatch exactly once. If it returns nil without calling dispatch, +// task processing completes successfully without sending the Activity to Matching. Hook errors are +// returned to task processing; returning an error after dispatch succeeds may cause the task to +// retry and resend. Standby discard spills bypass the hook. +type DispatchTaskHook func( + ctx context.Context, + namespaceID string, + task *activitypb.ActivityDispatchTask, + dispatch func(context.Context) error, +) error + type activityDispatchTaskHandlerOptions struct { fx.In - MatchingClient resource.MatchingClient + MatchingClient resource.MatchingClient + DispatchTaskHook DispatchTaskHook `optional:"true"` } type activityDispatchTaskHandler struct { @@ -43,9 +57,31 @@ func (h *activityDispatchTaskHandler) Execute( ctx context.Context, activityRef chasm.ComponentRef, _ chasm.TaskAttributes, - _ *activitypb.ActivityDispatchTask, + task *activitypb.ActivityDispatchTask, ) error { - return h.pushToMatching(ctx, activityRef) + request, err := h.createMatchingRequest(ctx, activityRef) + if err != nil { + return err + } + + // Invoke the hook at the final dispatch boundary so it observes only validated + // active tasks and can wrap the Matching call. + if h.opts.DispatchTaskHook != nil { + return h.opts.DispatchTaskHook( + ctx, + activityRef.NamespaceID, + task, + func(ctx context.Context) error { + _, err := h.opts.MatchingClient.AddActivityTask(ctx, request) + + return err + }, + ) + } + + _, err = h.opts.MatchingClient.AddActivityTask(ctx, request) + + return err } // Discard spills the task to matching instead of silently discarding it on standby clusters when the activity @@ -56,26 +92,26 @@ func (h *activityDispatchTaskHandler) Discard( _ chasm.TaskAttributes, _ *activitypb.ActivityDispatchTask, ) error { - return h.pushToMatching(ctx, activityRef) + request, err := h.createMatchingRequest(ctx, activityRef) + if err != nil { + return err + } + + _, err = h.opts.MatchingClient.AddActivityTask(ctx, request) + + return err } -func (h *activityDispatchTaskHandler) pushToMatching( +func (h *activityDispatchTaskHandler) createMatchingRequest( ctx context.Context, activityRef chasm.ComponentRef, -) error { - request, err := chasm.ReadComponent( +) (*matchingservice.AddActivityTaskRequest, error) { + return chasm.ReadComponent( ctx, activityRef, (*Activity).createAddActivityTaskRequest, activityRef.NamespaceID, ) - if err != nil { - return err - } - - _, err = h.opts.MatchingClient.AddActivityTask(ctx, request) - - return err } type scheduleToStartTimeoutTaskHandler struct { diff --git a/chasm/lib/activity/tasks_test.go b/chasm/lib/activity/tasks_test.go index efedc49ee4a..5108f5bcfb6 100644 --- a/chasm/lib/activity/tasks_test.go +++ b/chasm/lib/activity/tasks_test.go @@ -2,13 +2,16 @@ package activity import ( "context" + "errors" "testing" "time" "github.com/stretchr/testify/require" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" taskqueuepb "go.temporal.io/api/taskqueue/v1" + "go.temporal.io/server/api/matchingservicemock/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/activity/gen/activitypb/v1" "go.temporal.io/server/common" @@ -16,10 +19,221 @@ import ( "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/metrics/metricstest" "go.temporal.io/server/common/retrypolicy" + "go.uber.org/mock/gomock" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/timestamppb" ) +func TestActivityDispatchTaskHook(t *testing.T) { + t.Run("wraps active dispatch", func(t *testing.T) { + controller := gomock.NewController(t) + engine := chasm.NewMockEngine(controller) + matchingClient := matchingservicemock.NewMockMatchingServiceClient(controller) + activityRef := chasm.NewComponentRef[*Activity](chasm.ExecutionKey{NamespaceID: "namespace-id"}) + task := &activitypb.ActivityDispatchTask{DispatchReason: activitypb.DISPATCH_REASON_RETRY} + chasmCtx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleRef: func(chasm.Component) ([]byte, error) { + return []byte("component-ref"), nil + }, + }, + } + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + }, + LastAttempt: chasm.NewDataField(chasmCtx, &activitypb.ActivityAttemptState{Stamp: 7}), + } + + engine.EXPECT().ReadComponent(gomock.Any(), activityRef, gomock.Any()).DoAndReturn( + func( + _ context.Context, + _ chasm.ComponentRef, + readFn func(chasm.Context, chasm.Component) error, + _ ...chasm.TransitionOption, + ) error { + return readFn(chasmCtx, activity) + }, + ) + matchingClient.EXPECT().AddActivityTask(gomock.Any(), gomock.Any()).Return(nil, nil) + + hookCalls := 0 + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{ + MatchingClient: matchingClient, + DispatchTaskHook: func( + ctx context.Context, + namespaceID string, + actualTask *activitypb.ActivityDispatchTask, + dispatch func(context.Context) error, + ) error { + hookCalls++ + require.Equal(t, "namespace-id", namespaceID) + require.Same(t, task, actualTask) + + return dispatch(ctx) + }, + }) + + err := handler.Execute(chasm.NewEngineContext(context.Background(), engine), activityRef, chasm.TaskAttributes{}, task) + require.NoError(t, err) + require.Equal(t, 1, hookCalls) + }) + + t.Run("honors hook result", func(t *testing.T) { + hookErr := errors.New("hook failed") + testCases := []struct { + name string + hookErr error + }{ + {name: "suppresses dispatch"}, + {name: "propagates error", hookErr: hookErr}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + controller := gomock.NewController(t) + engine := chasm.NewMockEngine(controller) + activityRef := chasm.NewComponentRef[*Activity](chasm.ExecutionKey{NamespaceID: "namespace-id"}) + chasmCtx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleRef: func(chasm.Component) ([]byte, error) { + return []byte("component-ref"), nil + }, + }, + } + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + }, + LastAttempt: chasm.NewDataField(chasmCtx, &activitypb.ActivityAttemptState{Stamp: 7}), + } + + engine.EXPECT().ReadComponent(gomock.Any(), activityRef, gomock.Any()).DoAndReturn( + func( + _ context.Context, + _ chasm.ComponentRef, + readFn func(chasm.Context, chasm.Component) error, + _ ...chasm.TransitionOption, + ) error { + return readFn(chasmCtx, activity) + }, + ) + + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{ + MatchingClient: matchingservicemock.NewMockMatchingServiceClient(controller), + DispatchTaskHook: func( + context.Context, + string, + *activitypb.ActivityDispatchTask, + func(context.Context) error, + ) error { + return tc.hookErr + }, + }) + + err := handler.Execute( + chasm.NewEngineContext(context.Background(), engine), + activityRef, + chasm.TaskAttributes{}, + &activitypb.ActivityDispatchTask{}, + ) + if tc.hookErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tc.hookErr) + } + }) + } + }) + + t.Run("skipped when request construction fails", func(t *testing.T) { + controller := gomock.NewController(t) + engine := chasm.NewMockEngine(controller) + activityRef := chasm.NewComponentRef[*Activity](chasm.ExecutionKey{NamespaceID: "namespace-id"}) + validationErr := serviceerror.NewNotFound("task is no longer valid") + + engine.EXPECT().ReadComponent(gomock.Any(), activityRef, gomock.Any()).Return(validationErr) + + hookCalls := 0 + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{ + MatchingClient: matchingservicemock.NewMockMatchingServiceClient(controller), + DispatchTaskHook: func( + context.Context, + string, + *activitypb.ActivityDispatchTask, + func(context.Context) error, + ) error { + hookCalls++ + return nil + }, + }) + + err := handler.Execute( + chasm.NewEngineContext(context.Background(), engine), + activityRef, + chasm.TaskAttributes{}, + &activitypb.ActivityDispatchTask{}, + ) + require.ErrorIs(t, err, validationErr) + require.Zero(t, hookCalls) + }) + + t.Run("bypassed on standby discard", func(t *testing.T) { + controller := gomock.NewController(t) + engine := chasm.NewMockEngine(controller) + matchingClient := matchingservicemock.NewMockMatchingServiceClient(controller) + activityRef := chasm.NewComponentRef[*Activity](chasm.ExecutionKey{NamespaceID: "namespace-id"}) + chasmCtx := &chasm.MockMutableContext{ + MockContext: chasm.MockContext{ + HandleRef: func(chasm.Component) ([]byte, error) { + return []byte("component-ref"), nil + }, + }, + } + activity := &Activity{ + ActivityState: &activitypb.ActivityState{ + TaskQueue: &taskqueuepb.TaskQueue{Name: "task-queue"}, + }, + LastAttempt: chasm.NewDataField(chasmCtx, &activitypb.ActivityAttemptState{Stamp: 7}), + } + + engine.EXPECT().ReadComponent(gomock.Any(), activityRef, gomock.Any()).DoAndReturn( + func( + _ context.Context, + _ chasm.ComponentRef, + readFn func(chasm.Context, chasm.Component) error, + _ ...chasm.TransitionOption, + ) error { + return readFn(chasmCtx, activity) + }, + ) + matchingClient.EXPECT().AddActivityTask(gomock.Any(), gomock.Any()).Return(nil, nil) + + hookCalls := 0 + handler := newActivityDispatchTaskHandler(activityDispatchTaskHandlerOptions{ + MatchingClient: matchingClient, + DispatchTaskHook: func( + context.Context, + string, + *activitypb.ActivityDispatchTask, + func(context.Context) error, + ) error { + hookCalls++ + return nil + }, + }) + + err := handler.Discard( + chasm.NewEngineContext(context.Background(), engine), + activityRef, + chasm.TaskAttributes{}, + &activitypb.ActivityDispatchTask{DispatchReason: activitypb.DISPATCH_REASON_RETRY}, + ) + require.NoError(t, err) + require.Zero(t, hookCalls) + }) +} + func TestScheduleToCloseTimeoutTaskValidateStamp(t *testing.T) { handler := newScheduleToCloseTimeoutTaskHandler() From 055370b3f08fe765f483e0f6215fd1c1a6218150 Mon Sep 17 00:00:00 2001 From: Brian VanLoo Date: Thu, 3 Sep 2026 10:34:55 -0700 Subject: [PATCH 127/139] New metric for retry attempts (#11885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? Adds a new metric, `task_alertable_attempt`, alongside the existing `task_attempt` histogram in `service/history/queues/executable.go`, to distinguish system-caused retry loops from namespace/customer-caused ones. - New predicate, counting: any `ResourceExhausted` error with `Scope != NAMESPACE` (system-caused throttling, including causes that never set `Scope` at all, e.g. `CIRCUIT_BREAKER_OPEN`), plus a deliberate `BUSY_WORKFLOW` carve-out; and everything else not explicitly excluded. Also excludes `NamespaceNotActive`, `ErrDependencyTaskNotCompleted`, `ErrTaskRetry`, `ErrNamespaceHandover`. - Two emission sites, mirroring `task_attempt`'s: a terminal sample in `Ack()` (unconditional) and an in-flight sample gated by the `task_attempt` 30-attempt limit. - Two tags on both sites: `last_attempt_cause` (the specific `ResourceExhausted` cause, or the Go error type for the catchall case) and `stage` (`terminal` vs. `in_flight`). ## Why? `task_attempt` and its alert can't tell a system-caused retry storm apart from a namespace/customer-caused one — e.g. a task throttled by its own namespace's APS limit climbs into the same 100+ attempt range as a genuinely stuck task, firing the same cluster-wide stuck-task alert and turning it into noise. Full investigation and design rationale in `task-attempt-metric-noise-v2.md`. ## How did you test it? - [x] built - [x] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) --- common/metrics/metric_defs.go | 6 + common/metrics/tags.go | 16 ++ service/history/queues/executable.go | 36 ++++ service/history/queues/executable_test.go | 218 ++++++++++++++++++++++ 4 files changed, 276 insertions(+) diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 78433aa0e50..24b9652a757 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -34,6 +34,8 @@ const ( ArchetypeTagName = "archetype" ChasmTaskTypeTagName = "chasm_task_type" timeoutTypeTagName = "timeout_type" + LastAttemptCauseTagName = "last_attempt_cause" + AttemptStageTagName = "attempt_stage" ) // This package should hold all the metrics and tags for temporal @@ -891,6 +893,10 @@ var ( "task_attempt", WithDescription("The number of attempts took to complete a history task."), ) + TaskAlertableAttempt = NewDimensionlessHistogramDef( + "task_alertable_attempt", + WithDescription("The number of attempts, among a history task's total, caused by an alertable (system-side) error."), + ) TaskFailures = NewCounterDef( "task_errors", WithDescription("The number of unexpected history task processing errors."), diff --git a/common/metrics/tags.go b/common/metrics/tags.go index 7d764f8f6a4..9bb0ba19982 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -441,6 +441,22 @@ func ResourceExhaustedScopeTag(scope enumspb.ResourceExhaustedScope) Tag { return Tag{Key: resourceExhaustedScopeTag, Value: scope.String()} } +func LastAttemptCauseTag(value string) Tag { + return Tag{Key: LastAttemptCauseTagName, Value: value} +} + +// Values for AttemptStageTagName, identifying whether TaskAlertableAttempt was recorded +// mid-retry or at the attempt's final resolution. +const ( + AttemptStageInFlight = "in_flight" + AttemptStageTerminal = "terminal" +) + +var ( + AttemptStageInFlightTag = Tag{Key: AttemptStageTagName, Value: AttemptStageInFlight} + AttemptStageTerminalTag = Tag{Key: AttemptStageTagName, Value: AttemptStageTerminal} +) + func ServiceNameTag(value primitives.ServiceName) Tag { return Tag{Key: serviceName, Value: string(value)} } diff --git a/service/history/queues/executable.go b/service/history/queues/executable.go index a6820920298..855dd2d0f22 100644 --- a/service/history/queues/executable.go +++ b/service/history/queues/executable.go @@ -141,6 +141,8 @@ type ( dlqEnabled dynamicconfig.BoolPropertyFn terminalFailureCause error unexpectedErrorAttempts int + alertableAttempts int + alertableErrorCauseTag metrics.Tag maxUnexpectedErrorAttempts dynamicconfig.IntPropertyFn dlqInternalErrors dynamicconfig.BoolPropertyFn dlqErrorPattern dynamicconfig.StringPropertyFn @@ -240,6 +242,7 @@ func NewExecutable( maxUnexpectedErrorAttempts: params.MaxUnexpectedErrorAttempts, dlqInternalErrors: params.DLQInternalErrors, dlqErrorPattern: params.DLQErrorPattern, + alertableErrorCauseTag: metrics.LastAttemptCauseTag("none"), } e.refreshMetricsHandlers(nil) e.attempt.Store(1) @@ -467,6 +470,28 @@ func (e *executableImpl) isSafeToDropError(err error) bool { return false } +// classifyAlertableError answers both whether err should count toward alertableAttempts and, +// if so, what cause tag it should carry. +func classifyAlertableError(err error) (alertable bool, causeTag metrics.Tag) { + if resourceExhaustedErr, ok := errors.AsType[*serviceerror.ResourceExhausted](err); ok { + alertable = resourceExhaustedErr.Scope != enumspb.RESOURCE_EXHAUSTED_SCOPE_NAMESPACE || + resourceExhaustedErr.Cause == enumspb.RESOURCE_EXHAUSTED_CAUSE_BUSY_WORKFLOW + if alertable { + causeTag = metrics.LastAttemptCauseTag(resourceExhaustedErr.Cause.String()) + } + return alertable, causeTag + } + if _, ok := err.(*serviceerror.NamespaceNotActive); ok { + return false, causeTag + } + if err == consts.ErrDependencyTaskNotCompleted || + err == consts.ErrTaskRetry || + err.Error() == consts.ErrNamespaceHandover.Error() { + return false, causeTag + } + return true, metrics.LastAttemptCauseTag(metrics.ServiceErrorTypeTag(err).Value) +} + // Returns true when the error is expected and should be retried. You're expected to return // an error in this case, as that possible-rewritten-error is what we'll return func (e *executableImpl) isExpectedRetryableError(err error) (isRetryable bool, retErr error) { @@ -572,6 +597,15 @@ func (e *executableImpl) HandleErr(err error) (retErr error) { e.incAttempt() + if alertable, causeTag := classifyAlertableError(err); alertable { + e.alertableAttempts++ + e.alertableErrorCauseTag = causeTag + if e.attempt.Load() > taskCriticalLogMetricAttempts { + metrics.TaskAlertableAttempt.With(e.chasmMetricsHandler).Record( + int64(e.alertableAttempts), causeTag, metrics.AttemptStageInFlightTag) + } + } + if ok, rewrittenErr := e.isExpectedRetryableError(err); ok { return rewrittenErr } @@ -695,6 +729,8 @@ func (e *executableImpl) Ack() { } metrics.TaskAttempt.With(e.chasmMetricsHandler).Record(e.attempt.Load()) + metrics.TaskAlertableAttempt.With(e.chasmMetricsHandler).Record( + int64(e.alertableAttempts), e.alertableErrorCauseTag, metrics.AttemptStageTerminalTag) priorityTaggedProvider := e.chasmMetricsHandler.WithTags(metrics.TaskPriorityTag(e.priority.String())) metrics.TaskLatency.With(priorityTaggedProvider).Record(e.inMemoryNoUserLatency) diff --git a/service/history/queues/executable_test.go b/service/history/queues/executable_test.go index 2b4a4c901d6..049e72b9820 100644 --- a/service/history/queues/executable_test.go +++ b/service/history/queues/executable_test.go @@ -26,6 +26,7 @@ import ( "go.temporal.io/server/common/persistence/serialization" ctasks "go.temporal.io/server/common/tasks" "go.temporal.io/server/common/telemetry" + "go.temporal.io/server/common/util" "go.temporal.io/server/service/history/consts" "go.temporal.io/server/service/history/queues" "go.temporal.io/server/service/history/queues/queuestest" @@ -543,6 +544,223 @@ func (s *executableSuite) TestExecuteHandleErr_ResetAttempt() { } } +// Expected-retryable errors skip the DLQ threshold entirely, but should still behave like any +// other attempt otherwise. +func (s *executableSuite) TestHandleErr_ExpectedRetryableError_AttemptAndMetricsUnaffected() { + testCases := []struct { + name string + taskErr error + }{ + { + name: "SystemScopedResourceExhausted", + taskErr: &serviceerror.ResourceExhausted{ + Cause: enumspb.RESOURCE_EXHAUSTED_CAUSE_SYSTEM_OVERLOADED, + Scope: enumspb.RESOURCE_EXHAUSTED_SCOPE_SYSTEM, + Message: "test", + }, + }, + { + name: "BusyWorkflow", + taskErr: consts.ErrResourceExhaustedBusyWorkflow, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + // attempt starts at 1, so 30 HandleErr calls land it at exactly 31 - one past + // taskCriticalLogMetricAttempts (30). + const numAttempts = 30 + const finalAttempt = numAttempts + 1 + + queueWriter := &queuestest.FakeQueueWriter{} + executable := s.newTestExecutable(func(p *params) { + p.dlqWriter = queues.NewDLQWriter(queueWriter, metrics.NoopMetricsHandler, log.NewTestLogger(), s.mockNamespaceRegistry, s.chasmRegistry) + p.dlqEnabled = func() bool { return true } + p.maxUnexpectedErrorAttempts = func() int { return 1 } + }) + s.mockExecutor.EXPECT().Execute(gomock.Any(), executable).Return(queues.ExecuteResponse{ + ExecutionMetricTags: nil, + ExecutedAsActive: true, + ExecutionErr: tc.taskErr, + }).Times(numAttempts) + + capture := s.metricsHandler.StartCapture() + for range numAttempts { + err := executable.Execute() + s.Error(err) + handledErr := executable.HandleErr(err) + s.Error(handledErr) + s.NotErrorIs(handledErr, queues.ErrTerminalTaskFailure) + } + + // unexpectedErrorAttempts never climbed: no DLQ write despite maxUnexpectedErrorAttempts == 1. + s.Empty(queueWriter.EnqueueTaskRequests) + + // attempt still climbs, and TaskAttempt still records once it crosses the floor. + inFlight := capture.Snapshot()[metrics.TaskAttempt.Name()] + s.Len(inFlight, 1) + s.EqualValues(finalAttempt, inFlight[0].Value) + + executable.Ack() + snapshot := capture.Snapshot() + s.metricsHandler.StopCapture(capture) + terminal := snapshot[metrics.TaskAttempt.Name()] + s.Len(terminal, 2) + s.EqualValues(finalAttempt, terminal[1].Value) + }) + } +} + +func (s *executableSuite) TestHandleErr_IsAlertableError() { + testCases := []struct { + name string + taskErr error + expectedAlertable bool + expectedCauseTagValue string + }{ + { + name: "ResourceExhausted_NamespaceScoped_NotBusyWorkflow", + taskErr: consts.ErrResourceExhaustedAPSLimit, + expectedAlertable: false, + }, + { + name: "ResourceExhausted_BusyWorkflow", + taskErr: consts.ErrResourceExhaustedBusyWorkflow, + expectedAlertable: true, + expectedCauseTagValue: enumspb.RESOURCE_EXHAUSTED_CAUSE_BUSY_WORKFLOW.String(), + }, + { + name: "ResourceExhausted_SystemScoped", + taskErr: &serviceerror.ResourceExhausted{ + Cause: enumspb.RESOURCE_EXHAUSTED_CAUSE_SYSTEM_OVERLOADED, + Scope: enumspb.RESOURCE_EXHAUSTED_SCOPE_SYSTEM, + Message: "test", + }, + expectedAlertable: true, + expectedCauseTagValue: enumspb.RESOURCE_EXHAUSTED_CAUSE_SYSTEM_OVERLOADED.String(), + }, + { + name: "ResourceExhausted_ScopeUnspecified", + taskErr: &serviceerror.ResourceExhausted{ + Cause: enumspb.RESOURCE_EXHAUSTED_CAUSE_CIRCUIT_BREAKER_OPEN, + Message: "test", + }, + expectedAlertable: true, + expectedCauseTagValue: enumspb.RESOURCE_EXHAUSTED_CAUSE_CIRCUIT_BREAKER_OPEN.String(), + }, + { + name: "UnclassifiedCatchall", + taskErr: errors.New("some random error"), + expectedAlertable: true, + expectedCauseTagValue: util.ErrorType(errors.New("some random error")), + }, + { + name: "NamespaceNotActive", + taskErr: serviceerror.NewNamespaceNotActive("", "", ""), + expectedAlertable: false, + }, + { + name: "ErrDependencyTaskNotCompleted", + taskErr: consts.ErrDependencyTaskNotCompleted, + expectedAlertable: false, + }, + { + name: "ErrTaskRetry", + taskErr: consts.ErrTaskRetry, + expectedAlertable: false, + }, + { + name: "ErrNamespaceHandover", + taskErr: consts.ErrNamespaceHandover, + expectedAlertable: false, + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + executable := s.newTestExecutable() + s.mockExecutor.EXPECT().Execute(gomock.Any(), executable).Return(queues.ExecuteResponse{ + ExecutionMetricTags: nil, + ExecutedAsActive: true, + ExecutionErr: tc.taskErr, + }) + + err := executable.Execute() + s.Error(err) + s.Error(executable.HandleErr(err)) + + capture := s.metricsHandler.StartCapture() + executable.Ack() + snapshot := capture.Snapshot() + s.metricsHandler.StopCapture(capture) + recording := snapshot[metrics.TaskAlertableAttempt.Name()][0] + isAlertable := recording.Value == int64(1) + s.Equal(tc.expectedAlertable, isAlertable, "isAlertableError(%v)", tc.taskErr) + if tc.expectedAlertable { + s.Equal(tc.expectedCauseTagValue, recording.Tags[metrics.LastAttemptCauseTagName], "cause tag for %v", tc.taskErr) + } + }) + } +} + +func (s *executableSuite) TestAck_AlertableAttemptStageTag_Terminal() { + executable := s.newTestExecutable() + s.mockExecutor.EXPECT().Execute(gomock.Any(), executable).Return(queues.ExecuteResponse{ + ExecutionMetricTags: nil, + ExecutedAsActive: true, + ExecutionErr: consts.ErrResourceExhaustedBusyWorkflow, + }) + + err := executable.Execute() + s.Error(err) + s.Error(executable.HandleErr(err)) + + capture := s.metricsHandler.StartCapture() + executable.Ack() + snapshot := capture.Snapshot() + s.metricsHandler.StopCapture(capture) + + recording := snapshot[metrics.TaskAlertableAttempt.Name()][0] + s.Equal(metrics.AttemptStageTerminal, recording.Tags[metrics.AttemptStageTagName]) +} + +// Guards against a zero-value (empty) cause tag when no alertable error ever occurred. +func (s *executableSuite) TestAck_AlertableAttemptCauseTag_NoAlertableError() { + executable := s.newTestExecutable() + + capture := s.metricsHandler.StartCapture() + executable.Ack() + snapshot := capture.Snapshot() + s.metricsHandler.StopCapture(capture) + + recording := snapshot[metrics.TaskAlertableAttempt.Name()][0] + s.Contains(recording.Tags, metrics.LastAttemptCauseTagName) + s.NotEmpty(recording.Tags[metrics.LastAttemptCauseTagName]) +} + +func (s *executableSuite) TestHandleErr_AlertableAttemptStageTag_InFlight() { + executable := s.newTestExecutable() + s.mockExecutor.EXPECT().Execute(gomock.Any(), executable).Return(queues.ExecuteResponse{ + ExecutionMetricTags: nil, + ExecutedAsActive: true, + ExecutionErr: consts.ErrResourceExhaustedBusyWorkflow, + }).Times(30) // attempt starts at 1; 30 HandleErr calls land it at 31, crossing the floor (30) once. + + capture := s.metricsHandler.StartCapture() + for range 30 { + err := executable.Execute() + s.Error(err) + s.Error(executable.HandleErr(err)) + } + snapshot := capture.Snapshot() + s.metricsHandler.StopCapture(capture) + + recordings := snapshot[metrics.TaskAlertableAttempt.Name()] + s.Len(recordings, 1) + s.Equal(metrics.AttemptStageInFlight, recordings[0].Tags[metrics.AttemptStageTagName]) + s.Equal(enumspb.RESOURCE_EXHAUSTED_CAUSE_BUSY_WORKFLOW.String(), recordings[0].Tags[metrics.LastAttemptCauseTagName]) +} + func (s *executableSuite) TestExecuteHandleErr_Corrupted() { queueWriter := &queuestest.FakeQueueWriter{} executable := s.newTestExecutable(func(p *params) { From 02f5c1aa78b83254bc7060d0e06ca984bdad297e Mon Sep 17 00:00:00 2001 From: Chris Smith Date: Thu, 3 Sep 2026 13:07:31 -0700 Subject: [PATCH 128/139] Fix mislabeled "system payload" issue (#11906) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > ⚠️ I'm a little out of my depth when it comes to the System Nexus Endpoint, and the current requirements around "system payloads". So please review this carefully to keep me honest. > > I'm not sure if this is the right way we want to address this problem, of if instead we should just relax the check in the SDK. (@stuart-wells opened https://github.com/temporalio/api-go/pull/309, which we may also want to take. Independently of this.) ## Context The Temporal server was labeling _everything_ that came through the History Service's `StartNexusOperation` as a system payload. (https://github.com/temporalio/temporal/pull/10948) However, when we landed support for visiting nested payloads in the Golang SDK (https://github.com/temporalio/api-go/pull/297) it asserts that all system payloads have BOTH a `"encoding":"binary/protobuf"` AND `"messageType"` metadata key. That's a problem. Now, if the Temporal server were to pick up the latest `api-go` bits, it will introduce test failures. Because we have tests that return payloads encoded with `plain/json` that will fail when ran through the SDK's payload visitor at runtime. ## What changed? ~~This PR makes the requirements surrounding a system payload clearer, and ONLY flags a System Nexus Endpoint payload as a "system payload" IFF if is a properly labeled protobuf message. Otherwise, it doesn't set the system payload tag at all.~~ This PR now adds a check that the payload sent to the System Nexus Endpoint is a protobuf with message type available. It also updates a testcase that was sending `plain/json` responses to the SNE and added a new testcase. ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks We've already shipped code that includes the "label non-Protobuf payloads as system payloads". So it's possible that we've persisted those somewhere, and updating the `api-go` dependency (unless patched there) would cause problems. --- chasm/lib/tests/nexus_service.go | 20 +++++++++++++++++--- service/history/handler.go | 16 ++++++++++++++++ service/history/handler_test.go | 28 ++++++++++++++++++++++++++++ tests/nexus_workflow_test.go | 19 +++++++++++++++---- 4 files changed, 76 insertions(+), 7 deletions(-) diff --git a/chasm/lib/tests/nexus_service.go b/chasm/lib/tests/nexus_service.go index 0ae0d42905f..386073e1ec6 100644 --- a/chasm/lib/tests/nexus_service.go +++ b/chasm/lib/tests/nexus_service.go @@ -9,9 +9,14 @@ import ( "go.temporal.io/server/common/payload" ) -var TestOperation = nexus.NewSyncOperation("TestOperation", func(ctx context.Context, input string, options nexus.StartOperationOptions) (string, error) { - return "Hello, " + input, nil -}) +// TestOperation returns a failurepb.Failure so that it is a proper protobuf type. Its used for +// testing the System Nexus Endpoint, which only accepts protobuf-encoded payloads. +var TestOperation = nexus.NewSyncOperation( + "TestOperation", + func(ctx context.Context, input string, options nexus.StartOperationOptions) (*commonpb.DataBlob, error) { + d := []byte("Hello, " + input) + return &commonpb.DataBlob{Data: d}, nil + }) // TestOperationWithPayload is identical to TestOperation, except its response embeds a // nested *commonpb.Payload. It exists to exercise the commonnexus.SystemPayloadMetadataKey @@ -20,10 +25,18 @@ var TestOperationWithPayload = nexus.NewSyncOperation("TestOperationWithPayload" return &commonpb.Payloads{Payloads: []*commonpb.Payload{payload.EncodeString("Hello, " + input)}}, nil }) +// TestOperationStringOutput returns a string, which the data converter encodes as JSON rather +// than protobuf. It exists to exercise the System Nexus Endpoint's rejection of non-protobuf +// responses in service/history/handler.go's StartNexusOperation. +var TestOperationStringOutput = nexus.NewSyncOperation("TestOperationStringOutput", func(ctx context.Context, input string, options nexus.StartOperationOptions) (string, error) { + return "Hello, " + input, nil +}) + func NewTestServiceNexusService() *nexus.Service { service := nexus.NewService("TestService") service.MustRegister(TestOperation) service.MustRegister(TestOperationWithPayload) + service.MustRegister(TestOperationStringOutput) return service } @@ -43,5 +56,6 @@ func NewTestServiceNexusServiceProcessor() *chasm.NexusServiceProcessor { sp := chasm.NewNexusServiceProcessor("TestService") sp.MustRegisterOperation("TestOperation", chasm.NewRegisterableNexusOperationProcessor(testOperationProcessor{})) sp.MustRegisterOperation("TestOperationWithPayload", chasm.NewRegisterableNexusOperationProcessor(testOperationProcessor{})) + sp.MustRegisterOperation("TestOperationStringOutput", chasm.NewRegisterableNexusOperationProcessor(testOperationProcessor{})) return sp } diff --git a/service/history/handler.go b/service/history/handler.go index 42e88cfc4d5..64cba0eba80 100644 --- a/service/history/handler.go +++ b/service/history/handler.go @@ -2540,6 +2540,9 @@ func (h *Handler) UnpauseWorkflowExecution(ctx context.Context, request *history return unpauseResp, nil } +// StartNexusOperation is the History Service's StartNexusOperation endpoint is for dispatching requests +// to the System Nexus Endpoint, distinct from the Frontend Service's `nexus_handler.go` which +// starts Nexus operations by sending Nexus tasks directly to workers via the Matching Service. func (h *Handler) StartNexusOperation( ctx context.Context, req *historyservice.StartNexusOperationRequest, @@ -2602,16 +2605,29 @@ func (h *Handler) StartNexusOperation( h.logger.Error("failed to encode payload", tag.Error(err), tag.RequestID(requestID)) return nil, serviceerror.NewInternal("internal error (request ID: " + requestID + ")") } + var payload *commonpb.Payload if len(ps.GetPayloads()) == 1 { payload = ps.GetPayloads()[0] } + // Responses from the System Nexus Endpoint are server generated, so we must mark them as system payloads. if payload != nil { if payload.Metadata == nil { payload.Metadata = make(map[string][]byte, 1) } + + // For now, we require all responess from the System Nexus Endpoint be protobufs. + encoding := string(payload.Metadata["encoding"]) + if encoding != "binary/protobuf" { + return nil, serviceerror.NewFailedPreconditionf("system payload must be encoded as binary/protobuf but got %s", encoding) + } + if _, ok := payload.Metadata["messageType"]; !ok { + return nil, serviceerror.NewFailedPrecondition("system payload missing messageType metadata key") + } + payload.Metadata[commonnexus.SystemPayloadMetadataKey] = []byte("true") } + response.Variant = &nexuspb.StartOperationResponse_SyncSuccess{ SyncSuccess: &nexuspb.StartOperationResponse_Sync{ Payload: payload, diff --git a/service/history/handler_test.go b/service/history/handler_test.go index e0c3df1b9f0..3394f59206e 100644 --- a/service/history/handler_test.go +++ b/service/history/handler_test.go @@ -450,3 +450,31 @@ func TestStartNexusOperation_SystemNexusEndpointPayloadMetadataFlag(t *testing.T }) } } + +func TestStartNexusOperation_SystemNexusEndpointRejectsNonProtoResponse(t *testing.T) { + registry := nexus.NewServiceRegistry() + registry.MustRegister(chasmtests.NewTestServiceNexusService()) + nexusHandler, err := registry.NewHandler() + require.NoError(t, err) + + h := Handler{ + logger: log.NewNoopLogger(), + nexusHandler: nexusHandler, + } + + // TestOperationStringOutput returns a string, which the data converter encodes as JSON + // instead of protobuf. The System Nexus Endpoint only accepts protobuf-encoded responses. + resp, err := h.StartNexusOperation(context.Background(), &historyservice.StartNexusOperationRequest{ + Request: &nexuspb.StartOperationRequest{ + Service: "TestService", + Operation: "TestOperationStringOutput", + RequestId: "test-request-id", + Payload: payload.EncodeString("Temporal"), + }, + }) + require.Nil(t, resp) + + var failedPrecondition *serviceerror.FailedPrecondition + require.ErrorAs(t, err, &failedPrecondition) + require.ErrorContains(t, err, "system payload must be encoded as binary/protobuf but got json/plain") +} diff --git a/tests/nexus_workflow_test.go b/tests/nexus_workflow_test.go index 699395d9735..abef91f61e1 100644 --- a/tests/nexus_workflow_test.go +++ b/tests/nexus_workflow_test.go @@ -55,6 +55,12 @@ import ( "google.golang.org/protobuf/types/known/durationpb" ) +const ( + dataBlobMessageType = "temporal.api.common.v1.DataBlob" + payloadsMessageType = "temporal.api.common.v1.Payloads" + protobufEncoding = "binary/protobuf" +) + type NexusWorkflowTestSuite struct { parallelsuite.Suite[*NexusWorkflowTestSuite] } @@ -3314,6 +3320,9 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationSystemEndpoint(chasmEnabled b result := completedEvent.GetNexusOperationCompletedEventAttributes().Result s.NotNil(result) s.Equal([]byte("true"), result.GetMetadata()[commonnexus.SystemPayloadMetadataKey]) + // TestOperation returns a proto message, so the result must be proto encoded, not JSON. + s.Equal([]byte(dataBlobMessageType), result.GetMetadata()["messageType"]) + s.Equal([]byte(protobufEncoding), result.GetMetadata()["encoding"]) // Complete the workflow _, err = env.FrontendClient().RespondWorkflowTaskCompleted(ctx, &workflowservice.RespondWorkflowTaskCompletedRequest{ @@ -3333,9 +3342,10 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationSystemEndpoint(chasmEnabled b }, }) s.NoError(err) - var response string + var response commonpb.DataBlob s.NoError(run.Get(ctx, &response)) - s.Equal("Hello, Temporal", response) + data := response.Data + s.Equal("Hello, Temporal", string(data)) } // NOTE: This test cannot use the SDK workflow package because there is a restriction that prevents setting the @@ -3392,9 +3402,10 @@ func (s *NexusWorkflowTestSuite) TestNexusOperationSystemEndpoint_PayloadMetadat completedEvent := s.RequireHistoryEvent(pollResp.History.Events, enumspb.EVENT_TYPE_NEXUS_OPERATION_COMPLETED) result := completedEvent.GetNexusOperationCompletedEventAttributes().Result s.NotNil(result) - // TestOperationWithPayload's response embeds a nested Payload, so the system payload metadata - // flag must be set. + // TestOperationWithPayload's response embeds a nested Payload, so the system payload metadata flag must be set. s.Equal([]byte("true"), result.GetMetadata()[commonnexus.SystemPayloadMetadataKey]) + s.Equal([]byte(payloadsMessageType), result.GetMetadata()["messageType"]) + s.Equal([]byte(protobufEncoding), result.GetMetadata()["encoding"]) // Complete the workflow _, err = env.FrontendClient().RespondWorkflowTaskCompleted(s.Context(), &workflowservice.RespondWorkflowTaskCompletedRequest{ From 1f8442bf502e78fccaa4b3f7614f40b422047f79 Mon Sep 17 00:00:00 2001 From: Kannan Date: Thu, 3 Sep 2026 13:27:45 -0700 Subject: [PATCH 129/139] Tune default values for dispatching worker command (#11900) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Change default dynamic config values for worker command dispatch: - `WorkerCommandsDispatchTimeout`: 10s → 5s - `WorkerCommandsMaxAttempts`: 3 → 30 ## Why - **Shorter timeout (5s):** Each dispatch attempt holds an outbound executor thread waiting for a poller. If the worker is gone, that thread is blocked for nothing. Cutting from 10s to 5s means we detect missing workers faster and free up the thread sooner. - **More attempts (30):** The main reason for retrying is to survive matching server restarts — during a rolling restart, dispatch RPCs can fail with transient transport errors until the new pod is ready. With only 3 attempts, commands could be dropped permanently during a routine restart. 30 attempts with the default backoff (initial=1s, coefficient=1.1) spreads retries over ~2 minutes, enough to ride out a restart. Transport errors fail fast (no blocking wait), so more attempts are cheap. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 4.6 (1M context) --- common/dynamicconfig/constants.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index b6252268ca3..ec2c614ed40 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -212,13 +212,19 @@ in the consistent hash ring used by ringpop. Changing it may cause service disru ) WorkerCommandsDispatchTimeout = NewGlobalDurationSetting( "system.workerCommandsDispatchTimeout", - 10*time.Second*debug.TimeoutMultiplier, - `WorkerCommandsDispatchTimeout is the timeout for dispatching worker commands to a worker via Nexus.`, + 5*time.Second*debug.TimeoutMultiplier, + `WorkerCommandsDispatchTimeout is the timeout for dispatching worker commands to a worker via Nexus.`+ + ` A small value is used to detect missing workers sooner — otherwise the outbound executor`+ + ` thread is held waiting for a poller that will never arrive.`, ) WorkerCommandsMaxAttempts = NewGlobalIntSetting( "system.workerCommandsMaxAttempts", - 3, - `WorkerCommandsMaxAttempts is the maximum number of dispatch attempts for a worker commands task before dropping it.`, + 30, + `WorkerCommandsMaxAttempts is the maximum number of dispatch attempts for a worker commands task before dropping it.`+ + ` This only applies to transport errors (e.g. matching server unavailable) — missing poller`+ + ` timeouts are not retried. Set high enough to ride out matching server rolling restarts —`+ + ` with the default backoff (initial=1s, coefficient=1.1), 30 attempts spreads retries over`+ + ` ~2 minutes. Transport errors fail fast, so more attempts are cheap.`, ) NamespaceMinRetentionGlobal = NewGlobalDurationSetting( "system.namespaceMinRetentionGlobal", From e8713b63ec054b4c3a2bf7f174c16144f6dbfa56 Mon Sep 17 00:00:00 2001 From: Kannan Date: Thu, 3 Sep 2026 13:32:06 -0700 Subject: [PATCH 130/139] Fix a race condition that can leave a poll to not get cancelled during shutdown (#11841) ## What changed? Register worker polls before checking the shutdown cache. ## Why? Prevent a shutdown request from missing a poll that is still starting. The previous implementation handled polls that arrived after shutdown had populated the cache. It did not handle the case where the poll and shutdown calls are interleaved. - The poll checks the cache and sees the worker as active. - Shutdown records the worker and cancels registered polls, but this poll is not registered yet. - The poll registers afterward and remains outstanding. ## How did you test it? Tried adding a unit test to verify this exact race; but this gets into the guts of the impl and makes it hard to read. - [x] existing unit test(s) --- service/matching/matching_engine.go | 45 ++++++++++++++--------------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/service/matching/matching_engine.go b/service/matching/matching_engine.go index 9333d491712..af4595c23b5 100644 --- a/service/matching/matching_engine.go +++ b/service/matching/matching_engine.go @@ -1272,6 +1272,7 @@ func (e *matchingEngineImpl) CancelOutstandingWorkerPolls( } // TODO: Delete this code path after EnableMatchingFanOutForPollCancellation is rolled out. if request.WorkerInstanceKey != "" { + // Keep Put before CancelAll; poll registration uses the inverse order to avoid missed polls. e.shutdownWorkers.Put(request.WorkerInstanceKey, struct{}{}) } cancelledCount := e.workerInstancePollers.CancelAll(request.WorkerInstanceKey) @@ -1416,6 +1417,7 @@ func (e *matchingEngineImpl) CancelOutstandingWorkerPollsPartition( var cancelledCount int32 for _, worker := range request.GetWorkers() { if worker.GetWorkerInstanceKey() != "" { + // Keep Put before CancelAll; poll registration uses the inverse order to avoid missed polls. e.shutdownWorkers.Put(worker.GetWorkerInstanceKey(), struct{}{}) } cancelledCount += e.workerInstancePollers.CancelAll(worker.GetWorkerInstanceKey()) @@ -3058,17 +3060,6 @@ func (e *matchingEngineImpl) pollTask( // reached, instead of emptyTask, context timeout error is returned to the frontend by the rpc stack, // which counts against our SLO. By shortening the timeout by a very small amount, the emptyTask can be // returned to the handler before a context timeout error is generated. - workerInstanceKey := pollMetadata.workerInstanceKey - if workerInstanceKey != "" && e.shutdownWorkers.Get(workerInstanceKey) != nil { - e.logger.Info("Rejecting poll from recently-shutdown worker", - tag.WorkflowNamespaceID(partition.NamespaceId()), - tag.WorkflowTaskQueueName(partition.TaskQueue().Name()), - tag.WorkflowTaskQueueType(partition.TaskType()), - tag.NewStringTag("worker-instance-key", workerInstanceKey), - ) - return nil, false, errNoTasks - } - // For non-forwarded polls, subtract a proportional random jitter to spread expiration // times across pollers and prevent thundering herd reconnects. Jitter is capped so the // interval never falls below forwardedPollMinInterval. @@ -3085,22 +3076,28 @@ func (e *matchingEngineImpl) pollTask( ctx, cancel := contextutil.WithDeadlineBuffer(ctx, longPollInterval, returnEmptyTaskTimeBudget) defer cancel() - if pollerID, ok := ctx.Value(pollerIDKey).(string); ok && pollerID != "" { - e.outstandingPollers.Set(pollerID, cancel) - - // Also track by worker instance key for bulk cancellation during shutdown. - // Use UUID (not pollerID) because pollerID is reused when forwarded. + if workerInstanceKey := pollMetadata.workerInstanceKey; workerInstanceKey != "" { + // Register before checking shutdownWorkers. The shutdown path does the reverse: + // it populates shutdownWorkers before calling CancelAll. So either the check + // sees the shutdown, or CancelAll sees the registration. pollerTrackerKey := uuid.NewString() - if workerInstanceKey != "" { - e.workerInstancePollers.Add(workerInstanceKey, pollerTrackerKey, cancel) + e.workerInstancePollers.Add(workerInstanceKey, pollerTrackerKey, cancel) + defer e.workerInstancePollers.Remove(workerInstanceKey, pollerTrackerKey) + + if e.shutdownWorkers.Get(workerInstanceKey) != nil { + e.logger.Debug("Rejecting poll from recently-shutdown worker", + tag.WorkflowNamespaceID(partition.NamespaceId()), + tag.WorkflowTaskQueueName(partition.TaskQueue().Name()), + tag.WorkflowTaskQueueType(partition.TaskType()), + tag.NewStringTag("worker-instance-key", workerInstanceKey), + ) + return nil, false, errNoTasks } + } - defer func() { - e.outstandingPollers.Delete(pollerID) - if workerInstanceKey != "" { - e.workerInstancePollers.Remove(workerInstanceKey, pollerTrackerKey) - } - }() + if pollerID, ok := ctx.Value(pollerIDKey).(string); ok && pollerID != "" { + e.outstandingPollers.Set(pollerID, cancel) + defer e.outstandingPollers.Delete(pollerID) } return pm.PollTask(ctx, pollMetadata) } From 4fe1c3d05f0cf88fd5301104b7db8ac4b94c2ce2 Mon Sep 17 00:00:00 2001 From: Will Duan Date: Thu, 3 Sep 2026 16:18:16 -0700 Subject: [PATCH 131/139] Fix workflow update count during state replication (#11911) ## What changed? Removed the extra UpdateCount increment when replicated UpdateInfo entries are inserted into mutable state. Extended mutation and snapshot replication tests to cover a newly replicated workflow update and verify the source count is preserved. ## Why? UpdateCount is already synchronized from the source ExecutionInfo. Incrementing it again while applying UpdatedUpdateInfos caused passive mutable state to report 2 updates when the active mutable state reported 1. ## How did you test it? - [x] run locally and tested manually - [x] added new unit test(s) - [x] added new functional test(s) Ran the focused MutableState ApplyMutation, ApplySnapshot, and UpdateInfos tests. Also ran snapshot-only passive-path replication against the OMES throughput_stress scenario with Workflow Updates, Nexus, child workflows, and continue-as-new; 283 passive applies completed without task or mutable-state differences. ## Potential risks Low. The change only stops passive replication from incrementing a cumulative counter that is already copied from the source artifact. --- .../history/workflow/mutable_state_impl.go | 4 ++-- .../workflow/mutable_state_impl_test.go | 20 +++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index f5eaf1b06ad..a62154fd167 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -9548,14 +9548,14 @@ func (ms *MutableStateImpl) applyUpdatesToUpdateInfos( } } + // UpdateCount is the source's cumulative count and is synchronized with + // ExecutionInfo; inserting replicated map entries must not increment it again. for updateID, ui := range updatedUpdateInfos { if existing, ok := ms.executionInfo.UpdateInfos[updateID]; ok { if transitionhistory.Compare(existing.GetLastUpdateVersionedTransition(), ui.GetLastUpdateVersionedTransition()) == 0 { continue } ms.approximateSize -= existing.Size() + len(updateID) - } else { - ms.executionInfo.UpdateCount++ } ms.executionInfo.UpdateInfos[updateID] = ui ms.approximateSize += ui.Size() + len(updateID) diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index ba74313aa81..ab0d17b8deb 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -5836,6 +5836,8 @@ func (s *mutableStateSuite) buildSnapshot(state *MutableStateImpl) *persistences }, SignalRequestIdsLastUpdateVersionedTransition: &persistencespb.VersionedTransition{TransitionCount: 1025}, WorkflowTaskLastUpdateVersionedTransition: state.executionInfo.WorkflowTaskLastUpdateVersionedTransition, + UpdateInfos: state.executionInfo.UpdateInfos, + UpdateCount: state.executionInfo.UpdateCount, }, ExecutionState: &persistencespb.WorkflowExecutionState{ RunId: state.executionState.RunId, @@ -5900,6 +5902,14 @@ func (s *mutableStateSuite) TestApplySnapshot() { currentMS.chasmTree = currentMockChasmTree state = s.buildWorkflowMutableState() + state.ExecutionInfo.UpdateCount = 1 + state.ExecutionInfo.UpdateInfos = map[string]*persistencespb.UpdateInfo{ + "replicated-update": { + Value: &persistencespb.UpdateInfo_Acceptance{ + Acceptance: &persistencespb.UpdateAcceptanceInfo{EventId: 100}, + }, + }, + } state.ActivityInfos[91] = &persistencespb.ActivityInfo{ ActivityId: "activity_id_91", } @@ -5975,12 +5985,14 @@ func (s *mutableStateSuite) buildMutation( ) *persistencespb.WorkflowMutableStateMutation { executionInfoClone := common.CloneProto(state.executionInfo) executionInfoClone.SubStateMachineTombstoneBatches = nil + executionInfoClone.UpdateInfos = nil mutation := &persistencespb.WorkflowMutableStateMutation{ UpdatedActivityInfos: state.pendingActivityInfoIDs, UpdatedTimerInfos: state.pendingTimerInfoIDs, UpdatedChildExecutionInfos: state.pendingChildExecutionInfoIDs, UpdatedRequestCancelInfos: state.pendingRequestCancelInfoIDs, UpdatedSignalInfos: state.pendingSignalInfoIDs, + UpdatedUpdateInfos: state.executionInfo.UpdateInfos, UpdatedChasmNodes: state.chasmTree.Snapshot(nil).Nodes, SignalRequestedIds: state.GetPendingSignalRequestedIds(), SubStateMachineTombstoneBatches: tombstones, @@ -6051,6 +6063,14 @@ func (s *mutableStateSuite) TestApplyMutation() { currentMS.GetExecutionInfo().SubStateMachineTombstoneBatches = tombstones state = s.buildWorkflowMutableState() + state.ExecutionInfo.UpdateCount = 1 + state.ExecutionInfo.UpdateInfos = map[string]*persistencespb.UpdateInfo{ + "replicated-update": { + Value: &persistencespb.UpdateInfo_Acceptance{ + Acceptance: &persistencespb.UpdateAcceptanceInfo{EventId: 100}, + }, + }, + } targetMS, err := NewMutableStateFromDB(s.mockShard, s.mockEventsCache, s.logger, tests.LocalNamespaceEntry, state, 123) s.NoError(err) From 90c5573ea8acef235b4fbcb110da5b76bf68348f Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 3 Sep 2026 17:07:09 -0700 Subject: [PATCH 132/139] Compare weekly CI report metrics (#11890) This changes the weekly CI report to show deltas to help clearly see if sth improved or got worse. --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- tools/ci-notify/app.go | 4 +- tools/ci-notify/app_test.go | 14 +++--- tools/ci-notify/digest.go | 36 ++++++++++---- tools/ci-notify/slack.go | 98 +++++++++++++++++++++---------------- tools/ci-notify/types.go | 21 +++++--- 5 files changed, 106 insertions(+), 67 deletions(-) diff --git a/tools/ci-notify/app.go b/tools/ci-notify/app.go index ab61242cdea..4694cc9a799 100644 --- a/tools/ci-notify/app.go +++ b/tools/ci-notify/app.go @@ -295,9 +295,9 @@ func runDigestCommand(c *cli.Context) error { // Handle dry-run mode if dryRun { logger.Info("Dry-run mode: printing report to stdout") - fmt.Println(FormatReportForDebug(report)) - fmt.Println("\n--- Slack JSON Payload ---") message := BuildSuccessReportMessage(report) + fmt.Println(message.RenderMarkdown()) + fmt.Println("\n--- Slack JSON Payload ---") payload, err := marshalIndent(message) if err != nil { logger.Error("Failed to marshal message for display", zap.Error(err)) diff --git a/tools/ci-notify/app_test.go b/tools/ci-notify/app_test.go index d3246fd4a1d..961ad9ae730 100644 --- a/tools/ci-notify/app_test.go +++ b/tools/ci-notify/app_test.go @@ -247,12 +247,14 @@ func TestFilterCompleted(t *testing.T) { func TestSlowestRuns(t *testing.T) { report := &DigestReport{ - Runs: []github.Run{ - {DisplayTitle: "medium", Duration: 20 * time.Minute}, - {DisplayTitle: "ignored", Duration: 0}, - {DisplayTitle: "slowest", Duration: 45 * time.Minute}, - {DisplayTitle: "fastest", Duration: 5 * time.Minute}, - {DisplayTitle: "second slowest", Duration: 30 * time.Minute}, + DigestPeriod: DigestPeriod{ + Runs: []github.Run{ + {DisplayTitle: "medium", Duration: 20 * time.Minute}, + {DisplayTitle: "ignored", Duration: 0}, + {DisplayTitle: "slowest", Duration: 45 * time.Minute}, + {DisplayTitle: "fastest", Duration: 5 * time.Minute}, + {DisplayTitle: "second slowest", Duration: 30 * time.Minute}, + }, }, } diff --git a/tools/ci-notify/digest.go b/tools/ci-notify/digest.go index 3eebd7ded41..7464d765409 100644 --- a/tools/ci-notify/digest.go +++ b/tools/ci-notify/digest.go @@ -87,7 +87,7 @@ func getWorkflowRuns(branch, workflowName string, since time.Time) ([]github.Run Branch: branch, Workflow: workflowName, Created: ">=" + sinceDate, - Limit: 1000, + Limit: 2000, }) if err != nil { return nil, fmt.Errorf("failed to get workflow runs: %w", err) @@ -103,18 +103,39 @@ func getWorkflowRuns(branch, workflowName string, since time.Time) ([]github.Run return runs, nil } -// BuildDigest builds a digest report for the specified time range +// BuildDigest builds a digest report for the specified time range. func BuildDigest(branch, workflowName string, days int) (*DigestReport, error) { // Calculate the start date endDate := time.Now() startDate := endDate.AddDate(0, 0, -days) + previousStartDate := startDate.AddDate(0, 0, -days) // Fetch workflow runs - runs, err := getWorkflowRuns(branch, workflowName, startDate) + runs, err := getWorkflowRuns(branch, workflowName, previousStartDate) if err != nil { return nil, err } + var currentRuns, previousRuns []github.Run + for _, run := range runs { + if !run.CreatedAt.Before(startDate) { + currentRuns = append(currentRuns, run) + } else if !run.CreatedAt.Before(previousStartDate) { + previousRuns = append(previousRuns, run) + } + } + + return &DigestReport{ + Branch: branch, + WorkflowName: workflowName, + StartDate: startDate, + EndDate: endDate, + DigestPeriod: summarizeDigestPeriod(currentRuns), + Previous: summarizeDigestPeriod(previousRuns), + }, nil +} + +func summarizeDigestPeriod(runs []github.Run) DigestPeriod { // Filter to only completed runs completedRuns := filterCompleted(runs) @@ -146,20 +167,17 @@ func BuildDigest(branch, workflowName string, days int) (*DigestReport, error) { under25 := calculatePercentUnder(durations, 25*time.Minute) under30 := calculatePercentUnder(durations, 30*time.Minute) - return &DigestReport{ - Branch: branch, - WorkflowName: workflowName, - StartDate: startDate, - EndDate: endDate, + return DigestPeriod{ TotalRuns: totalRuns, SuccessfulRuns: successCount, FailedRuns: failureCount, SuccessRate: successRate, + DurationSamples: len(durations), AverageDuration: calculateAverage(durations), MedianDuration: calculateMedian(durations), Under20MinutesPercent: under20, Under25MinutesPercent: under25, Under30MinutesPercent: under30, Runs: completedRuns, - }, nil + } } diff --git a/tools/ci-notify/slack.go b/tools/ci-notify/slack.go index 9deb03b58e2..c631c6c8a88 100644 --- a/tools/ci-notify/slack.go +++ b/tools/ci-notify/slack.go @@ -1,15 +1,22 @@ package cinotify import ( + "cmp" "fmt" + "math" "strconv" "strings" + "time" "go.temporal.io/server/tools/common/github" "go.temporal.io/server/tools/common/slack" ) -const maxFailures = 5 +const ( + maxFailures = 5 + successRateComparisonTolerance = 0.1 + durationComparisonTolerance = 30 * time.Second +) // BuildFailureMessage creates a Slack message for CI failure func BuildFailureMessage(report *FailureReport) *slack.Message { @@ -118,6 +125,39 @@ func raceLink(runID string, race DataRace) string { return github.RunURL(temporalRepository, runID) } +func formatComparison(direction int, difference string) string { + switch { + case direction > 0: + return fmt.Sprintf("↑ %s", difference) + case direction < 0: + return fmt.Sprintf("↓ %s", difference) + default: + return "— no change" + } +} + +func formatPercentagePointComparison(current, previous float64, available bool) string { + if !available { + return "N/A" + } + difference := math.Abs(current - previous) + if difference <= successRateComparisonTolerance { + return formatComparison(0, "") + } + return formatComparison(cmp.Compare(current, previous), fmt.Sprintf("%.1f pp", difference)) +} + +func formatDurationComparison(current, previous time.Duration, available bool) string { + if !available { + return "N/A" + } + difference := (current - previous).Abs() + if difference <= durationComparisonTolerance { + return formatComparison(0, "") + } + return formatComparison(cmp.Compare(current, previous), formatDuration(difference)) +} + // BuildSuccessReportMessage creates a Slack message for success report func BuildSuccessReportMessage(report *DigestReport) *slack.Message { message := slack.NewMessage(fmt.Sprintf("Weekly CI Report - %s Branch", report.Branch)) @@ -127,13 +167,21 @@ func BuildSuccessReportMessage(report *DigestReport) *slack.Message { report.StartDate.Format("Jan 2, 2006"), report.EndDate.Format("Jan 2, 2006"), )) + durationsComparable := report.DurationSamples > 0 && report.Previous.DurationSamples > 0 message.AddFields( - fmt.Sprintf("*Success Rate:*\n%.1f%%", report.SuccessRate), - fmt.Sprintf("*Total Runs:*\n%d", report.TotalRuns), - fmt.Sprintf("*Failed Runs:*\n%d", report.FailedRuns), - fmt.Sprintf("*Successful Runs:*\n%d", report.SuccessfulRuns), - fmt.Sprintf("*Average Duration:*\n%s", formatDuration(report.AverageDuration)), - fmt.Sprintf("*Median Duration:*\n%s", formatDuration(report.MedianDuration)), + fmt.Sprintf("*Success Rate:*\n%.1f%% (%s)", report.SuccessRate, + formatPercentagePointComparison( + report.SuccessRate, report.Previous.SuccessRate, + report.TotalRuns > 0 && report.Previous.TotalRuns > 0)), + fmt.Sprintf("*Failed Runs:*\n%d/%d", report.FailedRuns, report.TotalRuns), + fmt.Sprintf("*Average Duration:*\n%s (%s)", formatDuration(report.AverageDuration), + formatDurationComparison( + report.AverageDuration, report.Previous.AverageDuration, + durationsComparable)), + fmt.Sprintf("*Median Duration:*\n%s (%s)", formatDuration(report.MedianDuration), + formatDurationComparison( + report.MedianDuration, report.Previous.MedianDuration, + durationsComparable)), ) message.AddSection(fmt.Sprintf( "*Run Duration Distribution:*\n"+ @@ -161,39 +209,3 @@ func BuildSuccessReportMessage(report *DigestReport) *slack.Message { return message } - -// FormatReportForDebug formats the success report for console output -func FormatReportForDebug(report *DigestReport) string { - var sb strings.Builder - fmt.Fprintf(&sb, "📊 Weekly CI Report - %s Branch\n\n", report.Branch) - fmt.Fprintf(&sb, "Report Period: %s to %s\n\n", - report.StartDate.Format("Jan 2, 2006"), - report.EndDate.Format("Jan 2, 2006")) - fmt.Fprintln(&sb, "Metrics:") - fmt.Fprintf(&sb, " Success Rate: %.1f%%\n", report.SuccessRate) - fmt.Fprintf(&sb, " Total Runs: %d\n", report.TotalRuns) - fmt.Fprintf(&sb, " Successful Runs: %d\n", report.SuccessfulRuns) - fmt.Fprintf(&sb, " Failed Runs: %d\n", report.FailedRuns) - fmt.Fprintf(&sb, " Average Duration: %s\n", formatDuration(report.AverageDuration)) - fmt.Fprintf(&sb, " Median Duration: %s\n", formatDuration(report.MedianDuration)) - - fmt.Fprintln(&sb, "\nRun Duration Distribution:") - fmt.Fprintf(&sb, " Under 20 minutes: %.1f%%\n", report.Under20MinutesPercent) - fmt.Fprintf(&sb, " Under 25 minutes: %.1f%%\n", report.Under25MinutesPercent) - fmt.Fprintf(&sb, " Under 30 minutes: %.1f%%\n", report.Under30MinutesPercent) - - slowestRuns := report.slowestRuns(3) - if len(slowestRuns) > 0 { - fmt.Fprintln(&sb, "\nSlowest Runs:") - for _, run := range slowestRuns { - fmt.Fprintf(&sb, " %s (%s): %s\n %s\n", - formatDuration(run.Duration), - run.Conclusion, - run.ShortSHA(), - run.URL, - ) - } - } - - return sb.String() -} diff --git a/tools/ci-notify/types.go b/tools/ci-notify/types.go index 14a411ca183..4f3e1d1b703 100644 --- a/tools/ci-notify/types.go +++ b/tools/ci-notify/types.go @@ -7,7 +7,7 @@ import ( "go.temporal.io/server/tools/common/github" ) -// FailureReport aggregates all failure information +// FailureReport aggregates all failure information. type FailureReport struct { Run github.Run FailedJobs []github.Job @@ -15,16 +15,13 @@ type FailureReport struct { TotalJobs int } -// DigestReport aggregates success metrics for a time period -type DigestReport struct { - Branch string - WorkflowName string - StartDate time.Time - EndDate time.Time +// DigestPeriod aggregates success metrics for a time period. +type DigestPeriod struct { TotalRuns int SuccessfulRuns int FailedRuns int SuccessRate float64 + DurationSamples int AverageDuration time.Duration MedianDuration time.Duration Under20MinutesPercent float64 @@ -33,6 +30,16 @@ type DigestReport struct { Runs []github.Run } +// DigestReport aggregates success metrics for a time period. +type DigestReport struct { + Branch string + WorkflowName string + StartDate time.Time + EndDate time.Time + DigestPeriod + Previous DigestPeriod +} + func (r *DigestReport) slowestRuns(limit int) []github.Run { if limit <= 0 { return nil From 2792bc86cbcc957edda27e3b2eefafca31d46ae8 Mon Sep 17 00:00:00 2001 From: Stephan Behnke Date: Thu, 3 Sep 2026 17:07:26 -0700 Subject: [PATCH 133/139] Fix SQLite transaction cancellation test race (#11922) Fix a test data race caused by passing the address of `sql.ErrNoRows` to `assert.ErrorAs`. Since `errors.As` writes the matching error into its target, the assertion could mutate the package-level sentinel while parallel persistence tests read it. Use `require.ErrorIs` to verify sentinel identity without mutating global state. --- common/persistence/tests/sqlite_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/persistence/tests/sqlite_test.go b/common/persistence/tests/sqlite_test.go index a1deb1fde65..b8ed04c01cc 100644 --- a/common/persistence/tests/sqlite_test.go +++ b/common/persistence/tests/sqlite_test.go @@ -1597,6 +1597,5 @@ func TestSQLiteTransactionContextCancellation(t *testing.T) { RangeHash: 0, TaskQueueID: []byte("test-queue"), }, sqlplugin.MatchingTaskVersion1) - assert.NotContains(t, err.Error(), "no such table") - assert.ErrorAs(t, err, &gosql.ErrNoRows) + require.ErrorIs(t, err, gosql.ErrNoRows) } From eb8f340caf59be69dab0e8c887ba2db836e86f3f Mon Sep 17 00:00:00 2001 From: Shahab Tajik Date: Thu, 3 Sep 2026 17:48:17 -0700 Subject: [PATCH 134/139] Prioritize worker versioning workflows (#11930) ## What changed? - assign priority key 1 to Worker Deployment and Worker Deployment Version workflow starts - cover the shared start-request builder with a unit test ## Why? Worker versioning workflows run on the per-namespace worker task queue alongside other system workflows. Giving their initial workflow tasks higher-than-default priority prevents versioning operations from being delayed behind default-priority work. ## How did you test it? - `go test -tags test_dep ./service/worker/workerdeployment -run TestMakeStartRequestSetsHighPriority` - `go vet -tags test_dep ./service/worker/workerdeployment` --- service/worker/workerdeployment/util.go | 4 +++- service/worker/workerdeployment/util_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/service/worker/workerdeployment/util.go b/service/worker/workerdeployment/util.go index f03fd1e580a..b87691fc1b6 100644 --- a/service/worker/workerdeployment/util.go +++ b/service/worker/workerdeployment/util.go @@ -40,7 +40,8 @@ const ( WorkerDeploymentWorkflowType = "temporal-sys-worker-deployment-workflow" // Namespace division - WorkerDeploymentNamespaceDivision = "TemporalWorkerDeployment" + WorkerDeploymentNamespaceDivision = "TemporalWorkerDeployment" + workerDeploymentWorkflowPriorityKey = 1 // Updates RegisterWorkerInDeploymentVersion = "register-task-queue-worker" // for Worker Deployment Version wf @@ -430,6 +431,7 @@ func makeStartRequest( SearchAttributes: buildSearchAttributes(), Memo: memo, Identity: identity, + Priority: &commonpb.Priority{PriorityKey: workerDeploymentWorkflowPriorityKey}, } } diff --git a/service/worker/workerdeployment/util_test.go b/service/worker/workerdeployment/util_test.go index cac6a752766..675cfe59baa 100644 --- a/service/worker/workerdeployment/util_test.go +++ b/service/worker/workerdeployment/util_test.go @@ -370,6 +370,18 @@ func decodeAndValidateMemo(t *testing.T, filePath, deploymentName, buildID strin require.Equal(t, buildID, result.RoutingConfig.GetCurrentDeploymentVersion().GetBuildId()) } +func TestMakeStartRequestSetsHighPriority(t *testing.T) { + ns := namespace.NewLocalNamespaceForTest( + &persistencespb.NamespaceInfo{Name: testNamespace}, + nil, + "", + ) + + request := makeStartRequest("request-id", "workflow-id", "identity", "workflow-type", ns, nil, nil) + + require.Equal(t, int32(workerDeploymentWorkflowPriorityKey), request.GetPriority().GetPriorityKey()) +} + func TestIsRetryableUpdateError(t *testing.T) { t.Run("returns true for errUpdateInProgress", func(t *testing.T) { require.True(t, isRetryableUpdateError(errUpdateInProgress)) From 9979699190e0da4559a420fc749fb5383e8ac391 Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Thu, 3 Sep 2026 18:29:00 -0700 Subject: [PATCH 135/139] [Visibility] Fix legacy visibility query converter (#11910) ## What changed? - Fix parsing negative double values. - Consolidated the function to resolve alias (there were two, and not behaving the same way, the legacy was missing some cases). - Added more unit tests to verify the legacy query converter behaves the same as in the new one except for the expected differences. ## Why? Fixes to legacy visibility query converter for SQL and Elasticsearch. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- .../store/elasticsearch/query_interceptors.go | 12 +- .../visibility_store_read_test.go | 80 ++- .../visibility/store/query/converter.go | 82 +-- .../store/query/converter_legacy.go | 38 ++ .../visibility/store/query/converter_test.go | 251 +-------- .../visibility/store/query/errors.go | 1 + .../visibility/store/query/resolve.go | 149 ++--- .../visibility/store/query/resolve_test.go | 526 ++++++++++-------- .../store/sql/query_converter_legacy.go | 58 +- .../store/sql/query_converter_legacy_test.go | 206 ++++++- .../store/sql/visibility_store_test.go | 2 +- .../tests/query_converter_legacy_test.go | 424 ++++++++++++++ .../store/tests/query_converter_test.go | 16 +- service/worker/scheduler/query_test.go | 4 +- 14 files changed, 1162 insertions(+), 687 deletions(-) create mode 100644 common/persistence/visibility/store/tests/query_converter_legacy_test.go diff --git a/common/persistence/visibility/store/elasticsearch/query_interceptors.go b/common/persistence/visibility/store/elasticsearch/query_interceptors.go index c6a181bad5e..9f1f23ed6a3 100644 --- a/common/persistence/visibility/store/elasticsearch/query_interceptors.go +++ b/common/persistence/visibility/store/elasticsearch/query_interceptors.go @@ -78,8 +78,14 @@ func (ni *nameInterceptor) Name(name string, usage query.FieldNameUsage) (string if err != nil { return "", err } - fieldName, fieldType, err := query.ResolveSearchAttributeAlias(name, ni.namespace, mapper, - ni.searchAttributesTypeMap, ni.chasmMapper) + fieldName, fieldType, err := query.ResolveSearchAttributeAlias( + name, + ni.namespace, + mapper, + ni.searchAttributesTypeMap, + ni.chasmMapper, + ni.archetypeID, + ) if err != nil { // Check for special aliases that require archetypeID context. if ni.archetypeID != chasm.SchedulerArchetypeID || name != "TemporalSystemExecutionStatus" { @@ -127,7 +133,7 @@ func (vi *valuesInterceptor) Values(name string, fieldName string, values ...any fieldType, err = vi.saTypeMap.GetType(fieldName) if err != nil { - return nil, query.NewConverterError("invalid search attribute: %s", name) + return nil, query.NewConverterError("%s: %s", query.InvalidSearchAttribute, name) } var result []any diff --git a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go index c07882f6778..f30908df759 100644 --- a/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go +++ b/common/persistence/visibility/store/elasticsearch/visibility_store_read_test.go @@ -411,6 +411,82 @@ func (s *ESVisibilitySuite) Test_convertQueryLegacy() { s.Equal(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"should":[{"range":{"ExecutionTime":{"from":null,"include_lower":true,"include_upper":false,"to":"1970-01-01T00:00:00.001Z"}}},{"range":{"ExecutionTime":{"from":"1970-01-01T00:00:00.002Z","include_lower":false,"include_upper":true,"to":null}}}]}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) s.Nil(queryParams.Sorter) + // The SQL parser folds the sign into integer values, but represents signed floats as an + // unary expression. + query = `CustomIntField = -10` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"match":{"CustomIntField":{"query":-10}}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + query = `CustomDoubleField = -1.5` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"match":{"CustomDoubleField":{"query":-1.5}}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + query = `CustomDoubleField = +1.5` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"match":{"CustomDoubleField":{"query":1.5}}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + query = `CustomDoubleField > -1.5` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"range":{"CustomDoubleField":{"from":-1.5,"include_lower":false,"include_upper":true,"to":null}}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + query = `CustomDoubleField between -2.5 and -1.5` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"range":{"CustomDoubleField":{"from":-2.5,"include_lower":true,"include_upper":true,"to":-1.5}}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + query = `CustomDoubleField in (-1.5, 2.5)` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.NoError(err) + s.JSONEq(`{"bool":{"filter":[{"term":{"NamespaceId":"bfd5c907-f899-4baf-a7b2-2ab85e623ebd"}},{"bool":{"filter":{"terms":{"CustomDoubleField":[-1.5,2.5]}}}}],"must_not":{"exists":{"field":"TemporalNamespaceDivision"}}}}`, s.queryToJSON(queryParams.Query)) + s.Nil(queryParams.Sorter) + + // Only a literal value can be signed, not another unary expression. + query = `CustomDoubleField = - -1.5` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.Error(err) + var invalidArgumentErr *serviceerror.InvalidArgument + s.ErrorAs(err, &invalidArgumentErr) + s.Equal( + `invalid query: unable to convert filter expression: `+ + `unable to convert right side of "CustomDoubleField = - -1.5": `+ + `invalid expression: unary operator not supported in "- -1.5"`, + err.Error(), + ) + s.Nil(queryParams) + + query = `CustomKeywordField = -'foo'` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.Error(err) + s.ErrorAs(err, &invalidArgumentErr) + s.Equal( + `invalid query: unable to convert filter expression: `+ + `unable to convert right side of "CustomKeywordField = -'foo'": `+ + `invalid expression: unary operator not supported in "-'foo'"`, + err.Error(), + ) + s.Nil(queryParams) + + query = `CustomIntField = ~1` + queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) + s.Error(err) + s.ErrorAs(err, &invalidArgumentErr) + s.Equal( + `invalid query: unable to convert filter expression: `+ + `unable to convert right side of "CustomIntField = ~1": `+ + `operation is not supported: unary operator "~"`, + err.Error(), + ) + s.Nil(queryParams) + query = `order by ExecutionTime` queryParams, err = s.visibilityStore.convertQueryLegacy(testNamespace, testNamespaceID, query, nil, chasm.UnspecifiedArchetypeID) s.NoError(err) @@ -625,7 +701,7 @@ func (s *ESVisibilitySuite) Test_convertQuery() { { name: "invalid custom search attributes", query: "WorkflowId = 'wid' AND InvalidField = 'foo'", - err: query.InvalidExpressionErrMessage, + err: query.InvalidSearchAttribute, }, } @@ -2088,7 +2164,7 @@ func (s *ESVisibilitySuite) Test_convertQuery_ChasmMapper() { { name: "invalid chasm attribute", query: "UnknownChasmField = 'value'", - err: query.InvalidExpressionErrMessage, + err: query.InvalidSearchAttribute, }, } diff --git a/common/persistence/visibility/store/query/converter.go b/common/persistence/visibility/store/query/converter.go index 2394061d435..c60106d2983 100644 --- a/common/persistence/visibility/store/query/converter.go +++ b/common/persistence/visibility/store/query/converter.go @@ -492,7 +492,14 @@ func (c *QueryConverter[ExprT]) convertColName(in sqlparser.Expr) (*SAColumn, er ) } saAlias := strings.ReplaceAll(sqlparser.String(expr), "`", "") - saFieldName, saType, err := c.resolveSearchAttributeAlias(saAlias) + saFieldName, saType, err := ResolveSearchAttributeAlias( + saAlias, + c.namespaceName, + c.saMapper, + c.saTypeMap, + c.chasmMapper, + c.archetypeID, + ) if err != nil { return nil, err } @@ -508,79 +515,6 @@ func (c *QueryConverter[ExprT]) convertColName(in sqlparser.Expr) (*SAColumn, er return colName, nil } -func (c *QueryConverter[ExprT]) resolveSearchAttributeAlias( - alias string, -) (fieldName string, fieldType enumspb.IndexedValueType, retErr error) { - // resolveCSA only returns true if `alias` is a custom search attribute. - resolveCSA := func(alias string) bool { - fn, err := c.saMapper.GetFieldName(alias, c.namespaceName.String()) - if err != nil { - return false - } - ft, err := c.saTypeMap.GetType(fn) - if err != nil { - return false - } - fieldName, fieldType = fn, ft - return true - } - // resolveChasmSA only returns true if `alias` is a CHASM search attribute. - resolveChasmSA := func(alias string) bool { - if c.chasmMapper == nil { - return false - } - fn, err := c.chasmMapper.Field(alias) - if err != nil { - return false - } - ft, err := c.chasmMapper.ValueType(fn) - if err != nil { - return false - } - fieldName, fieldType = fn, ft - return true - } - - var err error - fieldName = alias - // First, check if it's a custom search attribute. - if sadefs.IsMappable(alias) && resolveCSA(alias) { - return - } - // Second, check if it's a CHASM search attribute. - if resolveChasmSA(alias) { - return - } - // Third, check if it's a system/reserved search attribute. - fieldType, err = c.saTypeMap.GetType(fieldName) - if err == nil { - return - } - // Fourth, check for special aliases or adding/removing the `Temporal` prefix. - if strings.TrimPrefix(alias, sadefs.ReservedPrefix) == sadefs.ScheduleID { - fieldName = sadefs.WorkflowID - } else if c.archetypeID == chasm.SchedulerArchetypeID && alias == "TemporalSystemExecutionStatus" { - // To support querying Workflow based schedulers and CHASM based schedulers, we need to translate - // TemporalSystemExecutionStatus as an alias to the system search attribute ExecutionStatus. - fieldName = "ExecutionStatus" - } else if strings.HasPrefix(fieldName, sadefs.ReservedPrefix) { - fieldName = fieldName[len(sadefs.ReservedPrefix):] - } else { - fieldName = sadefs.ReservedPrefix + fieldName - } - fieldType, err = c.saTypeMap.GetType(fieldName) - if err == nil { - return - } - - retErr = NewConverterError( - "%s: column name '%s' is not a valid search attribute", - InvalidExpressionErrMessage, - alias, - ) - return -} - func (c *QueryConverter[ExprT]) parseValueExpr( expr sqlparser.Expr, saName string, diff --git a/common/persistence/visibility/store/query/converter_legacy.go b/common/persistence/visibility/store/query/converter_legacy.go index ce2b99f5b6c..82328b270f7 100644 --- a/common/persistence/visibility/store/query/converter_legacy.go +++ b/common/persistence/visibility/store/query/converter_legacy.go @@ -534,6 +534,44 @@ func convertComparisonExprValue(expr sqlparser.Expr) (any, error) { result = append(result, v) } return result, nil + case *sqlparser.UnaryExpr: + // Negative value may be parsed as UnaryExpr + if e.Operator != sqlparser.UPlusStr && e.Operator != sqlparser.UMinusStr { + return nil, NewConverterError( + "%s: unary operator %q", + NotSupportedErrMessage, + e.Operator, + ) + } + if value, ok := e.Expr.(*sqlparser.SQLVal); !ok || value.Type == sqlparser.StrVal { + return nil, NewConverterError( + "%s: unary operator not supported in %q", + InvalidExpressionErrMessage, + sqlparser.String(expr), + ) + } + value, err := convertComparisonExprValue(e.Expr) + if err != nil { + return nil, err + } + switch v := value.(type) { + case int64: + if e.Operator == sqlparser.UMinusStr { + value = -v + } + case float64: + if e.Operator == sqlparser.UMinusStr { + value = -v + } + default: + // This should never happen, but here to catch any unexpected case. + return nil, NewConverterError( + "%s: unary expression %q", + InvalidExpressionErrMessage, + sqlparser.String(expr), + ) + } + return value, nil case *sqlparser.GroupConcatExpr: return nil, NewConverterError("%s: 'group_concat'", NotSupportedErrMessage) case *sqlparser.FuncExpr: diff --git a/common/persistence/visibility/store/query/converter_test.go b/common/persistence/visibility/store/query/converter_test.go index 5b6c8aa6671..112f780841a 100644 --- a/common/persistence/visibility/store/query/converter_test.go +++ b/common/persistence/visibility/store/query/converter_test.go @@ -508,13 +508,13 @@ func TestQueryConverter_ConvertSelectStmt(t *testing.T) { { name: "fail invalid group by field", in: "select * from t group by InvalidField", - err: InvalidExpressionErrMessage, + err: InvalidSearchAttribute, }, { name: "fail invalid order by field", in: "select * from t order by InvalidField", - err: InvalidExpressionErrMessage, + err: InvalidSearchAttribute, }, { @@ -1319,7 +1319,7 @@ func TestQueryConverter_ConvertComparisonExprFail(t *testing.T) { { name: "invalid col name", in: "InvalidField = 'foo'", - err: InvalidExpressionErrMessage, + err: InvalidSearchAttribute, }, { @@ -1430,7 +1430,7 @@ func TestQueryConverter_ConvertRangeCond(t *testing.T) { { name: "fail invalid col name", in: "InvalidField BETWEEN '123' AND '456'", - err: InvalidExpressionErrMessage, + err: InvalidSearchAttribute, }, { @@ -1537,7 +1537,7 @@ func TestQueryConverter_ConvertIsExpr(t *testing.T) { { name: "fail invalid col name", in: "InvalidField IS NOT NULL", - err: InvalidExpressionErrMessage, + err: InvalidSearchAttribute, }, { @@ -1680,10 +1680,7 @@ func TestQueryConverter_ConvertColName(t *testing.T) { in: &sqlparser.ColName{ Name: sqlparser.NewColIdent("InvalidField"), }, - err: fmt.Sprintf( - "%s: column name 'InvalidField' is not a valid search attribute", - InvalidExpressionErrMessage, - ), + err: fmt.Sprintf("%s: InvalidField", InvalidSearchAttribute), }, } @@ -1713,166 +1710,6 @@ func TestQueryConverter_ConvertColName(t *testing.T) { } } -func TestQueryConverter_ResolveSearchAttributeAlias(t *testing.T) { - t.Parallel() - - testCases := []struct { - name string - in string - withCustomScheduleID bool - useNoopMapper bool - outFn string - outFt enumspb.IndexedValueType - err string - }{ - { - name: "success system StartTime", - in: "StartTime", - outFn: "StartTime", - outFt: enumspb.INDEXED_VALUE_TYPE_DATETIME, - }, - - { - name: "success system WorkflowId", - in: "WorkflowId", - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success reserved BuildIds", - in: "BuildIds", - outFn: "BuildIds", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, - }, - - { - name: "success reserved TemporalBuildIds", - in: "TemporalBuildIds", - outFn: "BuildIds", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, - }, - - { - name: "success reserved TemporalWorkerDeployment", - in: "TemporalWorkerDeployment", - outFn: "TemporalWorkerDeployment", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success reserved WorkerDeployment", - in: "WorkerDeployment", - outFn: "TemporalWorkerDeployment", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success custom AliasForInt01", - in: "AliasForInt01", - outFn: "Int01", - outFt: enumspb.INDEXED_VALUE_TYPE_INT, - }, - - { - name: "success custom noop mapper Int01", - in: "Int01", - useNoopMapper: true, - outFn: "Int01", - outFt: enumspb.INDEXED_VALUE_TYPE_INT, - }, - - { - name: "success special ScheduleId", - in: "ScheduleId", - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success special TemporalScheduleId", - in: "TemporalScheduleId", - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success custom ScheduleId", - in: "ScheduleId", - withCustomScheduleID: true, - outFn: searchattribute.TestScheduleIDFieldName, - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success custom ScheduleId reserved TemporalScheduleId", - in: "TemporalScheduleId", - withCustomScheduleID: true, - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success noop mapper ScheduleId", - in: "ScheduleId", - withCustomScheduleID: false, - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "success noop mapper TemporalScheduleId", - in: "TemporalScheduleId", - withCustomScheduleID: false, - outFn: "WorkflowId", - outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - - { - name: "invalid search attribute", - in: "Foo", - err: fmt.Sprintf( - "%s: column name 'Foo' is not a valid search attribute", - InvalidExpressionErrMessage, - ), - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - r := require.New(t) - ctrl := gomock.NewController(t) - storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - queryConverter := NewQueryConverter( - storeQCMock, - testNamespaceName, - searchattribute.TestNameTypeMap(), - &searchattribute.TestMapper{ - WithCustomScheduleID: tc.withCustomScheduleID, - }, - metrics.NewMockHandler(ctrl), - log.NewNoopLogger(), - ) - - if tc.useNoopMapper { - queryConverter.saMapper = &searchattribute.NoopMapper{} - } - - fn, ft, err := queryConverter.resolveSearchAttributeAlias(tc.in) - if tc.err != "" { - r.Error(err) - r.ErrorContains(err, tc.err) - var expectedErr *ConverterError - r.ErrorAs(err, &expectedErr) - } else { - r.NoError(err) - r.Equal(tc.outFn, fn) - r.Equal(tc.outFt, ft) - } - }) - } -} - func TestQueryConverter_ParseValueExpr(t *testing.T) { t.Parallel() @@ -2534,84 +2371,10 @@ func TestQueryConverter_TemporalSystemExecutionStatus(t *testing.T) { } _, err := queryConverter.convertColName(in) r.Error(err) - r.ErrorContains(err, "not a valid search attribute") + r.ErrorContains(err, InvalidSearchAttribute) }) } -func TestQueryConverter_ResolveSearchAttributeAlias_WithChasmMapper(t *testing.T) { - t.Parallel() - ctrl := gomock.NewController(t) - storeQCMock := NewMockStoreQueryConverter[sqlparser.Expr](ctrl) - - chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{ - "TemporalBool01": "ChasmCompleted", - "TemporalKeyword01": "ChasmStatus", - }, - map[string]enumspb.IndexedValueType{ - "TemporalBool01": enumspb.INDEXED_VALUE_TYPE_BOOL, - "TemporalKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }, - ) - - queryConverter := newTestQueryConverter(storeQCMock). - WithChasmMapper(chasmMapper) - - testCases := []struct { - name string - expectedFieldName string - expectedFieldType enumspb.IndexedValueType - expectedErr bool - expectNamespaceDivision bool - }{ - { - name: "ChasmCompleted", - expectedFieldName: "TemporalBool01", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_BOOL, - expectedErr: false, - expectNamespaceDivision: false, - }, - { - name: "ChasmStatus", - expectedFieldName: "TemporalKeyword01", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - expectedErr: false, - expectNamespaceDivision: false, - }, - { - name: "TemporalNamespaceDivision", - expectedFieldName: "TemporalNamespaceDivision", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - expectedErr: false, - expectNamespaceDivision: true, - }, - { - name: "NonExistentChasmAlias", - expectedFieldName: "", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, - expectedErr: true, - expectNamespaceDivision: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - r := require.New(t) - fieldName, fieldType, err := queryConverter.resolveSearchAttributeAlias(tc.name) - if tc.expectedErr { - r.Error(err) - // Note: fieldName may have been set during resolution attempts - r.Equal(enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, fieldType) - } else { - r.NoError(err) - r.Equal(tc.expectedFieldName, fieldName) - r.Equal(tc.expectedFieldType, fieldType) - // Note: seenNamespaceDivision is only set in convertColName, not resolveSearchAttributeAlias - } - }) - } -} - func TestQueryConverter_CapturePanic(t *testing.T) { t.Parallel() r := require.New(t) diff --git a/common/persistence/visibility/store/query/errors.go b/common/persistence/visibility/store/query/errors.go index d662d0821f7..13f584ada29 100644 --- a/common/persistence/visibility/store/query/errors.go +++ b/common/persistence/visibility/store/query/errors.go @@ -19,6 +19,7 @@ var ( MalformedSqlQueryErrMessage = "malformed SQL query" NotSupportedErrMessage = "operation is not supported" InvalidExpressionErrMessage = "invalid expression" + InvalidSearchAttribute = "invalid search attribute" ) func NewConverterError(format string, a ...any) error { diff --git a/common/persistence/visibility/store/query/resolve.go b/common/persistence/visibility/store/query/resolve.go index d94cab8dcef..69218a57c77 100644 --- a/common/persistence/visibility/store/query/resolve.go +++ b/common/persistence/visibility/store/query/resolve.go @@ -10,109 +10,76 @@ import ( "go.temporal.io/server/common/searchattribute/sadefs" ) -// ResolveSearchAttributeAlias resolves the search attribute alias for the given name. The process is: -// 1. If the name has the "Temporal", skip mapping to a custom search attribute. -// 2. If the search attribute exists in the visibility mapper, pass it through. -// 3. If the search attribute exists in the CHASM mapper, pass it through. -// 4. If it exists as a system / predefined attribute, map it. -// 4.1 Some pre-defined attributes are already defined with the Temporal prefix, so both options need to be checked. func ResolveSearchAttributeAlias( - name string, - ns namespace.Name, - mapper searchattribute.Mapper, + alias string, + namespaceName namespace.Name, + saMapper searchattribute.Mapper, saTypeMap searchattribute.NameTypeMap, chasmMapper *chasm.VisibilitySearchAttributesMapper, -) (string, enumspb.IndexedValueType, error) { - if sadefs.IsMappable(name) { - // First check if the visibility mapper can handle this field (e.g., custom search attributes) - fieldName, fieldType := tryVisibilityMapper(name, ns, mapper, saTypeMap) - if fieldName != "" { - return fieldName, fieldType, nil + archetypeID chasm.ArchetypeID, +) (fieldName string, fieldType enumspb.IndexedValueType, retErr error) { + // resolveCSA only returns true if `alias` is a custom search attribute. + resolveCSA := func(alias string) bool { + fn, err := saMapper.GetFieldName(alias, namespaceName.String()) + if err != nil { + return false } - - // Handle ScheduleID → WorkflowID transformation, but only if ScheduleID is not defined as a custom search attribute - // This fallback only applies when the visibility mapper doesn't handle the field - if name == sadefs.ScheduleID { - // ScheduleID is not defined, transform to WorkflowID - saType, _ := saTypeMap.GetType(sadefs.WorkflowID) - return sadefs.WorkflowID, saType, nil + ft, err := saTypeMap.GetType(fn) + if err != nil { + return false } + fieldName, fieldType = fn, ft + return true } - - fieldName, fieldType := tryChasmMapper(name, chasmMapper) - if fieldName != "" { - return fieldName, fieldType, nil - } - - fieldName, fieldType, found := tryDirectAndPrefixedLookup(name, saTypeMap) - if found { - return fieldName, fieldType, nil - } - - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, NewConverterError("invalid search attribute: %s", name) -} - -// tryVisibilityMapper might find a successful match in which case we return it -// otherwise we continue with the fallback logic. -func tryVisibilityMapper( - name string, - ns namespace.Name, - mapper searchattribute.Mapper, - saTypeMap searchattribute.NameTypeMap, -) (string, enumspb.IndexedValueType) { - if mapper == nil { - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED - } - - fieldName, err := mapper.GetFieldName(name, ns.String()) - if err != nil { - // If there is an error, we need to continue with the fallback logic - // because this search attribute is not defined in the mapper, but might - // exist with a different name in the namespace. - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED - } - - // Mapper successfully resolved the field name, now check if it exists in the type map - fieldType, err := saTypeMap.GetType(fieldName) - if err != nil { - // If the mapped field doesn't exist in type map, allow fallback to direct/prefixed lookup. - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED - } - - return fieldName, fieldType -} - -func tryChasmMapper(name string, chasmMapper *chasm.VisibilitySearchAttributesMapper) (string, enumspb.IndexedValueType) { - if chasmMapper == nil { - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED + // resolveChasmSA only returns true if `alias` is a CHASM search attribute. + resolveChasmSA := func(alias string) bool { + if chasmMapper == nil { + return false + } + fn, err := chasmMapper.Field(alias) + if err != nil { + return false + } + ft, err := chasmMapper.ValueType(fn) + if err != nil { + return false + } + fieldName, fieldType = fn, ft + return true } - fieldName, err := chasmMapper.Field(name) - if err != nil { - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED + var err error + fieldName = alias + // First, check if it's a custom search attribute. + if sadefs.IsMappable(alias) && resolveCSA(alias) { + return } - - fieldType, err := chasmMapper.ValueType(fieldName) - if err != nil { - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED + // Second, check if it's a CHASM search attribute. + if resolveChasmSA(alias) { + return } - return fieldName, fieldType -} - -func tryDirectAndPrefixedLookup(name string, saTypeMap searchattribute.NameTypeMap) (string, enumspb.IndexedValueType, bool) { - if saType, err := saTypeMap.GetType(name); err == nil { - return name, saType, true + // Third, check if it's a system/reserved search attribute. + fieldType, err = saTypeMap.GetType(fieldName) + if err == nil { + return } - - prefixedName := sadefs.ReservedPrefix + name - if saType, err := saTypeMap.GetType(prefixedName); err == nil { - return prefixedName, saType, true + // Fourth, check for special aliases or adding/removing the `Temporal` prefix. + if strings.TrimPrefix(alias, sadefs.ReservedPrefix) == sadefs.ScheduleID { + fieldName = sadefs.WorkflowID + } else if archetypeID == chasm.SchedulerArchetypeID && alias == "TemporalSystemExecutionStatus" { + // To support querying Workflow based schedulers and CHASM based schedulers, we need to translate + // TemporalSystemExecutionStatus as an alias to the system search attribute ExecutionStatus. + fieldName = sadefs.ExecutionStatus + } else if strings.HasPrefix(fieldName, sadefs.ReservedPrefix) { + fieldName = fieldName[len(sadefs.ReservedPrefix):] + } else { + fieldName = sadefs.ReservedPrefix + fieldName } - - strippedName := strings.TrimPrefix(name, sadefs.ReservedPrefix) - if saType, err := saTypeMap.GetType(strippedName); err == nil { - return strippedName, saType, true + fieldType, err = saTypeMap.GetType(fieldName) + if err == nil { + return } - return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, false + return "", enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, + NewConverterError("%s: %s", InvalidSearchAttribute, alias) } diff --git a/common/persistence/visibility/store/query/resolve_test.go b/common/persistence/visibility/store/query/resolve_test.go index 47db20105a4..61649b40836 100644 --- a/common/persistence/visibility/store/query/resolve_test.go +++ b/common/persistence/visibility/store/query/resolve_test.go @@ -1,314 +1,350 @@ package query import ( - "fmt" "testing" "github.com/stretchr/testify/require" enumspb "go.temporal.io/api/enums/v1" - "go.temporal.io/api/serviceerror" "go.temporal.io/server/chasm" - "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/searchattribute" - "go.temporal.io/server/common/searchattribute/sadefs" ) +// mapAllMapper is a mapper that resolves every alias to the same field name, to assert that +// search attributes that are not mappable are not resolved through the namespace mapper. +type mapAllMapper string + +var _ searchattribute.Mapper = mapAllMapper("") + +func (m mapAllMapper) GetAlias(fieldName string, _ string) (string, error) { + return fieldName, nil +} + +func (m mapAllMapper) GetFieldName(_ string, _ string) (string, error) { + return string(m), nil +} + func TestResolveSearchAttributeAlias(t *testing.T) { - cases := []struct { - name string - expectedFieldName string - expectedFieldType enumspb.IndexedValueType - expectedErr bool + t.Parallel() + + testCases := []struct { + name string + in string + withCustomScheduleID bool + // saMapper overrides the default searchattribute.TestMapper. + saMapper searchattribute.Mapper + archetypeID chasm.ArchetypeID + outFn string + outFt enumspb.IndexedValueType + err string }{ - {name: "MyCustomField", expectedFieldName: "MyCustomField", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, expectedErr: false}, - {name: "ExecutionStatus", expectedFieldName: "ExecutionStatus", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, expectedErr: false}, - {name: "ScheduledStartTime", expectedFieldName: "TemporalScheduledStartTime", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_DATETIME, expectedErr: false}, - {name: "SchedulePaused", expectedFieldName: "TemporalSchedulePaused", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_BOOL, expectedErr: false}, - {name: "BuildIds", expectedFieldName: "BuildIds", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, expectedErr: false}, - {name: "TemporalBuildIds", expectedFieldName: "BuildIds", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, expectedErr: false}, - {name: "TemporalPauseInfo", expectedFieldName: "TemporalPauseInfo", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, expectedErr: false}, - {name: "NonExistentField", expectedFieldName: "", expectedFieldType: enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, expectedErr: true}, - {name: sadefs.ScheduleID, expectedFieldName: sadefs.WorkflowID, expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, expectedErr: false}, - } + { + name: "success system StartTime", + in: "StartTime", + outFn: "StartTime", + outFt: enumspb.INDEXED_VALUE_TYPE_DATETIME, + }, - ns := namespace.Name("test-namespace") - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - "MyCustomField": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - "ExecutionStatus": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }) - mapper := customMapper{ - fieldToAlias: map[string]string{ - "MyCustomField": "some-alias", - "ExecutionStatus": "some-alias2", + { + name: "success system WorkflowId", + in: "WorkflowId", + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, }, - aliasToField: map[string]string{ - "some-alias": "MyCustomField", - "some-alias2": "ExecutionStatus", + + { + name: "success reserved BuildIds", + in: "BuildIds", + outFn: "BuildIds", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - fieldName, fieldType, err := ResolveSearchAttributeAlias(tc.name, ns, mapper, saTypeMap, nil) - require.Equal(t, tc.expectedFieldName, fieldName) - require.Equal(t, tc.expectedFieldType, fieldType) - require.Equal(t, tc.expectedErr, err != nil) - }) - } -} + { + name: "success reserved TemporalBuildIds", + in: "TemporalBuildIds", + outFn: "BuildIds", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, + }, -func TestResolveSearchAttributeAlias_CustomScheduleID(t *testing.T) { - ns := namespace.Name("test-namespace") + { + name: "success reserved TemporalWorkerDeployment", + in: "TemporalWorkerDeployment", + outFn: "TemporalWorkerDeployment", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, - // Test case where ScheduleID is defined as a custom search attribute - saTypeMapWithCustomScheduleID := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - sadefs.ScheduleID: enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, - }) + { + name: "success reserved WorkerDeployment", + in: "WorkerDeployment", + outFn: "TemporalWorkerDeployment", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, - mapper := customMapper{ - fieldToAlias: map[string]string{ - sadefs.ScheduleID: sadefs.ScheduleID, + { + name: "success custom AliasForInt01", + in: "AliasForInt01", + outFn: "Int01", + outFt: enumspb.INDEXED_VALUE_TYPE_INT, }, - aliasToField: map[string]string{ - sadefs.ScheduleID: sadefs.ScheduleID, + + { + name: "success custom noop mapper Int01", + in: "Int01", + saMapper: &searchattribute.NoopMapper{}, + outFn: "Int01", + outFt: enumspb.INDEXED_VALUE_TYPE_INT, }, - } - // When ScheduleID is a custom search attribute, it should use the custom attribute, not transform to WorkflowID - fieldName, fieldType, err := ResolveSearchAttributeAlias(sadefs.ScheduleID, ns, mapper, saTypeMapWithCustomScheduleID, nil) - require.NoError(t, err) - require.Equal(t, sadefs.ScheduleID, fieldName) - require.Equal(t, enumspb.INDEXED_VALUE_TYPE_KEYWORD_LIST, fieldType) -} + { + // System search attributes are not mappable, so the namespace mapper must not + // be able to resolve them to another field. + name: "success system not resolved by mapper", + in: "WorkflowId", + saMapper: mapAllMapper("Int01"), + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, -type customMapper struct { - fieldToAlias map[string]string - aliasToField map[string]string -} + { + // Predefined search attributes are not mappable either. + name: "success predefined not resolved by mapper", + in: "TemporalSchedulePaused", + saMapper: mapAllMapper("Int01"), + outFn: "TemporalSchedulePaused", + outFt: enumspb.INDEXED_VALUE_TYPE_BOOL, + }, -func (m customMapper) GetAlias(fieldName string, ns string) (string, error) { - if fn, ok := m.fieldToAlias[fieldName]; ok { - return fn, nil - } - return "", serviceerror.NewInvalidArgument( - fmt.Sprintf("Namespace %s has no mapping defined for field name %s", ns, fieldName), - ) -} + { + name: "success special ScheduleId", + in: "ScheduleId", + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, -func (m customMapper) GetFieldName(alias string, ns string) (string, error) { - if fn, ok := m.aliasToField[alias]; ok { - return fn, nil - } - return "", serviceerror.NewInvalidArgument( - fmt.Sprintf("Namespace %s has no mapping defined for search attribute %s", ns, alias), - ) -} + { + name: "success special TemporalScheduleId", + in: "TemporalScheduleId", + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, -func TestResolveSearchAttributeAlias_WithChasmMapper(t *testing.T) { - ns := namespace.Name("test-namespace") - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - "ExecutionStatus": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - "StartTime": enumspb.INDEXED_VALUE_TYPE_DATETIME, - }) - mapper := customMapper{ - fieldToAlias: map[string]string{}, - aliasToField: map[string]string{}, - } + { + name: "success custom ScheduleId", + in: "ScheduleId", + withCustomScheduleID: true, + outFn: searchattribute.TestScheduleIDFieldName, + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, - chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{ - "TemporalBool01": "ChasmCompleted", - "TemporalKeyword01": "ChasmStatus", + { + name: "success custom ScheduleId reserved TemporalScheduleId", + in: "TemporalScheduleId", + withCustomScheduleID: true, + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, }, - map[string]enumspb.IndexedValueType{ - "TemporalBool01": enumspb.INDEXED_VALUE_TYPE_BOOL, - "TemporalKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + + { + name: "success noop mapper ScheduleId", + in: "ScheduleId", + saMapper: &searchattribute.NoopMapper{}, + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, + + { + name: "success noop mapper TemporalScheduleId", + in: "TemporalScheduleId", + saMapper: &searchattribute.NoopMapper{}, + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, + + { + // The ScheduleId alias is resolved before the scheduler archetype aliases. + name: "success scheduler archetype ScheduleId", + in: "ScheduleId", + archetypeID: chasm.SchedulerArchetypeID, + outFn: "WorkflowId", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, + + { + // TemporalSystemExecutionStatus allows querying Workflow based and CHASM based + // schedulers with the same alias. + name: "success scheduler archetype TemporalSystemExecutionStatus", + in: "TemporalSystemExecutionStatus", + archetypeID: chasm.SchedulerArchetypeID, + outFn: "ExecutionStatus", + outFt: enumspb.INDEXED_VALUE_TYPE_KEYWORD, }, - ) - testCases := []struct { - name string - chasmMapper *chasm.VisibilitySearchAttributesMapper - expectedFieldName string - expectedFieldType enumspb.IndexedValueType - expectedErr bool - }{ { - name: "ChasmCompleted", - chasmMapper: chasmMapper, - expectedFieldName: "TemporalBool01", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_BOOL, - expectedErr: false, + name: "invalid TemporalSystemExecutionStatus without archetype", + in: "TemporalSystemExecutionStatus", + err: "invalid search attribute: TemporalSystemExecutionStatus", }, + + { + name: "invalid TemporalSystemExecutionStatus with other archetype", + in: "TemporalSystemExecutionStatus", + archetypeID: chasm.SchedulerArchetypeID + 1, + err: "invalid search attribute: TemporalSystemExecutionStatus", + }, + { - name: "ChasmStatus", - chasmMapper: chasmMapper, - expectedFieldName: "TemporalKeyword01", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - expectedErr: false, + name: "invalid search attribute", + in: "Foo", + err: "invalid search attribute: Foo", }, + { - name: "ExecutionStatus", - chasmMapper: chasmMapper, - expectedFieldName: "ExecutionStatus", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - expectedErr: false, + // The mapper resolves the alias, but the field is not in the type map. + name: "invalid custom search attribute unknown field", + in: "AliasForFoo", + err: "invalid search attribute: AliasForFoo", }, + { - name: "NonExistentChasmAlias", - chasmMapper: chasmMapper, - expectedFieldName: "", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, - expectedErr: true, + name: "invalid search attribute with reserved prefix", + in: "TemporalFoo", + err: "invalid search attribute: TemporalFoo", }, + { - name: "ChasmCompleted", - chasmMapper: nil, - expectedFieldName: "", - expectedFieldType: enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, - expectedErr: true, + name: "invalid empty search attribute", + in: "", + err: "invalid search attribute: ", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fieldName, fieldType, err := ResolveSearchAttributeAlias(tc.name, ns, mapper, saTypeMap, tc.chasmMapper) - require.Equal(t, tc.expectedFieldName, fieldName) - require.Equal(t, tc.expectedFieldType, fieldType) - require.Equal(t, tc.expectedErr, err != nil) + r := require.New(t) + saMapper := tc.saMapper + if saMapper == nil { + saMapper = &searchattribute.TestMapper{ + WithCustomScheduleID: tc.withCustomScheduleID, + } + } + + fn, ft, err := ResolveSearchAttributeAlias( + tc.in, + testNamespaceName, + saMapper, + searchattribute.TestNameTypeMap(), + nil, // chasmMapper + tc.archetypeID, + ) + if tc.err != "" { + r.Error(err) + r.ErrorContains(err, tc.err) + var expectedErr *ConverterError + r.ErrorAs(err, &expectedErr) + r.Empty(fn) + r.Equal(enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, ft) + } else { + r.NoError(err) + r.Equal(tc.outFn, fn) + r.Equal(tc.outFt, ft) + } }) } } -func TestResolveSearchAttributeAlias_ChasmOverridesSystemAttribute(t *testing.T) { - ns := namespace.Name("test-namespace") - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - "ExecutionStatus": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }) - mapper := customMapper{ - fieldToAlias: map[string]string{}, - aliasToField: map[string]string{}, - } +func TestResolveSearchAttributeAlias_WithChasmMapper(t *testing.T) { + t.Parallel() chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( map[string]string{ - "TemporalLowCardinalityKeyword01": "ExecutionStatus", + "TemporalBool01": "ChasmCompleted", + "TemporalKeyword01": "ChasmStatus", + // Shadows the ExecutionStatus system search attribute. + "TemporalKeyword02": "ExecutionStatus", + // Shadows the alias of the Keyword01 custom search attribute. + "TemporalKeyword03": "AliasForKeyword01", }, map[string]enumspb.IndexedValueType{ - "TemporalLowCardinalityKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + "TemporalBool01": enumspb.INDEXED_VALUE_TYPE_BOOL, + "TemporalKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + "TemporalKeyword02": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + "TemporalKeyword03": enumspb.INDEXED_VALUE_TYPE_KEYWORD, }, ) - fieldName, fieldType, err := ResolveSearchAttributeAlias("ExecutionStatus", ns, mapper, saTypeMap, chasmMapper) - require.NoError(t, err) - require.Equal(t, "TemporalLowCardinalityKeyword01", fieldName) - require.Equal(t, enumspb.INDEXED_VALUE_TYPE_KEYWORD, fieldType) -} - -func TestResolveSearchAttributeAlias_ChasmPriority(t *testing.T) { - ns := namespace.Name("test-namespace") - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - "MyCustomField": enumspb.INDEXED_VALUE_TYPE_KEYWORD, - "StartTime": enumspb.INDEXED_VALUE_TYPE_DATETIME, - }) - mapper := customMapper{ - fieldToAlias: map[string]string{ - "MyCustomField": "MyCustomField", + testCases := []struct { + name string + expectedFieldName string + expectedFieldType enumspb.IndexedValueType + expectedErr bool + expectNamespaceDivision bool + }{ + { + name: "ChasmCompleted", + expectedFieldName: "TemporalBool01", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_BOOL, + expectedErr: false, + expectNamespaceDivision: false, }, - aliasToField: map[string]string{ - "MyCustomField": "MyCustomField", + { + name: "ChasmStatus", + expectedFieldName: "TemporalKeyword01", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + expectedErr: false, + expectNamespaceDivision: false, }, - } - - chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{ - "TemporalKeyword01": "MyCustomField", + { + name: "TemporalNamespaceDivision", + expectedFieldName: "TemporalNamespaceDivision", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + expectedErr: false, + expectNamespaceDivision: true, }, - map[string]enumspb.IndexedValueType{ - "TemporalKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + { + // A CHASM search attribute takes precedence over a system search attribute, + // which is not mappable. + name: "ExecutionStatus", + expectedFieldName: "TemporalKeyword02", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + expectedErr: false, + expectNamespaceDivision: false, }, - ) - - // Custom SA should take precedence over CHASM SA - fieldName, fieldType, err := ResolveSearchAttributeAlias("MyCustomField", ns, mapper, saTypeMap, chasmMapper) - require.NoError(t, err) - require.Equal(t, "MyCustomField", fieldName) - require.Equal(t, enumspb.INDEXED_VALUE_TYPE_KEYWORD, fieldType) - - // CHASM SA should be used when custom SA doesn't exist - chasmMapper2 := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{ - "TemporalKeyword01": "ChasmOnlyField", + { + // A custom search attribute takes precedence over a CHASM search attribute. + name: "AliasForKeyword01", + expectedFieldName: "Keyword01", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_KEYWORD, + expectedErr: false, + expectNamespaceDivision: false, }, - map[string]enumspb.IndexedValueType{ - "TemporalKeyword01": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + { + name: "NonExistentChasmAlias", + expectedFieldName: "", + expectedFieldType: enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, + expectedErr: true, + expectNamespaceDivision: false, }, - ) - fieldName2, fieldType2, err2 := ResolveSearchAttributeAlias("ChasmOnlyField", ns, mapper, saTypeMap, chasmMapper2) - require.NoError(t, err2) - require.Equal(t, "TemporalKeyword01", fieldName2) - require.Equal(t, enumspb.INDEXED_VALUE_TYPE_KEYWORD, fieldType2) -} - -func TestResolveSearchAttributeAlias_OverridableSystemResolvesViaSystemColumn(t *testing.T) { - ns := namespace.Name("test-namespace") - mapper := customMapper{ - fieldToAlias: map[string]string{}, - aliasToField: map[string]string{}, } - // System search attribute overrides are recorded only in the mapper's overriddenSystemFields - // and are NOT added to the CHASM alias/field maps. On the query path they resolve via the - // system type map (which always includes system search attributes), preserving the column name. - chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{}, - map[string]enumspb.IndexedValueType{}, - ) - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - sadefs.ExecutionTime: enumspb.INDEXED_VALUE_TYPE_DATETIME, - sadefs.TaskQueue: enumspb.INDEXED_VALUE_TYPE_KEYWORD, - }) - - cases := []struct { - name string - typ enumspb.IndexedValueType - }{ - {sadefs.ExecutionTime, enumspb.INDEXED_VALUE_TYPE_DATETIME}, - {sadefs.TaskQueue, enumspb.INDEXED_VALUE_TYPE_KEYWORD}, - } - for _, tc := range cases { + for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - fieldName, fieldType, err := ResolveSearchAttributeAlias(tc.name, ns, mapper, saTypeMap, chasmMapper) - require.NoError(t, err) - require.Equal(t, tc.name, fieldName) - require.Equal(t, tc.typ, fieldType) + r := require.New(t) + fieldName, fieldType, err := ResolveSearchAttributeAlias( + tc.name, + testNamespaceName, + &searchattribute.TestMapper{}, + searchattribute.TestNameTypeMap(), + chasmMapper, + chasm.UnspecifiedArchetypeID, + ) + if tc.expectedErr { + r.Error(err) + r.Empty(fieldName) + r.Equal(enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, fieldType) + } else { + r.NoError(err) + r.Equal(tc.expectedFieldName, fieldName) + r.Equal(tc.expectedFieldType, fieldType) + } }) } } - -func TestResolveSearchAttributeAlias_ChasmMissingType(t *testing.T) { - ns := namespace.Name("test-namespace") - saTypeMap := searchattribute.NewNameTypeMapStub(map[string]enumspb.IndexedValueType{ - "StartTime": enumspb.INDEXED_VALUE_TYPE_DATETIME, - }) - mapper := customMapper{ - fieldToAlias: map[string]string{}, - aliasToField: map[string]string{}, - } - - // CHASM mapper with field that doesn't exist in type map - chasmMapper := chasm.NewTestVisibilitySearchAttributesMapper( - map[string]string{ - "TemporalKeyword01": "ChasmField", - }, - map[string]enumspb.IndexedValueType{ - // Missing TemporalKeyword01 in type map - }, - ) - - // Should fall back to system SA resolution - fieldName, fieldType, err := ResolveSearchAttributeAlias("ChasmField", ns, mapper, saTypeMap, chasmMapper) - require.Error(t, err) - require.Empty(t, fieldName) - require.Equal(t, enumspb.INDEXED_VALUE_TYPE_UNSPECIFIED, fieldType) -} diff --git a/common/persistence/visibility/store/sql/query_converter_legacy.go b/common/persistence/visibility/store/sql/query_converter_legacy.go index 89e9d18849c..b640b9beb7c 100644 --- a/common/persistence/visibility/store/sql/query_converter_legacy.go +++ b/common/persistence/visibility/store/sql/query_converter_legacy.go @@ -450,19 +450,16 @@ func (c *QueryConverterLegacy) convertColName(exprRef *sqlparser.Expr) (*saColNa } saAlias := strings.ReplaceAll(sqlparser.String(expr), "`", "") - saFieldName, saType, err := query.ResolveSearchAttributeAlias(saAlias, c.namespaceName, c.saMapper, c.saTypeMap, c.chasmMapper) + saFieldName, saType, err := query.ResolveSearchAttributeAlias( + saAlias, + c.namespaceName, + c.saMapper, + c.saTypeMap, + c.chasmMapper, + c.archetypeID, + ) if err != nil { - if c.archetypeID != chasm.SchedulerArchetypeID || saAlias != "TemporalSystemExecutionStatus" { - return nil, query.NewConverterError( - "%s: column name '%s' is not a valid search attribute", - query.InvalidExpressionErrMessage, - saAlias, - ) - } - // To support querying Workflow based schedulers and CHASM based schedulers, we need to translate - // TemporalSystemExecutionStatus as an alias to the system search attribute ExecutionStatus. - saFieldName = sadefs.ExecutionStatus - saType, _ = c.saTypeMap.GetType(saFieldName) + return nil, err } if saFieldName == sadefs.TemporalNamespaceDivision { c.seenNamespaceDivision = true @@ -526,6 +523,43 @@ func (c *QueryConverterLegacy) convertValueExpr( } } return nil + case *sqlparser.UnaryExpr: + // Negative value may be parsed as UnaryExpr + if e.Operator != sqlparser.UPlusStr && e.Operator != sqlparser.UMinusStr { + return query.NewConverterError( + "%s: unary operator %q", + query.NotSupportedErrMessage, + e.Operator, + ) + } + if value, ok := e.Expr.(*sqlparser.SQLVal); !ok || value.Type == sqlparser.StrVal { + return query.NewConverterError( + "%s: unary operator not supported in %q", + query.InvalidExpressionErrMessage, + sqlparser.String(expr), + ) + } + err := c.convertValueExpr(&e.Expr, name, saFieldName, saType) + if err != nil { + return err + } + if value, ok := e.Expr.(*sqlparser.SQLVal); ok && (value.Type == sqlparser.IntVal || value.Type == sqlparser.FloatVal) { + if e.Operator == sqlparser.UMinusStr && len(value.Val) > 0 { + if value.Val[0] == '-' { + value.Val = value.Val[1:] + } else { + value.Val = append([]byte{'-'}, value.Val...) + } + } + *exprRef = e.Expr + return nil + } + // This should never happen, but here to catch any unexpected case. + return query.NewConverterError( + "%s: unary expression %q", + query.InvalidExpressionErrMessage, + sqlparser.String(expr), + ) case *sqlparser.GroupConcatExpr: return query.NewConverterError("%s: 'group_concat'", query.NotSupportedErrMessage) case *sqlparser.FuncExpr: diff --git a/common/persistence/visibility/store/sql/query_converter_legacy_test.go b/common/persistence/visibility/store/sql/query_converter_legacy_test.go index 5b17266a47c..28010043144 100644 --- a/common/persistence/visibility/store/sql/query_converter_legacy_test.go +++ b/common/persistence/visibility/store/sql/query_converter_legacy_test.go @@ -523,11 +523,7 @@ func (s *queryConverterSuite) TestConvertColName() { input: "InvalidName", output: "", retValue: nil, - err: query.NewConverterError( - "%s: column name '%s' is not a valid search attribute", - query.InvalidExpressionErrMessage, - "InvalidName", - ), + err: query.NewConverterError("%s: %s", query.InvalidSearchAttribute, "InvalidName"), }, { name: "valid system search attribute: ExecutionStatus", @@ -785,6 +781,206 @@ func (s *queryConverterSuite) TestConvertValueExpr() { output: "('foo', 'bar')", err: nil, }, + // The SQL parser folds the sign into integer values, but represents signed floats + // as an unary expression. + { + name: "valid negative integer", + input: "-123", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "-123", + err: nil, + }, + { + // The parser folds both signs into the value, so this is not an unary expression. + name: "valid double negative integer", + input: "- -123", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "123", + err: nil, + }, + { + name: "valid positive negative integer", + input: "+ -123", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "-123", + err: nil, + }, + { + name: "valid negative positive integer", + input: "- +123", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "-123", + err: nil, + }, + { + name: "valid negative float", + input: "-1.5", + args: map[string]any{ + "saName": "AliasForDouble01", + "saFieldName": "Double01", + "saType": enumspb.INDEXED_VALUE_TYPE_DOUBLE, + }, + output: "-1.5", + err: nil, + }, + { + name: "valid positive sign float", + input: "+1.5", + args: map[string]any{ + "saName": "AliasForDouble01", + "saFieldName": "Double01", + "saType": enumspb.INDEXED_VALUE_TYPE_DOUBLE, + }, + output: "1.5", + err: nil, + }, + { + name: "valid negative float with trailing zeros", + input: "-1.500", + args: map[string]any{ + "saName": "AliasForDouble01", + "saFieldName": "Double01", + "saType": enumspb.INDEXED_VALUE_TYPE_DOUBLE, + }, + output: "-1.5", + err: nil, + }, + { + name: "valid negative float in tuple", + input: "(-1.5, 2.5)", + args: map[string]any{ + "saName": "AliasForDouble01", + "saFieldName": "Double01", + "saType": enumspb.INDEXED_VALUE_TYPE_DOUBLE, + }, + output: "(-1.5, 2.5)", + err: nil, + }, + { + // Only a literal value can be signed, not another unary expression. + name: "nested unary expression", + input: "- -1.5", + args: map[string]any{ + "saName": "AliasForDouble01", + "saFieldName": "Double01", + "saType": enumspb.INDEXED_VALUE_TYPE_DOUBLE, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator not supported in %q", + query.InvalidExpressionErrMessage, + "- -1.5", + ), + }, + { + name: "unary expression with string value", + input: "-'foo'", + args: map[string]any{ + "saName": "AliasForKeyword01", + "saFieldName": "Keyword01", + "saType": enumspb.INDEXED_VALUE_TYPE_KEYWORD, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator not supported in %q", + query.InvalidExpressionErrMessage, + "-'foo'", + ), + }, + { + name: "unary expression with bool value", + input: "-true", + args: map[string]any{ + "saName": "AliasForBool01", + "saFieldName": "Bool01", + "saType": enumspb.INDEXED_VALUE_TYPE_BOOL, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator not supported in %q", + query.InvalidExpressionErrMessage, + "-true", + ), + }, + { + name: "unary expression with parenthesized value", + input: "-(10)", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator not supported in %q", + query.InvalidExpressionErrMessage, + "-(10)", + ), + }, + { + name: "unsupported unary operator: bitwise not", + input: "~1", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator %q", + query.NotSupportedErrMessage, + sqlparser.TildaStr, + ), + }, + { + name: "unsupported unary operator: logical not", + input: "!1", + args: map[string]any{ + "saName": "AliasForInt01", + "saFieldName": "Int01", + "saType": enumspb.INDEXED_VALUE_TYPE_INT, + }, + output: "", + err: query.NewConverterError( + "%s: unary operator %q", + query.NotSupportedErrMessage, + sqlparser.BangStr, + ), + }, + { + // The value is converted before the sign is applied, so the errors of the + // underlying value are returned as is. + name: "unary expression with invalid value type", + input: "-1.5", + args: map[string]any{ + "saName": "StartTime", + "saFieldName": "StartTime", + "saType": enumspb.INDEXED_VALUE_TYPE_DATETIME, + }, + output: "", + err: query.NewConverterError( + "%s: unexpected value type %T for search attribute %s", + query.InvalidExpressionErrMessage, + float64(1.5), + "StartTime", + ), + }, } for _, tc := range tests { diff --git a/common/persistence/visibility/store/sql/visibility_store_test.go b/common/persistence/visibility/store/sql/visibility_store_test.go index b3714181c6b..3d7c1e9476e 100644 --- a/common/persistence/visibility/store/sql/visibility_store_test.go +++ b/common/persistence/visibility/store/sql/visibility_store_test.go @@ -54,7 +54,7 @@ func TestBuildQueryParams(t *testing.T) { { name: "fail invalid custom search attribute", query: "AliasForFoo = 'foo'", - err: "invalid expression: column name 'AliasForFoo' is not a valid search attribute", + err: "invalid search attribute: AliasForFoo", }, { name: "fail order by not supported", diff --git a/common/persistence/visibility/store/tests/query_converter_legacy_test.go b/common/persistence/visibility/store/tests/query_converter_legacy_test.go new file mode 100644 index 00000000000..042f2d6f03b --- /dev/null +++ b/common/persistence/visibility/store/tests/query_converter_legacy_test.go @@ -0,0 +1,424 @@ +package tests + +import ( + "errors" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/metrics" + "go.temporal.io/server/common/namespace" + "go.temporal.io/server/common/persistence/sql/sqlplugin" + "go.temporal.io/server/common/persistence/visibility/store/elasticsearch" + "go.temporal.io/server/common/persistence/visibility/store/query" + vissql "go.temporal.io/server/common/persistence/visibility/store/sql" + "go.temporal.io/server/common/searchattribute" +) + +const ( + testNamespaceID = namespace.ID("test-namespace-id") + testPageSize = 10 +) + +// legacyDivergence describes a query from queryConverterTestCases that the legacy query +// converter accepts or rejects differently from the unified one. The legacy converters build +// the whole statement (SQL) or leave the namespace filters to the store (Elasticsearch), so +// their output can't be compared against the expectations in queryConverterTestCases; only +// whether the query is accepted is. +type legacyDivergence struct { + // accepted is whether the legacy query converter accepts the query, and plugins + // overrides it for the SQL plugins that behave differently. + accepted bool + plugins map[string]bool + // why documents why the legacy query converter behaves differently. + why string +} + +func (d legacyDivergence) isAccepted(store string) bool { + if accepted, ok := d.plugins[store]; ok { + return accepted + } + return d.accepted +} + +// legacySQLDivergences maps a test case name from queryConverterTestCases to how the legacy +// SQL query converter handles it. Every entry is a behavior change intentionally introduced +// by the unified query converter: queries that used to be silently accepted are now validated, +// and queries that used to be rejected are now supported. +var legacySQLDivergences = map[string]legacyDivergence{ + // The legacy query converter doesn't validate the value against the search attribute type. + "Keyword invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Keyword search attributes", + }, + "Keyword with bool value": { + accepted: true, + why: "legacy doesn't validate the value type of Keyword search attributes", + }, + "KeywordList invalid value type": { + accepted: true, + // SQLite requires a string to build the full text search query. + plugins: map[string]bool{sqliteStore: false}, + why: "legacy doesn't validate the value type of KeywordList search attributes", + }, + "Int invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Int search attributes", + }, + "Int mixed types in list": { + accepted: true, + why: "legacy doesn't validate the value type of the items of a list", + }, + "Double invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Double search attributes", + }, + "Bool invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Bool search attributes", + }, + "Datetime with bool value": { + accepted: true, + why: "legacy doesn't validate the value type of Datetime search attributes", + }, + "ExecutionStatus invalid code": { + accepted: true, + why: "legacy doesn't validate that the ExecutionStatus code is a valid enum value", + }, + "ExecutionDuration with bool value": { + accepted: true, + why: "legacy doesn't validate the value type of ExecutionDuration", + }, + "Text invalid value type": { + accepted: true, + // PostgreSQL and SQLite require a string to build the full text search query. + plugins: map[string]bool{postgresStore: false, sqliteStore: false}, + why: "legacy doesn't validate the value type of Text search attributes", + }, + "Text empty value": { + accepted: true, + // PostgreSQL and SQLite tokenize the value to build the full text search query. + plugins: map[string]bool{postgresStore: false, sqliteStore: false}, + why: "legacy only rejects a blank Text value on the stores that tokenize it", + }, + "Text blank value": { + accepted: true, + plugins: map[string]bool{postgresStore: false, sqliteStore: false}, + why: "legacy only rejects a blank Text value on the stores that tokenize it", + }, + + // The legacy query converter rejects ORDER BY for every SQL plugin. The unified query + // converter parses it, and the SQL visibility store rejects it instead. + "order by": { + why: "legacy rejects 'ORDER BY' in the query converter instead of the store", + }, + "order by desc": { + why: "legacy rejects 'ORDER BY' in the query converter instead of the store", + }, + "order by multiple search attributes": { + why: "legacy rejects 'ORDER BY' in the query converter instead of the store", + }, + "order by with where clause": { + why: "legacy rejects 'ORDER BY' in the query converter instead of the store", + }, + "order by custom search attribute": { + why: "legacy rejects 'ORDER BY' in the query converter instead of the store", + }, +} + +func newLegacyQueryConverter(plugin string, queryString string) *vissql.QueryConverterLegacy { + return vissql.NewQueryConverterLegacy( + plugin, + testNamespaceName, + testNamespaceID, + searchattribute.TestNameTypeMap(), + &searchattribute.TestMapper{}, + queryString, + nil, // chasmMapper + chasm.UnspecifiedArchetypeID, + ) +} + +func TestSQLQueryConverterLegacy(t *testing.T) { + t.Parallel() + for _, plugin := range sqlPlugins { + t.Run(plugin, func(t *testing.T) { + t.Parallel() + for _, tc := range queryConverterTestCases { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + + // The legacy query converter output can't be compared against the + // expectations in queryConverterTestCases, so the test only asserts that + // both query converters accept the same queries, except for the + // differences listed in legacySQLDivergences. + _, unifiedErr := tc.expected(plugin) + accepted := unifiedErr == "" + reason := "unified query converter and legacy query converter must agree" + if divergence, ok := legacySQLDivergences[tc.name]; ok { + accepted = divergence.isAccepted(plugin) + reason = divergence.why + } + + // Count queries support the GROUP BY clause. + countFilter, err := newLegacyQueryConverter(plugin, tc.in).BuildCountStmt() + assertLegacyOutcome(r, countFilter, err, accepted, reason) + if accepted { + r.Equal(tc.groupBy, countFilter.GroupBy) + } + + // List queries don't support the GROUP BY clause. + selectFilter, err := newLegacyQueryConverter(plugin, tc.in). + BuildSelectStmt(testPageSize, nil) + if accepted && len(tc.groupBy) > 0 { + r.Error(err) + r.EqualError(err, "operation is not supported: 'GROUP BY' clause") + r.Nil(selectFilter) + return + } + assertLegacyOutcome(r, selectFilter, err, accepted, reason) + }) + } + }) + } +} + +// TestSQLQueryConverterLegacyDivergences keeps legacySQLDivergences honest: an entry that +// names a test case that no longer exists, or that no longer differs from the unified query +// converter, silently weakens TestSQLQueryConverterLegacy. +func TestSQLQueryConverterLegacyDivergences(t *testing.T) { + t.Parallel() + assertDivergencesAreCurrent(t, "legacySQLDivergences", legacySQLDivergences, sqlPlugins) +} + +func assertDivergencesAreCurrent( + t *testing.T, + varName string, + divergences map[string]legacyDivergence, + stores []string, +) { + t.Helper() + r := require.New(t) + + testCases := make(map[string]queryConverterTestCase, len(queryConverterTestCases)) + for _, tc := range queryConverterTestCases { + testCases[tc.name] = tc + } + + for name, divergence := range divergences { + tc, ok := testCases[name] + r.True(ok, "%s has an entry for unknown test case %q", varName, name) + r.NotEmpty(divergence.why, "%s entry %q must document why", varName, name) + + diverges := false + for _, store := range stores { + _, unifiedErr := tc.expected(store) + diverges = diverges || divergence.isAccepted(store) != (unifiedErr == "") + } + r.True( + diverges, + "%s entry %q agrees with the unified query converter for every store: remove it", + varName, + name, + ) + } +} + +// legacyESDivergences maps a test case name from queryConverterTestCases to how the legacy +// Elasticsearch query converter handles it. Every entry is a behavior change intentionally +// introduced by the unified query converter: queries that used to be silently accepted are +// now validated, and queries that used to be rejected are now supported. +var legacyESDivergences = map[string]legacyDivergence{ + // The legacy query converter doesn't validate the value against the search attribute type. + "Keyword invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Keyword search attributes", + }, + "Keyword with bool value": { + accepted: true, + why: "legacy doesn't validate the value type of Keyword search attributes", + }, + "KeywordList invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of KeywordList search attributes", + }, + "Text invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of Text search attributes", + }, + "Text empty value": { + accepted: true, + why: "legacy doesn't reject a blank Text value", + }, + "Text blank value": { + accepted: true, + why: "legacy doesn't reject a blank Text value", + }, + "ExecutionStatus invalid name": { + accepted: true, + why: "legacy doesn't validate that the ExecutionStatus name is a valid enum value", + }, + "ExecutionStatus invalid value in list": { + accepted: true, + why: "legacy doesn't validate that the ExecutionStatus name is a valid enum value", + }, + "ExecutionStatus invalid value type": { + accepted: true, + why: "legacy doesn't validate the value type of ExecutionStatus", + }, + + // The legacy query converter doesn't validate the operator against the search attribute + // type, and relies on Elasticsearch to reject the query instead. + "KeywordList unsupported operator": { + accepted: true, + why: "legacy doesn't validate the operator of KeywordList search attributes", + }, + "KeywordList starts with not supported": { + accepted: true, + why: "legacy doesn't validate the operator of KeywordList search attributes", + }, + "Text unsupported operator": { + accepted: true, + why: "legacy doesn't validate the operator of Text search attributes", + }, + "Text in not supported": { + accepted: true, + why: "legacy doesn't validate the operator of Text search attributes", + }, + "Text range condition not supported": { + accepted: true, + why: "legacy doesn't validate the type of a range condition", + }, + + // The legacy query converter doesn't support the NOT operator. + "not expression": { + why: "legacy doesn't support the 'NOT' operator", + }, + "not parenthesized and expression": { + why: "legacy doesn't support the 'NOT' operator", + }, + "not parenthesized or expression": { + why: "legacy doesn't support the 'NOT' operator", + }, + "not is null expression": { + why: "legacy doesn't support the 'NOT' operator", + }, + "complex query": { + why: "legacy doesn't support the 'NOT' operator", + }, +} + +func newLegacyESQueryConverter() *query.ConverterLegacy { + saTypeMap := searchattribute.TestNameTypeMap() + return elasticsearch.NewQueryConverterLegacy( + elasticsearch.NewNameInterceptor( + testNamespaceName, + saTypeMap, + searchattribute.NewTestMapperProvider(&searchattribute.TestMapper{}), + nil, // chasmMapper + chasm.UnspecifiedArchetypeID, + ), + elasticsearch.NewValuesInterceptor( + testNamespaceName, + saTypeMap, + nil, // chasmMapper + metrics.NoopMetricsHandler, + log.NewNoopLogger(), + ), + saTypeMap, + nil, // chasmMapper + ) +} + +func TestElasticsearchQueryConverterLegacy(t *testing.T) { + t.Parallel() + for _, tc := range queryConverterTestCases { + t.Run(tc.name, func(t *testing.T) { + r := require.New(t) + + // The legacy query converter doesn't add the namespace and namespace division + // filters, which the visibility store adds instead, so its output can't be + // compared against the expectations in queryConverterTestCases. The test only + // asserts that both query converters accept the same queries, except for the + // differences listed in legacyESDivergences. + _, unifiedErr := tc.expected(esStore) + accepted := unifiedErr == "" + reason := "unified query converter and legacy query converter must agree" + if divergence, ok := legacyESDivergences[tc.name]; ok { + accepted = divergence.isAccepted(esStore) + reason = divergence.why + } + + queryParams, err := newLegacyESQueryConverter().ConvertWhereOrderBy(tc.in) + if !accepted { + r.Error(err, reason) + // The visibility store converts a ConverterError into an InvalidArgument + // and returns any other error as is, so a rejected query must fail with + // one of the two to reach the caller as an invalid argument. + var converterErr *query.ConverterError + var invalidArgumentErr *serviceerror.InvalidArgument + r.True( + errors.As(err, &converterErr) || errors.As(err, &invalidArgumentErr), + "expected a ConverterError or an InvalidArgument, got %T: %v", + err, + err, + ) + r.Nil(queryParams) + return + } + r.NoError(err, reason) + + // Unlike the SQL stores, Elasticsearch supports both clauses. + r.Equal(tc.groupBy, queryParams.GroupBy) + if tc.orderBy == "" { + r.Empty(queryParams.Sorter) + } else { + r.NotEmpty(queryParams.Sorter) + } + + if hasFilter(tc.in) { + r.NotNil(queryParams.Query) + } else { + r.Nil(queryParams.Query) + } + }) + } +} + +// TestElasticsearchQueryConverterLegacyDivergences keeps legacyESDivergences honest: an entry +// that names a test case that no longer exists, or that no longer differs from the unified +// query converter, silently weakens TestElasticsearchQueryConverterLegacy. +func TestElasticsearchQueryConverterLegacyDivergences(t *testing.T) { + t.Parallel() + assertDivergencesAreCurrent(t, "legacyESDivergences", legacyESDivergences, []string{esStore}) +} + +// hasFilter returns whether the query has a filtering clause, following the same rule the +// query converters use to tell a filter from a standalone GROUP BY or ORDER BY clause. +func hasFilter(queryString string) bool { + queryString = strings.ToLower(strings.TrimSpace(queryString)) + return queryString != "" && + !strings.HasPrefix(queryString, "group by") && + !strings.HasPrefix(queryString, "order by") +} + +func assertLegacyOutcome( + r *require.Assertions, + filter *sqlplugin.VisibilitySelectFilter, + err error, + accepted bool, + reason string, +) { + if !accepted { + r.Error(err, reason) + var converterErr *query.ConverterError + r.ErrorAs(err, &converterErr) + r.Nil(filter) + return + } + r.NoError(err, reason) + r.NotEmpty(filter.Query) +} diff --git a/common/persistence/visibility/store/tests/query_converter_test.go b/common/persistence/visibility/store/tests/query_converter_test.go index 12875c1da33..ec0e74b3cd7 100644 --- a/common/persistence/visibility/store/tests/query_converter_test.go +++ b/common/persistence/visibility/store/tests/query_converter_test.go @@ -235,7 +235,7 @@ var queryConverterTestCases = []queryConverterTestCase{ { name: "search attribute name is case sensitive", in: "workflowid = 'wid'", - err: "invalid expression: column name 'workflowid' is not a valid search attribute", + err: "invalid search attribute: workflowid", }, // Int type search attributes. @@ -908,12 +908,12 @@ var queryConverterTestCases = []queryConverterTestCase{ { name: "unknown search attribute", in: "Foo = 'bar'", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, { name: "reserved field name is not a search attribute", in: "NamespaceId = 'foo'", - err: "invalid expression: column name 'NamespaceId' is not a valid search attribute", + err: "invalid search attribute: NamespaceId", }, // Namespace division. @@ -1036,17 +1036,17 @@ var queryConverterTestCases = []queryConverterTestCase{ { name: "error inside and expression", in: "WorkflowId = 'wid' AND Foo = 'bar'", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, { name: "error inside or expression", in: "Foo = 'bar' OR WorkflowId = 'wid'", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, { name: "error inside not expression", in: "NOT Foo = 'bar'", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, // GROUP BY clause. @@ -1082,7 +1082,7 @@ var queryConverterTestCases = []queryConverterTestCase{ { name: "group by unknown search attribute", in: "GROUP BY Foo", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, // ORDER BY clause. @@ -1129,7 +1129,7 @@ var queryConverterTestCases = []queryConverterTestCase{ { name: "order by unknown search attribute", in: "ORDER BY Foo", - err: "invalid expression: column name 'Foo' is not a valid search attribute", + err: "invalid search attribute: Foo", }, // Malformed and unsupported queries. diff --git a/service/worker/scheduler/query_test.go b/service/worker/scheduler/query_test.go index 8d951391d93..917c701280f 100644 --- a/service/worker/scheduler/query_test.go +++ b/service/worker/scheduler/query_test.go @@ -250,7 +250,7 @@ func TestGetQueryFields(t *testing.T) { name: "invalid custom search attribute", input: "Foo = 'bar'", expectedFields: nil, - expectedErrMsg: "'Foo' is not a valid search attribute", + expectedErrMsg: "invalid search attribute: Foo", }, } @@ -334,7 +334,7 @@ func TestValidateVisibilityQuery(t *testing.T) { { name: "invalid custom search attribute", input: "Foo = foo", - expectedErrMsg: "'Foo' is not a valid search attribute", + expectedErrMsg: "invalid search attribute: Foo", }, } From a370f4dced0c128d5e155a905bc923543a708e18 Mon Sep 17 00:00:00 2001 From: Veeral Patel Date: Thu, 3 Sep 2026 20:36:52 -0700 Subject: [PATCH 136/139] Add improved poller scaling signals (opt-in) (#11909) This PR replaces the two existing scaling signals for poller autoscaling with improved versions (on a opt-in basis): * Backlog age signal: Instead of checking if backlog age is >200ms (default threshold) to scale pollers, check if overall dispatch latency is greater than the threshold * Ratio signal: Instead of checking if addRate/dispatchRate ratio > 1.2 (default threshold), check if addRate/syncMatchRate > the threshold. This will let us add more pollers if the sync match rate is too low. Both improved signals are only enabled if the `matching.useSignalsV2ForPollerScaling` flag is true - set to `false` by default. ## How we tested it - added unit tests - ran benchmarks in a test cell, with multiple omes workloads (see below) --------- Co-authored-by: Kannan Rajah Co-authored-by: Claude Opus 4.6 --- common/dynamicconfig/constants.go | 14 +- common/metrics/metric_defs.go | 3 +- common/metrics/tags.go | 4 +- service/matching/config.go | 6 + .../matching/physical_task_queue_manager.go | 87 ++++-- .../physical_task_queue_manager_interface.go | 3 +- .../physical_task_queue_manager_mock.go | 9 +- .../physical_task_queue_manager_test.go | 248 ++++++++++++++++-- .../matching/task_queue_partition_manager.go | 2 +- 9 files changed, 319 insertions(+), 57 deletions(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index ec2c614ed40..dacf561bf65 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -1560,7 +1560,9 @@ these log lines can be noisy, we want to be able to turn on and sample selective "matching.pollerScalingMinimumBacklog", 200*time.Millisecond, `MatchingPollerScalingBacklogAgeScaleUp is the minimum backlog age that must be accumulated before -a decision to scale up the number of pollers will be issued`, +a decision to scale up the number of pollers will be issued. If MatchingUseSignalsV2ForPollerScaling is true, +this is instead the maximum age of a dispatched task (measured from its create time) above which a scale-up +will be issued.`, ) MatchingPollerScalingWaitTime = NewTaskQueueDurationSetting( "matching.pollerScalingWaitTime", @@ -1578,7 +1580,8 @@ second per poller by one physical queue manager`, "matching.pollerScalingTaskAddToDispatchRatio", 1.2, `MatchingPollerScalingTaskAddToDispatchRatio is the ratio of task add rate to task -dispatch rate above which a decision to scale up the number of pollers will be issued`, +dispatch rate above which a decision to scale up the number of pollers will be issued. If MatchingUseSignalsV2ForPollerScaling +is true, this is instead the ratio of task add rate to task sync match rate.`, ) MatchingEnablePollerScalingDecisionMetrics = NewTaskQueueBoolSetting( "matching.enablePollerScalingDecisionMetrics", @@ -1586,6 +1589,13 @@ dispatch rate above which a decision to scale up the number of pollers will be i `MatchingEnablePollerScalingDecisionMetrics, when enabled, causes matching to emit the poller_scale_decision metric describing why pollers are scaled up, down, or held for a physical task queue. This is opt-in and can be scoped by namespace and/or task queue.`, + ) + MatchingUseSignalsV2ForPollerScaling = NewTaskQueueBoolSetting( + "matching.useSignalsV2ForPollerScaling", + false, + `MatchingUseSignalsV2ForPollerScaling, when enabled, uses v2 scaling signals for poller autoscaling: +(1) sync match rate instead of total dispatch rate for the add-to-dispatch ratio check, and +(2) task dispatch latency instead of backlog age stats for the backlog scale-up check.`, ) MatchingUseNewMatcher = NewTaskQueueTypedSettingWithConverter( "matching.useNewMatcher", diff --git a/common/metrics/metric_defs.go b/common/metrics/metric_defs.go index 24b9652a757..cddaf220c80 100644 --- a/common/metrics/metric_defs.go +++ b/common/metrics/metric_defs.go @@ -1429,7 +1429,8 @@ var ( WithDescription( "Count of poller scaling decisions made by a physical task queue manager. Emitted only when the opt-in "+ "dynamic config matching.enablePollerScalingDecisionMetrics is enabled. Dimensions: namespace, taskqueue, "+ - "task_type, partition, decision (scale_up/scale_down/hold), reason (idle/backlog/task_rate/rate_limited)"), + "task_type, partition, decision (scale_up/scale_down/hold), "+ + "reason (idle/delay/ratio/rate_limited/task_queue_rate_limited)"), ) // ---------------------------------------------------------------------------------------------------------------- diff --git a/common/metrics/tags.go b/common/metrics/tags.go index 9bb0ba19982..c91a1fb64d5 100644 --- a/common/metrics/tags.go +++ b/common/metrics/tags.go @@ -336,8 +336,8 @@ const ( const ( PollerScaleReasonIdle ReasonString = "idle" - PollerScaleReasonBacklog ReasonString = "backlog" - PollerScaleReasonTaskRate ReasonString = "task_rate" + PollerScaleReasonDelay ReasonString = "delay" + PollerScaleReasonRatio ReasonString = "ratio" PollerScaleReasonRateLimited ReasonString = "rate_limited" PollerScaleReasonTaskQueueRateLimited ReasonString = "task_queue_rate_limited" ) diff --git a/service/matching/config.go b/service/matching/config.go index 763bfb972f4..6dae5bc2273 100644 --- a/service/matching/config.go +++ b/service/matching/config.go @@ -138,6 +138,7 @@ type ( PollerScalingDecisionsPerSecond dynamicconfig.FloatPropertyFnWithTaskQueueFilter PollerScalingTaskAddToDispatchRatio dynamicconfig.FloatPropertyFnWithTaskQueueFilter EnablePollerScalingDecisionMetrics dynamicconfig.BoolPropertyFnWithTaskQueueFilter + UseSignalsV2ForPollerScaling dynamicconfig.BoolPropertyFnWithTaskQueueFilter FairnessCounter dynamicconfig.TypedPropertyFnWithTaskQueueFilter[counter.CounterParams] FairnessPassDither dynamicconfig.BoolPropertyFnWithTaskQueueFilter @@ -232,6 +233,7 @@ type ( PollerScalingDecisionsPerSecond func() float64 PollerScalingTaskAddToDispatchRatio func() float64 EnablePollerScalingDecisionMetrics func() bool + UseSignalsV2ForPollerScaling func() bool FairnessCounter func() counter.CounterParams FairnessPassDither func() bool @@ -385,6 +387,7 @@ func NewConfig( PollerScalingDecisionsPerSecond: dynamicconfig.MatchingPollerScalingDecisionsPerSecond.Get(dc), PollerScalingTaskAddToDispatchRatio: dynamicconfig.MatchingPollerScalingTaskAddToDispatchRatio.Get(dc), EnablePollerScalingDecisionMetrics: dynamicconfig.MatchingEnablePollerScalingDecisionMetrics.Get(dc), + UseSignalsV2ForPollerScaling: dynamicconfig.MatchingUseSignalsV2ForPollerScaling.Get(dc), FairnessCounter: dynamicconfig.MatchingFairnessCounter.Get(dc), FairnessPassDither: dynamicconfig.MatchingFairnessPassDither.Get(dc), @@ -564,6 +567,9 @@ func newTaskQueueConfig(tq *tqid.TaskQueue, config *Config, ns namespace.Name) * EnablePollerScalingDecisionMetrics: func() bool { return config.EnablePollerScalingDecisionMetrics(ns.String(), taskQueueName, taskType) }, + UseSignalsV2ForPollerScaling: func() bool { + return config.UseSignalsV2ForPollerScaling(ns.String(), taskQueueName, taskType) + }, FairnessCounter: func() counter.CounterParams { return config.FairnessCounter(ns.String(), taskQueueName, taskType) }, diff --git a/service/matching/physical_task_queue_manager.go b/service/matching/physical_task_queue_manager.go index c19517be803..c625f71eb38 100644 --- a/service/matching/physical_task_queue_manager.go +++ b/service/matching/physical_task_queue_manager.go @@ -95,6 +95,10 @@ type ( taskTrackerLock sync.Mutex tasksAdded map[priorityKey]*taskTracker tasksDispatched map[priorityKey]*taskTracker + // tasksSyncMatched is kept out of GetStatsByPriority because TaskQueueStats is a public API + // proto and this rate is only used internally, for poller scaling decisions. Exposing it via + // DescribeTaskQueue would mean moving it there and adding a proto field. + tasksSyncMatched map[priorityKey]*taskTracker // tasksRateLimited tracks rate-limit events in a sliding window for stats reporting. tasksRateLimited *taskTracker } @@ -162,6 +166,7 @@ func newPhysicalTaskQueueManager( metricsHandler: taggedMetricsHandler, tasksAdded: make(map[priorityKey]*taskTracker), tasksDispatched: make(map[priorityKey]*taskTracker), + tasksSyncMatched: make(map[priorityKey]*taskTracker), tasksRateLimited: e.newTaskTracker(), pollerScalingRateLimiter: quotas.NewDefaultOutgoingRateLimiter(pollerScalingRateLimitFn), deploymentRegistrationCh: make(chan struct{}, 1), @@ -530,7 +535,13 @@ func (c *physicalTaskQueueManagerImpl) PollTask( task.backlogCountHint = c.backlogCountHint if pollMetadata.forwardedFrom == "" { // track the task on the child, not where a poll was forwarded to - c.incTaskTracker(c.tasksDispatched, priorityKey(task.getPriority().GetPriorityKey()), 1) + pri := priorityKey(task.getPriority().GetPriorityKey()) + c.incTaskTracker(c.tasksDispatched, pri, 1) + // Mirror the !task.isForwarded() guard on tasksAdded in TrySyncMatch. A task forwarded + // in from a child is not counted as added here, so it is not counted as matched here. + if task.source == enumsspb.TASK_SOURCE_HISTORY && !task.isForwarded() { + c.incTaskTracker(c.tasksSyncMatched, pri, 1) + } } return task, nil } @@ -881,16 +892,16 @@ func (c *physicalTaskQueueManagerImpl) GetFairnessWeightOverrides() fairnessWeig func (c *physicalTaskQueueManagerImpl) MakePollerScalingDecision( ctx context.Context, pollStartTime time.Time, - taskSource enumsspb.TaskSource, + task *internalTask, ) *taskqueuepb.PollerScalingDecision { - return c.makePollerScalingDecisionImpl(pollStartTime, taskSource, func() *taskqueuepb.TaskQueueStats { + return c.makePollerScalingDecisionImpl(pollStartTime, task, func() *taskqueuepb.TaskQueueStats { return c.partitionMgr.GetPhysicalQueueAdjustedStats(ctx, c) }) } func (c *physicalTaskQueueManagerImpl) makePollerScalingDecisionImpl( pollStartTime time.Time, - taskSource enumsspb.TaskSource, + task *internalTask, statsFn func() *taskqueuepb.TaskQueueStats, ) *taskqueuepb.PollerScalingDecision { pollWaitTime := c.partitionMgr.engine.timeSource.Since(pollStartTime) @@ -899,7 +910,7 @@ func (c *physicalTaskQueueManagerImpl) makePollerScalingDecisionImpl( // idle — there are more pollers than needed. For backlog tasks, the wait most likely // comes from the DB read path (write → reader → matcher), not excess pollers. Skip // the scale-down and fall through to the scale-up checks below. - if pollWaitTime >= c.partitionMgr.config.PollerScalingWaitTime() && taskSource != enumsspb.TASK_SOURCE_DB_BACKLOG { + if pollWaitTime >= c.partitionMgr.config.PollerScalingWaitTime() && task.source != enumsspb.TASK_SOURCE_DB_BACKLOG { c.recordPollerScaleDecision(metrics.PollerScaleDecisionDown, metrics.PollerScaleReasonIdle) return &taskqueuepb.PollerScalingDecision{ PollRequestDeltaSuggestion: -1, @@ -925,24 +936,18 @@ func (c *physicalTaskQueueManagerImpl) makePollerScalingDecisionImpl( c.recordPollerScaleDecision(metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonTaskQueueRateLimited) return nil } - if stats.GetApproximateBacklogCount() > 0 && - stats.GetApproximateBacklogAge().AsDuration() > c.partitionMgr.config.PollerScalingBacklogAgeScaleUp() { - // Always increase when there is a backlog, even if we're a partition. It's also important to increase for - // sticky queues. + + if c.delaySignalFiring(stats, task) { delta = 1 - reason = metrics.PollerScaleReasonBacklog + reason = metrics.PollerScaleReasonDelay } else if c.queue.Partition().Kind() != enumspb.TASK_QUEUE_KIND_STICKY && !c.queue.Partition().IsRoot() { // Non-root partitions don't have an appropriate view of the data to make decisions beyond backlog. // Sticky queues are exempt: they aren't considered root but do have a complete view of their data, // as they have only 1 partition. return nil - } else { - if float64(stats.GetTasksAddRate())/float64(stats.GetTasksDispatchRate()) > c.partitionMgr.config.PollerScalingTaskAddToDispatchRatio() { - // Increase if we're adding tasks faster than we're dispatching them. Particularly useful for Nexus tasks, - // since those (currently) don't get backlogged. - delta = 1 - reason = metrics.PollerScaleReasonTaskRate - } + } else if c.ratioSignalFiring(stats) { + delta = 1 + reason = metrics.PollerScaleReasonRatio } if delta == 0 { @@ -966,6 +971,53 @@ func (c *physicalTaskQueueManagerImpl) recordPollerScaleDecision(decision string Record(1, metrics.PollerScaleDecisionTag(decision), metrics.ReasonTag(reason)) } +// delaySignalFiring reports whether tasks are taking too long to reach a worker, which warrants +// scaling pollers up. Under UseSignalsV2ForPollerScaling, this measures task dispatch latency, +// which is wider than backlog delay alone. +func (c *physicalTaskQueueManagerImpl) delaySignalFiring(stats *taskqueuepb.TaskQueueStats, task *internalTask) bool { + maxAge := c.partitionMgr.config.PollerScalingBacklogAgeScaleUp() + + if c.partitionMgr.config.UseSignalsV2ForPollerScaling() { + return c.partitionMgr.engine.timeSource.Since(task.getCreateTime().AsTime()) > maxAge + } + + return stats.GetApproximateBacklogCount() > 0 && stats.GetApproximateBacklogAge().AsDuration() > maxAge +} + +// ratioSignalFiring reports whether we're adding tasks faster than we're dispatching them, which +// warrants scaling pollers up. Under UseSignalsV2ForPollerScaling, this measures if we are adding +// tasks faster than we can sync match them. +// Particularly useful for Nexus tasks, since those (currently) don't get backlogged. +func (c *physicalTaskQueueManagerImpl) ratioSignalFiring(stats *taskqueuepb.TaskQueueStats) bool { + maxRatio := c.partitionMgr.config.PollerScalingTaskAddToDispatchRatio() + + if c.partitionMgr.config.UseSignalsV2ForPollerScaling() { + // If the syncMatch rate is 0 while tasks are being added, the division is +Inf and this + // always fires -- intentional. Note that when neither tracker has recorded anything the + // division is 0/0 = NaN instead, and NaN > maxRatio is false, so it does not fire. + addRate, syncMatchRate := c.getAggregateRates() + return float64(addRate)/float64(syncMatchRate) > maxRatio + } + + return float64(stats.GetTasksAddRate())/float64(stats.GetTasksDispatchRate()) > maxRatio +} + +// getAggregateRates returns the task add and sync match rates, each summed across all priorities. +// Both are read under one lock so they form a consistent snapshot: incTaskTracker takes the same +// lock on every add and every dispatch, so reading them separately could mix two instants. +func (c *physicalTaskQueueManagerImpl) getAggregateRates() (addRate, syncMatchRate float32) { + c.taskTrackerLock.Lock() + defer c.taskTrackerLock.Unlock() + + for _, tt := range c.tasksAdded { + addRate += tt.rate() + } + for _, tt := range c.tasksSyncMatched { + syncMatchRate += tt.rate() + } + return addRate, syncMatchRate +} + func (c *physicalTaskQueueManagerImpl) UpdateRemotePriorityBacklogs(backlogs remotePriorityBacklogSet) { if c.priMatcher != nil { c.priMatcher.UpdateRemotePriorityBacklogs(backlogs) @@ -995,6 +1047,7 @@ func (c *physicalTaskQueueManagerImpl) incTaskTracker( // Initialize all task trackers together; or the timeframes won't line up. c.tasksAdded[priorityKey] = c.partitionMgr.engine.newTaskTracker() c.tasksDispatched[priorityKey] = c.partitionMgr.engine.newTaskTracker() + c.tasksSyncMatched[priorityKey] = c.partitionMgr.engine.newTaskTracker() tracker = intervals[priorityKey] } tracker.inc(n) diff --git a/service/matching/physical_task_queue_manager_interface.go b/service/matching/physical_task_queue_manager_interface.go index 3c2cd8e3f3e..3809bcff194 100644 --- a/service/matching/physical_task_queue_manager_interface.go +++ b/service/matching/physical_task_queue_manager_interface.go @@ -8,7 +8,6 @@ import ( enumspb "go.temporal.io/api/enums/v1" taskqueuepb "go.temporal.io/api/taskqueue/v1" - enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservice/v1" persistencespb "go.temporal.io/server/api/persistence/v1" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" @@ -68,7 +67,7 @@ type ( QueueKey() *PhysicalTaskQueueKey // MakePollerScalingDecision makes a decision on whether to scale pollers up or down based on the current state // of the task queue and the task about to be returned. - MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, taskSource enumsspb.TaskSource) *taskqueuepb.PollerScalingDecision + MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, task *internalTask) *taskqueuepb.PollerScalingDecision // GetFairnessWeightOverrides returns current fairness weight overrides for this queue. GetFairnessWeightOverrides() fairnessWeightOverrides UpdateRemotePriorityBacklogs(remotePriorityBacklogSet) diff --git a/service/matching/physical_task_queue_manager_mock.go b/service/matching/physical_task_queue_manager_mock.go index cbf42a03414..0996b1c6933 100644 --- a/service/matching/physical_task_queue_manager_mock.go +++ b/service/matching/physical_task_queue_manager_mock.go @@ -16,7 +16,6 @@ import ( enums "go.temporal.io/api/enums/v1" taskqueue "go.temporal.io/api/taskqueue/v1" - enums0 "go.temporal.io/server/api/enums/v1" matchingservice "go.temporal.io/server/api/matchingservice/v1" persistence "go.temporal.io/server/api/persistence/v1" taskqueue0 "go.temporal.io/server/api/taskqueue/v1" @@ -216,17 +215,17 @@ func (mr *MockphysicalTaskQueueManagerMockRecorder) LegacyDescribeTaskQueue(incl } // MakePollerScalingDecision mocks base method. -func (m *MockphysicalTaskQueueManager) MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, taskSource enums0.TaskSource) *taskqueue.PollerScalingDecision { +func (m *MockphysicalTaskQueueManager) MakePollerScalingDecision(ctx context.Context, pollStartTime time.Time, task *internalTask) *taskqueue.PollerScalingDecision { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakePollerScalingDecision", ctx, pollStartTime, taskSource) + ret := m.ctrl.Call(m, "MakePollerScalingDecision", ctx, pollStartTime, task) ret0, _ := ret[0].(*taskqueue.PollerScalingDecision) return ret0 } // MakePollerScalingDecision indicates an expected call of MakePollerScalingDecision. -func (mr *MockphysicalTaskQueueManagerMockRecorder) MakePollerScalingDecision(ctx, pollStartTime, taskSource any) *gomock.Call { +func (mr *MockphysicalTaskQueueManagerMockRecorder) MakePollerScalingDecision(ctx, pollStartTime, task any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakePollerScalingDecision", reflect.TypeOf((*MockphysicalTaskQueueManager)(nil).MakePollerScalingDecision), ctx, pollStartTime, taskSource) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakePollerScalingDecision", reflect.TypeOf((*MockphysicalTaskQueueManager)(nil).MakePollerScalingDecision), ctx, pollStartTime, task) } // MarkAlive mocks base method. diff --git a/service/matching/physical_task_queue_manager_test.go b/service/matching/physical_task_queue_manager_test.go index ef90aa30ffa..40e733222e2 100644 --- a/service/matching/physical_task_queue_manager_test.go +++ b/service/matching/physical_task_queue_manager_test.go @@ -17,7 +17,9 @@ import ( enumsspb "go.temporal.io/server/api/enums/v1" "go.temporal.io/server/api/matchingservicemock/v1" persistencespb "go.temporal.io/server/api/persistence/v1" + taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/common" + "go.temporal.io/server/common/clock" "go.temporal.io/server/common/cluster" "go.temporal.io/server/common/dynamicconfig" "go.temporal.io/server/common/goro" @@ -67,6 +69,11 @@ func TestPhysicalTaskQueueManager_Fair_TestSuite(t *testing.T) { suite.Run(t, &PhysicalTaskQueueManagerTestSuite{newMatcher: true, fairness: true}) } +// SetupSubTest rebuilds the manager for each subtest so table cases cannot leak tracker state. +func (s *PhysicalTaskQueueManagerTestSuite) SetupSubTest() { + s.SetupTest() +} + func (s *PhysicalTaskQueueManagerTestSuite) SetupTest() { s.config = defaultTestConfig() s.controller = gomock.NewController(s.T()) @@ -426,6 +433,19 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestTQMInterruptsPollOnClose() { s.Less(time.Since(pollStart), 4*time.Second) } +// pollScalingTask builds a minimal task carrying just the source and create time that +// makePollerScalingDecisionImpl reads off it. +func pollScalingTask(source enumsspb.TaskSource, createTime time.Time) *internalTask { + return &internalTask{ + source: source, + event: &genericTaskInfo{ + AllocatedTaskInfo: &persistencespb.AllocatedTaskInfo{ + Data: &persistencespb.TaskInfo{CreateTime: timestamppb.New(createTime)}, + }, + }, + } +} + func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpOnBacklog() { rl := quotas.NewMockRateLimiter(s.controller) rl.EXPECT().AllowN(gomock.Any(), gomock.Any()).Return(true).AnyTimes() @@ -436,7 +456,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpOnBacklog() { ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -451,7 +471,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpAddRateExceedsDispa TasksDispatchRate: 10, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -465,12 +485,12 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpUsesDefaultRatioThr // 110/100 = 1.1, below the 1.2 default. belowThreshold := &taskqueuepb.TaskQueueStats{TasksAddRate: 110, TasksDispatchRate: 100} - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return belowThreshold }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return belowThreshold }) s.Nil(decision) // 130/100 = 1.3, above the 1.2 default. aboveThreshold := &taskqueuepb.TaskQueueStats{TasksAddRate: 130, TasksDispatchRate: 100} - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return aboveThreshold }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return aboveThreshold }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -484,14 +504,14 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingUpRatioThresholdIsCon // above the actual ratio should suppress the scale-up. highRatio := &taskqueuepb.TaskQueueStats{TasksAddRate: 100, TasksDispatchRate: 10} s.tqMgr.partitionMgr.config.PollerScalingTaskAddToDispatchRatio = func() float64 { return 20 } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return highRatio }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return highRatio }) s.Nil(decision) // 110/100 = 1.1, which would not scale up under the default 1.2 threshold. Lowering the // threshold below the actual ratio should trigger a scale-up. lowRatio := &taskqueuepb.TaskQueueStats{TasksAddRate: 110, TasksDispatchRate: 100} s.tqMgr.partitionMgr.config.PollerScalingTaskAddToDispatchRatio = func() float64 { return 1.0 } - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return lowRatio }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return lowRatio }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -501,7 +521,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingNoChangeOnNoBacklogFa ApproximateBacklogCount: 0, ApproximateBacklogAge: durationpb.New(0), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -520,12 +540,12 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingNonRootPartition() { ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) fakeStats.ApproximateBacklogCount = 0 - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -548,7 +568,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingStickyQueue() { TasksAddRate: 100, TasksDispatchRate: 10, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) } @@ -557,7 +577,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDownOnLongSyncMatch() fakeStats := &taskqueuepb.TaskQueueStats{ ApproximateBacklogCount: 0, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.LessOrEqual(decision.PollRequestDeltaSuggestion, int32(-1)) } @@ -566,7 +586,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingHoldOnLongBacklogMatc fakeStats := &taskqueuepb.TaskQueueStats{ ApproximateBacklogCount: 0, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_DB_BACKLOG, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -576,7 +596,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingBacklogFallsThruToSca ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_DB_BACKLOG, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.NotNil(decision) s.Equal(int32(1), decision.PollRequestDeltaSuggestion) } @@ -588,7 +608,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingHoldOnTaskQueueRateLi ApproximateBacklogAge: durationpb.New(1 * time.Minute), RateLimitingActive: true, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_DB_BACKLOG, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -602,7 +622,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricTaskQue ApproximateBacklogAge: durationpb.New(1 * time.Minute), RateLimitingActive: true, } - s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_DB_BACKLOG, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_DB_BACKLOG, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonTaskQueueRateLimited) } @@ -616,10 +636,10 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionsAreRateLimit ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.GreaterOrEqual(decision.PollRequestDeltaSuggestion, int32(1)) - decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision = s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) } @@ -648,11 +668,11 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleDo defer handler.StopCapture(capture) fakeStats := &taskqueuepb.TaskQueueStats{ApproximateBacklogCount: 0} - s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionDown, metrics.PollerScaleReasonIdle) } -func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUpBacklog() { +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUpDelay() { rl := quotas.NewMockRateLimiter(s.controller) rl.EXPECT().AllowN(gomock.Any(), gomock.Any()).Return(true).AnyTimes() s.tqMgr.pollerScalingRateLimiter = rl @@ -664,11 +684,11 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUp ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) - s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonBacklog) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonDelay) } -func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUpTaskRate() { +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUpRatio() { rl := quotas.NewMockRateLimiter(s.controller) rl.EXPECT().AllowN(gomock.Any(), gomock.Any()).Return(true).AnyTimes() s.tqMgr.pollerScalingRateLimiter = rl @@ -680,8 +700,8 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricScaleUp TasksAddRate: 100, TasksDispatchRate: 10, } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) - s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonTaskRate) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionUp, metrics.PollerScaleReasonRatio) } func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricHoldRateLimited() { @@ -696,7 +716,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricHoldRat ApproximateBacklogCount: 100, ApproximateBacklogAge: durationpb.New(1 * time.Minute), } - s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.assertPollerScaleDecision(capture, metrics.PollerScaleDecisionHold, metrics.PollerScaleReasonRateLimited) } @@ -715,7 +735,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricNoSigna TasksAddRate: 10, TasksDispatchRate: 100, } - decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + decision := s.tqMgr.makePollerScalingDecisionImpl(time.Now(), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Nil(decision) s.Empty(capture.Snapshot()[metrics.PollerScaleDecisionCounter.Name()]) } @@ -728,7 +748,7 @@ func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingDecisionMetricDisable defer handler.StopCapture(capture) fakeStats := &taskqueuepb.TaskQueueStats{ApproximateBacklogCount: 0} - s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), enumsspb.TASK_SOURCE_HISTORY, func() *taskqueuepb.TaskQueueStats { return fakeStats }) + s.tqMgr.makePollerScalingDecisionImpl(time.Now().Add(-2*time.Second), pollScalingTask(enumsspb.TASK_SOURCE_HISTORY, time.Now()), func() *taskqueuepb.TaskQueueStats { return fakeStats }) s.Empty(capture.Snapshot()[metrics.PollerScaleDecisionCounter.Name()]) } @@ -836,3 +856,177 @@ func TestDrainCompletionNoReloadDraining(t *testing.T) { assert.Equal(t, prevPriStats.updateCount, priQueueData.persistenceStats().updateCount, "no new UpdateTaskQueue calls should be made to drained table after reload") } + +// syncMatchOneTask offers a task to a waiting local poller and blocks until PollTask has returned, +// which is when the task trackers are updated. +func (s *PhysicalTaskQueueManagerTestSuite) syncMatchOneTask(forwardInfo *taskqueuespb.TaskForwardInfo) { + poller, pollResult := runOneShotPoller(context.Background(), s.tqMgr) + defer poller.Cancel() + + task := newInternalTaskForSyncMatch(&persistencespb.TaskInfo{ + CreateTime: timestamppb.New(time.Now()), + }, forwardInfo, 0, nil) + + outcome, err := s.tqMgr.TrySyncMatch(context.Background(), task) + s.Require().NoError(err) + s.Require().Equal(syncMatchSuccess, outcome) + s.Require().IsType(&internalTask{}, <-pollResult, "poller should have received the task") +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestSyncMatchRateCountsLocallyAddedTasks() { + // Counterpart to TestSyncMatchRateExcludesForwardedTasks: a locally added task is counted. + s.syncMatchOneTask(nil) + + addRate, syncMatchRate := s.tqMgr.getAggregateRates() + s.Positive(addRate) + s.Positive(syncMatchRate) +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestSyncMatchRateExcludesForwardedTasks() { + // TrySyncMatch does not count a forwarded task as added on this partition, so a match of one + // must not count toward the sync match rate either. + s.syncMatchOneTask(&taskqueuespb.TaskForwardInfo{ + SourcePartition: "/_sys/" + taskQueueName + "/1", + TaskSource: enumsspb.TASK_SOURCE_HISTORY, + }) + + addRate, syncMatchRate := s.tqMgr.getAggregateRates() + s.Zero(addRate, "a forwarded task is not an add on this partition") + s.Zero(syncMatchRate, "a forwarded task must not count toward the sync match rate") +} + +// useSignalsV2ForPollerScaling sets matching.useSignalsV2ForPollerScaling for the manager under +// test and removes the decision rate limit so every call yields a decision. +func (s *PhysicalTaskQueueManagerTestSuite) useSignalsV2ForPollerScaling(enabled bool) { + rl := quotas.NewMockRateLimiter(s.controller) + rl.EXPECT().AllowN(gomock.Any(), gomock.Any()).Return(true).AnyTimes() + s.tqMgr.pollerScalingRateLimiter = rl + s.tqMgr.partitionMgr.config.UseSignalsV2ForPollerScaling = func() bool { return enabled } +} + +// freezePollScalingTime installs a controllable time source. The backlog threshold is 200ms, so +// poll scaling decisions race wall time without it. Call it before any task tracker is created: +// newTaskTracker captures the engine time source. +func (s *PhysicalTaskQueueManagerTestSuite) freezePollScalingTime() *clock.EventTimeSource { + ts := clock.NewEventTimeSource() + ts.Update(time.Now()) + s.tqMgr.partitionMgr.engine.timeSource = ts + return ts +} + +// seedTaskTrackers records task counts and advances the clock so rate() has a nonzero divisor. The +// advance must stay under the 5s bucket size or the counts rotate out of the circular buffer. +func (s *PhysicalTaskQueueManagerTestSuite) seedTaskTrackers(ts *clock.EventTimeSource, added, syncMatched int) { + pri := s.tqMgr.config.DefaultPriorityKey + s.tqMgr.incTaskTracker(s.tqMgr.tasksAdded, pri, added) + s.tqMgr.incTaskTracker(s.tqMgr.tasksSyncMatched, pri, syncMatched) + ts.Update(ts.Now().Add(time.Second)) +} + +func (s *PhysicalTaskQueueManagerTestSuite) TestPollScalingSignalsV2() { + cases := []struct { + name string + v2Enabled bool + taskAge time.Duration + taskSource enumsspb.TaskSource + added int + syncMatched int + stats *taskqueuepb.TaskQueueStats + wantDelta int32 + wantDecision string + wantReason metrics.ReasonString + }{ + { + name: "dispatch latency above threshold scales up", + v2Enabled: true, + taskAge: 500 * time.Millisecond, + taskSource: enumsspb.TASK_SOURCE_DB_BACKLOG, + stats: &taskqueuepb.TaskQueueStats{}, + wantDelta: 1, + wantDecision: metrics.PollerScaleDecisionUp, + wantReason: metrics.PollerScaleReasonDelay, + }, + { + name: "dispatch latency below threshold holds", + v2Enabled: true, + taskAge: 50 * time.Millisecond, + taskSource: enumsspb.TASK_SOURCE_HISTORY, + stats: &taskqueuepb.TaskQueueStats{TasksAddRate: 10, TasksDispatchRate: 100}, + }, + { + name: "add rate outpacing sync match rate scales up", + v2Enabled: true, + taskSource: enumsspb.TASK_SOURCE_HISTORY, + added: 300, + syncMatched: 100, + stats: &taskqueuepb.TaskQueueStats{TasksAddRate: 1, TasksDispatchRate: 100}, + wantDelta: 1, + wantDecision: metrics.PollerScaleDecisionUp, + wantReason: metrics.PollerScaleReasonRatio, + }, + { + name: "zero sync match rate scales up", + v2Enabled: true, + taskSource: enumsspb.TASK_SOURCE_HISTORY, + added: 100, + stats: &taskqueuepb.TaskQueueStats{TasksAddRate: 10, TasksDispatchRate: 100}, + wantDelta: 1, + wantDecision: metrics.PollerScaleDecisionUp, + wantReason: metrics.PollerScaleReasonRatio, + }, + { + name: "no local rates holds and ignores stats", + v2Enabled: true, + taskSource: enumsspb.TASK_SOURCE_HISTORY, + stats: &taskqueuepb.TaskQueueStats{TasksAddRate: 100, TasksDispatchRate: 10}, + }, + { + name: "task queue rate limit suppresses a latency scale up", + v2Enabled: true, + taskAge: 500 * time.Millisecond, + taskSource: enumsspb.TASK_SOURCE_DB_BACKLOG, + stats: &taskqueuepb.TaskQueueStats{RateLimitingActive: true}, + wantDecision: metrics.PollerScaleDecisionHold, + wantReason: metrics.PollerScaleReasonTaskQueueRateLimited, + }, + { + name: "disabled ignores task create time", + v2Enabled: false, + taskAge: time.Hour, + taskSource: enumsspb.TASK_SOURCE_DB_BACKLOG, + stats: &taskqueuepb.TaskQueueStats{ApproximateBacklogCount: 0}, + }, + } + + for _, tc := range cases { + s.Run(tc.name, func() { + s.useSignalsV2ForPollerScaling(tc.v2Enabled) + ts := s.freezePollScalingTime() + handler := s.enablePollerScaleDecisionMetrics() + capture := handler.StartCapture() + defer handler.StopCapture(capture) + + if tc.added != 0 || tc.syncMatched != 0 { + s.seedTaskTrackers(ts, tc.added, tc.syncMatched) + } + + decision := s.tqMgr.makePollerScalingDecisionImpl( + ts.Now(), pollScalingTask(tc.taskSource, ts.Now().Add(-tc.taskAge)), + func() *taskqueuepb.TaskQueueStats { return tc.stats }, + ) + + if tc.wantDelta == 0 { + s.Nil(decision) + } else { + s.Require().NotNil(decision) + s.Equal(tc.wantDelta, decision.PollRequestDeltaSuggestion) + } + + if tc.wantDecision == "" { + s.Empty(capture.Snapshot()[metrics.PollerScaleDecisionCounter.Name()]) + } else { + s.assertPollerScaleDecision(capture, tc.wantDecision, tc.wantReason) + } + }) + } +} diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index 49e166dbbde..c4859cfe980 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -847,7 +847,7 @@ func (pm *taskQueuePartitionManagerImpl) PollTask( task, err := dbq.PollTask(ctx, pollMetadata) if task != nil { - task.pollerScalingDecision = dbq.MakePollerScalingDecision(ctx, pollMetadata.localPollStartTime, task.source) + task.pollerScalingDecision = dbq.MakePollerScalingDecision(ctx, pollMetadata.localPollStartTime, task) } // Update poller timestamp when poll ends, unless cancelled (e.g., shutdown/disconnect). From 2220587dea828d938fc99253e178a13d1cb658c5 Mon Sep 17 00:00:00 2001 From: Carly de Frondeville Date: Thu, 3 Sep 2026 21:55:01 -0700 Subject: [PATCH 137/139] dynamic partitioning: use probability of AddTask to root partition to estimate overall add task rate (#11699) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What changed? - use probability of AddTask to root partition to estimate overall add task rate, instead of assuming uniform distribution - to enable this, change the AddTask loadbalancing to ensure that there is at least 1% chance of hitting the root, regardless of how much backlog the root has ## Why? A previous PR changed AddTask load-balancing from uniform random to backlog-aware ## How did you test it? - [x] built - [x] run locally and tested manually - [ ] covered by existing tests - [x] added new unit test(s) - [ ] added new functional test(s) ## Potential risks Changes AddTask computation and AddTask load balancing, but improves the estimation of the former, and we are ok with the tradeoff for the latter --- > [!NOTE] > **Medium Risk** > Touches AddTask write routing and dynamic partition scaling inputs; behavior changes when backlog-aware routing heavily favors child partitions, though a 1% root floor and fallback estimates limit worst-case drift. > > **Overview** > Backlog-aware write routing no longer assumes one root add implies uniform load on every partition. The matching client now returns an **estimated tasks across all partitions** when load-balancing picks the **root** partition, sends it on gRPC metadata (`etap`), and the root partition scaler uses that instead of multiplying by write partition count. > > **Write load balancing** enforces at least a **1% chance** of routing to the root (`writePartitionRootProbabilityFloor`) so sampling stays viable when the root backlog is full. When the root is chosen, the estimate is derived from gap-weighted routing (`randomRound(total/gap0)`); non-root picks carry zero estimate. Fallback paths (uniform random, forced partition via test hooks) still return `partitionCount` on root samples. > > **Scale manager** batches and wakes the partition scaler on **estimated** queue-wide task volume (`AddedTasks(estimatedTasksAllPartitions)`), with threshold `batchSize × currentWrite` once write count is known. > > Reviewed by [Cursor Bugbot](https://cursor.com/bugbot) for commit 1ee95df694c22cb1cfd93068ec36e808d84fb5fd. Bugbot is set up for automated code reviews on this repo. Configure [here](https://www.cursor.com/dashboard/bugbot). --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- client/matching/client.go | 20 ++- client/matching/loadbalancer.go | 100 +++++++++----- client/matching/loadbalancer_test.go | 122 +++++++++++++++++- client/matching/partition_counts.go | 21 +++ .../matching/partition_scaler_interface.go | 2 +- service/matching/scale_manager.go | 24 +++- service/matching/scale_manager_test.go | 87 ++++++++----- .../matching/task_queue_partition_manager.go | 28 ++-- 8 files changed, 308 insertions(+), 96 deletions(-) diff --git a/client/matching/client.go b/client/matching/client.go index d14c4f90ccb..013546d99c1 100644 --- a/client/matching/client.go +++ b/client/matching/client.go @@ -175,10 +175,11 @@ func (c *clientImpl) addActivityTask( opts []grpc.CallOption, ) (*matchingservice.AddActivityTaskResponse, error) { request = common.CloneProto(request) - client, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) + client, estimatedTasksAllPartitions, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) if err != nil { return nil, err } + ctx = appendEstimatedTasksAllPartitions(ctx, estimatedTasksAllPartitions) ctx, cancel := c.createContext(ctx) defer cancel() @@ -207,10 +208,11 @@ func (c *clientImpl) addWorkflowTask( opts []grpc.CallOption, ) (*matchingservice.AddWorkflowTaskResponse, error) { request = common.CloneProto(request) - client, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) + client, estimatedTasksAllPartitions, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) if err != nil { return nil, err } + ctx = appendEstimatedTasksAllPartitions(ctx, estimatedTasksAllPartitions) ctx, cancel := c.createContext(ctx) defer cancel() return client.AddWorkflowTask(ctx, request, opts...) @@ -317,10 +319,11 @@ func (c *clientImpl) queryWorkflow( ForwardInfo: request.ForwardInfo, Priority: request.Priority, } - client, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) + client, estimatedTasksAllPartitions, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) if err != nil { return nil, err } + ctx = appendEstimatedTasksAllPartitions(ctx, estimatedTasksAllPartitions) ctx, cancel := c.createContext(ctx) defer cancel() return client.QueryWorkflow(ctx, request, opts...) @@ -355,10 +358,11 @@ func (c *clientImpl) dispatchNexusTask( Request: request.Request, ForwardInfo: request.ForwardInfo, } - client, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) + client, estimatedTasksAllPartitions, err := c.pickClientForWrite(request.GetTaskQueue(), p, loadBalance, pc) if err != nil { return nil, err } + ctx = appendEstimatedTasksAllPartitions(ctx, estimatedTasksAllPartitions) ctx, cancel := c.createContext(ctx) defer cancel() return client.DispatchNexusTask(ctx, request, opts...) @@ -422,12 +426,14 @@ func (c *clientImpl) pickClientForWrite( p tqid.Partition, loadBalance bool, pc PartitionCounts, -) (matchingservice.MatchingServiceClient, error) { +) (matchingservice.MatchingServiceClient, int, error) { + estimatedTasksAllPartitions := 0 if loadBalance { - p = c.loadBalancer.PickWritePartition(p.TaskQueue(), pc) + p, estimatedTasksAllPartitions = c.loadBalancer.PickWritePartition(p.TaskQueue(), pc) } proto.Name = p.RpcName() - return c.getClientForTaskQueuePartition(p) + client, err := c.getClientForTaskQueuePartition(p) + return client, estimatedTasksAllPartitions, err } // pickClientForRead mutates the given proto. Callers should copy the proto before if necessary. diff --git a/client/matching/loadbalancer.go b/client/matching/loadbalancer.go index d0c8294e963..204ceaf4af4 100644 --- a/client/matching/loadbalancer.go +++ b/client/matching/loadbalancer.go @@ -1,6 +1,7 @@ package matching import ( + "math" "math/rand" "sync" @@ -18,25 +19,26 @@ import ( // when another partition has encoded-greater-than-zero backlog. 2x feels like a good ratio. var readPartitionWeightFloor = number.DecodeCompact8(1) +// Keep a small root sample for estimating total write rate when backlog-aware routing would +// otherwise make the root probability too small. +const writePartitionRootProbabilityFloor = 0.01 + type ( // LoadBalancer is the interface for implementers of // component that distributes add/poll api calls across // available task queue partitions when possible LoadBalancer interface { - // PickWritePartition returns the task queue partition for adding - // an activity or workflow task. The input is the name of the - // original task queue (with no partition info). When forwardedFrom - // is non-empty, this call is forwardedFrom from a child partition - // to a parent partition in which case, no load balancing should be - // performed + // PickWritePartition returns the task queue partition for adding an + // activity or workflow task and the estimated number of tasks added + // across all partitions per root task. The input is the name of the + // original task queue (with no partition info). PickWritePartition( taskQueue *tqid.TaskQueue, pc PartitionCounts, - ) *tqid.NormalPartition + ) (*tqid.NormalPartition, int) // PickReadPartition returns the task queue partition to send a poller to. - // Input is name of the original task queue as specified by caller. When - // forwardedFrom is non-empty, no load balancing should be done. + // Input is name of the original task queue as specified by caller. PickReadPartition( taskQueue *tqid.TaskQueue, pc PartitionCounts, @@ -86,28 +88,37 @@ func NewLoadBalancer( func (lb *defaultLoadBalancer) PickWritePartition( taskQueue *tqid.TaskQueue, pc PartitionCounts, -) *tqid.NormalPartition { - if n, ok := testhooks.Get(lb.testHooks, testhooks.MatchingLBForceWritePartition, namespace.ID(taskQueue.NamespaceId())); ok { - return taskQueue.NormalPartition(n) - } - - nsName, err := lb.namespaceIDToName(namespace.ID(taskQueue.NamespaceId())) - if err != nil { - return taskQueue.RootPartition() - } - +) (*tqid.NormalPartition, int) { var partitionCount int if pc.Write > 0 { partitionCount = int(pc.Write) } else { + nsName, err := lb.namespaceIDToName(namespace.ID(taskQueue.NamespaceId())) + if err != nil { + return taskQueue.RootPartition(), 1 + } partitionCount = max(1, lb.nWritePartitions(nsName.String(), taskQueue.Name(), taskQueue.TaskType())) } - return taskQueue.NormalPartition(pickWritePartitionByGap( + if n, ok := testhooks.Get(lb.testHooks, testhooks.MatchingLBForceWritePartition, namespace.ID(taskQueue.NamespaceId())); ok { + partition := taskQueue.NormalPartition(n) + if partition.IsRoot() { + return partition, partitionCount + } + return partition, 0 + } + + partitionID, estimatedTasksAllPartitions := pickWritePartitionByGap( pc.BacklogCount, partitionCount, number.DecodeCompact8(pc.BacklogCap), - )) + ) + partition := taskQueue.NormalPartition(partitionID) + if !partition.IsRoot() { + estimatedTasksAllPartitions = 0 + } + + return partition, estimatedTasksAllPartitions } // pickWritePartitionByGap picks a partition with probability proportional to how far its backlog @@ -115,10 +126,14 @@ func (lb *defaultLoadBalancer) PickWritePartition( // - every partition is at or above the backlogCap // - backlogCap is 0 // - when backlog data is not available for all write partitions -func pickWritePartitionByGap(counts []number.Compact8, partitionCount int, backlogCap int64) int { +func pickWritePartitionByGap( + counts []number.Compact8, + partitionCount int, + backlogCap int64, +) (partitionID int, estimatedTasksAllPartitions int) { if backlogCap == 0 || len(counts) < partitionCount { - return rand.Intn(partitionCount) + return rand.Intn(partitionCount), partitionCount } var total int64 @@ -128,20 +143,47 @@ func pickWritePartitionByGap(counts []number.Compact8, partitionCount int, backl } } if total <= 0 { // all partitions are at or above cap - return rand.Intn(partitionCount) + return rand.Intn(partitionCount), partitionCount } + + count0 := number.DecodeCompact8(counts[0]) + // p(root) = gap_root / (others + gap_root) + // we want to force p(root) >= writePartitionRootProbabilityFloor + // => gap_root >= others * floor / (1 - floor) + others := total - max(int64(0), backlogCap-count0) + minRootGap := math.Ceil(float64(others) * writePartitionRootProbabilityFloor / (1 - writePartitionRootProbabilityFloor)) + gap0 := max(backlogCap-count0, int64(minRootGap)) + total = others + gap0 + if partitionID := pickPartitionByGap(counts[:partitionCount], gap0, backlogCap, total); partitionID != 0 { + return partitionID, 0 + } + return 0, randomRound(float64(total) / float64(gap0)) +} + +// randomRound rounds without biasing the expected value. +func randomRound(x float64) int { + n := math.Floor(x) + if rand.Float64() < x-n { + n++ + } + return int(n) +} + +func pickPartitionByGap(counts []number.Compact8, gap0, backlogCap, total int64) int { r := rand.Int63n(total) - for i := range partitionCount { - gap := backlogCap - number.DecodeCompact8(counts[i]) - if gap <= 0 { // this partition is at or above cap - continue + for i, count := range counts { + var gap int64 + if i == 0 { + gap = gap0 + } else { + gap = max(int64(0), backlogCap-number.DecodeCompact8(count)) } if r < gap { // more likely to be true the bigger this partition's gap is return i } r -= gap } - return partitionCount - 1 // unreachable in practice; guard against compact8 rounding + return len(counts) - 1 } // PickReadPartition picks a partition for poller to poll task from, and keeps load balanced between partitions. diff --git a/client/matching/loadbalancer_test.go b/client/matching/loadbalancer_test.go index 83ecaf00edd..b739df471a1 100644 --- a/client/matching/loadbalancer_test.go +++ b/client/matching/loadbalancer_test.go @@ -1,6 +1,7 @@ package matching import ( + "math" "math/rand" "sync" "testing" @@ -10,6 +11,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/server/common/namespace" "go.temporal.io/server/common/number" + "go.temporal.io/server/common/testing/testhooks" "go.temporal.io/server/common/tqid" ) @@ -344,6 +346,42 @@ func TestPickReadPartition_IncompleteBacklogFallsBack(t *testing.T) { require.Equal(t, []int{2, 2, 2, 2}, tqlb.pollerCounts) } +func TestPickWritePartitionRootFloorPreservesNonRootWeights(t *testing.T) { + f, err := tqid.NewTaskQueueFamily("fake-namespace-id", "fake-taskqueue") + require.NoError(t, err) + taskQueue := f.TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY) + lb := &defaultLoadBalancer{} + + const partitionCount = 32 + backlogCap := number.EncodeCompact8(1000) + backlogCounts := make([]number.Compact8, partitionCount) + backlogCounts[0] = backlogCap + backlogCounts[partitionCount-1] = number.EncodeCompact8(number.DecodeCompact8(backlogCap) - 100) + pc := PartitionCounts{ + Write: partitionCount, + BacklogCap: backlogCap, + BacklogCount: backlogCounts, + } + + decodedCap := number.DecodeCompact8(backlogCap) + lastGap := decodedCap - number.DecodeCompact8(backlogCounts[partitionCount-1]) + nonRootTotal := int64(partitionCount-2)*decodedCap + lastGap + rootGap := int64(math.Ceil(float64(nonRootTotal) * writePartitionRootProbabilityFloor / + (1 - writePartitionRootProbabilityFloor))) + require.Less(t, lastGap, rootGap) + + const attempts = 100_000 + picks := 0 + for range attempts { + partition, _ := lb.PickWritePartition(taskQueue, pc) + if partition.PartitionId() == partitionCount-1 { + picks++ + } + } + expectedPicks := float64(attempts) * float64(lastGap) / float64(nonRootTotal+rootGap) + require.InDelta(t, expectedPicks, picks, expectedPicks*0.3) +} + func TestPickWritePartition_BacklogAware(t *testing.T) { f, err := tqid.NewTaskQueueFamily("fake-namespace-id", "fake-taskqueue") require.NoError(t, err) @@ -365,18 +403,24 @@ func TestPickWritePartition_BacklogAware(t *testing.T) { BacklogCap: 200, BacklogCount: []number.Compact8{0, number.EncodeCompact8(13_000_000)}, } + gap0 := backlogCap - number.DecodeCompact8(0) + gap1 := backlogCap - number.DecodeCompact8(number.EncodeCompact8(13_000_000)) counts := make([]int, 2) + estimatedTasks := 0 const n = 3000 for range n { - p := lb.PickWritePartition(taskQueue, pc) + p, estimatedTasksAllPartitions := lb.PickWritePartition(taskQueue, pc) counts[p.PartitionId()]++ + if p.IsRoot() { + estimatedTasks += estimatedTasksAllPartitions + } } require.Greater(t, counts[0], counts[1], "emptier partition should receive more writes") require.Positive(t, counts[1], "the below-cap partition should still receive some writes") - gap0 := backlogCap - number.DecodeCompact8(0) - gap1 := backlogCap - number.DecodeCompact8(number.EncodeCompact8(13_000_000)) require.InDelta(t, float64(n)*float64(gap0)/float64(gap0+gap1), counts[0], float64(n)*0.05, "writes split in proportion to each partition's gap to cap") + require.InDelta(t, n, estimatedTasks, float64(n)*0.05, + "root samples should estimate total writes without bias") // Now, every partition at/above cap -> no gap to weight by, so the picker declines and the caller // falls back to uniform random. @@ -388,14 +432,56 @@ func TestPickWritePartition_BacklogAware(t *testing.T) { } atCap := make([]int, 2) for range n { - p := lb.PickWritePartition(taskQueue, pcAtCap) + p, estimatedTasksAllPartitions := lb.PickWritePartition(taskQueue, pcAtCap) atCap[p.PartitionId()]++ + if p.IsRoot() { + require.Equal(t, 2, estimatedTasksAllPartitions) + } else { + require.Zero(t, estimatedTasksAllPartitions) + } } for i := range atCap { require.InDelta(t, n/2, atCap[i], float64(n)*0.1, "at-cap partition %d roughly uniform", i) } } +func TestPickWritePartition_RootProbabilityFloor(t *testing.T) { + f, err := tqid.NewTaskQueueFamily("fake-namespace-id", "fake-taskqueue") + require.NoError(t, err) + taskQueue := f.TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY) + lb := &defaultLoadBalancer{ + namespaceIDToName: func(namespace.ID) (namespace.Name, error) { return "fake-namespace", nil }, + taskQueueLBs: make(map[tqid.TaskQueue]*tqLoadBalancer), + } + + pc := PartitionCounts{ + Read: 2, + Write: 2, + BacklogCap: number.EncodeCompact8(50), + BacklogCount: []number.Compact8{number.EncodeCompact8(50), 0}, + } + const attempts = 100_000 + rootPicks := 0 + estimatedTasks := 0 + backlogCap := number.DecodeCompact8(pc.BacklogCap) + total := backlogCap - number.DecodeCompact8(pc.BacklogCount[1]) + gap0 := int64(math.Ceil(float64(total) * writePartitionRootProbabilityFloor)) + expectedProbability := float64(gap0) / float64(total) + expectedTasksAllPartitions := float64(total) / float64(gap0) + for range attempts { + partition, estimatedTasksAllPartitions := lb.PickWritePartition(taskQueue, pc) + if partition.IsRoot() { + rootPicks++ + require.InDelta(t, expectedTasksAllPartitions, estimatedTasksAllPartitions, 1) + estimatedTasks += estimatedTasksAllPartitions + } else { + require.Zero(t, estimatedTasksAllPartitions) + } + } + require.InDelta(t, attempts*expectedProbability, rootPicks, attempts*expectedProbability*0.2) + require.InDelta(t, attempts, estimatedTasks, attempts*0.1) +} + func TestPickWritePartition_NoBacklogUniform(t *testing.T) { f, err := tqid.NewTaskQueueFamily("fake-namespace-id", "fake-taskqueue") require.NoError(t, err) @@ -411,14 +497,40 @@ func TestPickWritePartition_NoBacklogUniform(t *testing.T) { counts := make([]int, 4) const n = 4000 for range n { - p := lb.PickWritePartition(taskQueue, pc) + p, estimatedTasksAllPartitions := lb.PickWritePartition(taskQueue, pc) counts[p.PartitionId()]++ + if p.IsRoot() { + require.Equal(t, 4, estimatedTasksAllPartitions) + } else { + require.Zero(t, estimatedTasksAllPartitions) + } } for i := range counts { require.InDelta(t, n/4, counts[i], float64(n)*0.1, "partition %d roughly uniform", i) } } +func TestPickWritePartition_Forced(t *testing.T) { + f, err := tqid.NewTaskQueueFamily("fake-namespace-id", "fake-taskqueue") + require.NoError(t, err) + taskQueue := f.TaskQueue(enumspb.TASK_QUEUE_TYPE_ACTIVITY) + testHooks := testhooks.NewTestHooks() + lb := &defaultLoadBalancer{ + namespaceIDToName: func(namespace.ID) (namespace.Name, error) { return "fake-namespace", nil }, + testHooks: testHooks, + } + pc := PartitionCounts{Write: 4} + + for partitionID, expectedEstimate := range map[int]int{0: 4, 1: 0} { + cleanup := testhooks.Set(testHooks, testhooks.MatchingLBForceWritePartition, partitionID, namespace.ID(taskQueue.NamespaceId())) + partition, estimatedTasksAllPartitions := lb.PickWritePartition(taskQueue, pc) + cleanup() + + require.Equal(t, partitionID, partition.PartitionId()) + require.Equal(t, expectedEstimate, estimatedTasksAllPartitions) + } +} + func maxPollerCount(tqlb *tqLoadBalancer) int { res := -1 for _, c := range tqlb.pollerCounts { diff --git a/client/matching/partition_counts.go b/client/matching/partition_counts.go index a1a6fc893a8..61428853496 100644 --- a/client/matching/partition_counts.go +++ b/client/matching/partition_counts.go @@ -5,6 +5,7 @@ import ( "context" "errors" "slices" + "strconv" taskqueuespb "go.temporal.io/server/api/taskqueue/v1" "go.temporal.io/server/common/log" @@ -20,6 +21,7 @@ import ( // The "-bin" suffix instructs grpc to base64-encode the value, so we can use binary. const partitionCountsHeaderName = "pcnt-bin" const partitionCountsTrailerName = "pcnt-bin" +const estimatedTasksAllPartitionsHeaderName = "etap" // PartitionCounts is a smaller version of taskqueuespb.ClientPartitionCounts that we can more // easily pass around and put in a map. @@ -94,6 +96,25 @@ func ParsePartitionCountsFromIncomingContext(ctx context.Context) (PartitionCoun return parsePartitionCounts(vals[0]) } +func appendEstimatedTasksAllPartitions(ctx context.Context, estimatedTasksAllPartitions int) context.Context { + if estimatedTasksAllPartitions <= 0 { + return ctx + } + return metadata.AppendToOutgoingContext(ctx, estimatedTasksAllPartitionsHeaderName, strconv.Itoa(estimatedTasksAllPartitions)) +} + +func ParseEstimatedTasksAllPartitions(ctx context.Context) int { + vals := metadata.ValueFromIncomingContext(ctx, estimatedTasksAllPartitionsHeaderName) + if len(vals) == 0 { + return 0 + } + estimatedTasksAllPartitions, err := strconv.Atoi(vals[0]) + if err != nil || estimatedTasksAllPartitions <= 0 { + return 0 + } + return estimatedTasksAllPartitions +} + func parsePartitionCountsFromTrailer(trailer metadata.MD) (PartitionCounts, error) { vals := trailer.Get(partitionCountsTrailerName) if len(vals) == 0 { diff --git a/service/matching/partition_scaler_interface.go b/service/matching/partition_scaler_interface.go index 16dfd4ba9f4..05a0242bc05 100644 --- a/service/matching/partition_scaler_interface.go +++ b/service/matching/partition_scaler_interface.go @@ -38,7 +38,7 @@ type PartitionScaler interface { } type PartitionScalerInput struct { - NumTasks int // new tasks added since last call + NumTasks int // estimated tasks added since last call CurrentTarget int BacklogCounts []byte PrivateState *anypb.Any diff --git a/service/matching/scale_manager.go b/service/matching/scale_manager.go index a307ad344e0..3332fe7cd34 100644 --- a/service/matching/scale_manager.go +++ b/service/matching/scale_manager.go @@ -32,8 +32,8 @@ import ( // All scaler/state work is funneled through a single background goroutine. That // goroutine owns scaleState/scaleDB/lastDecision and is the only caller of // partitionScaler, so the scaler implementation can rely on serial calls and -// scaleState needs no lock. AddedTasks talks to the worker only via the atomic -// batch counter and the wakeup channel, so it never blocks. +// scaleState needs no lock. AddedTasks talks to the worker only via atomics and +// the wakeup channel, so it never blocks. type scaleManager struct { partition tqid.Partition logger log.Logger @@ -55,6 +55,10 @@ type scaleManager struct { nextShadowLog time.Time prevShadowTarget int32 + // store separately from scaleState to avoid data race + currentWrite atomic.Int32 + + // batch counts estimated tasks across all partitions in between calls to the scaler batch atomic.Int64 wakeup chan struct{} } @@ -117,16 +121,21 @@ func (sm *scaleManager) Start(scaleState *persistencespb.PartitionScaleState, sc sm.background.Go(sm.backgroundWork) } -// AddedTasks is called on a batch of tasks added. +// AddedTasks records one root sample representing estimated queue-wide task additions. // This is called in the task add path, so it shouldn't block. -func (sm *scaleManager) AddedTasks(numTasks int) { +func (sm *scaleManager) AddedTasks(estimatedTasksAllPartitions int) { if sm == nil { return } - // scale target batch size by numTasks (since numTasks is scaled by partitions) - batchSize := int64(numTasks) * sm.batchSize - if sm.batch.Add(int64(numTasks)) < batchSize { + // Wake once ~batchSize tasks per write partition have accumulated. Before the first + // scaler decision we don't know the write count, so use the per-sample estimate, which + // scales with partitions the same way. + threshold := int64(estimatedTasksAllPartitions) * sm.batchSize + if currentWrite := sm.currentWrite.Load(); currentWrite > 0 { + threshold = int64(currentWrite) * sm.batchSize + } + if sm.batch.Add(int64(estimatedTasksAllPartitions)) < threshold { return // not enough for a batch yet } @@ -302,6 +311,7 @@ func (sm *scaleManager) setState(newState *persistencespb.PartitionScaleState, s sm.scaleState = newState newInfo := scaleStateToInfo(sm.scaleState, settings) + sm.currentWrite.Store(newInfo.GetWrite()) // only push ephemeral data if _info_ changed, not on any state change if !proto.Equal(prevInfo, newInfo) { diff --git a/service/matching/scale_manager_test.go b/service/matching/scale_manager_test.go index 2f7d91ce558..ae11a336e63 100644 --- a/service/matching/scale_manager_test.go +++ b/service/matching/scale_manager_test.go @@ -66,6 +66,29 @@ func TestScaleManagerSuite(t *testing.T) { suite.Run(t, new(ScaleManagerSuite)) } +func TestScaleManagerAddedTasksBatchThreshold(t *testing.T) { + for _, tc := range []struct { + name string + currentWrite int32 + callsBeforeWake int + }{ + {name: "current write", currentWrite: 2, callsBeforeWake: 3}, + {name: "estimate fallback", callsBeforeWake: 4}, + } { + t.Run(tc.name, func(t *testing.T) { + sm := &scaleManager{batchSize: 5, wakeup: make(chan struct{}, 1)} + sm.currentWrite.Store(tc.currentWrite) + for range tc.callsBeforeWake { + sm.AddedTasks(3) + } + require.Empty(t, sm.wakeup) + + sm.AddedTasks(3) + require.Len(t, sm.wakeup, 1) + }) + } +} + func (s *ScaleManagerSuite) SetupTest() { s.ProtoAssertions = protorequire.New(s.T()) s.userDataValue = nil @@ -290,7 +313,7 @@ func (s *ScaleManagerSuite) TestUnavailablePartitionPreservesBacklogAndDrainStat } // TestAddedTasksWakesScalerOnFullBatch verifies that the scaler is only called -// once cumulative batch reaches numTasks*BatchSize. +// once cumulative batch reaches BatchSize*numPartitions. func (s *ScaleManagerSuite) TestAddedTasksWakesScalerOnFullBatch() { s.settings.BatchSize = 5 @@ -302,13 +325,13 @@ func (s *ScaleManagerSuite) TestAddedTasksWakesScalerOnFullBatch() { s.startManager(4, nil) for range 4 { - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) } assertNoRecv(s, inputs, 30*time.Millisecond, "scaler called below threshold") - s.sm.AddedTasks(1) // cumulative 5 hits threshold + s.sm.AddedTasks(4) // cumulative 20 hits threshold in := waitRecv(s, inputs, "scaler never called") - s.Equal(5, in.NumTasks, "all 5 tasks should accumulate before firing") + s.Equal(20, in.NumTasks, "all 20 tasks should accumulate before firing") } // TestPeriodicTimerCallsScaler verifies that the scaler is called periodically @@ -354,7 +377,7 @@ func (s *ScaleManagerSuite) TestDecisionPersistsAndUpdatesEphemeralData() { s.startManager(4, nil) // 4 write partitions in dynamic config - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) state := waitRecv(s, dbWrites, "no db write") s.Equal(int32(2), state.Target) s.Equal(int32(2), state.MaxTarget) @@ -388,7 +411,7 @@ func (s *ScaleManagerSuite) TestEmitsGaugeMetrics() { s.startManager(4, nil) // 4 write partitions in dynamic config - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, dbWrites, "no db write") // setState records the gauges after the ephemeral push, so poll the snapshot @@ -475,8 +498,8 @@ func (s *ScaleManagerSuite) TestShadowModeEmitsExpectedGauges() { defer s.capture.StopCapture(capt) // call AddedTasks 2x (# of tasks added does not impact decision, decision is mocked) - for i := range 2 { - s.sm.AddedTasks(1) + for i, tasks := range []int{5, 4} { + s.sm.AddedTasks(tasks) waitRecv(s, inputs, "shadow call missing") // wait for log indicating nextDecision was set before we advance @@ -486,7 +509,7 @@ func (s *ScaleManagerSuite) TestShadowModeEmitsExpectedGauges() { s.timeSource.Advance(110 * time.Millisecond) } // call a third time, but don't expect the decision applied log, because shadow target didn't change - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "third shadow call missing") // Wait for three scale events (the first two record gauges, the last does not). @@ -522,7 +545,7 @@ func (s *ScaleManagerSuite) TestNonPositiveShadowLogIntervalDisabled() { s.startManager(4, nil) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) state := waitRecv(s, dbWrites, "no db write") s.Equal(int32(2), state.Target) info := waitRecv(s, scaleInfos, "no ephemeral data update") @@ -604,7 +627,7 @@ func (s *ScaleManagerSuite) TestShadowModeColdStartsScalerFromBaseline() { s.awaitDecisionApplied(1) // barrier: nextDecision set before we advance s.timeSource.Advance(110 * time.Millisecond) // past the 100ms cooldown - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) in2 := waitRecv(s, inputs, "second shadow call missing") s.Equal(0, in2.CurrentTarget) s.Nil(in2.PrivateState) @@ -625,7 +648,7 @@ func (s *ScaleManagerSuite) TestShadowModeDoesNotLogNoChange() { s.startManagerWithLogger(logger, 4, nil) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "shadow scaler call missing") assertNoNewLogs(s, s.newTarget, 30*time.Millisecond, "NoChange decision should not log") } @@ -654,24 +677,24 @@ func (s *ScaleManagerSuite) TestShadowLoggingCadence() { s.startManagerWithLogger(logger, 4, nil) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "first shadow call missing") waitLogMatches(s, s.newTarget, 1, "first shadow log missing") - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) assertNoRecv(s, inputs, 30*time.Millisecond, "shadow scaler called inside cooldown") s.timeSource.Advance(110 * time.Millisecond) // past the 100ms cooldown - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "second shadow call missing") assertNoNewLogs(s, s.newTarget, 30*time.Millisecond, "shadow log repeated before cadence") s.timeSource.Advance(time.Minute) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "third shadow call missing") assertNoNewLogs(s, s.newTarget, 30*time.Millisecond, "unchanged shadow decision logged after cadence") - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "fourth shadow call missing") waitLogMatches(s, s.newTarget, 2, "second shadow log missing") } @@ -690,7 +713,7 @@ func (s *ScaleManagerSuite) TestShadowModeDoesNotLogDisabledScaler() { s.userData.EXPECT().SetPartitionScale(gomock.Any()).Times(2) s.startManagerWithLogger(logger, 4, &persistencespb.PartitionScaleState{Target: 2}) - s.sm.AddedTasks(1) + s.sm.AddedTasks(2) waitRecv(s, inputs, "shadow scaler call missing") assertNoNewLogs(s, s.newTarget, 30*time.Millisecond, "disabled scaler should not log") } @@ -778,7 +801,7 @@ func (s *ScaleManagerSuite) TestShadowModeReleasesManagedTargetToBaseline() { } s.startManager(4, initial) - s.sm.AddedTasks(1) + s.sm.AddedTasks(10) w := waitRecv(s, dbWrites, "release write missing") s.Equal(int32(0), w.Target) @@ -819,17 +842,17 @@ func (s *ScaleManagerSuite) TestShadowModeLogsOscillationFromBaseline() { s.startManagerWithLogger(logger, 4, &persistencespb.PartitionScaleState{Target: 2}) - s.sm.AddedTasks(1) + s.sm.AddedTasks(2) waitRecv(s, inputs, "first shadow call missing") waitLogMatches(s, s.newTarget, 1, "shadow target 3 not logged") s.timeSource.Advance(time.Minute) // past cooldown and cadence - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "second shadow call missing") waitLogMatches(s, s.newTarget, 2, "shadow target 2 not logged") s.timeSource.Advance(time.Minute) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "third shadow call missing") waitLogMatches(s, s.newTarget, 3, "shadow target 3 after oscillation not logged") } @@ -864,9 +887,9 @@ func (s *ScaleManagerSuite) TestNoChangeDecisionSkipsWrite() { time.Second, time.Millisecond) baseline := scalePushes.Load() - s.sm.AddedTasks(1) + s.sm.AddedTasks(3) waitRecv(s, inputs, "first decision never delivered") - s.sm.AddedTasks(1) + s.sm.AddedTasks(3) waitRecv(s, inputs, "second decision never delivered") s.Require().Never(func() bool { @@ -901,9 +924,9 @@ func (s *ScaleManagerSuite) TestCooldown() { s.startManager(4, nil) // First decision lands immediately (lastDecision is zero, no cooldown). - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) in1 := waitRecv(s, inputs, "first decision never delivered") - s.Equal(1, in1.NumTasks) + s.Equal(4, in1.NumTasks) waitRecv(s, dbWrites, "first db write missing") // Within cooldown: another wakeup carrying 7 tasks. Scaler must NOT be @@ -949,14 +972,14 @@ func (s *ScaleManagerSuite) TestPrivateStatePropagation() { s.startManager(4, nil) - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) in1 := waitRecv(s, inputs, "first call missing") s.Nil(in1.PrivateState, "first call should have no private state") waitRecv(s, dbWrites, "first db write missing") s.awaitDecisionApplied(1) // barrier: nextDecision set before we advance s.timeSource.Advance(110 * time.Millisecond) // past 100ms cooldown - s.sm.AddedTasks(1) + s.sm.AddedTasks(2) in2 := waitRecv(s, inputs, "second call missing") s.ProtoEqual(priv1, in2.PrivateState) w2 := waitRecv(s, dbWrites, "second db write missing") @@ -1366,14 +1389,14 @@ func (s *ScaleManagerSuite) TestDBWriteFailureKeepsState() { // First decision: scaler called, DB write fails. lastDecision is not set, // so cooldown is not engaged. - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "first decision missing") // No ephemeral push yet; the failed write must not have advanced state. assertNoRecv(s, scaleInfos, 30*time.Millisecond, "ephemeral data pushed after failed write") // Recovery: trigger another decision; this one persists. - s.sm.AddedTasks(1) + s.sm.AddedTasks(4) waitRecv(s, inputs, "second decision missing") state := waitRecv(s, dbWrites, "second db write missing") s.Equal(int32(3), state.Target, "second decision's target landed (first was dropped)") @@ -1503,7 +1526,7 @@ func (s *ScaleManagerSuite) TestBacklogCapChangePersists() { // Current target is already 3, so only the backlog cap differs from the decision. s.startManager(4, &persistencespb.PartitionScaleState{Target: 3, MaxTarget: 3}) - s.sm.AddedTasks(1) + s.sm.AddedTasks(3) state := waitRecv(s, dbWrites, "cap-only change never persisted") s.Equal(int32(3), state.Target, "target unchanged; the cap change alone must trigger the write") s.Equal(int32(number.EncodeCompact8(1000)), state.BacklogCap) @@ -1538,7 +1561,7 @@ func (s *ScaleManagerSuite) TestSameTargetAndCapSkipsWrite() { } s.startManager(4, initial) - s.sm.AddedTasks(1) + s.sm.AddedTasks(3) in := waitRecv(s, inputs, "scaler never called") s.Equal([]byte{c500, c500, c500}, in.BacklogCounts, "stored backlog counts must be fed to the scaler") diff --git a/service/matching/task_queue_partition_manager.go b/service/matching/task_queue_partition_manager.go index c4859cfe980..65eddd4393f 100644 --- a/service/matching/task_queue_partition_manager.go +++ b/service/matching/task_queue_partition_manager.go @@ -436,22 +436,20 @@ func validatePartitionScaleDrift( // signalPartitionScaler sends a signal to the partition scaler that a new task has arrived // (directly from history, not forwarded). -func (pm *taskQueuePartitionManagerImpl) signalPartitionScaler() { +func (pm *taskQueuePartitionManagerImpl) signalPartitionScaler(ctx context.Context) { if pm.scaleManager == nil { return // only run on root partition } - scaleInfo := pm.userDataManager.PartitionScale() - effectiveWrite := int(scaleInfo.GetWrite()) - // if no target is set yet, get effective count from dynamic config (matches client behavior) - if effectiveWrite == 0 { - effectiveWrite = max(1, pm.config.NumWritePartitions()) + estimatedTasksAllPartitions := matching.ParseEstimatedTasksAllPartitions(ctx) + if estimatedTasksAllPartitions == 0 { + scaleInfo := pm.userDataManager.PartitionScale() + effectiveWrite := int(scaleInfo.GetWrite()) + if effectiveWrite == 0 { + effectiveWrite = max(1, pm.config.NumWritePartitions()) + } + estimatedTasksAllPartitions = effectiveWrite } - // we assume that tasks are balanced uniformly across partitions, so if the root has - // seen 1 task then all have seen ~1 task, so the whole queue has seen 'effective' - // tasks in total. - // TODO(dp): this will change when we add non-uniform load balancing. we should eventually - // aggregate real stats instead of assuming - pm.scaleManager.AddedTasks(effectiveWrite) + pm.scaleManager.AddedTasks(estimatedTasksAllPartitions) } func (pm *taskQueuePartitionManagerImpl) sendPartitionCountTrailer(ctx context.Context) { @@ -563,7 +561,7 @@ func (pm *taskQueuePartitionManagerImpl) AddTask( return "", false, err } if params.forwardInfo == nil { - pm.signalPartitionScaler() + pm.signalPartitionScaler(ctx) } var spoolQueue, syncMatchQueue physicalTaskQueueManager @@ -1023,7 +1021,7 @@ func (pm *taskQueuePartitionManagerImpl) DispatchQueryTask( return nil, err } if request.ForwardInfo == nil { - pm.signalPartitionScaler() + pm.signalPartitionScaler(ctx) } task := newInternalQueryTask(taskID, request) @@ -1093,7 +1091,7 @@ func (pm *taskQueuePartitionManagerImpl) DispatchNexusTask( return nil, err } if request.ForwardInfo == nil { - pm.signalPartitionScaler() + pm.signalPartitionScaler(ctx) } deadline, _ := ctx.Deadline() // If not set by user, our client will set a default. From ba0ae78d19f890a0d1037c3c897f27aa6ee62dd4 Mon Sep 17 00:00:00 2001 From: Brian VanLoo Date: Fri, 4 Sep 2026 09:51:09 -0700 Subject: [PATCH 138/139] Deprecate unused single-axis outbound predicate types (#11781) ## What changed? Marks three `Predicate` oneof fields, and the messages they hold, `[deprecated = true]` in `predicates.proto`, with a comment on each pointing at what replaced them: `TaskTypePredicateAttributes`, `DestinationPredicateAttributes`, and `OutboundTaskGroupPredicateAttributes`. Regenerated `api/persistence/v1/predicates.pb.go` accordingly. On top of the annotation, also removes the dead code those three types left behind: - The `*TypePredicate`/`*DestinationPredicate`/`*OutboundTaskGroupPredicate` case blocks in `AndPredicates`/`OrPredicates` (`service/history/tasks/predicates.go`). Combining two of these now falls through to the existing generic `predicates.And`/`predicates.Or` wrapper instead of a type-specific simplification. - `// Deprecated: ...` GoDoc comments on `NewTypePredicate`, `NewDestinationPredicate`, and `NewOutboundTaskGroupPredicate`, so `staticcheck` flags any new caller. - Test fixtures that used one of these three types incidentally are swapped to a still-supported type. The tests that cover these types are untouched, so that old stored shards work correctly if they ever do happen to be reloaded. ## Why? These three types are never constructed anywhere in the server today. The only place any of them is referenced outside their own message definitions and unit tests is the read-path switch in `service/history/queues/convert.go`, which exists to deserialize a variant if it happens to be stored in a very old shard. - `TaskTypePredicateAttributes` was added speculatively to be able to create task-keyed predicates. - `DestinationPredicateAttributes` and `OutboundTaskGroupPredicateAttributes` both were introduced during the outbounb queue's development. Two months later they were unified into one composite predicate, `OutboundTaskPredicateAttributes.Group{task_group, namespace_id, destination}` Removing their `AndPredicates`/`OrPredicates` case blocks is safe because both functions already fall back to a generic, correct wrapper (`predicates.And`/`predicates.Or`) for any pair they don't specifically simplify. Losing a type's branch only costs simplification precision and not correctness. ## How did you test it? - [x] built - [ ] run locally and tested manually - [x] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) `make proto` regenerated only `predicates.pb.go`. `go build ./...` and `go test ./service/history/tasks/... ./service/history/queues/... ./api/persistence/...` all pass. --- api/persistence/v1/predicates.pb.go | 31 +++++-- .../api/persistence/v1/predicates.proto | 18 +++- service/history/queues/convert.go | 9 ++ service/history/queues/convert_test.go | 38 ++++---- service/history/queues/queue_base_test.go | 12 +-- service/history/queues/scope_test.go | 4 +- service/history/tasks/predicates.go | 87 ++----------------- service/history/tasks/predicates_test.go | 75 ++++------------ 8 files changed, 102 insertions(+), 172 deletions(-) diff --git a/api/persistence/v1/predicates.pb.go b/api/persistence/v1/predicates.pb.go index eef8df764a6..36e34021b5d 100644 --- a/api/persistence/v1/predicates.pb.go +++ b/api/persistence/v1/predicates.pb.go @@ -141,6 +141,7 @@ func (x *Predicate) GetNamespaceIdPredicateAttributes() *NamespaceIdPredicateAtt return nil } +// Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. func (x *Predicate) GetTaskTypePredicateAttributes() *TaskTypePredicateAttributes { if x != nil { if x, ok := x.Attributes.(*Predicate_TaskTypePredicateAttributes); ok { @@ -150,6 +151,7 @@ func (x *Predicate) GetTaskTypePredicateAttributes() *TaskTypePredicateAttribute return nil } +// Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. func (x *Predicate) GetDestinationPredicateAttributes() *DestinationPredicateAttributes { if x != nil { if x, ok := x.Attributes.(*Predicate_DestinationPredicateAttributes); ok { @@ -159,6 +161,7 @@ func (x *Predicate) GetDestinationPredicateAttributes() *DestinationPredicateAtt return nil } +// Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. func (x *Predicate) GetOutboundTaskGroupPredicateAttributes() *OutboundTaskGroupPredicateAttributes { if x != nil { if x, ok := x.Attributes.(*Predicate_OutboundTaskGroupPredicateAttributes); ok { @@ -206,14 +209,28 @@ type Predicate_NamespaceIdPredicateAttributes struct { } type Predicate_TaskTypePredicateAttributes struct { + // Deprecated: never built by any queue's grouper. Kept only so a shard that still holds one + // from before the outbound queue existed, and hasn't recompacted since, still deserializes + // correctly. + // + // Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. TaskTypePredicateAttributes *TaskTypePredicateAttributes `protobuf:"bytes,8,opt,name=task_type_predicate_attributes,json=taskTypePredicateAttributes,proto3,oneof"` } type Predicate_DestinationPredicateAttributes struct { + // Deprecated: superseded by outbound_task_predicate_attributes (#6445), which combines + // task group, namespace and destination into one predicate. Kept only so a shard checkpointed + // before that change, and never checkpointed since, still deserializes correctly. + // + // Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. DestinationPredicateAttributes *DestinationPredicateAttributes `protobuf:"bytes,9,opt,name=destination_predicate_attributes,json=destinationPredicateAttributes,proto3,oneof"` } type Predicate_OutboundTaskGroupPredicateAttributes struct { + // Deprecated: superseded by outbound_task_predicate_attributes (#6445); see above. Not to be + // confused with that message's own task_group field, which is very much in use. + // + // Deprecated: Marked as deprecated in temporal/server/api/persistence/v1/predicates.proto. OutboundTaskGroupPredicateAttributes *OutboundTaskGroupPredicateAttributes `protobuf:"bytes,10,opt,name=outbound_task_group_predicate_attributes,json=outboundTaskGroupPredicateAttributes,proto3,oneof"` } @@ -489,6 +506,7 @@ func (x *NamespaceIdPredicateAttributes) GetNamespaceIds() []string { return nil } +// Deprecated: see the deprecated task_type_predicate_attributes field on Predicate. type TaskTypePredicateAttributes struct { state protoimpl.MessageState `protogen:"open.v1"` TaskTypes []v1.TaskType `protobuf:"varint,1,rep,packed,name=task_types,json=taskTypes,proto3,enum=temporal.server.api.enums.v1.TaskType" json:"task_types,omitempty"` @@ -533,6 +551,7 @@ func (x *TaskTypePredicateAttributes) GetTaskTypes() []v1.TaskType { return nil } +// Deprecated: see the deprecated destination_predicate_attributes field on Predicate. type DestinationPredicateAttributes struct { state protoimpl.MessageState `protogen:"open.v1"` Destinations []string `protobuf:"bytes,1,rep,name=destinations,proto3" json:"destinations,omitempty"` @@ -577,6 +596,8 @@ func (x *DestinationPredicateAttributes) GetDestinations() []string { return nil } +// Deprecated: see the deprecated outbound_task_group_predicate_attributes field on Predicate. +// Not to be confused with OutboundTaskPredicateAttributes.Group.task_group, which is in use. type OutboundTaskGroupPredicateAttributes struct { state protoimpl.MessageState `protogen:"open.v1"` Groups []string `protobuf:"bytes,1,rep,name=groups,proto3" json:"groups,omitempty"` @@ -729,7 +750,7 @@ var File_temporal_server_api_persistence_v1_predicates_proto protoreflect.FileDe const file_temporal_server_api_persistence_v1_predicates_proto_rawDesc = "" + "\n" + - "3temporal/server/api/persistence/v1/predicates.proto\x12\"temporal.server.api.persistence.v1\x1a,temporal/server/api/enums/v1/predicate.proto\x1a'temporal/server/api/enums/v1/task.proto\"\xc1\v\n" + + "3temporal/server/api/persistence/v1/predicates.proto\x12\"temporal.server.api.persistence.v1\x1a,temporal/server/api/enums/v1/predicate.proto\x1a'temporal/server/api/enums/v1/task.proto\"\xcd\v\n" + "\tPredicate\x12R\n" + "\x0epredicate_type\x18\x01 \x01(\x0e2+.temporal.server.api.enums.v1.PredicateTypeR\rpredicateType\x12\x88\x01\n" + "\x1euniversal_predicate_attributes\x18\x02 \x01(\v2@.temporal.server.api.persistence.v1.UniversalPredicateAttributesH\x00R\x1cuniversalPredicateAttributes\x12|\n" + @@ -737,11 +758,11 @@ const file_temporal_server_api_persistence_v1_predicates_proto_rawDesc = "" + "\x18and_predicate_attributes\x18\x04 \x01(\v2:.temporal.server.api.persistence.v1.AndPredicateAttributesH\x00R\x16andPredicateAttributes\x12s\n" + "\x17or_predicate_attributes\x18\x05 \x01(\v29.temporal.server.api.persistence.v1.OrPredicateAttributesH\x00R\x15orPredicateAttributes\x12v\n" + "\x18not_predicate_attributes\x18\x06 \x01(\v2:.temporal.server.api.persistence.v1.NotPredicateAttributesH\x00R\x16notPredicateAttributes\x12\x8f\x01\n" + - "!namespace_id_predicate_attributes\x18\a \x01(\v2B.temporal.server.api.persistence.v1.NamespaceIdPredicateAttributesH\x00R\x1enamespaceIdPredicateAttributes\x12\x86\x01\n" + - "\x1etask_type_predicate_attributes\x18\b \x01(\v2?.temporal.server.api.persistence.v1.TaskTypePredicateAttributesH\x00R\x1btaskTypePredicateAttributes\x12\x8e\x01\n" + - " destination_predicate_attributes\x18\t \x01(\v2B.temporal.server.api.persistence.v1.DestinationPredicateAttributesH\x00R\x1edestinationPredicateAttributes\x12\xa2\x01\n" + + "!namespace_id_predicate_attributes\x18\a \x01(\v2B.temporal.server.api.persistence.v1.NamespaceIdPredicateAttributesH\x00R\x1enamespaceIdPredicateAttributes\x12\x8a\x01\n" + + "\x1etask_type_predicate_attributes\x18\b \x01(\v2?.temporal.server.api.persistence.v1.TaskTypePredicateAttributesB\x02\x18\x01H\x00R\x1btaskTypePredicateAttributes\x12\x92\x01\n" + + " destination_predicate_attributes\x18\t \x01(\v2B.temporal.server.api.persistence.v1.DestinationPredicateAttributesB\x02\x18\x01H\x00R\x1edestinationPredicateAttributes\x12\xa6\x01\n" + "(outbound_task_group_predicate_attributes\x18\n" + - " \x01(\v2H.temporal.server.api.persistence.v1.OutboundTaskGroupPredicateAttributesH\x00R$outboundTaskGroupPredicateAttributes\x12\x92\x01\n" + + " \x01(\v2H.temporal.server.api.persistence.v1.OutboundTaskGroupPredicateAttributesB\x02\x18\x01H\x00R$outboundTaskGroupPredicateAttributes\x12\x92\x01\n" + "\"outbound_task_predicate_attributes\x18\v \x01(\v2C.temporal.server.api.persistence.v1.OutboundTaskPredicateAttributesH\x00R\x1foutboundTaskPredicateAttributesB\f\n" + "\n" + "attributes\"\x1e\n" + diff --git a/proto/internal/temporal/server/api/persistence/v1/predicates.proto b/proto/internal/temporal/server/api/persistence/v1/predicates.proto index b6d2c71f39d..8428afb549a 100644 --- a/proto/internal/temporal/server/api/persistence/v1/predicates.proto +++ b/proto/internal/temporal/server/api/persistence/v1/predicates.proto @@ -16,9 +16,17 @@ message Predicate { OrPredicateAttributes or_predicate_attributes = 5; NotPredicateAttributes not_predicate_attributes = 6; NamespaceIdPredicateAttributes namespace_id_predicate_attributes = 7; - TaskTypePredicateAttributes task_type_predicate_attributes = 8; - DestinationPredicateAttributes destination_predicate_attributes = 9; - OutboundTaskGroupPredicateAttributes outbound_task_group_predicate_attributes = 10; + // Deprecated: never built by any queue's grouper. Kept only so a shard that still holds one + // from before the outbound queue existed, and hasn't recompacted since, still deserializes + // correctly. + TaskTypePredicateAttributes task_type_predicate_attributes = 8 [deprecated = true]; + // Deprecated: superseded by outbound_task_predicate_attributes (#6445), which combines + // task group, namespace and destination into one predicate. Kept only so a shard checkpointed + // before that change, and never checkpointed since, still deserializes correctly. + DestinationPredicateAttributes destination_predicate_attributes = 9 [deprecated = true]; + // Deprecated: superseded by outbound_task_predicate_attributes (#6445); see above. Not to be + // confused with that message's own task_group field, which is very much in use. + OutboundTaskGroupPredicateAttributes outbound_task_group_predicate_attributes = 10 [deprecated = true]; OutboundTaskPredicateAttributes outbound_task_predicate_attributes = 11; } } @@ -43,14 +51,18 @@ message NamespaceIdPredicateAttributes { repeated string namespace_ids = 1; } +// Deprecated: see the deprecated task_type_predicate_attributes field on Predicate. message TaskTypePredicateAttributes { repeated temporal.server.api.enums.v1.TaskType task_types = 1; } +// Deprecated: see the deprecated destination_predicate_attributes field on Predicate. message DestinationPredicateAttributes { repeated string destinations = 1; } +// Deprecated: see the deprecated outbound_task_group_predicate_attributes field on Predicate. +// Not to be confused with OutboundTaskPredicateAttributes.Group.task_group, which is in use. message OutboundTaskGroupPredicateAttributes { repeated string groups = 1; } diff --git a/service/history/queues/convert.go b/service/history/queues/convert.go index 4fa216632e8..58e6e11d34b 100644 --- a/service/history/queues/convert.go +++ b/service/history/queues/convert.go @@ -300,6 +300,9 @@ func ToPersistenceTaskTypePredicate( func FromPersistenceTaskTypePredicate( attributes *persistencespb.TaskTypePredicateAttributes, ) tasks.Predicate { + //nolint:staticcheck // NewTypePredicate is deprecated for new construction; this is the one + // legitimate caller, deserializing a shard from before any grouper stopped building this + // predicate type (no grouper ever has). return tasks.NewTypePredicate(attributes.TaskTypes) } @@ -319,6 +322,9 @@ func ToPersistenceDestinationPredicate( func FromPersistenceDestinationPredicate( attributes *persistencespb.DestinationPredicateAttributes, ) tasks.Predicate { + //nolint:staticcheck // NewDestinationPredicate is deprecated for new construction; this is + // the one legitimate caller, deserializing a shard checkpointed before #6445 unified this + // predicate. return tasks.NewDestinationPredicate(attributes.Destinations) } @@ -338,6 +344,9 @@ func ToPersistenceOutboundTaskGroupPredicate( func FromPersistenceOutboundTaskGroupPredicate( attributes *persistencespb.OutboundTaskGroupPredicateAttributes, ) tasks.Predicate { + //nolint:staticcheck // NewOutboundTaskGroupPredicate is deprecated for new construction; this + // is the one legitimate caller, deserializing a shard checkpointed before #6445 unified this + // predicate. return tasks.NewOutboundTaskGroupPredicate(attributes.Groups) } diff --git a/service/history/queues/convert_test.go b/service/history/queues/convert_test.go index 87120580c83..19498f9a85b 100644 --- a/service/history/queues/convert_test.go +++ b/service/history/queues/convert_test.go @@ -52,11 +52,11 @@ func (s *convertSuite) TestConvertPredicate_And() { tasks.NewNamespacePredicate([]string{uuid.NewString()}), ), predicates.Or[tasks.Task]( - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, }), - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g2", NamespaceID: "n2", Destination: "d2"}, }), ), ), @@ -71,8 +71,8 @@ func (s *convertSuite) TestConvertPredicate_And() { predicates.Not(predicates.Empty[tasks.Task]()), predicates.And[tasks.Task]( tasks.NewNamespacePredicate([]string{uuid.NewString()}), - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, }), ), ), @@ -95,11 +95,11 @@ func (s *convertSuite) TestConvertPredicate_Or() { tasks.NewNamespacePredicate([]string{uuid.NewString()}), ), predicates.And[tasks.Task]( - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, }), - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g2", NamespaceID: "n2", Destination: "d2"}, }), ), ), @@ -114,8 +114,8 @@ func (s *convertSuite) TestConvertPredicate_Or() { predicates.Not(predicates.Empty[tasks.Task]()), predicates.And[tasks.Task]( tasks.NewNamespacePredicate([]string{uuid.NewString()}), - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, }), ), ), @@ -132,16 +132,16 @@ func (s *convertSuite) TestConvertPredicate_Not() { predicates.Not(predicates.Empty[tasks.Task]()), predicates.Not(predicates.And[tasks.Task]( tasks.NewNamespacePredicate([]string{uuid.NewString()}), - tasks.NewTypePredicate([]enumsspb.TaskType{}), + tasks.NewOutboundTaskPredicate(nil), )), predicates.Not(predicates.Or[tasks.Task]( tasks.NewNamespacePredicate([]string{uuid.NewString()}), - tasks.NewTypePredicate([]enumsspb.TaskType{}), + tasks.NewOutboundTaskPredicate(nil), )), predicates.Not(predicates.Not(predicates.Empty[tasks.Task]())), predicates.Not[tasks.Task](tasks.NewNamespacePredicate([]string{uuid.NewString()})), - predicates.Not[tasks.Task](tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER, + predicates.Not[tasks.Task](tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, })), } @@ -219,9 +219,9 @@ func (s *convertSuite) TestConvertQueueState() { ), NewScope( NewRandomRange(), - tasks.NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER, + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{ + {TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}, + {TaskGroup: "g2", NamespaceID: "n2", Destination: "d2"}, }), ), }, diff --git a/service/history/queues/queue_base_test.go b/service/history/queues/queue_base_test.go index a7c0891f806..f22e789241c 100644 --- a/service/history/queues/queue_base_test.go +++ b/service/history/queues/queue_base_test.go @@ -139,10 +139,12 @@ func (s *queueBaseSuite) TestNewProcessBase_WithPreviousState_RestoreSucceed() { ExclusiveMax: &persistencespb.TaskKey{FireTime: timestamppb.New(tasks.DefaultFireTime), TaskId: 3000}, }, Predicate: &persistencespb.Predicate{ - PredicateType: enumsspb.PREDICATE_TYPE_TASK_TYPE, - Attributes: &persistencespb.Predicate_TaskTypePredicateAttributes{ - TaskTypePredicateAttributes: &persistencespb.TaskTypePredicateAttributes{ - TaskTypes: []enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}, + PredicateType: enumsspb.PREDICATE_TYPE_OUTBOUND_TASK, + Attributes: &persistencespb.Predicate_OutboundTaskPredicateAttributes{ + OutboundTaskPredicateAttributes: &persistencespb.OutboundTaskPredicateAttributes{ + Groups: []*persistencespb.OutboundTaskPredicateAttributes_Group{ + {TaskGroup: "g1", NamespaceId: "n1", Destination: "d1"}, + }, }, }, }, @@ -557,7 +559,7 @@ func (s *queueBaseSuite) TestCheckPoint_SlicePredicateAction() { exclusiveReaderHighWatermark := tasks.MaximumKey scopes := NewRandomScopes(3) scopes[0].Predicate = tasks.NewNamespacePredicate([]string{uuid.NewString()}) - scopes[2].Predicate = tasks.NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}) + scopes[2].Predicate = tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{{TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}}) initialQueueState := &queueState{ readerScopes: map[int64][]Scope{ DefaultReaderId: scopes, diff --git a/service/history/queues/scope_test.go b/service/history/queues/scope_test.go index e7990f9d703..bac1f6ca169 100644 --- a/service/history/queues/scope_test.go +++ b/service/history/queues/scope_test.go @@ -198,7 +198,7 @@ func (s *scopeSuite) TestCanMergeByRange() { predicate, tasks.NewNamespacePredicate(namespaceIDs), tasks.NewNamespacePredicate([]string{uuid.NewString(), uuid.NewString(), uuid.NewString(), uuid.NewString()}), - tasks.NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}), + tasks.NewOutboundTaskPredicate([]tasks.TaskGroupNamespaceIDAndDestination{{TaskGroup: "g1", NamespaceID: "n1", Destination: "d1"}}), } s.True(predicate.Equals(testPredicates[0])) s.True(predicate.Equals(testPredicates[1])) @@ -284,7 +284,7 @@ func (s *scopeSuite) TestCanMergeByPredicate() { s.True(scope.CanMergeByPredicate(scope)) s.True(scope.CanMergeByPredicate(NewScope(r, predicate))) - s.True(scope.CanMergeByPredicate(NewScope(r, tasks.NewTypePredicate([]enumsspb.TaskType{})))) + s.True(scope.CanMergeByPredicate(NewScope(r, tasks.NewOutboundTaskPredicate(nil)))) s.False(scope.CanMergeByPredicate(NewScope(NewRandomRange(), predicate))) s.False(scope.CanMergeByPredicate(NewScope(NewRandomRange(), predicates.Universal[tasks.Task]()))) diff --git a/service/history/tasks/predicates.go b/service/history/tasks/predicates.go index 7d5e1c3c900..feda86f197d 100644 --- a/service/history/tasks/predicates.go +++ b/service/history/tasks/predicates.go @@ -69,6 +69,8 @@ func (n *NamespacePredicate) Size() int { return size } +// Deprecated: superseded by NewOutboundTaskPredicate (#6445). Only called from +// service/history/queues/convert.go to deserialize a shard checkpointed before that change. func NewDestinationPredicate( destinations []string, ) *DestinationPredicate { @@ -108,6 +110,8 @@ func (n *DestinationPredicate) Size() int { return size } +// Deprecated: superseded by NewOutboundTaskPredicate (#6445). Only called from +// service/history/queues/convert.go to deserialize a shard checkpointed before that change. func NewOutboundTaskGroupPredicate( groups []string, ) *OutboundTaskGroupPredicate { @@ -147,6 +151,8 @@ func (n *OutboundTaskGroupPredicate) Size() int { return size } +// Deprecated: never constructed by any queue's grouper. Only called from +// service/history/queues/convert.go to deserialize a legacy shard. func NewTypePredicate( types []enumsspb.TaskType, ) *TypePredicate { @@ -265,69 +271,6 @@ func AndPredicates(a Predicate, b Predicate) Predicate { } } } - case *TypePredicate: - switch b := b.(type) { - case *TypePredicate: - intersection := intersect(a.Types, b.Types) - if len(intersection) == 0 { - return predicates.Empty[Task]() - } - return &TypePredicate{ - Types: intersection, - } - case *predicates.NotImpl[Task]: - if exclude, ok := b.Predicate.(*TypePredicate); ok { - difference := difference(a.Types, exclude.Types) - if len(difference) == 0 { - return predicates.Empty[Task]() - } - return &TypePredicate{ - Types: difference, - } - } - } - case *DestinationPredicate: - switch b := b.(type) { - case *DestinationPredicate: - intersection := intersect(a.Destinations, b.Destinations) - if len(intersection) == 0 { - return predicates.Empty[Task]() - } - return &DestinationPredicate{ - Destinations: intersection, - } - case *predicates.NotImpl[Task]: - if exclude, ok := b.Predicate.(*DestinationPredicate); ok { - difference := difference(a.Destinations, exclude.Destinations) - if len(difference) == 0 { - return predicates.Empty[Task]() - } - return &DestinationPredicate{ - Destinations: difference, - } - } - } - case *OutboundTaskGroupPredicate: - switch b := b.(type) { - case *OutboundTaskGroupPredicate: - intersection := intersect(a.Groups, b.Groups) - if len(intersection) == 0 { - return predicates.Empty[Task]() - } - return &OutboundTaskGroupPredicate{ - Groups: intersection, - } - case *predicates.NotImpl[Task]: - if exclude, ok := b.Predicate.(*OutboundTaskGroupPredicate); ok { - difference := difference(a.Groups, exclude.Groups) - if len(difference) == 0 { - return predicates.Empty[Task]() - } - return &OutboundTaskGroupPredicate{ - Groups: difference, - } - } - } case *OutboundTaskPredicate: switch b := b.(type) { case *OutboundTaskPredicate: @@ -383,24 +326,6 @@ func OrPredicates(a Predicate, b Predicate) Predicate { NamespaceIDs: union(a.NamespaceIDs, b.NamespaceIDs), } } - case *TypePredicate: - if b, ok := b.(*TypePredicate); ok { - return &TypePredicate{ - Types: union(a.Types, b.Types), - } - } - case *DestinationPredicate: - if b, ok := b.(*DestinationPredicate); ok { - return &DestinationPredicate{ - Destinations: union(a.Destinations, b.Destinations), - } - } - case *OutboundTaskGroupPredicate: - if b, ok := b.(*OutboundTaskGroupPredicate); ok { - return &OutboundTaskGroupPredicate{ - Groups: union(a.Groups, b.Groups), - } - } case *OutboundTaskPredicate: if b, ok := b.(*OutboundTaskPredicate); ok { return &OutboundTaskPredicate{ diff --git a/service/history/tasks/predicates_test.go b/service/history/tasks/predicates_test.go index b9f8a330e49..43c69c14556 100644 --- a/service/history/tasks/predicates_test.go +++ b/service/history/tasks/predicates_test.go @@ -66,7 +66,7 @@ func (s *predicatesSuite) TestNamespacePredicate_Equals() { s.True(p.Equals(NewNamespacePredicate(namespaceIDs))) s.False(p.Equals(NewNamespacePredicate([]string{uuid.NewString(), uuid.NewString()}))) - s.False(p.Equals(NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}))) + s.False(p.Equals(NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{{"g1", "n1", "d1"}}))) s.False(p.Equals(predicates.Universal[Task]())) } @@ -175,7 +175,7 @@ func (s *predicatesSuite) TestDestinationPredicate_Equals() { s.True(p.Equals(NewDestinationPredicate(destinations))) s.False(p.Equals(NewDestinationPredicate([]string{uuid.NewString(), uuid.NewString()}))) - s.False(p.Equals(NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}))) + s.False(p.Equals(NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{{"g1", "n1", "d1"}}))) s.False(p.Equals(predicates.Universal[Task]())) } @@ -217,7 +217,7 @@ func (s *predicatesSuite) TestOutboundTaskGroupPredicate_Equals() { s.True(p.Equals(NewOutboundTaskGroupPredicate(groups))) s.False(p.Equals(NewOutboundTaskGroupPredicate([]string{"3", "4"}))) - s.False(p.Equals(NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}))) + s.False(p.Equals(NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{{"g1", "n1", "d1"}}))) s.False(p.Equals(predicates.Universal[Task]())) } @@ -297,7 +297,7 @@ func (s *predicatesSuite) TestOutboundTaskPredicate_Equals() { {"g1", "n1", "d3"}, {"g2", "n2", "d4"}, }))) - s.False(p.Equals(NewTypePredicate([]enumsspb.TaskType{enumsspb.TASK_TYPE_ACTIVITY_RETRY_TIMER}))) + s.False(p.Equals(NewNamespacePredicate([]string{uuid.NewString()}))) s.False(p.Equals(predicates.Universal[Task]())) } @@ -322,33 +322,11 @@ func (s *predicatesSuite) TestAndPredicates() { predicateB: NewNamespacePredicate([]string{"namespace2", "namespace3"}), expectedResult: NewNamespacePredicate([]string{"namespace2"}), }, - { - predicateA: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - }), - predicateB: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, - }), - expectedResult: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - }), - }, - { - predicateA: NewDestinationPredicate([]string{"dest1", "dest2"}), - predicateB: NewDestinationPredicate([]string{"dest2", "dest3"}), - expectedResult: NewDestinationPredicate([]string{"dest2"}), - }, { predicateA: NewNamespacePredicate([]string{"namespace1", "namespace2"}), predicateB: NewNamespacePredicate([]string{"namespace3"}), expectedResult: predicates.Empty[Task](), }, - { - predicateA: NewOutboundTaskGroupPredicate([]string{"g1", "g2"}), - predicateB: NewOutboundTaskGroupPredicate([]string{"g3"}), - expectedResult: predicates.Empty[Task](), - }, { predicateA: NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ {"g1", "n1", "d1"}, @@ -360,14 +338,17 @@ func (s *predicatesSuite) TestAndPredicates() { expectedResult: predicates.Empty[Task](), }, { + // Cross-type combination that no case-specific branch simplifies, so it falls back + // to the generic wrapper. NamespacePredicate/OutboundTaskPredicate are both still + // live types; either would do here. predicateA: NewNamespacePredicate([]string{"namespace1"}), - predicateB: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, + predicateB: NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ + {"g1", "n1", "d1"}, }), expectedResult: predicates.And( NewNamespacePredicate([]string{"namespace1"}), - NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, + NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ + {"g1", "n1", "d1"}, }), ), }, @@ -409,29 +390,6 @@ func (s *predicatesSuite) TestOrPredicates() { predicateB: NewNamespacePredicate([]string{"namespace2", "namespace3"}), expectedResult: NewNamespacePredicate([]string{"namespace1", "namespace2", "namespace3"}), }, - { - predicateA: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - }), - predicateB: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, - }), - expectedResult: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, - enumsspb.TASK_TYPE_DELETE_HISTORY_EVENT, - }), - }, - { - predicateA: NewDestinationPredicate([]string{"dest1", "dest2"}), - predicateB: NewDestinationPredicate([]string{"dest2", "dest3"}), - expectedResult: NewDestinationPredicate([]string{"dest1", "dest2", "dest3"}), - }, - { - predicateA: NewOutboundTaskGroupPredicate([]string{"g1", "g2"}), - predicateB: NewOutboundTaskGroupPredicate([]string{"g2", "g3"}), - expectedResult: NewOutboundTaskGroupPredicate([]string{"g1", "g2", "g3"}), - }, { predicateA: NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ {"g1", "n1", "d1"}, @@ -447,14 +405,17 @@ func (s *predicatesSuite) TestOrPredicates() { }), }, { + // Cross-type combination that no case-specific branch simplifies, so it falls back + // to the generic wrapper. NamespacePredicate/OutboundTaskPredicate are both still + // live types; either would do here. predicateA: NewNamespacePredicate([]string{"namespace1"}), - predicateB: NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, + predicateB: NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ + {"g1", "n1", "d1"}, }), expectedResult: predicates.Or( NewNamespacePredicate([]string{"namespace1"}), - NewTypePredicate([]enumsspb.TaskType{ - enumsspb.TASK_TYPE_ACTIVITY_TIMEOUT, + NewOutboundTaskPredicate([]TaskGroupNamespaceIDAndDestination{ + {"g1", "n1", "d1"}, }), ), }, From ad7b2298dcce00688cfe3b83ab02819abf9249e8 Mon Sep 17 00:00:00 2001 From: Rodrigo Zhou Date: Fri, 4 Sep 2026 10:56:35 -0700 Subject: [PATCH 139/139] [Visibility] Enable unified query converter by default (#11935) ## What changed? Enable the new Visibility unified query converter introduced in v1.30.0 by default. ## Why? The unified query converter is stable, and ready to replace the old one. ## How did you test it? - [ ] built - [ ] run locally and tested manually - [ ] covered by existing tests - [ ] added new unit test(s) - [ ] added new functional test(s) ## Potential risks --- common/dynamicconfig/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/dynamicconfig/constants.go b/common/dynamicconfig/constants.go index dacf561bf65..188bc799d41 100644 --- a/common/dynamicconfig/constants.go +++ b/common/dynamicconfig/constants.go @@ -95,7 +95,7 @@ values in system search attributes.`, ) VisibilityEnableUnifiedQueryConverter = NewGlobalBoolSetting( "system.visibilityEnableUnifiedQueryConverter", - false, + true, `VisibilityEnableUnifiedQueryConverter enables the unified query converter for parsing the query.`, )