From b148da8b016822acd44962235700ff2b9ad34e97 Mon Sep 17 00:00:00 2001 From: chaptersix <13949480+chaptersix@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:15:44 -0500 Subject: [PATCH] Generalize CHASM scheduler action and overlap policy contracts --- api/schedule/v1/message.pb.go | 207 +++++++---- chasm/chasmtest/test_engine_test.go | 92 +++-- chasm/lib/scheduler/action.go | 83 +++++ chasm/lib/scheduler/action_policy.go | 54 +++ chasm/lib/scheduler/action_results.go | 19 + chasm/lib/scheduler/action_workflow.go | 105 ++++++ chasm/lib/scheduler/backfiller_tasks.go | 29 +- chasm/lib/scheduler/buffer_planner.go | 81 +++-- .../scheduler/buffer_processor_legacy_test.go | 15 +- .../lib/scheduler/execution_compatibility.go | 43 +++ chasm/lib/scheduler/export_test.go | 93 +++-- .../gen/schedulerpb/v1/message.pb.go | 92 +++-- chasm/lib/scheduler/generator_tasks.go | 2 +- chasm/lib/scheduler/helper_test.go | 202 ++++++----- .../lib/scheduler/internal/buffer_planner.go | 195 ++++++----- .../scheduler/internal/buffer_planner_test.go | 12 +- .../lib/scheduler/internal/buffered_start.go | 17 +- chasm/lib/scheduler/internal/execution.go | 47 +++ chasm/lib/scheduler/internal/policy.go | 150 ++++++++ chasm/lib/scheduler/internal/policy_test.go | 40 +++ chasm/lib/scheduler/internal/request_id.go | 8 +- .../lib/scheduler/internal/request_id_test.go | 6 +- chasm/lib/scheduler/invoker.go | 119 ++++--- .../scheduler/invoker_execute_phase_test.go | 8 +- chasm/lib/scheduler/invoker_execute_task.go | 324 ++++++++---------- .../invoker_process_buffer_task_test.go | 9 +- chasm/lib/scheduler/invoker_tasks.go | 13 +- .../lib/scheduler/migration/identity_test.go | 2 +- chasm/lib/scheduler/migration/migration.go | 27 +- chasm/lib/scheduler/proto/v1/message.proto | 3 + chasm/lib/scheduler/scheduler.go | 72 ++-- chasm/lib/scheduler/spec_processor.go | 24 +- chasm/lib/scheduler/spec_processor_mock.go | 12 +- .../server/api/schedule/v1/message.proto | 8 + 34 files changed, 1492 insertions(+), 721 deletions(-) create mode 100644 chasm/lib/scheduler/action.go create mode 100644 chasm/lib/scheduler/action_policy.go create mode 100644 chasm/lib/scheduler/action_results.go create mode 100644 chasm/lib/scheduler/action_workflow.go create mode 100644 chasm/lib/scheduler/execution_compatibility.go create mode 100644 chasm/lib/scheduler/internal/execution.go create mode 100644 chasm/lib/scheduler/internal/policy.go create mode 100644 chasm/lib/scheduler/internal/policy_test.go diff --git a/api/schedule/v1/message.pb.go b/api/schedule/v1/message.pb.go index 513c6c1d61d..d18b20db71d 100644 --- a/api/schedule/v1/message.pb.go +++ b/api/schedule/v1/message.pb.go @@ -11,10 +11,10 @@ import ( sync "sync" unsafe "unsafe" - v12 "go.temporal.io/api/common/v1" + v11 "go.temporal.io/api/common/v1" v1 "go.temporal.io/api/enums/v1" v13 "go.temporal.io/api/failure/v1" - v11 "go.temporal.io/api/schedule/v1" + v12 "go.temporal.io/api/schedule/v1" v14 "go.temporal.io/api/workflowservice/v1" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" @@ -69,7 +69,15 @@ type BufferedStart struct { // True when a running BufferedStart is known to have a Nexus callback // attached. False when a BufferedStart originated from a migrated V1 // workflow. Only used by CHASM scheduler, for migration from V1. - HasCallback bool `protobuf:"varint,13,opt,name=has_callback,json=hasCallback,proto3" json:"has_callback,omitempty"` + HasCallback bool `protobuf:"varint,13,opt,name=has_callback,json=hasCallback,proto3" json:"has_callback,omitempty"` + // Generic execution identity. Workflow fields remain a compatibility projection. + Execution *v11.Execution `protobuf:"bytes,14,opt,name=execution,proto3" json:"execution,omitempty"` + Completion *v11.ActionExecutionResult `protobuf:"bytes,15,opt,name=completion,proto3" json:"completion,omitempty"` + CompletionTime *timestamppb.Timestamp `protobuf:"bytes,16,opt,name=completion_time,json=completionTime,proto3" json:"completion_time,omitempty"` + CustomOverlapPolicy *v12.CustomOverlapPolicy `protobuf:"bytes,17,opt,name=custom_overlap_policy,json=customOverlapPolicy,proto3" json:"custom_overlap_policy,omitempty"` + // Assigned on enqueue, independent of request IDs and buffer position. + OccurrenceId string `protobuf:"bytes,18,opt,name=occurrence_id,json=occurrenceId,proto3" json:"occurrence_id,omitempty"` + StartAccepted bool `protobuf:"varint,19,opt,name=start_accepted,json=startAccepted,proto3" json:"start_accepted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -195,6 +203,48 @@ func (x *BufferedStart) GetHasCallback() bool { return false } +func (x *BufferedStart) GetExecution() *v11.Execution { + if x != nil { + return x.Execution + } + return nil +} + +func (x *BufferedStart) GetCompletion() *v11.ActionExecutionResult { + if x != nil { + return x.Completion + } + return nil +} + +func (x *BufferedStart) GetCompletionTime() *timestamppb.Timestamp { + if x != nil { + return x.CompletionTime + } + return nil +} + +func (x *BufferedStart) GetCustomOverlapPolicy() *v12.CustomOverlapPolicy { + if x != nil { + return x.CustomOverlapPolicy + } + return nil +} + +func (x *BufferedStart) GetOccurrenceId() string { + if x != nil { + return x.OccurrenceId + } + return "" +} + +func (x *BufferedStart) GetStartAccepted() bool { + if x != nil { + return x.StartAccepted + } + return false +} + // Result when a workflow execution has completed. // Only used by the CHASM scheduler. type CompletedResult struct { @@ -258,9 +308,9 @@ type InternalState struct { ScheduleId string `protobuf:"bytes,8,opt,name=schedule_id,json=scheduleId,proto3" json:"schedule_id,omitempty"` LastProcessedTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=last_processed_time,json=lastProcessedTime,proto3" json:"last_processed_time,omitempty"` BufferedStarts []*BufferedStart `protobuf:"bytes,4,rep,name=buffered_starts,json=bufferedStarts,proto3" json:"buffered_starts,omitempty"` - OngoingBackfills []*v11.BackfillRequest `protobuf:"bytes,10,rep,name=ongoing_backfills,json=ongoingBackfills,proto3" json:"ongoing_backfills,omitempty"` + OngoingBackfills []*v12.BackfillRequest `protobuf:"bytes,10,rep,name=ongoing_backfills,json=ongoingBackfills,proto3" json:"ongoing_backfills,omitempty"` // last completion/failure - LastCompletionResult *v12.Payloads `protobuf:"bytes,5,opt,name=last_completion_result,json=lastCompletionResult,proto3" json:"last_completion_result,omitempty"` + LastCompletionResult *v11.Payloads `protobuf:"bytes,5,opt,name=last_completion_result,json=lastCompletionResult,proto3" json:"last_completion_result,omitempty"` ContinuedFailure *v13.Failure `protobuf:"bytes,6,opt,name=continued_failure,json=continuedFailure,proto3" json:"continued_failure,omitempty"` // conflict token is implemented as simple sequence number ConflictToken int64 `protobuf:"varint,7,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` @@ -335,14 +385,14 @@ func (x *InternalState) GetBufferedStarts() []*BufferedStart { return nil } -func (x *InternalState) GetOngoingBackfills() []*v11.BackfillRequest { +func (x *InternalState) GetOngoingBackfills() []*v12.BackfillRequest { if x != nil { return x.OngoingBackfills } return nil } -func (x *InternalState) GetLastCompletionResult() *v12.Payloads { +func (x *InternalState) GetLastCompletionResult() *v11.Payloads { if x != nil { return x.LastCompletionResult } @@ -379,9 +429,9 @@ func (x *InternalState) GetPendingMigration() bool { type StartScheduleArgs struct { state protoimpl.MessageState `protogen:"open.v1"` - Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` - Info *v11.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` - InitialPatch *v11.SchedulePatch `protobuf:"bytes,3,opt,name=initial_patch,json=initialPatch,proto3" json:"initial_patch,omitempty"` + Schedule *v12.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Info *v12.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` + InitialPatch *v12.SchedulePatch `protobuf:"bytes,3,opt,name=initial_patch,json=initialPatch,proto3" json:"initial_patch,omitempty"` State *InternalState `protobuf:"bytes,4,opt,name=state,proto3" json:"state,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -417,21 +467,21 @@ func (*StartScheduleArgs) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{3} } -func (x *StartScheduleArgs) GetSchedule() *v11.Schedule { +func (x *StartScheduleArgs) GetSchedule() *v12.Schedule { if x != nil { return x.Schedule } return nil } -func (x *StartScheduleArgs) GetInfo() *v11.ScheduleInfo { +func (x *StartScheduleArgs) GetInfo() *v12.ScheduleInfo { if x != nil { return x.Info } return nil } -func (x *StartScheduleArgs) GetInitialPatch() *v11.SchedulePatch { +func (x *StartScheduleArgs) GetInitialPatch() *v12.SchedulePatch { if x != nil { return x.InitialPatch } @@ -447,9 +497,9 @@ func (x *StartScheduleArgs) GetState() *InternalState { type FullUpdateRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Schedule *v12.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` ConflictToken int64 `protobuf:"varint,2,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` - SearchAttributes *v12.SearchAttributes `protobuf:"bytes,3,opt,name=search_attributes,json=searchAttributes,proto3" json:"search_attributes,omitempty"` + SearchAttributes *v11.SearchAttributes `protobuf:"bytes,3,opt,name=search_attributes,json=searchAttributes,proto3" json:"search_attributes,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -484,7 +534,7 @@ func (*FullUpdateRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{4} } -func (x *FullUpdateRequest) GetSchedule() *v11.Schedule { +func (x *FullUpdateRequest) GetSchedule() *v12.Schedule { if x != nil { return x.Schedule } @@ -498,7 +548,7 @@ func (x *FullUpdateRequest) GetConflictToken() int64 { return 0 } -func (x *FullUpdateRequest) GetSearchAttributes() *v12.SearchAttributes { +func (x *FullUpdateRequest) GetSearchAttributes() *v11.SearchAttributes { if x != nil { return x.SearchAttributes } @@ -507,8 +557,8 @@ func (x *FullUpdateRequest) GetSearchAttributes() *v12.SearchAttributes { type DescribeResponse struct { state protoimpl.MessageState `protogen:"open.v1"` - Schedule *v11.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` - Info *v11.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` + Schedule *v12.Schedule `protobuf:"bytes,1,opt,name=schedule,proto3" json:"schedule,omitempty"` + Info *v12.ScheduleInfo `protobuf:"bytes,2,opt,name=info,proto3" json:"info,omitempty"` ConflictToken int64 `protobuf:"varint,3,opt,name=conflict_token,json=conflictToken,proto3" json:"conflict_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -544,14 +594,14 @@ func (*DescribeResponse) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{5} } -func (x *DescribeResponse) GetSchedule() *v11.Schedule { +func (x *DescribeResponse) GetSchedule() *v12.Schedule { if x != nil { return x.Schedule } return nil } -func (x *DescribeResponse) GetInfo() *v11.ScheduleInfo { +func (x *DescribeResponse) GetInfo() *v12.ScheduleInfo { if x != nil { return x.Info } @@ -569,7 +619,7 @@ type WatchWorkflowRequest struct { state protoimpl.MessageState `protogen:"open.v1"` // Note: this will be sent to the activity with empty execution.run_id, and // the run id that we started in first_execution_run_id. - Execution *v12.WorkflowExecution `protobuf:"bytes,3,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v11.WorkflowExecution `protobuf:"bytes,3,opt,name=execution,proto3" json:"execution,omitempty"` FirstExecutionRunId string `protobuf:"bytes,4,opt,name=first_execution_run_id,json=firstExecutionRunId,proto3" json:"first_execution_run_id,omitempty"` LongPoll bool `protobuf:"varint,5,opt,name=long_poll,json=longPoll,proto3" json:"long_poll,omitempty"` unknownFields protoimpl.UnknownFields @@ -606,7 +656,7 @@ func (*WatchWorkflowRequest) Descriptor() ([]byte, []int) { return file_temporal_server_api_schedule_v1_message_proto_rawDescGZIP(), []int{6} } -func (x *WatchWorkflowRequest) GetExecution() *v12.WorkflowExecution { +func (x *WatchWorkflowRequest) GetExecution() *v11.WorkflowExecution { if x != nil { return x.Execution } @@ -685,7 +735,7 @@ func (x *WatchWorkflowResponse) GetResultFailure() isWatchWorkflowResponse_Resul return nil } -func (x *WatchWorkflowResponse) GetResult() *v12.Payloads { +func (x *WatchWorkflowResponse) GetResult() *v11.Payloads { if x != nil { if x, ok := x.ResultFailure.(*WatchWorkflowResponse_Result); ok { return x.Result @@ -715,7 +765,7 @@ type isWatchWorkflowResponse_ResultFailure interface { } type WatchWorkflowResponse_Result struct { - Result *v12.Payloads `protobuf:"bytes,2,opt,name=result,proto3,oneof"` + Result *v11.Payloads `protobuf:"bytes,2,opt,name=result,proto3,oneof"` } type WatchWorkflowResponse_Failure struct { @@ -835,7 +885,7 @@ type CancelWorkflowRequest struct { RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` Identity string `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` // Note: run id in execution is first execution run id - Execution *v12.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v11.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -885,7 +935,7 @@ func (x *CancelWorkflowRequest) GetIdentity() string { return "" } -func (x *CancelWorkflowRequest) GetExecution() *v12.WorkflowExecution { +func (x *CancelWorkflowRequest) GetExecution() *v11.WorkflowExecution { if x != nil { return x.Execution } @@ -904,7 +954,7 @@ type TerminateWorkflowRequest struct { RequestId string `protobuf:"bytes,3,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` Identity string `protobuf:"bytes,4,opt,name=identity,proto3" json:"identity,omitempty"` // Note: run id in execution is first execution run id - Execution *v12.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` + Execution *v11.WorkflowExecution `protobuf:"bytes,5,opt,name=execution,proto3" json:"execution,omitempty"` Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -954,7 +1004,7 @@ func (x *TerminateWorkflowRequest) GetIdentity() string { return "" } -func (x *TerminateWorkflowRequest) GetExecution() *v12.WorkflowExecution { +func (x *TerminateWorkflowRequest) GetExecution() *v11.WorkflowExecution { if x != nil { return x.Execution } @@ -1054,7 +1104,7 @@ var File_temporal_server_api_schedule_v1_message_proto protoreflect.FileDescript const file_temporal_server_api_schedule_v1_message_proto_rawDesc = "" + "\n" + - "-temporal/server/api/schedule/v1/message.proto\x12\x1ftemporal.server.api.schedule.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/schedule.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a&temporal/api/schedule/v1/message.proto\x1a6temporal/api/workflowservice/v1/request_response.proto\"\x95\x05\n" + + "-temporal/server/api/schedule/v1/message.proto\x12\x1ftemporal.server.api.schedule.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a$temporal/api/enums/v1/schedule.proto\x1a$temporal/api/enums/v1/workflow.proto\x1a%temporal/api/failure/v1/message.proto\x1a&temporal/api/schedule/v1/message.proto\x1a6temporal/api/workflowservice/v1/request_response.proto\"\x99\b\n" + "\rBufferedStart\x12=\n" + "\fnominal_time\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\vnominalTime\x12;\n" + "\vactual_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\n" + @@ -1073,7 +1123,15 @@ const file_temporal_server_api_schedule_v1_message_proto_rawDesc = "" + "\n" + "start_time\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tstartTime\x12N\n" + "\tcompleted\x18\f \x01(\v20.temporal.server.api.schedule.v1.CompletedResultR\tcompleted\x12!\n" + - "\fhas_callback\x18\r \x01(\bR\vhasCallback\"\x94\x01\n" + + "\fhas_callback\x18\r \x01(\bR\vhasCallback\x12?\n" + + "\texecution\x18\x0e \x01(\v2!.temporal.api.common.v1.ExecutionR\texecution\x12M\n" + + "\n" + + "completion\x18\x0f \x01(\v2-.temporal.api.common.v1.ActionExecutionResultR\n" + + "completion\x12C\n" + + "\x0fcompletion_time\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\x0ecompletionTime\x12a\n" + + "\x15custom_overlap_policy\x18\x11 \x01(\v2-.temporal.api.schedule.v1.CustomOverlapPolicyR\x13customOverlapPolicy\x12#\n" + + "\roccurrence_id\x18\x12 \x01(\tR\foccurrenceId\x12%\n" + + "\x0estart_accepted\x18\x13 \x01(\bR\rstartAccepted\"\x94\x01\n" + "\x0fCompletedResult\x12F\n" + "\x06status\x18\x01 \x01(\x0e2..temporal.api.enums.v1.WorkflowExecutionStatusR\x06status\x129\n" + "\n" + @@ -1172,16 +1230,19 @@ var file_temporal_server_api_schedule_v1_message_proto_goTypes = []any{ (*NextTimeCache)(nil), // 12: temporal.server.api.schedule.v1.NextTimeCache (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp (v1.ScheduleOverlapPolicy)(0), // 14: temporal.api.enums.v1.ScheduleOverlapPolicy - (v1.WorkflowExecutionStatus)(0), // 15: temporal.api.enums.v1.WorkflowExecutionStatus - (*v11.BackfillRequest)(nil), // 16: temporal.api.schedule.v1.BackfillRequest - (*v12.Payloads)(nil), // 17: temporal.api.common.v1.Payloads - (*v13.Failure)(nil), // 18: temporal.api.failure.v1.Failure - (*v11.Schedule)(nil), // 19: temporal.api.schedule.v1.Schedule - (*v11.ScheduleInfo)(nil), // 20: temporal.api.schedule.v1.ScheduleInfo - (*v11.SchedulePatch)(nil), // 21: temporal.api.schedule.v1.SchedulePatch - (*v12.SearchAttributes)(nil), // 22: temporal.api.common.v1.SearchAttributes - (*v12.WorkflowExecution)(nil), // 23: temporal.api.common.v1.WorkflowExecution - (*v14.StartWorkflowExecutionRequest)(nil), // 24: temporal.api.workflowservice.v1.StartWorkflowExecutionRequest + (*v11.Execution)(nil), // 15: temporal.api.common.v1.Execution + (*v11.ActionExecutionResult)(nil), // 16: temporal.api.common.v1.ActionExecutionResult + (*v12.CustomOverlapPolicy)(nil), // 17: temporal.api.schedule.v1.CustomOverlapPolicy + (v1.WorkflowExecutionStatus)(0), // 18: temporal.api.enums.v1.WorkflowExecutionStatus + (*v12.BackfillRequest)(nil), // 19: temporal.api.schedule.v1.BackfillRequest + (*v11.Payloads)(nil), // 20: temporal.api.common.v1.Payloads + (*v13.Failure)(nil), // 21: temporal.api.failure.v1.Failure + (*v12.Schedule)(nil), // 22: temporal.api.schedule.v1.Schedule + (*v12.ScheduleInfo)(nil), // 23: temporal.api.schedule.v1.ScheduleInfo + (*v12.SchedulePatch)(nil), // 24: temporal.api.schedule.v1.SchedulePatch + (*v11.SearchAttributes)(nil), // 25: temporal.api.common.v1.SearchAttributes + (*v11.WorkflowExecution)(nil), // 26: temporal.api.common.v1.WorkflowExecution + (*v14.StartWorkflowExecutionRequest)(nil), // 27: temporal.api.workflowservice.v1.StartWorkflowExecutionRequest } var file_temporal_server_api_schedule_v1_message_proto_depIdxs = []int32{ 13, // 0: temporal.server.api.schedule.v1.BufferedStart.nominal_time:type_name -> google.protobuf.Timestamp @@ -1191,36 +1252,40 @@ var file_temporal_server_api_schedule_v1_message_proto_depIdxs = []int32{ 13, // 4: temporal.server.api.schedule.v1.BufferedStart.backoff_time:type_name -> google.protobuf.Timestamp 13, // 5: temporal.server.api.schedule.v1.BufferedStart.start_time:type_name -> google.protobuf.Timestamp 1, // 6: temporal.server.api.schedule.v1.BufferedStart.completed:type_name -> temporal.server.api.schedule.v1.CompletedResult - 15, // 7: temporal.server.api.schedule.v1.CompletedResult.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 13, // 8: temporal.server.api.schedule.v1.CompletedResult.close_time:type_name -> google.protobuf.Timestamp - 13, // 9: temporal.server.api.schedule.v1.InternalState.last_processed_time:type_name -> google.protobuf.Timestamp - 0, // 10: temporal.server.api.schedule.v1.InternalState.buffered_starts:type_name -> temporal.server.api.schedule.v1.BufferedStart - 16, // 11: temporal.server.api.schedule.v1.InternalState.ongoing_backfills:type_name -> temporal.api.schedule.v1.BackfillRequest - 17, // 12: temporal.server.api.schedule.v1.InternalState.last_completion_result:type_name -> temporal.api.common.v1.Payloads - 18, // 13: temporal.server.api.schedule.v1.InternalState.continued_failure:type_name -> temporal.api.failure.v1.Failure - 19, // 14: temporal.server.api.schedule.v1.StartScheduleArgs.schedule:type_name -> temporal.api.schedule.v1.Schedule - 20, // 15: temporal.server.api.schedule.v1.StartScheduleArgs.info:type_name -> temporal.api.schedule.v1.ScheduleInfo - 21, // 16: temporal.server.api.schedule.v1.StartScheduleArgs.initial_patch:type_name -> temporal.api.schedule.v1.SchedulePatch - 2, // 17: temporal.server.api.schedule.v1.StartScheduleArgs.state:type_name -> temporal.server.api.schedule.v1.InternalState - 19, // 18: temporal.server.api.schedule.v1.FullUpdateRequest.schedule:type_name -> temporal.api.schedule.v1.Schedule - 22, // 19: temporal.server.api.schedule.v1.FullUpdateRequest.search_attributes:type_name -> temporal.api.common.v1.SearchAttributes - 19, // 20: temporal.server.api.schedule.v1.DescribeResponse.schedule:type_name -> temporal.api.schedule.v1.Schedule - 20, // 21: temporal.server.api.schedule.v1.DescribeResponse.info:type_name -> temporal.api.schedule.v1.ScheduleInfo - 23, // 22: temporal.server.api.schedule.v1.WatchWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 15, // 23: temporal.server.api.schedule.v1.WatchWorkflowResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus - 17, // 24: temporal.server.api.schedule.v1.WatchWorkflowResponse.result:type_name -> temporal.api.common.v1.Payloads - 18, // 25: temporal.server.api.schedule.v1.WatchWorkflowResponse.failure:type_name -> temporal.api.failure.v1.Failure - 13, // 26: temporal.server.api.schedule.v1.WatchWorkflowResponse.close_time:type_name -> google.protobuf.Timestamp - 24, // 27: temporal.server.api.schedule.v1.StartWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.StartWorkflowExecutionRequest - 13, // 28: temporal.server.api.schedule.v1.StartWorkflowResponse.real_start_time:type_name -> google.protobuf.Timestamp - 23, // 29: temporal.server.api.schedule.v1.CancelWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 23, // 30: temporal.server.api.schedule.v1.TerminateWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution - 13, // 31: temporal.server.api.schedule.v1.NextTimeCache.start_time:type_name -> google.protobuf.Timestamp - 32, // [32:32] is the sub-list for method output_type - 32, // [32:32] is the sub-list for method input_type - 32, // [32:32] is the sub-list for extension type_name - 32, // [32:32] is the sub-list for extension extendee - 0, // [0:32] is the sub-list for field type_name + 15, // 7: temporal.server.api.schedule.v1.BufferedStart.execution:type_name -> temporal.api.common.v1.Execution + 16, // 8: temporal.server.api.schedule.v1.BufferedStart.completion:type_name -> temporal.api.common.v1.ActionExecutionResult + 13, // 9: temporal.server.api.schedule.v1.BufferedStart.completion_time:type_name -> google.protobuf.Timestamp + 17, // 10: temporal.server.api.schedule.v1.BufferedStart.custom_overlap_policy:type_name -> temporal.api.schedule.v1.CustomOverlapPolicy + 18, // 11: temporal.server.api.schedule.v1.CompletedResult.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 13, // 12: temporal.server.api.schedule.v1.CompletedResult.close_time:type_name -> google.protobuf.Timestamp + 13, // 13: temporal.server.api.schedule.v1.InternalState.last_processed_time:type_name -> google.protobuf.Timestamp + 0, // 14: temporal.server.api.schedule.v1.InternalState.buffered_starts:type_name -> temporal.server.api.schedule.v1.BufferedStart + 19, // 15: temporal.server.api.schedule.v1.InternalState.ongoing_backfills:type_name -> temporal.api.schedule.v1.BackfillRequest + 20, // 16: temporal.server.api.schedule.v1.InternalState.last_completion_result:type_name -> temporal.api.common.v1.Payloads + 21, // 17: temporal.server.api.schedule.v1.InternalState.continued_failure:type_name -> temporal.api.failure.v1.Failure + 22, // 18: temporal.server.api.schedule.v1.StartScheduleArgs.schedule:type_name -> temporal.api.schedule.v1.Schedule + 23, // 19: temporal.server.api.schedule.v1.StartScheduleArgs.info:type_name -> temporal.api.schedule.v1.ScheduleInfo + 24, // 20: temporal.server.api.schedule.v1.StartScheduleArgs.initial_patch:type_name -> temporal.api.schedule.v1.SchedulePatch + 2, // 21: temporal.server.api.schedule.v1.StartScheduleArgs.state:type_name -> temporal.server.api.schedule.v1.InternalState + 22, // 22: temporal.server.api.schedule.v1.FullUpdateRequest.schedule:type_name -> temporal.api.schedule.v1.Schedule + 25, // 23: temporal.server.api.schedule.v1.FullUpdateRequest.search_attributes:type_name -> temporal.api.common.v1.SearchAttributes + 22, // 24: temporal.server.api.schedule.v1.DescribeResponse.schedule:type_name -> temporal.api.schedule.v1.Schedule + 23, // 25: temporal.server.api.schedule.v1.DescribeResponse.info:type_name -> temporal.api.schedule.v1.ScheduleInfo + 26, // 26: temporal.server.api.schedule.v1.WatchWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 18, // 27: temporal.server.api.schedule.v1.WatchWorkflowResponse.status:type_name -> temporal.api.enums.v1.WorkflowExecutionStatus + 20, // 28: temporal.server.api.schedule.v1.WatchWorkflowResponse.result:type_name -> temporal.api.common.v1.Payloads + 21, // 29: temporal.server.api.schedule.v1.WatchWorkflowResponse.failure:type_name -> temporal.api.failure.v1.Failure + 13, // 30: temporal.server.api.schedule.v1.WatchWorkflowResponse.close_time:type_name -> google.protobuf.Timestamp + 27, // 31: temporal.server.api.schedule.v1.StartWorkflowRequest.request:type_name -> temporal.api.workflowservice.v1.StartWorkflowExecutionRequest + 13, // 32: temporal.server.api.schedule.v1.StartWorkflowResponse.real_start_time:type_name -> google.protobuf.Timestamp + 26, // 33: temporal.server.api.schedule.v1.CancelWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 26, // 34: temporal.server.api.schedule.v1.TerminateWorkflowRequest.execution:type_name -> temporal.api.common.v1.WorkflowExecution + 13, // 35: temporal.server.api.schedule.v1.NextTimeCache.start_time:type_name -> google.protobuf.Timestamp + 36, // [36:36] is the sub-list for method output_type + 36, // [36:36] is the sub-list for method input_type + 36, // [36:36] is the sub-list for extension type_name + 36, // [36:36] is the sub-list for extension extendee + 0, // [0:36] is the sub-list for field type_name } func init() { file_temporal_server_api_schedule_v1_message_proto_init() } diff --git a/chasm/chasmtest/test_engine_test.go b/chasm/chasmtest/test_engine_test.go index dc8a018ed1d..a3ec62cf77d 100644 --- a/chasm/chasmtest/test_engine_test.go +++ b/chasm/chasmtest/test_engine_test.go @@ -19,6 +19,29 @@ import ( "go.temporal.io/server/service/history/tasks" ) +// TestTasksArePhysicallyGenerated: a task a component adds must reach the backend as a physical task, in +// the category its TaskAttributes imply, whether it was added while starting the execution or while +// updating it. +func TestTasksArePhysicallyGenerated(t *testing.T) { + const ttl = time.Hour + + t.Run("added while starting", func(t *testing.T) { + e, ref := startStore(t, ttl) + require.Equal(t, 1, countTasks(t, e, ref, tasks.CategoryTimer)) + }) + + t.Run("added while updating", func(t *testing.T) { + e, ref := startStore(t, 0) + require.Equal(t, 0, countTasks(t, e, ref, tasks.CategoryTimer)) + _, _, err := chasm.UpdateComponent(engineContext(e), ref, + func(s *tests.PayloadStore, mc chasm.MutableContext, _ any) (any, error) { + return nil, addPayload(s, mc, "second", ttl) + }, nil) + require.NoError(t, err) + require.Equal(t, 1, countTasks(t, e, ref, tasks.CategoryTimer)) + }) +} + func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { var calls []string check := func(name string) chasmtest.InvariantCheck { @@ -31,27 +54,35 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { } } - e, ref := startStoreWithEngineOptions(t, time.Hour, []chasmtest.EngineOption{ - chasmtest.WithInvariantCheck(check("first")), - chasmtest.WithInvariantCheck(check("second")), - }) + e, ref := startStoreWithEngineOptions( + t, + time.Hour, + []chasmtest.EngineOption{ + chasmtest.WithInvariantCheck(check("first")), + chasmtest.WithInvariantCheck(check("second")), + }, + ) require.Equal(t, []string{"first", "second"}, calls) _, _, err := chasm.UpdateComponent(engineContext(e), ref, - func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { return nil, nil }, nil, - chasm.WithRequestID("invariant-check")) + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, nil, chasm.WithRequestID("invariant-check")) require.NoError(t, err) require.Equal(t, []string{"first", "second", "first", "second"}, calls) _, _, err = chasm.UpdateComponent(engineContext(e), ref, - func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { return nil, nil }, nil, - chasm.WithRequestID("invariant-check")) + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, nil, chasm.WithRequestID("invariant-check")) require.ErrorIs(t, err, chasm.ErrRequestIDAlreadyUsed) require.Equal(t, []string{"first", "second", "first", "second"}, calls) transitionErr := errors.New("transition failed") _, _, err = chasm.UpdateComponent(engineContext(e), ref, - func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { return nil, transitionErr }, nil) + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, transitionErr + }, nil) require.ErrorIs(t, err, transitionErr) require.Equal(t, []string{"first", "second", "first", "second"}, calls) @@ -61,39 +92,27 @@ func TestInvariantCheckRunsAfterSuccessfulTransitions(t *testing.T) { require.Equal(t, []string{"first", "second", "first", "second", "first", "second"}, calls) key := chasm.ExecutionKey{NamespaceID: "test-ns", BusinessID: "update-with-start"} - result, err := chasm.UpdateWithStartExecution(engineContext(e), key, - func(mc chasm.MutableContext, _ any) (*tests.PayloadStore, error) { return tests.NewPayloadStore(mc) }, - func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { return nil, nil }, nil) + result, err := chasm.UpdateWithStartExecution( + engineContext(e), + key, + func(mc chasm.MutableContext, _ any) (*tests.PayloadStore, error) { + return tests.NewPayloadStore(mc) + }, + func(*tests.PayloadStore, chasm.MutableContext, any) (any, error) { + return nil, nil + }, + nil, + ) require.NoError(t, err) require.True(t, result.Created) require.Equal(t, []string{ - "first", "second", "first", "second", "first", "second", "first", "second", + "first", "second", + "first", "second", + "first", "second", + "first", "second", }, calls) } -// TestTasksArePhysicallyGenerated: a task a component adds must reach the backend as a physical task, in -// the category its TaskAttributes imply, whether it was added while starting the execution or while -// updating it. -func TestTasksArePhysicallyGenerated(t *testing.T) { - const ttl = time.Hour - - t.Run("added while starting", func(t *testing.T) { - e, ref := startStore(t, ttl) - require.Equal(t, 1, countTasks(t, e, ref, tasks.CategoryTimer)) - }) - - t.Run("added while updating", func(t *testing.T) { - e, ref := startStore(t, 0) - require.Equal(t, 0, countTasks(t, e, ref, tasks.CategoryTimer)) - _, _, err := chasm.UpdateComponent(engineContext(e), ref, - func(s *tests.PayloadStore, mc chasm.MutableContext, _ any) (any, error) { - return nil, addPayload(s, mc, "second", ttl) - }, nil) - require.NoError(t, err) - require.Equal(t, 1, countTasks(t, e, ref, tasks.CategoryTimer)) - }) -} - func TestFireSideEffectTasks(t *testing.T) { const ttl = time.Hour e, ref := startStore(t, ttl) @@ -150,6 +169,7 @@ func TestUpdateComponentDeduplicatesRequestID(t *testing.T) { updatedRef, err = update() var failedPrecondition *serviceerror.FailedPrecondition require.ErrorAs(t, err, &failedPrecondition) + require.ErrorIs(t, err, chasm.ErrRequestIDAlreadyUsed) require.Nil(t, updatedRef) require.Equal(t, 1, updateCount) } diff --git a/chasm/lib/scheduler/action.go b/chasm/lib/scheduler/action.go new file mode 100644 index 00000000000..36606727f3e --- /dev/null +++ b/chasm/lib/scheduler/action.go @@ -0,0 +1,83 @@ +package scheduler + +import ( + "context" + "time" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" + "go.temporal.io/server/common/resource" +) + +type occurrenceContext struct { + NominalTime, ScheduledTime time.Time + Manual bool + RequestID string +} + +type actionMetadata struct { + Kind enumspb.ExecutionType + Type, IDBase, TaskQueue string + SearchAttributes *commonpb.SearchAttributes +} + +type actionStartInput struct { + Scheduler *Scheduler + Occurrence *schedulespb.BufferedStart + Callback *commonpb.Callback + Previous *schedulerpb.LastCompletionResult + EnableVersioningOverride bool +} + +type actionClients struct { + Frontend workflowservice.WorkflowServiceClient + History resource.HistoryClient +} + +type actionCompletion struct { + Result *commonpb.ActionExecutionResult + Failed bool +} + +type actionImplementation interface { + Metadata(*schedulepb.ScheduleAction) actionMetadata + Validate(*schedulepb.ScheduleAction) error + Policies() *internal.PolicyRegistry + GenerateTargetID(string, occurrenceContext) string + Start(context.Context, actionClients, actionStartInput) (string, error) + Cancel(context.Context, actionClients, *Scheduler, *commonpb.Execution) error + Terminate(context.Context, actionClients, *Scheduler, *commonpb.Execution) error + Completion(*persistencespb.ChasmNexusCompletion, *commonpb.Execution) actionCompletion + ParticipatesInCompletionHistory() bool +} + +func implementation(_ *schedulepb.ScheduleAction) actionImplementation { return workflowAction{} } + +func (s *Scheduler) actionMetadata() actionMetadata { + return implementation(s.Schedule.GetAction()).Metadata(s.Schedule.GetAction()) +} + +func (s *Scheduler) newBufferedExecution(start *schedulespb.BufferedStart, base string) { + action := implementation(s.Schedule.GetAction()) + id := action.GenerateTargetID(base, occurrenceContext{NominalTime: start.GetNominalTime().AsTime(), ScheduledTime: start.GetActualTime().AsTime(), Manual: start.GetManual(), RequestID: start.GetRequestId()}) + start.Execution = &commonpb.Execution{Type: action.Metadata(s.Schedule.GetAction()).Kind, BusinessId: id} + if start.Execution.Type == enumspb.EXECUTION_TYPE_WORKFLOW { + start.WorkflowId = id + } +} + +func (s *Scheduler) targetIDBase() string { return s.actionMetadata().IDBase } + +func validateActionKind(previous, next *schedulepb.ScheduleAction) error { + if implementation(previous).Metadata(previous).Kind != implementation(next).Metadata(next).Kind { + return serviceerror.NewInvalidArgument("schedule action kind cannot be changed") + } + return nil +} diff --git a/chasm/lib/scheduler/action_policy.go b/chasm/lib/scheduler/action_policy.go new file mode 100644 index 00000000000..30e2ca21542 --- /dev/null +++ b/chasm/lib/scheduler/action_policy.go @@ -0,0 +1,54 @@ +package scheduler + +import ( + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/chasm/lib/scheduler/internal" +) + +func policySelection(builtin enumspb.ScheduleOverlapPolicy, custom *schedulepb.CustomOverlapPolicy) internal.PolicyIdentity { + return internal.PolicyIdentity{Builtin: builtin, Custom: custom.GetName()} +} + +func actionPolicies(action *schedulepb.ScheduleAction) *internal.PolicyRegistry { + return implementation(action).Policies() +} + +// ValidateScheduleActionPolicies validates action-specific selections independently of legacy validation settings. +func ValidateScheduleActionPolicies(schedule *schedulepb.Schedule, patch *schedulepb.SchedulePatch) error { + return ValidateActionPolicyOverrides(schedule.GetAction(), schedule.GetPolicies(), patch) +} + +func ValidateActionPolicyOverrides(action *schedulepb.ScheduleAction, policies *schedulepb.SchedulePolicies, patch *schedulepb.SchedulePatch) error { + registry := actionPolicies(action) + configured := policySelection(policies.GetOverlapPolicy(), policies.GetCustomOverlapPolicy()) + if policies.GetCustomOverlapPolicy() != nil && (configured.Custom == "" || configured.Builtin != 0) { + return serviceerror.NewInvalidArgument("custom overlap policy requires a name and cannot be combined with overlap_policy") + } + if _, err := registry.Resolve(internal.PolicyIdentity{}, configured); err != nil { + return err + } + validate := func(builtin enumspb.ScheduleOverlapPolicy, custom *schedulepb.CustomOverlapPolicy) error { + if custom != nil && (custom.GetName() == "" || builtin != 0) { + return serviceerror.NewInvalidArgument("custom overlap policy requires a name and cannot be combined with overlap_policy") + } + _, err := registry.Resolve(policySelection(builtin, custom), configured) + return err + } + if trigger := patch.GetTriggerImmediately(); trigger != nil { + if err := validate(trigger.GetOverlapPolicy(), trigger.GetCustomOverlapPolicy()); err != nil { + return err + } + } + for _, backfill := range patch.GetBackfillRequest() { + if err := validate(backfill.GetOverlapPolicy(), backfill.GetCustomOverlapPolicy()); err != nil { + return err + } + } + return nil +} + +func (s *Scheduler) resolvedPolicy(builtin enumspb.ScheduleOverlapPolicy, custom *schedulepb.CustomOverlapPolicy) (internal.PolicyIdentity, error) { + return actionPolicies(s.Schedule.GetAction()).Resolve(policySelection(builtin, custom), policySelection(s.Schedule.GetPolicies().GetOverlapPolicy(), s.Schedule.GetPolicies().GetCustomOverlapPolicy())) +} diff --git a/chasm/lib/scheduler/action_results.go b/chasm/lib/scheduler/action_results.go new file mode 100644 index 00000000000..e0705500d26 --- /dev/null +++ b/chasm/lib/scheduler/action_results.go @@ -0,0 +1,19 @@ +package scheduler + +import ( + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" + "go.temporal.io/server/common" +) + +func actionResult(start *schedulespb.BufferedStart) *schedulepb.ScheduleActionResult { + result := &schedulepb.ScheduleActionResult{ScheduleTime: start.GetActualTime(), ActualTime: start.GetStartTime(), ActionExecutionResult: common.CloneProto(internal.ExecutionResult(start)), CloseTime: internal.CompletionTime(start)} + if result.ActionExecutionResult.Execution.GetType() == enumspb.EXECUTION_TYPE_WORKFLOW { + result.StartWorkflowResult = &commonpb.WorkflowExecution{WorkflowId: internal.TargetID(start), RunId: internal.RunID(start)} + result.StartWorkflowStatus = result.ActionExecutionResult.GetWorkflowStatus() + } + return result +} diff --git a/chasm/lib/scheduler/action_workflow.go b/chasm/lib/scheduler/action_workflow.go new file mode 100644 index 00000000000..bdeb4e57b0c --- /dev/null +++ b/chasm/lib/scheduler/action_workflow.go @@ -0,0 +1,105 @@ +package scheduler + +import ( + "context" + + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/api/workflowservice/v1" + "go.temporal.io/server/api/historyservice/v1" + persistencespb "go.temporal.io/server/api/persistence/v1" + "go.temporal.io/server/chasm/lib/scheduler/internal" +) + +type workflowAction struct{} + +func (workflowAction) Metadata(action *schedulepb.ScheduleAction) actionMetadata { + spec := action.GetStartWorkflow() + return actionMetadata{Kind: enumspb.EXECUTION_TYPE_WORKFLOW, Type: spec.GetWorkflowType().GetName(), IDBase: spec.GetWorkflowId(), TaskQueue: spec.GetTaskQueue().GetName(), SearchAttributes: spec.GetSearchAttributes()} +} +func (workflowAction) Validate(action *schedulepb.ScheduleAction) error { + if action.GetStartWorkflow() == nil { + return serviceerror.NewInvalidArgument("schedule must have an action") + } + return nil +} +func (workflowAction) Policies() *internal.PolicyRegistry { return internal.WorkflowPolicies() } +func (workflowAction) GenerateTargetID(base string, occurrence occurrenceContext) string { + return internal.GenerateTimestampTargetID(base, occurrence.NominalTime) +} +func (workflowAction) ParticipatesInCompletionHistory() bool { return true } +func (workflowAction) Start(ctx context.Context, clients actionClients, input actionStartInput) (string, error) { + scheduler, start, lastCompletionState := input.Scheduler, input.Occurrence, input.Previous + requestSpec := scheduler.Schedule.GetAction().GetStartWorkflow() + reusePolicy := enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE + if start.Manual { + reusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE + } + + tracksCompletionResult := internal.TracksCompletionResult(start.GetOverlapPolicy()) + var lcr []*commonpb.Payload + continuedFailure := lastCompletionState.Failure + if !tracksCompletionResult { + continuedFailure = nil + } + if tracksCompletionResult && lastCompletionState.Success != nil { + lcr = append(lcr, lastCompletionState.Success) + } + request := &workflowservice.StartWorkflowExecutionRequest{ + CompletionCallbacks: []*commonpb.Callback{input.Callback}, + Header: requestSpec.Header, + Identity: scheduler.identity(), + Input: requestSpec.Input, + Memo: requestSpec.Memo, + Namespace: scheduler.Namespace, + RequestId: start.RequestId, + RetryPolicy: requestSpec.RetryPolicy, + SearchAttributes: scheduler.startActionSearchAttributes(start.NominalTime.AsTime()), + TaskQueue: requestSpec.TaskQueue, + UserMetadata: requestSpec.UserMetadata, + WorkflowExecutionTimeout: requestSpec.WorkflowExecutionTimeout, + WorkflowId: internal.TargetID(start), + WorkflowIdReusePolicy: reusePolicy, + WorkflowRunTimeout: requestSpec.WorkflowRunTimeout, + WorkflowTaskTimeout: requestSpec.WorkflowTaskTimeout, + WorkflowType: requestSpec.WorkflowType, + Priority: requestSpec.Priority, + ContinuedFailure: continuedFailure, + LastCompletionResult: &commonpb.Payloads{ + Payloads: lcr, + }, + } + if input.EnableVersioningOverride { + request.VersioningOverride = requestSpec.VersioningOverride + } + + result, err := clients.Frontend.StartWorkflowExecution(ctx, request) + if err != nil { + return "", err + } + return result.RunId, nil +} +func (workflowAction) Cancel(ctx context.Context, clients actionClients, scheduler *Scheduler, target *commonpb.Execution) error { + _, err := clients.History.RequestCancelWorkflowExecution(ctx, &historyservice.RequestCancelWorkflowExecutionRequest{ + NamespaceId: scheduler.NamespaceId, CancelRequest: &workflowservice.RequestCancelWorkflowExecutionRequest{Namespace: scheduler.Namespace, WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: target.BusinessId}, Reason: "cancelled by schedule overlap policy", Identity: scheduler.identity(), FirstExecutionRunId: target.RunId}, + }) + return err +} +func (workflowAction) Terminate(ctx context.Context, clients actionClients, scheduler *Scheduler, target *commonpb.Execution) error { + _, err := clients.History.TerminateWorkflowExecution(ctx, &historyservice.TerminateWorkflowExecutionRequest{ + NamespaceId: scheduler.NamespaceId, TerminateRequest: &workflowservice.TerminateWorkflowExecutionRequest{Namespace: scheduler.Namespace, WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: target.BusinessId}, Reason: "terminated by schedule overlap policy", Identity: scheduler.identity(), FirstExecutionRunId: target.RunId}, + }) + return err +} +func (workflowAction) Completion(info *persistencespb.ChasmNexusCompletion, execution *commonpb.Execution) actionCompletion { + status := enumspb.WORKFLOW_EXECUTION_STATUS_FAILED + if _, ok := info.Outcome.(*persistencespb.ChasmNexusCompletion_Success); ok { + status = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED + } + if failure := info.GetFailure(); failure != nil { + status = executionStatusFromFailure(failure) + } + return actionCompletion{Result: &commonpb.ActionExecutionResult{Execution: execution, Status: &commonpb.ActionExecutionResult_WorkflowStatus{WorkflowStatus: status}}, Failed: countsAsFailureForPause(status)} +} diff --git a/chasm/lib/scheduler/backfiller_tasks.go b/chasm/lib/scheduler/backfiller_tasks.go index 082215cbfcc..911d70086a0 100644 --- a/chasm/lib/scheduler/backfiller_tasks.go +++ b/chasm/lib/scheduler/backfiller_tasks.go @@ -4,10 +4,10 @@ import ( "fmt" "time" + schedulepb "go.temporal.io/api/schedule/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" - schedulerinternal "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -191,7 +191,7 @@ func (b *BackfillerTaskHandler) processBackfill( startTime, endTime, request.GetOverlapPolicy(), - scheduler.WorkflowID(), + scheduler.targetIDBase(), backfiller.GetBackfillId(), true, &limit, @@ -209,6 +209,17 @@ func (b *BackfillerTaskHandler) processBackfill( result.LastProcessedTime = specResult.LastActionTime } result.BufferedStarts = specResult.BufferedStarts + policy, err := scheduler.resolvedPolicy(request.GetOverlapPolicy(), request.GetCustomOverlapPolicy()) + if err != nil { + return result, err + } + for _, start := range result.BufferedStarts { + start.OverlapPolicy = policy.Builtin + start.CustomOverlapPolicy = nil + if policy.Custom != "" { + start.CustomOverlapPolicy = &schedulepb.CustomOverlapPolicy{Name: policy.Custom} + } + } return } @@ -234,7 +245,11 @@ func (b *BackfillerTaskHandler) processTrigger( backfiller *Backfiller, ) (result backfillProgressResult, err error) { request := backfiller.GetTriggerRequest() - overlapPolicy := scheduler.resolveOverlapPolicy(request.GetOverlapPolicy()) + policy, err := scheduler.resolvedPolicy(request.GetOverlapPolicy(), request.GetCustomOverlapPolicy()) + if err != nil { + return result, err + } + overlapPolicy := policy.Builtin // Add a single manual start and mark the Backfiller as complete. For batch // backfill requests, a deterministic start time is trivial as they follow the @@ -244,7 +259,6 @@ func (b *BackfillerTaskHandler) processTrigger( nowpb := backfiller.GetLastProcessedTime() now := nowpb.AsTime() requestID := generateRequestID(scheduler, backfiller.GetBackfillId(), now, now) - workflowID := schedulerinternal.GenerateWorkflowID(scheduler.WorkflowID(), now) result.BufferedStarts = []*schedulespb.BufferedStart{ { NominalTime: nowpb, @@ -253,9 +267,14 @@ func (b *BackfillerTaskHandler) processTrigger( OverlapPolicy: overlapPolicy, Manual: true, RequestId: requestID, - WorkflowId: workflowID, }, } + for _, start := range result.BufferedStarts { + scheduler.newBufferedExecution(start, scheduler.targetIDBase()) + if policy.Custom != "" { + start.CustomOverlapPolicy = &schedulepb.CustomOverlapPolicy{Name: policy.Custom} + } + } result.Complete = true return diff --git a/chasm/lib/scheduler/buffer_planner.go b/chasm/lib/scheduler/buffer_planner.go index 3d7238358bf..49a978e3014 100644 --- a/chasm/lib/scheduler/buffer_planner.go +++ b/chasm/lib/scheduler/buffer_planner.go @@ -22,22 +22,24 @@ type appliedBufferPlan struct { func newBufferProcessingSnapshot(invoker *Invoker, scheduler *Scheduler, catchupWindow time.Duration) schedulerinternal.BufferProcessingSnapshot { state := scheduler.Schedule.GetState() snapshot := schedulerinternal.BufferProcessingSnapshot{ - Starts: make([]schedulerinternal.BufferedStartSnapshot, 0, len(invoker.GetBufferedStarts())), - DefaultOverlapPolicy: scheduler.overlapPolicy(), - CatchupWindow: catchupWindow, - MinimumCatchupWindow: startWorkflowMinDeadline, - Paused: state.GetPaused(), - LimitedActions: state.GetLimitedActions(), - RemainingActions: state.GetRemainingActions(), + Starts: make([]schedulerinternal.BufferedStartSnapshot, 0, len(invoker.GetBufferedStarts())), + DefaultOverlapPolicy: scheduler.Schedule.GetPolicies().GetOverlapPolicy(), + DefaultCustomOverlapPolicy: scheduler.Schedule.GetPolicies().GetCustomOverlapPolicy().GetName(), + Policies: actionPolicies(scheduler.Schedule.GetAction()), + CatchupWindow: catchupWindow, + MinimumCatchupWindow: startActionMinDeadline, + Paused: state.GetPaused(), + LimitedActions: state.GetLimitedActions(), + RemainingActions: state.GetRemainingActions(), } for index, start := range invoker.GetBufferedStarts() { projected := projectBufferedStart(start, index) snapshot.Starts = append(snapshot.Starts, projected) - if projected.RunID != "" && !projected.Completed && - schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { - snapshot.RunningWorkflows = append(snapshot.RunningWorkflows, schedulerinternal.WorkflowExecutionSnapshot{ - WorkflowID: projected.WorkflowID, - RunID: projected.RunID, + if (projected.RunID != "" || projected.Attempt > 0) && !projected.Completed && schedulerinternal.TracksExecution(start) { + snapshot.RunningExecutions = append(snapshot.RunningExecutions, schedulerinternal.ExecutionSnapshot{ + TargetID: projected.TargetID, + Kind: projected.Kind, + RunID: projected.RunID, }) } } @@ -46,16 +48,18 @@ func newBufferProcessingSnapshot(invoker *Invoker, scheduler *Scheduler, catchup func projectBufferedStart(start *schedulespb.BufferedStart, occurrence int) schedulerinternal.BufferedStartSnapshot { return schedulerinternal.BufferedStartSnapshot{ - Occurrence: occurrence, - RequestID: start.GetRequestId(), - WorkflowID: start.GetWorkflowId(), - RunID: start.GetRunId(), - Attempt: start.GetAttempt(), - Manual: start.GetManual(), - OverlapPolicy: start.GetOverlapPolicy(), - ActualTime: start.GetActualTime().AsTime(), - DesiredTime: start.GetDesiredTime().AsTime(), - Completed: start.GetCompleted() != nil, + Occurrence: occurrence, + RequestID: start.GetRequestId(), + TargetID: schedulerinternal.TargetID(start), + Kind: schedulerinternal.Execution(start).GetType(), + RunID: schedulerinternal.RunID(start), + Attempt: start.GetAttempt(), + Manual: start.GetManual(), + OverlapPolicy: start.GetOverlapPolicy(), + CustomOverlapPolicy: start.GetCustomOverlapPolicy().GetName(), + ActualTime: start.GetActualTime().AsTime(), + DesiredTime: start.GetDesiredTime().AsTime(), + Completed: schedulerinternal.IsCompleted(start), } } @@ -97,16 +101,17 @@ func applyBufferPlan( if applied.invalidatedDecisions == 0 { applied.result.overlapSkipped = plan.OverlapSkipped applied.result.overlapSkippedByPolicy = maps.Clone(plan.OverlapSkippedByPolicy) + applied.result.overlapSkippedByCustomPolicy = maps.Clone(plan.OverlapSkippedByCustomPolicy) } - for _, target := range plan.TerminateWorkflows { - if currentRunningWorkflow(invoker, target) { - applied.result.terminateWorkflows = append(applied.result.terminateWorkflows, workflowExecutionFromSnapshot(target)) + for _, target := range plan.TerminateExecutions { + if currentRunningExecution(invoker, target) { + applied.result.terminateExecutions = append(applied.result.terminateExecutions, executionFromSnapshot(target)) } } - for _, target := range plan.CancelWorkflows { - if currentRunningWorkflow(invoker, target) { - applied.result.cancelWorkflows = append(applied.result.cancelWorkflows, workflowExecutionFromSnapshot(target)) + for _, target := range plan.CancelExecutions { + if currentRunningExecution(invoker, target) { + applied.result.cancelExecutions = append(applied.result.cancelExecutions, executionFromSnapshot(target)) } } @@ -168,7 +173,7 @@ func applyBufferDecision(result *processBufferResult, decision schedulerinternal result.processedStarts[start] = true switch decision.Action { case schedulerinternal.BufferDecisionExecute: - result.startWorkflows = append(result.startWorkflows, start) + result.startActions = append(result.startActions, start) case schedulerinternal.BufferDecisionDiscard: result.discardStarts = append(result.discardStarts, start) default: @@ -191,24 +196,32 @@ func recordAppliedDecisionMetrics(result *processBufferResult, decision schedule func bufferProcessingSnapshotsEqual(left, right schedulerinternal.BufferProcessingSnapshot) bool { return left.DefaultOverlapPolicy == right.DefaultOverlapPolicy && + left.DefaultCustomOverlapPolicy == right.DefaultCustomOverlapPolicy && left.CatchupWindow == right.CatchupWindow && left.MinimumCatchupWindow == right.MinimumCatchupWindow && left.Paused == right.Paused && left.LimitedActions == right.LimitedActions && left.RemainingActions == right.RemainingActions && slices.Equal(left.Starts, right.Starts) && - slices.Equal(left.RunningWorkflows, right.RunningWorkflows) + slices.Equal(left.RunningExecutions, right.RunningExecutions) } -func currentRunningWorkflow(invoker *Invoker, target schedulerinternal.WorkflowExecutionSnapshot) bool { +func currentRunningExecution(invoker *Invoker, target schedulerinternal.ExecutionSnapshot) bool { + if target.RunID == "" { + return false + } for _, start := range invoker.GetBufferedStarts() { - if start.GetWorkflowId() == target.WorkflowID && start.GetRunId() == target.RunID && start.GetCompleted() == nil { + if schedulerinternal.TargetID(start) == target.TargetID && schedulerinternal.RunID(start) == target.RunID && !schedulerinternal.IsCompleted(start) { return true } } return false } -func workflowExecutionFromSnapshot(execution schedulerinternal.WorkflowExecutionSnapshot) *commonpb.WorkflowExecution { - return &commonpb.WorkflowExecution{WorkflowId: execution.WorkflowID, RunId: execution.RunID} +func executionFromSnapshot(execution schedulerinternal.ExecutionSnapshot) *commonpb.Execution { + kind := execution.Kind + if kind == 0 { + kind = enumspb.EXECUTION_TYPE_WORKFLOW + } + return &commonpb.Execution{Type: kind, BusinessId: execution.TargetID, RunId: execution.RunID} } diff --git a/chasm/lib/scheduler/buffer_processor_legacy_test.go b/chasm/lib/scheduler/buffer_processor_legacy_test.go index 06e2c1c1366..7a7b2ba00a3 100644 --- a/chasm/lib/scheduler/buffer_processor_legacy_test.go +++ b/chasm/lib/scheduler/buffer_processor_legacy_test.go @@ -4,6 +4,7 @@ import ( enumspb "go.temporal.io/api/enums/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" + "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common/util" legacyscheduler "go.temporal.io/server/service/worker/scheduler" ) @@ -15,6 +16,12 @@ func (h *InvokerProcessBufferTaskHandler) processBufferLegacy( ) (result processBufferResult) { runningWorkflows := invoker.runningWorkflowExecutions() isRunning := len(runningWorkflows) > 0 + // A selected start owns the overlap slot while its start RPC is in flight. + for _, start := range invoker.BufferedStarts { + if start.Attempt > 0 && !internal.IsCompleted(start) && internal.TracksExecution(start) { + isRunning = true + } + } result.missedCatchupByActionRunning = make(map[bool]int64) // Processing ignores starts that are already executing or backing off. An existing @@ -48,7 +55,7 @@ func (h *InvokerProcessBufferTaskHandler) processBufferLegacy( // checked before consumeScheduledAction so that a start past its catchup // window doesn't consume a LimitedActions slot. for _, start := range readyStarts { - deadline := h.startWorkflowDeadline(ctx, scheduler, start) + deadline := h.startActionDeadline(ctx, scheduler, start) if ctx.Now(invoker).After(deadline) { // Action was buffered in time but expired before execution // (e.g., due to overlap deferral, retries, or system delay). @@ -78,7 +85,7 @@ func (h *InvokerProcessBufferTaskHandler) processBufferLegacy( } keepStarts[start] = struct{}{} - result.startWorkflows = append(result.startWorkflows, start) + result.startActions = append(result.startActions, start) } result.discardStarts = util.FilterSlice(pendingBufferedStarts, func(start *schedulespb.BufferedStart) bool { @@ -88,9 +95,9 @@ func (h *InvokerProcessBufferTaskHandler) processBufferLegacy( // Terminate overrides cancel if both are requested. if action.NeedTerminate { - result.terminateWorkflows = runningWorkflows + result.terminateExecutions = workflowExecutions(runningWorkflows) } else if action.NeedCancel { - result.cancelWorkflows = runningWorkflows + result.cancelExecutions = workflowExecutions(runningWorkflows) } return diff --git a/chasm/lib/scheduler/execution_compatibility.go b/chasm/lib/scheduler/execution_compatibility.go new file mode 100644 index 00000000000..f5222bad200 --- /dev/null +++ b/chasm/lib/scheduler/execution_compatibility.go @@ -0,0 +1,43 @@ +package scheduler + +import ( + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" +) + +func workflowExecutions(executions []*commonpb.WorkflowExecution) []*commonpb.Execution { + result := make([]*commonpb.Execution, 0, len(executions)) + for _, execution := range executions { + result = append(result, &commonpb.Execution{Type: enumspb.EXECUTION_TYPE_WORKFLOW, BusinessId: execution.WorkflowId, RunId: execution.RunId}) + } + return result +} +func workflowProjection(executions []*commonpb.Execution) []*commonpb.WorkflowExecution { + var result []*commonpb.WorkflowExecution + for _, execution := range executions { + if execution.Type == enumspb.EXECUTION_TYPE_WORKFLOW { + result = append(result, &commonpb.WorkflowExecution{WorkflowId: execution.BusinessId, RunId: execution.RunId}) + } + } + return result +} +func (i *Invoker) terminationExecutions() []*commonpb.Execution { + if i.TerminateExecutions != nil { + return i.TerminateExecutions + } + return workflowExecutions(i.TerminateWorkflows) +} +func (i *Invoker) cancellationExecutions() []*commonpb.Execution { + if i.CancelExecutions != nil { + return i.CancelExecutions + } + return workflowExecutions(i.CancelWorkflows) +} +func (i *Invoker) setTerminationExecutions(executions []*commonpb.Execution) { + i.TerminateExecutions = executions + i.TerminateWorkflows = workflowProjection(executions) +} +func (i *Invoker) setCancellationExecutions(executions []*commonpb.Execution) { + i.CancelExecutions = executions + i.CancelWorkflows = workflowProjection(executions) +} diff --git a/chasm/lib/scheduler/export_test.go b/chasm/lib/scheduler/export_test.go index 98d1d887d62..dd8dc62a5d3 100644 --- a/chasm/lib/scheduler/export_test.go +++ b/chasm/lib/scheduler/export_test.go @@ -2,8 +2,6 @@ package scheduler import ( "context" - "fmt" - "slices" "time" schedulespb "go.temporal.io/server/api/schedule/v1" @@ -12,7 +10,6 @@ import ( schedulerinternal "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common/log" "go.temporal.io/server/common/metrics" - "go.temporal.io/server/common/util" queueerrors "go.temporal.io/server/service/history/queues/errors" legacyscheduler "go.temporal.io/server/service/worker/scheduler" ) @@ -60,18 +57,26 @@ func ContextWithTweakables(ctx chasm.Context, tweakables Tweakables) chasm.Conte // RecentActionCount exposes the completed-retention limit for tests. const RecentActionCount = recentActionCount +// ComputeLastEventTime exposes the non-monotonic, recompute-from-state value so +// tests can pin the regression it is subject to. func (s *Scheduler) ComputeLastEventTime(ctx chasm.Context) time.Time { return s.computeLastEventTime(ctx) } +// GetLastEventTimeFloored exposes the monotonic read path (recomputed value +// floored at the persisted high water mark). func (s *Scheduler) GetLastEventTimeFloored(ctx chasm.Context) time.Time { return s.getLastEventTime(ctx) } +// AdvanceLastEventTime exposes the high-water-mark write performed by the +// Generator tick. func (s *Scheduler) AdvanceLastEventTime(ctx chasm.MutableContext) time.Time { return s.advanceLastEventTime(ctx) } +// IdleDeadline exposes the deadline the idle task is armed against and that +// SchedulerIdleTaskHandler.Validate recomputes. func (s *Scheduler) IdleDeadline(ctx chasm.Context, idleTime time.Duration) time.Time { return s.idleDeadline(ctx, idleTime) } @@ -81,64 +86,43 @@ func (i *Invoker) ApplyCompletedRetention() { i.applyCompletedRetention() } -// RecordExecuteResult retains the pre-refactor request-ID matching behavior as -// a test-only oracle for existing event-log and race characterizations. +// RecordExecuteResult adapts the old test helper to the execution batch commit +// path. It intentionally matches starts by request ID because callers model +// the legacy frontend response shape. func (i *Invoker) RecordExecuteResult( ctx chasm.MutableContext, completed []*schedulespb.BufferedStart, retryable []*schedulespb.BufferedStart, ) (newlyStarted, droppedDuplicates int, startOnlyActions []*schedulespb.BufferedStart) { - completedByRequestID := make(map[string]*schedulespb.BufferedStart) - retryableByRequestID := make(map[string]*schedulespb.BufferedStart) - for _, start := range completed { - completedByRequestID[start.GetRequestId()] = start - } - for _, start := range retryable { - retryableByRequestID[start.GetRequestId()] = start - } - - retriedStarts := 0 - startedUntracked := make(map[string]struct{}) - var latestStartTime time.Time - for _, start := range i.GetBufferedStarts() { - if start.GetRunId() != "" { - if _, duplicate := completedByRequestID[start.GetRequestId()]; duplicate { - droppedDuplicates++ + result := executionBatchResult{} + for _, completion := range completed { + for index, start := range i.BufferedStarts { + if start.GetRequestId() == completion.GetRequestId() { + result.starts = append(result.starts, startExecutionResult{ + loaded: loadedBufferedStart{index: index, expected: start}, + outcome: startExecutionCompleted, + runID: completion.GetRunId(), + startTime: completion.GetStartTime(), + }) + break } - continue } - if completedStart, ok := completedByRequestID[start.GetRequestId()]; ok { - newlyStarted++ - latestStartTime = util.MaxTime(latestStartTime, completedStart.GetStartTime().AsTime()) - if !schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { - startOnlyActions = append(startOnlyActions, completedStart) - startedUntracked[start.GetRequestId()] = struct{}{} - continue + } + for _, retry := range retryable { + for index, start := range i.BufferedStarts { + if start.GetRequestId() == retry.GetRequestId() { + result.starts = append(result.starts, startExecutionResult{ + loaded: loadedBufferedStart{index: index, expected: start}, + outcome: startExecutionRetryable, + backoffTime: retry.GetBackoffTime(), + }) + break } - schedulerinternal.MarkStartStarted(start, completedStart.GetRunId(), completedStart.GetStartTime()) - start.HasCallback = true } - if retry, ok := retryableByRequestID[start.GetRequestId()]; ok { - schedulerinternal.MarkStartRetrying(start, start.GetAttempt()+1, retry.GetBackoffTime()) - retriedStarts++ - } - } - i.BufferedStarts = slices.DeleteFunc(i.BufferedStarts, func(start *schedulespb.BufferedStart) bool { - _, remove := startedUntracked[start.GetRequestId()] - return remove - }) - i.getOrCreateEventLog(ctx).LogEvent(ctx, - fmt.Sprintf("recordExecuteResult kicked off %d starts, removed 0 starts, retried %d starts", newlyStarted, retriedStarts)) - i.addTasks(ctx) - i.Scheduler.Get(ctx).advanceLastEventTimeTo(latestStartTime) - if newlyStarted > 0 { - i.Scheduler.Get(ctx).Generator.Get(ctx).Generate(ctx) } - return newlyStarted, droppedDuplicates, startOnlyActions -} - -func (s *Scheduler) RecordStartOnlyActions(ctx chasm.MutableContext, starts []*schedulespb.BufferedStart) { - s.recordStartOnlyActions(ctx, starts) + outcome := i.commitExecutionResult(ctx, result) + i.Scheduler.Get(ctx).advanceLastEventTimeTo(outcome.latestStartTime) + return outcome.appliedStarts, outcome.duplicateInvalidations, outcome.startOnlyActions } type ExecutionBatchForTest struct { @@ -222,6 +206,13 @@ func (h *InvokerProcessBufferTaskHandler) ExecuteProcessBufferLegacyForTest( return nil } +func (s *Scheduler) RecordStartOnlyActions( + ctx chasm.MutableContext, + starts []*schedulespb.BufferedStart, +) { + s.recordStartOnlyActions(ctx, starts) +} + func (b *BackfillerTaskHandler) ProcessBackfill( scheduler *Scheduler, backfiller *Backfiller, diff --git a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go index c1a61058df4..4cf52fadece 100644 --- a/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go +++ b/chasm/lib/scheduler/gen/schedulerpb/v1/message.pb.go @@ -305,9 +305,12 @@ type InvokerState struct { // generation will be consistent, regardless of when generation occurs, such // as after applying a replicated state (as opposed to evaluating based on // present time). - LastProcessedTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=last_processed_time,json=lastProcessedTime,proto3" json:"last_processed_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + LastProcessedTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=last_processed_time,json=lastProcessedTime,proto3" json:"last_processed_time,omitempty"` + CancelExecutions []*v12.Execution `protobuf:"bytes,7,rep,name=cancel_executions,json=cancelExecutions,proto3" json:"cancel_executions,omitempty"` + TerminateExecutions []*v12.Execution `protobuf:"bytes,8,rep,name=terminate_executions,json=terminateExecutions,proto3" json:"terminate_executions,omitempty"` + NextOccurrence int64 `protobuf:"varint,9,opt,name=next_occurrence,json=nextOccurrence,proto3" json:"next_occurrence,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InvokerState) Reset() { @@ -368,6 +371,27 @@ func (x *InvokerState) GetLastProcessedTime() *timestamppb.Timestamp { return nil } +func (x *InvokerState) GetCancelExecutions() []*v12.Execution { + if x != nil { + return x.CancelExecutions + } + return nil +} + +func (x *InvokerState) GetTerminateExecutions() []*v12.Execution { + if x != nil { + return x.TerminateExecutions + } + return nil +} + +func (x *InvokerState) GetNextOccurrence() int64 { + if x != nil { + return x.NextOccurrence + } + return 0 +} + // CHASM scheduler's Backfiller internal state. Backfill requests are 1:1 // with Backfiller nodes. Backfiller nodes also handle immediate trigger requests. type BackfillerState struct { @@ -768,12 +792,15 @@ const file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_rawDesc = "\x13pre_migration_notes\x18\x02 \x01(\tR\x11preMigrationNotes\"\xa8\x01\n" + "\x0eGeneratorState\x12J\n" + "\x13last_processed_time\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastProcessedTime\x12J\n" + - "\x13future_action_times\x18\x04 \x03(\v2\x1a.google.protobuf.TimestampR\x11futureActionTimes\"\xeb\x02\n" + + "\x13future_action_times\x18\x04 \x03(\v2\x1a.google.protobuf.TimestampR\x11futureActionTimes\"\xba\x04\n" + "\fInvokerState\x12W\n" + "\x0fbuffered_starts\x18\x02 \x03(\v2..temporal.server.api.schedule.v1.BufferedStartR\x0ebufferedStarts\x12T\n" + "\x10cancel_workflows\x18\x03 \x03(\v2).temporal.api.common.v1.WorkflowExecutionR\x0fcancelWorkflows\x12Z\n" + "\x13terminate_workflows\x18\x04 \x03(\v2).temporal.api.common.v1.WorkflowExecutionR\x12terminateWorkflows\x12J\n" + - "\x13last_processed_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastProcessedTimeJ\x04\b\x06\x10\a\"\xfa\x02\n" + + "\x13last_processed_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x11lastProcessedTime\x12N\n" + + "\x11cancel_executions\x18\a \x03(\v2!.temporal.api.common.v1.ExecutionR\x10cancelExecutions\x12T\n" + + "\x14terminate_executions\x18\b \x03(\v2!.temporal.api.common.v1.ExecutionR\x13terminateExecutions\x12'\n" + + "\x0fnext_occurrence\x18\t \x01(\x03R\x0enextOccurrenceJ\x04\b\x06\x10\a\"\xfa\x02\n" + "\x0fBackfillerState\x12V\n" + "\x10backfill_request\x18\x01 \x01(\v2).temporal.api.schedule.v1.BackfillRequestH\x00R\x0fbackfillRequest\x12^\n" + "\x0ftrigger_request\x18\x02 \x01(\v23.temporal.api.schedule.v1.TriggerImmediatelyRequestH\x00R\x0etriggerRequest\x12\x1f\n" + @@ -841,10 +868,11 @@ var file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_goTypes = [] (*timestamppb.Timestamp)(nil), // 14: google.protobuf.Timestamp (*v11.BufferedStart)(nil), // 15: temporal.server.api.schedule.v1.BufferedStart (*v12.WorkflowExecution)(nil), // 16: temporal.api.common.v1.WorkflowExecution - (*v1.BackfillRequest)(nil), // 17: temporal.api.schedule.v1.BackfillRequest - (*v1.TriggerImmediatelyRequest)(nil), // 18: temporal.api.schedule.v1.TriggerImmediatelyRequest - (*v12.Payload)(nil), // 19: temporal.api.common.v1.Payload - (*v13.Failure)(nil), // 20: temporal.api.failure.v1.Failure + (*v12.Execution)(nil), // 17: temporal.api.common.v1.Execution + (*v1.BackfillRequest)(nil), // 18: temporal.api.schedule.v1.BackfillRequest + (*v1.TriggerImmediatelyRequest)(nil), // 19: temporal.api.schedule.v1.TriggerImmediatelyRequest + (*v12.Payload)(nil), // 20: temporal.api.common.v1.Payload + (*v13.Failure)(nil), // 21: temporal.api.failure.v1.Failure } var file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_depIdxs = []int32{ 12, // 0: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState.schedule:type_name -> temporal.api.schedule.v1.Schedule @@ -858,28 +886,30 @@ var file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_depIdxs = [] 16, // 8: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.cancel_workflows:type_name -> temporal.api.common.v1.WorkflowExecution 16, // 9: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.terminate_workflows:type_name -> temporal.api.common.v1.WorkflowExecution 14, // 10: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.last_processed_time:type_name -> google.protobuf.Timestamp - 17, // 11: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.backfill_request:type_name -> temporal.api.schedule.v1.BackfillRequest - 18, // 12: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.trigger_request:type_name -> temporal.api.schedule.v1.TriggerImmediatelyRequest - 14, // 13: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.last_processed_time:type_name -> google.protobuf.Timestamp - 19, // 14: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.success:type_name -> temporal.api.common.v1.Payload - 20, // 15: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.failure:type_name -> temporal.api.failure.v1.Failure - 0, // 16: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.scheduler_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState - 2, // 17: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.generator_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState - 3, // 18: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.invoker_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.InvokerState - 9, // 19: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.backfillers:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry - 5, // 20: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.last_completion_result:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult - 10, // 21: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.search_attributes:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry - 11, // 22: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.memo:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry - 8, // 23: temporal.server.chasm.lib.scheduler.proto.v1.EventLog.events:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.Event - 14, // 24: temporal.server.chasm.lib.scheduler.proto.v1.Event.time:type_name -> google.protobuf.Timestamp - 4, // 25: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry.value:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState - 19, // 26: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload - 19, // 27: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry.value:type_name -> temporal.api.common.v1.Payload - 28, // [28:28] is the sub-list for method output_type - 28, // [28:28] is the sub-list for method input_type - 28, // [28:28] is the sub-list for extension type_name - 28, // [28:28] is the sub-list for extension extendee - 0, // [0:28] is the sub-list for field type_name + 17, // 11: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.cancel_executions:type_name -> temporal.api.common.v1.Execution + 17, // 12: temporal.server.chasm.lib.scheduler.proto.v1.InvokerState.terminate_executions:type_name -> temporal.api.common.v1.Execution + 18, // 13: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.backfill_request:type_name -> temporal.api.schedule.v1.BackfillRequest + 19, // 14: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.trigger_request:type_name -> temporal.api.schedule.v1.TriggerImmediatelyRequest + 14, // 15: temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState.last_processed_time:type_name -> google.protobuf.Timestamp + 20, // 16: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.success:type_name -> temporal.api.common.v1.Payload + 21, // 17: temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult.failure:type_name -> temporal.api.failure.v1.Failure + 0, // 18: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.scheduler_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerState + 2, // 19: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.generator_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.GeneratorState + 3, // 20: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.invoker_state:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.InvokerState + 9, // 21: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.backfillers:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry + 5, // 22: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.last_completion_result:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.LastCompletionResult + 10, // 23: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.search_attributes:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry + 11, // 24: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.memo:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry + 8, // 25: temporal.server.chasm.lib.scheduler.proto.v1.EventLog.events:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.Event + 14, // 26: temporal.server.chasm.lib.scheduler.proto.v1.Event.time:type_name -> google.protobuf.Timestamp + 4, // 27: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.BackfillersEntry.value:type_name -> temporal.server.chasm.lib.scheduler.proto.v1.BackfillerState + 20, // 28: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.SearchAttributesEntry.value:type_name -> temporal.api.common.v1.Payload + 20, // 29: temporal.server.chasm.lib.scheduler.proto.v1.SchedulerMigrationState.MemoEntry.value:type_name -> temporal.api.common.v1.Payload + 30, // [30:30] is the sub-list for method output_type + 30, // [30:30] is the sub-list for method input_type + 30, // [30:30] is the sub-list for extension type_name + 30, // [30:30] is the sub-list for extension extendee + 0, // [0:30] is the sub-list for field type_name } func init() { file_temporal_server_chasm_lib_scheduler_proto_v1_message_proto_init() } diff --git a/chasm/lib/scheduler/generator_tasks.go b/chasm/lib/scheduler/generator_tasks.go index 21993f02d26..d8eca4ce7b8 100644 --- a/chasm/lib/scheduler/generator_tasks.go +++ b/chasm/lib/scheduler/generator_tasks.go @@ -101,7 +101,7 @@ func (g *GeneratorTaskHandler) Execute( scheduler, t1, t2, scheduler.overlapPolicy(), - scheduler.WorkflowID(), + scheduler.targetIDBase(), "", false, limit, diff --git a/chasm/lib/scheduler/helper_test.go b/chasm/lib/scheduler/helper_test.go index a5994715f09..ae9e7f009cb 100644 --- a/chasm/lib/scheduler/helper_test.go +++ b/chasm/lib/scheduler/helper_test.go @@ -143,16 +143,19 @@ func newTestLibrary( // testEnv holds all components needed for scheduler tests. type testEnv struct { - t *testing.T - Ctrl *gomock.Controller - Registry *chasm.Registry - Node *chasm.Node - NodeBackend *chasm.MockNodeBackend - TimeSource *clock.EventTimeSource - Scheduler *scheduler.Scheduler - SpecProcessor scheduler.SpecProcessor - MockEngine *chasm.MockEngine - Logger log.Logger + t *testing.T + Ctrl *gomock.Controller + Registry *chasm.Registry + Node *chasm.Node + NodeBackend *chasm.MockNodeBackend + TimeSource *clock.EventTimeSource + Scheduler *scheduler.Scheduler + SpecProcessor scheduler.SpecProcessor + MockEngine *chasm.MockEngine + Logger log.Logger + + // allowStuckReason, when non-empty, suppresses the stuckness invariant + // asserted by CloseTransaction. See AllowStuck. allowStuckReason string } @@ -182,10 +185,18 @@ func withMockEngine() testEnvOption { } } +// withSchedule overrides defaultSchedule(), for tests that need a spec other +// than the package default 1-minute interval (e.g. an exhausted spec, which is +// what drives the Generator into its idle branch). func withSchedule(schedule *schedulepb.Schedule) testEnvOption { - return func(c *testEnvConfig) { c.schedule = schedule } + return func(c *testEnvConfig) { + c.schedule = schedule + } } +// expiredSchedule returns a schedule whose spec has already ended, so the +// Generator finds no next wakeup and takes its idle branch. This is the shape +// of a real schedule that has run to the end of its subscription window. func expiredSchedule(now time.Time) *schedulepb.Schedule { schedule := defaultSchedule() schedule.Spec.StartTime = timestamppb.New(now.Add(-2 * time.Hour)) @@ -248,6 +259,48 @@ func withEngineMetricsHandler(handler metrics.Handler) engineTestOption { } } +func newEngineTestConfig(opts ...engineTestOption) *engineTestConfig { + config := &engineTestConfig{} + for _, opt := range opts { + opt(config) + } + return config +} + +// newTestEngineContext builds a CHASM registry with the core and scheduler +// libraries registered, wraps it in a chasmtest.Engine, and returns the +// engine along with an engine-bound context ready for chasm.StartExecution / +// ReadComponent / etc. +func newTestEngineContext(t *testing.T, logger log.Logger, opts ...engineTestOption) (*chasmtest.Engine, context.Context) { + return newTestEngineContextFromConfig(t, logger, newEngineTestConfig(opts...)) +} + +func newTestEngineContextFromConfig( + t *testing.T, + logger log.Logger, + config *engineTestConfig, +) (*chasmtest.Engine, context.Context) { + specProcessor := config.specProcessor + if specProcessor == nil { + specProcessor = newRealSpecProcessor(gomock.NewController(t), logger) + } + + registry := chasm.NewRegistry(logger) + require.NoError(t, registry.Register(&chasm.CoreLibrary{})) + require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor, config.frontendClient))) + + config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( + func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { + requireValidSchedulerState(t, registry, node, root) + }, + )) + engine := chasmtest.NewEngine(t, registry, config.engineOpts...) + return engine, chasm.NewEngineContext(context.Background(), engine) +} + +// schedulerTestEngine bundles a CHASM test engine, a created Scheduler root +// component, and convenience accessors, for tests that drive a Scheduler +// through its actual task handlers rather than the newTestEnv rapid harness. type schedulerTestEngine struct { engine *chasmtest.Engine engineCtx context.Context @@ -256,6 +309,10 @@ type schedulerTestEngine struct { timeSource *clock.EventTimeSource } +// newSchedulerTestEngine builds a schedulerTestEngine and creates a schedule +// on it via the real CreateSchedule handler path. If no time source is +// supplied via withEngineTimeSource, a controllable one is created and +// wired in automatically. func newSchedulerTestEngine( t *testing.T, schedule *schedulepb.Schedule, @@ -294,6 +351,8 @@ func newSchedulerTestEngine( } } +// updateScheduler runs update against the Scheduler root component through +// the engine's UpdateComponent path. func (e *schedulerTestEngine) updateScheduler( update func(*scheduler.Scheduler, chasm.MutableContext) error, ) error { @@ -308,6 +367,8 @@ func (e *schedulerTestEngine) updateScheduler( return err } +// readScheduler runs read against the Scheduler root component through the +// engine's read-only ReadComponent path. func (e *schedulerTestEngine) readScheduler( read func(*scheduler.Scheduler, chasm.Context) error, ) error { @@ -322,70 +383,6 @@ func (e *schedulerTestEngine) readScheduler( return err } -type invokerExecuteEngine struct { - *schedulerTestEngine - handler *scheduler.InvokerExecuteTaskHandler - frontendClient *workflowservicemock.MockWorkflowServiceClient -} - -func newInvokerExecuteEngine(t *testing.T) *invokerExecuteEngine { - t.Helper() - - ctrl := gomock.NewController(t) - frontendClient := workflowservicemock.NewMockWorkflowServiceClient(ctrl) - env := newSchedulerTestEngine(t, defaultSchedule(), withEngineFrontendClient(frontendClient)) - handler := scheduler.NewInvokerExecuteTaskHandler(scheduler.InvokerTaskHandlerOptions{ - Config: defaultConfig(), - MetricsHandler: metrics.NoopMetricsHandler, - BaseLogger: env.logger, - FrontendClient: frontendClient, - }) - return &invokerExecuteEngine{ - schedulerTestEngine: env, - handler: handler, - frontendClient: frontendClient, - } -} - -func newEngineTestConfig(opts ...engineTestOption) *engineTestConfig { - config := &engineTestConfig{} - for _, opt := range opts { - opt(config) - } - return config -} - -// newTestEngineContext builds a CHASM registry with the core and scheduler -// libraries registered, wraps it in a chasmtest.Engine, and returns the -// engine along with an engine-bound context ready for chasm.StartExecution / -// ReadComponent / etc. -func newTestEngineContext(t *testing.T, logger log.Logger, opts ...engineTestOption) (*chasmtest.Engine, context.Context) { - return newTestEngineContextFromConfig(t, logger, newEngineTestConfig(opts...)) -} - -func newTestEngineContextFromConfig( - t *testing.T, - logger log.Logger, - config *engineTestConfig, -) (*chasmtest.Engine, context.Context) { - specProcessor := config.specProcessor - if specProcessor == nil { - specProcessor = newRealSpecProcessor(gomock.NewController(t), logger) - } - - registry := chasm.NewRegistry(logger) - require.NoError(t, registry.Register(&chasm.CoreLibrary{})) - require.NoError(t, registry.Register(newTestLibrary(logger, specProcessor, config.frontendClient))) - config.engineOpts = append(config.engineOpts, chasmtest.WithInvariantCheck( - func(t *testing.T, node *chasm.Node, root chasm.RootComponent) { - requireValidSchedulerState(t, registry, node, root) - }, - )) - - engine := chasmtest.NewEngine(t, registry, config.engineOpts...) - return engine, chasm.NewEngineContext(context.Background(), engine) -} - // newTestEnv creates a new test environment with the given options. func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { config := &testEnvConfig{} @@ -417,6 +414,11 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { now := time.Now() timeSource.Update(now) + schedule := config.schedule + if schedule == nil { + schedule = defaultSchedule() + } + tv := testvars.New(t) nodeBackend := &chasm.MockNodeBackend{ HandleNextTransitionCount: func() int64 { return 2 }, @@ -434,10 +436,6 @@ func newTestEnv(t *testing.T, opts ...testEnvOption) *testEnv { node := chasm.NewEmptyTree(registry, timeSource, nodeBackend, nodePathEncoder, logger, metrics.NoopMetricsHandler) ctx := chasm.NewMutableContext(context.Background(), node) - schedule := config.schedule - if schedule == nil { - schedule = defaultSchedule() - } sched, err := scheduler.NewScheduler(ctx, namespace, namespaceID, scheduleID, schedule, nil) if err != nil { t.Fatalf("failed to create scheduler: %v", err) @@ -484,8 +482,20 @@ func (e *testEnv) ReadContext() chasm.Context { return chasm.NewContext(context.Background(), e.Node) } +// CloseTransaction closes the current CHASM transaction and then asserts the +// stuckness invariant: a scheduler that is not in a terminal state must carry +// at least one live logical task, or nothing will ever wake it again. +// +// This runs on every test in the package that closes through testEnv, so the +// existing suite doubles as a stuckness detector at no extra cost. Tests that +// deliberately construct a state with no pending work must opt out explicitly +// via AllowStuck. +// +// The invariant is checked only on a successful close, since Node.Snapshot +// requires a clean tree. func (e *testEnv) CloseTransaction() error { e.t.Helper() + _, err := e.Node.CloseTransaction() if err != nil { return err @@ -497,10 +507,17 @@ func (e *testEnv) CloseTransaction() error { return nil } +// AllowStuck opts this test out of the stuckness invariant asserted by +// CloseTransaction, for tests that deliberately drive the scheduler into a +// state with no pending work. +// +// reason is mandatory: an unexplained opt-out is indistinguishable from a +// silently tolerated bug, and the set of tests that need one is itself a +// finding worth reviewing. func (e *testEnv) AllowStuck(reason string) { e.t.Helper() if reason == "" { - e.t.Fatal("AllowStuck requires a reason") + e.t.Fatal("AllowStuck requires a reason explaining why this test tolerates a stuck scheduler") } e.allowStuckReason = reason } @@ -661,3 +678,28 @@ func setupSentinelForTest(t *testing.T) (*scheduler.Scheduler, chasm.MutableCont ctx = chasm.NewMutableContext(context.Background(), infra.node) return sentinel, ctx, infra.node } + +type invokerExecuteEngine struct { + *schedulerTestEngine + handler *scheduler.InvokerExecuteTaskHandler + frontendClient *workflowservicemock.MockWorkflowServiceClient +} + +func newInvokerExecuteEngine(t *testing.T) *invokerExecuteEngine { + t.Helper() + + ctrl := gomock.NewController(t) + frontendClient := workflowservicemock.NewMockWorkflowServiceClient(ctrl) + env := newSchedulerTestEngine(t, defaultSchedule(), withEngineFrontendClient(frontendClient)) + handler := scheduler.NewInvokerExecuteTaskHandler(scheduler.InvokerTaskHandlerOptions{ + Config: defaultConfig(), + MetricsHandler: metrics.NoopMetricsHandler, + BaseLogger: env.logger, + FrontendClient: frontendClient, + }) + return &invokerExecuteEngine{ + schedulerTestEngine: env, + handler: handler, + frontendClient: frontendClient, + } +} diff --git a/chasm/lib/scheduler/internal/buffer_planner.go b/chasm/lib/scheduler/internal/buffer_planner.go index b915e959d50..6e17469c683 100644 --- a/chasm/lib/scheduler/internal/buffer_planner.go +++ b/chasm/lib/scheduler/internal/buffer_planner.go @@ -35,34 +35,40 @@ const ( // BufferedStartSnapshot is the value-only planner projection of a persisted BufferedStart. // Keeping protobuf pointers out of the planner prevents planning from mutating live CHASM state. type BufferedStartSnapshot struct { - Occurrence int - RequestID string - WorkflowID string - RunID string - Attempt int64 - Manual bool - OverlapPolicy enumspb.ScheduleOverlapPolicy - ActualTime time.Time - DesiredTime time.Time - Completed bool + Occurrence int + RequestID string + TargetID string + Kind enumspb.ExecutionType + RunID string + Attempt int64 + Manual bool + OverlapPolicy enumspb.ScheduleOverlapPolicy + CustomOverlapPolicy string + ActualTime time.Time + DesiredTime time.Time + Completed bool } -// WorkflowExecutionSnapshot identifies a running workflow without retaining a protobuf pointer. -type WorkflowExecutionSnapshot struct { - WorkflowID string - RunID string +// ExecutionSnapshot identifies a running workflow without retaining a protobuf pointer. +type ExecutionSnapshot struct { + TargetID string + Kind enumspb.ExecutionType + RunID string } // BufferProcessingSnapshot contains all state read by one buffer-planning pass. type BufferProcessingSnapshot struct { - Starts []BufferedStartSnapshot - RunningWorkflows []WorkflowExecutionSnapshot - DefaultOverlapPolicy enumspb.ScheduleOverlapPolicy - CatchupWindow time.Duration - MinimumCatchupWindow time.Duration - Paused bool - LimitedActions bool - RemainingActions int64 + Starts []BufferedStartSnapshot + RunningExecutions []ExecutionSnapshot + DefaultOverlapPolicy enumspb.ScheduleOverlapPolicy + DefaultCustomOverlapPolicy string + Policies *PolicyRegistry + CustomOverlapPolicy string + CatchupWindow time.Duration + MinimumCatchupWindow time.Duration + Paused bool + LimitedActions bool + RemainingActions int64 } // BufferDecision describes the planned outcome for one exact BufferedStart snapshot. @@ -86,22 +92,24 @@ func (d BufferDecision) MutatesState() bool { // BufferPlan is an ordered, value-based description of a buffer-processing pass. // Snapshot anchors whole-plan validation; each decision also carries its exact expected start. type BufferPlan struct { - Snapshot BufferProcessingSnapshot - Decisions []BufferDecision - CancelWorkflows []WorkflowExecutionSnapshot - TerminateWorkflows []WorkflowExecutionSnapshot - OverlapSkipped int64 - OverlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + Snapshot BufferProcessingSnapshot + Decisions []BufferDecision + CancelExecutions []ExecutionSnapshot + TerminateExecutions []ExecutionSnapshot + OverlapSkipped int64 + OverlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + OverlapSkippedByCustomPolicy map[string]int64 } type overlapAction struct { - OverlappingStarts []BufferedStartSnapshot - NonOverlappingStart BufferedStartSnapshot - NewBuffer []BufferedStartSnapshot - NeedCancel bool - NeedTerminate bool - OverlapSkipped int64 - OverlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + OverlappingStarts []BufferedStartSnapshot + NonOverlappingStart BufferedStartSnapshot + NewBuffer []BufferedStartSnapshot + NeedCancel bool + NeedTerminate bool + OverlapSkipped int64 + OverlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + OverlapSkippedByCustomPolicy map[string]int64 } // PlanBufferProcessing computes buffer outcomes without mutating snapshot or live scheduler state. @@ -122,13 +130,14 @@ func PlanBufferProcessing(snapshot BufferProcessingSnapshot, now time.Time) Buff } pending := pendingBufferedStarts(snapshot.Starts, resolveOverlapPolicy) - action := planOverlapActions(pending, len(snapshot.RunningWorkflows) > 0, resolveOverlapPolicy) + action := planRegisteredOverlapActions(pending, snapshot, now) plan.OverlapSkipped = action.OverlapSkipped plan.OverlapSkippedByPolicy = action.OverlapSkippedByPolicy + plan.OverlapSkippedByCustomPolicy = action.OverlapSkippedByCustomPolicy if action.NeedTerminate { - plan.TerminateWorkflows = append(plan.TerminateWorkflows, snapshot.RunningWorkflows...) + plan.TerminateExecutions = append(plan.TerminateExecutions, snapshot.RunningExecutions...) } else if action.NeedCancel { - plan.CancelWorkflows = append(plan.CancelWorkflows, snapshot.RunningWorkflows...) + plan.CancelExecutions = append(plan.CancelExecutions, snapshot.RunningExecutions...) } keep := make(map[BufferedStartSnapshot]struct{}, len(action.NewBuffer)+len(action.OverlappingStarts)+1) @@ -183,58 +192,6 @@ func PlanBufferProcessing(snapshot BufferProcessingSnapshot, now time.Time) Buff return plan } -func planOverlapActions( - buffer []BufferedStartSnapshot, - isRunning bool, - resolve func(enumspb.ScheduleOverlapPolicy) enumspb.ScheduleOverlapPolicy, -) overlapAction { - action := overlapAction{OverlapSkippedByPolicy: make(map[enumspb.ScheduleOverlapPolicy]int64)} - for _, start := range buffer { - overlapPolicy := resolve(start.OverlapPolicy) - if overlapPolicy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL { - action.OverlappingStarts = append(action.OverlappingStarts, start) - continue - } - if !isRunning && action.NonOverlappingStart.RequestID == "" { - action.NonOverlappingStart = start - continue - } - switch overlapPolicy { - case enumspb.SCHEDULE_OVERLAP_POLICY_SKIP: - action.OverlapSkipped++ - action.OverlapSkippedByPolicy[overlapPolicy]++ - case enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE: - if len(action.NewBuffer) == 0 { - action.NewBuffer = append(action.NewBuffer, start) - } else { - action.OverlapSkipped++ - action.OverlapSkippedByPolicy[overlapPolicy]++ - } - case enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL: - action.NewBuffer = append(action.NewBuffer, start) - case enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER: - if isRunning { - action.NeedCancel = true - action.NewBuffer = append(action.NewBuffer, start) - } else { - action.NonOverlappingStart = start - } - case enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER: - if isRunning { - action.NeedTerminate = true - action.NewBuffer = append(action.NewBuffer, start) - } else { - action.NonOverlappingStart = start - } - default: - } - } - if action.NeedCancel || action.NeedTerminate { - action.OverlappingStarts = nil - } - return action -} - func pendingBufferedStarts( starts []BufferedStartSnapshot, resolve func(enumspb.ScheduleOverlapPolicy) enumspb.ScheduleOverlapPolicy, @@ -254,7 +211,7 @@ func isPendingBufferedStart( ) bool { return start.Attempt == bufferedStartUnprocessedAttempt || (start.Attempt == bufferedStartDeferredAttempt && - resolve(start.OverlapPolicy) == enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE) + (resolve(start.OverlapPolicy) == enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE || start.CustomOverlapPolicy != "")) } func alreadyProcessedBufferDecision(start BufferedStartSnapshot) BufferDecision { @@ -290,7 +247,7 @@ func planReadyBufferDecision( decision.Reason = BufferDecisionReasonMissedCatchupWindow if canTakeScheduledAction { decision.MissedCatchupMetric = true - decision.MissedCatchupActionRunning = len(snapshot.RunningWorkflows) > 0 || start.DesiredTime.After(deadline) + decision.MissedCatchupActionRunning = len(snapshot.RunningExecutions) > 0 || start.DesiredTime.After(deadline) } return decision } @@ -323,6 +280,58 @@ func finalPendingBufferAction( func cloneBufferProcessingSnapshot(snapshot BufferProcessingSnapshot) BufferProcessingSnapshot { snapshot.Starts = slices.Clone(snapshot.Starts) - snapshot.RunningWorkflows = slices.Clone(snapshot.RunningWorkflows) + if snapshot.Policies != nil { + snapshot.Policies = snapshot.Policies.clone() + } + snapshot.RunningExecutions = slices.Clone(snapshot.RunningExecutions) return snapshot } + +func planRegisteredOverlapActions(buffer []BufferedStartSnapshot, snapshot BufferProcessingSnapshot, now time.Time) overlapAction { + registry := snapshot.Policies + if registry == nil { + registry = WorkflowPolicies() + } + action := overlapAction{OverlapSkippedByPolicy: make(map[enumspb.ScheduleOverlapPolicy]int64), OverlapSkippedByCustomPolicy: make(map[string]int64)} + recordSkip := func(start BufferedStartSnapshot) { + action.OverlapSkipped++ + if start.CustomOverlapPolicy != "" { + action.OverlapSkippedByCustomPolicy[start.CustomOverlapPolicy]++ + } else { + action.OverlapSkippedByPolicy[ResolveOverlapPolicy(start.OverlapPolicy, snapshot.DefaultOverlapPolicy)]++ + } + } + for _, start := range buffer { + id, err := registry.Resolve(PolicyIdentity{Builtin: start.OverlapPolicy, Custom: start.CustomOverlapPolicy}, PolicyIdentity{Builtin: snapshot.DefaultOverlapPolicy, Custom: snapshot.DefaultCustomOverlapPolicy}) + if err != nil { + recordSkip(start) + continue + } + var selected *BufferedStartSnapshot + if action.NonOverlappingStart.RequestID != "" { + value := action.NonOverlappingStart + selected = &value + } + decision := registry.plan(id, PolicySnapshot{Occurrence: start, Running: snapshot.RunningExecutions, Waiting: action.NewBuffer, Selected: selected, Now: now}) + for _, replaced := range decision.Replace { + action.NewBuffer = slices.DeleteFunc(action.NewBuffer, func(waiting BufferedStartSnapshot) bool { return waiting == replaced }) + recordSkip(replaced) + } + switch { + case decision.Start && decision.Overlap: + action.OverlappingStarts = append(action.OverlappingStarts, start) + case decision.Start: + action.NonOverlappingStart = start + case decision.Wait: + action.NewBuffer = append(action.NewBuffer, start) + default: + recordSkip(start) + } + action.NeedCancel = action.NeedCancel || decision.Cancel + action.NeedTerminate = action.NeedTerminate || decision.Terminate + } + if action.NeedCancel || action.NeedTerminate { + action.OverlappingStarts = nil + } + return action +} diff --git a/chasm/lib/scheduler/internal/buffer_planner_test.go b/chasm/lib/scheduler/internal/buffer_planner_test.go index 9cd71b1a02a..2b1c09feb4c 100644 --- a/chasm/lib/scheduler/internal/buffer_planner_test.go +++ b/chasm/lib/scheduler/internal/buffer_planner_test.go @@ -36,7 +36,7 @@ func TestPlanBufferProcessing_OverlapPolicies(t *testing.T) { ActualTime: now, DesiredTime: now, }}, - RunningWorkflows: []WorkflowExecutionSnapshot{{WorkflowID: "running", RunID: "run"}}, + RunningExecutions: []ExecutionSnapshot{{TargetID: "running", RunID: "run"}}, DefaultOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, CatchupWindow: time.Hour, MinimumCatchupWindow: 5 * time.Second, @@ -47,8 +47,8 @@ func TestPlanBufferProcessing_OverlapPolicies(t *testing.T) { require.Len(t, plan.Decisions, 1) require.Equal(t, test.action, plan.Decisions[0].Action) require.Equal(t, test.reason, plan.Decisions[0].Reason) - require.Len(t, plan.CancelWorkflows, test.cancelCount) - require.Len(t, plan.TerminateWorkflows, test.terminateCount) + require.Len(t, plan.CancelExecutions, test.cancelCount) + require.Len(t, plan.TerminateExecutions, test.terminateCount) require.Equal(t, test.overlapSkipped, plan.OverlapSkipped) }) } @@ -165,10 +165,10 @@ func TestPlanBufferProcessing_BufferOneDoesNotMergeDuplicateRequestIDs(t *testin now := time.Date(2026, 8, 19, 12, 0, 0, 0, time.UTC) snapshot := BufferProcessingSnapshot{ Starts: []BufferedStartSnapshot{ - {RequestID: "duplicate", WorkflowID: "first", Attempt: bufferedStartDeferredAttempt, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, ActualTime: now}, - {RequestID: "duplicate", WorkflowID: "second", Attempt: bufferedStartDeferredAttempt, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, ActualTime: now}, + {RequestID: "duplicate", TargetID: "first", Attempt: bufferedStartDeferredAttempt, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, ActualTime: now}, + {RequestID: "duplicate", TargetID: "second", Attempt: bufferedStartDeferredAttempt, OverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, ActualTime: now}, }, - RunningWorkflows: []WorkflowExecutionSnapshot{{WorkflowID: "running", RunID: "run"}}, + RunningExecutions: []ExecutionSnapshot{{TargetID: "running", RunID: "run"}}, DefaultOverlapPolicy: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, CatchupWindow: time.Hour, MinimumCatchupWindow: time.Second, diff --git a/chasm/lib/scheduler/internal/buffered_start.go b/chasm/lib/scheduler/internal/buffered_start.go index b60a3340ea2..4ad3baca4ee 100644 --- a/chasm/lib/scheduler/internal/buffered_start.go +++ b/chasm/lib/scheduler/internal/buffered_start.go @@ -3,6 +3,7 @@ package internal import ( "time" + enumspb "go.temporal.io/api/enums/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -35,8 +36,8 @@ func ClassifyBufferedStart( return BufferedStartStateInvalid } - hasRun := start.GetRunId() != "" - hasCompletion := start.GetCompleted() != nil + hasRun := RunID(start) != "" + hasCompletion := IsCompleted(start) hasBackoff := start.GetBackoffTime() != nil switch start.GetAttempt() { @@ -104,8 +105,18 @@ func MarkStartStarted( runID string, startTime *timestamppb.Timestamp, ) { - start.RunId = runID + if start.GetExecution() != nil { + start.Execution.RunId = runID + if start.Execution.GetType() == enumspb.EXECUTION_TYPE_WORKFLOW { + start.RunId = runID + } + } else { + start.RunId = runID + } start.StartTime = startTime + if start.GetCompletion() != nil { + start.Completion.Execution = Execution(start) + } } // MarkStartCompleted records a start's terminal workflow result. diff --git a/chasm/lib/scheduler/internal/execution.go b/chasm/lib/scheduler/internal/execution.go new file mode 100644 index 00000000000..c51f5d5e746 --- /dev/null +++ b/chasm/lib/scheduler/internal/execution.go @@ -0,0 +1,47 @@ +package internal + +import ( + commonpb "go.temporal.io/api/common/v1" + enumspb "go.temporal.io/api/enums/v1" + schedulespb "go.temporal.io/server/api/schedule/v1" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// Execution reads old workflow state without storing activity data in workflow fields. +func Execution(start *schedulespb.BufferedStart) *commonpb.Execution { + if start.GetExecution() != nil { + return start.GetExecution() + } + return &commonpb.Execution{Type: enumspb.EXECUTION_TYPE_WORKFLOW, BusinessId: start.GetWorkflowId(), RunId: start.GetRunId()} +} + +func RunID(start *schedulespb.BufferedStart) string { return Execution(start).GetRunId() } +func TargetID(start *schedulespb.BufferedStart) string { return Execution(start).GetBusinessId() } +func IsCompleted(start *schedulespb.BufferedStart) bool { + return start.GetCompletion() != nil || start.GetCompleted() != nil +} +func CompletionTime(start *schedulespb.BufferedStart) *timestamppb.Timestamp { + if start.GetCompletionTime() != nil { + return start.GetCompletionTime() + } + return start.GetCompleted().GetCloseTime() +} +func TracksExecution(start *schedulespb.BufferedStart) bool { + return start.GetOverlapPolicy() != enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL || start.GetCustomOverlapPolicy() != nil +} +func ExecutionResult(start *schedulespb.BufferedStart) *commonpb.ActionExecutionResult { + if start.GetCompletion() != nil { + return start.GetCompletion() + } + result := &commonpb.ActionExecutionResult{Execution: Execution(start)} + if result.Execution.Type == enumspb.EXECUTION_TYPE_ACTIVITY { + result.Status = &commonpb.ActionExecutionResult_ActivityStatus{ActivityStatus: enumspb.ACTIVITY_EXECUTION_STATUS_RUNNING} + } else { + status := enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING + if start.GetCompleted() != nil { + status = start.GetCompleted().GetStatus() + } + result.Status = &commonpb.ActionExecutionResult_WorkflowStatus{WorkflowStatus: status} + } + return result +} diff --git a/chasm/lib/scheduler/internal/policy.go b/chasm/lib/scheduler/internal/policy.go new file mode 100644 index 00000000000..52d96efc5af --- /dev/null +++ b/chasm/lib/scheduler/internal/policy.go @@ -0,0 +1,150 @@ +package internal + +import ( + "fmt" + "maps" + "slices" + "time" + + enumspb "go.temporal.io/api/enums/v1" + "go.temporal.io/api/serviceerror" + "go.temporal.io/server/common/log" + "go.temporal.io/server/common/log/tag" +) + +const BufferLatestPolicyName = "temporal.buffer_latest" + +type PolicyIdentity struct { + Builtin enumspb.ScheduleOverlapPolicy + Custom string +} + +func (p PolicyIdentity) String() string { + if p.Custom != "" { + return p.Custom + } + return p.Builtin.String() +} + +func (p PolicyIdentity) IsZero() bool { return p.Builtin == 0 && p.Custom == "" } + +type ExecutionOperations struct{ Cancel, Terminate bool } + +type PolicySnapshot struct { + Occurrence BufferedStartSnapshot + Running []ExecutionSnapshot + Waiting []BufferedStartSnapshot + Selected *BufferedStartSnapshot + Now time.Time +} + +type PolicyDecision struct { + Start, Overlap, Wait, Cancel, Terminate bool + Replace []BufferedStartSnapshot +} + +type PolicyDefinition struct { + Identity PolicyIdentity + Requires ExecutionOperations + Plan func(PolicySnapshot) PolicyDecision +} + +type PolicyRegistry struct { + policies map[PolicyIdentity]PolicyDefinition + defaultPolicy PolicyIdentity +} + +func NewPolicyRegistry(definitions []PolicyDefinition, defaultPolicy PolicyIdentity, operations ExecutionOperations) (*PolicyRegistry, error) { + r := &PolicyRegistry{policies: make(map[PolicyIdentity]PolicyDefinition), defaultPolicy: defaultPolicy} + for _, policy := range definitions { + id := policy.Identity + if id.IsZero() || (id.Builtin != 0 && id.Custom != "") || policy.Plan == nil { + return nil, fmt.Errorf("invalid overlap policy registration %s", id) + } + if _, ok := r.policies[id]; ok { + return nil, fmt.Errorf("duplicate overlap policy %s", id) + } + if policy.Requires.Cancel && !operations.Cancel || policy.Requires.Terminate && !operations.Terminate { + return nil, fmt.Errorf("overlap policy %s requires unsupported execution operations", id) + } + r.policies[id] = policy + } + if !defaultPolicy.IsZero() { + if _, ok := r.policies[defaultPolicy]; !ok { + return nil, fmt.Errorf("unregistered default overlap policy %s", defaultPolicy) + } + } + return r, nil +} + +func (r *PolicyRegistry) Resolve(override, configured PolicyIdentity) (PolicyIdentity, error) { + for _, id := range []PolicyIdentity{override, configured} { + if id.Builtin != 0 && id.Custom != "" { + return PolicyIdentity{}, serviceerror.NewInvalidArgument("only one overlap policy selector may be set") + } + } + id := override + if id.IsZero() { + id = configured + } + if id.IsZero() { + id = r.defaultPolicy + } + if _, ok := r.policies[id]; !ok { + return PolicyIdentity{}, serviceerror.NewInvalidArgumentf("unsupported or missing overlap policy %s", id) + } + return id, nil +} + +func (r *PolicyRegistry) plan(id PolicyIdentity, snapshot PolicySnapshot) PolicyDecision { + snapshot.Running = slices.Clone(snapshot.Running) + snapshot.Waiting = slices.Clone(snapshot.Waiting) + if snapshot.Selected != nil { + selected := *snapshot.Selected + snapshot.Selected = &selected + } + return r.policies[id].Plan(snapshot) +} + +func BuiltinPolicy(policy enumspb.ScheduleOverlapPolicy) PolicyDefinition { + return PolicyDefinition{ + Identity: PolicyIdentity{Builtin: policy}, + Requires: ExecutionOperations{Cancel: policy == enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, Terminate: policy == enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER}, + Plan: func(s PolicySnapshot) PolicyDecision { + if policy == enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL { + return PolicyDecision{Start: true, Overlap: true} + } + if len(s.Running) == 0 && s.Selected == nil { + return PolicyDecision{Start: true} + } + switch policy { + case enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE: + return PolicyDecision{Wait: len(s.Waiting) == 0} + case enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL: + return PolicyDecision{Wait: true} + case enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER: + return PolicyDecision{Start: len(s.Running) == 0, Wait: len(s.Running) > 0, Cancel: len(s.Running) > 0} + case enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER: + return PolicyDecision{Start: len(s.Running) == 0, Wait: len(s.Running) > 0, Terminate: len(s.Running) > 0} + default: + return PolicyDecision{} + } + }, + } +} + +func WorkflowPolicies() *PolicyRegistry { + definitions := make([]PolicyDefinition, 0, 6) + for _, policy := range []enumspb.ScheduleOverlapPolicy{enumspb.SCHEDULE_OVERLAP_POLICY_SKIP, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ONE, enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL, enumspb.SCHEDULE_OVERLAP_POLICY_CANCEL_OTHER, enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER, enumspb.SCHEDULE_OVERLAP_POLICY_ALLOW_ALL} { + definitions = append(definitions, BuiltinPolicy(policy)) + } + r, err := NewPolicyRegistry(definitions, PolicyIdentity{Builtin: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP}, ExecutionOperations{Cancel: true, Terminate: true}) + if err != nil { + log.NewCLILogger().Fatal("invalid scheduler overlap policy registry", tag.Error(err)) + } + return r +} + +func (r *PolicyRegistry) clone() *PolicyRegistry { + return &PolicyRegistry{policies: maps.Clone(r.policies), defaultPolicy: r.defaultPolicy} +} diff --git a/chasm/lib/scheduler/internal/policy_test.go b/chasm/lib/scheduler/internal/policy_test.go new file mode 100644 index 00000000000..f8846b9d213 --- /dev/null +++ b/chasm/lib/scheduler/internal/policy_test.go @@ -0,0 +1,40 @@ +package internal + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + enumspb "go.temporal.io/api/enums/v1" +) + +func TestPolicyRegistrySubsetWithoutDefault(t *testing.T) { + only := PolicyIdentity{Builtin: enumspb.SCHEDULE_OVERLAP_POLICY_BUFFER_ALL} + registry, err := NewPolicyRegistry([]PolicyDefinition{BuiltinPolicy(only.Builtin)}, PolicyIdentity{}, ExecutionOperations{}) + require.NoError(t, err) + _, err = registry.Resolve(PolicyIdentity{}, PolicyIdentity{}) + require.Error(t, err) + selected, err := registry.Resolve(PolicyIdentity{}, only) + require.NoError(t, err) + require.Equal(t, only, selected) + _, err = registry.Resolve(PolicyIdentity{Builtin: enumspb.SCHEDULE_OVERLAP_POLICY_SKIP}, only) + require.Error(t, err) + _, err = NewPolicyRegistry([]PolicyDefinition{BuiltinPolicy(enumspb.SCHEDULE_OVERLAP_POLICY_TERMINATE_OTHER)}, PolicyIdentity{}, ExecutionOperations{}) + require.Error(t, err) + _, err = NewPolicyRegistry([]PolicyDefinition{BuiltinPolicy(only.Builtin)}, PolicyIdentity{Custom: "unknown"}, ExecutionOperations{}) + require.Error(t, err) +} + +func TestPolicyPlannerReceivesDetachedSnapshots(t *testing.T) { + id := PolicyIdentity{Custom: "test.only"} + registry, err := NewPolicyRegistry([]PolicyDefinition{{Identity: id, Plan: func(snapshot PolicySnapshot) PolicyDecision { + snapshot.Running[0].RunID = "changed" + return PolicyDecision{Wait: true} + }}}, id, ExecutionOperations{}) + require.NoError(t, err) + snapshot := BufferProcessingSnapshot{Policies: registry, Starts: []BufferedStartSnapshot{{RequestID: "pending"}}, RunningExecutions: []ExecutionSnapshot{{RunID: "original"}}} + plan := PlanBufferProcessing(snapshot, time.Time{}) + require.Equal(t, "original", snapshot.RunningExecutions[0].RunID) + require.Equal(t, "original", plan.Snapshot.RunningExecutions[0].RunID) + require.Equal(t, BufferDecisionDefer, plan.Decisions[0].Action) +} diff --git a/chasm/lib/scheduler/internal/request_id.go b/chasm/lib/scheduler/internal/request_id.go index d84c29abd2a..ebb3245eaf3 100644 --- a/chasm/lib/scheduler/internal/request_id.go +++ b/chasm/lib/scheduler/internal/request_id.go @@ -41,11 +41,11 @@ func GenerateRequestID( ) } -// GenerateWorkflowID generates a deterministic workflow ID for a buffered -// action by combining the base workflow ID with the truncated nominal time. -func GenerateWorkflowID(baseWorkflowID string, nominalTime time.Time) string { +// GenerateTimestampTargetID generates a deterministic target ID for a buffered +// action by combining the base target ID with the truncated nominal time. +func GenerateTimestampTargetID(baseTargetID string, nominalTime time.Time) string { nominalTimeSec := nominalTime.Truncate(time.Second) - return fmt.Sprintf("%s-%s", baseWorkflowID, nominalTimeSec.UTC().Format(time.RFC3339)) + return fmt.Sprintf("%s-%s", baseTargetID, nominalTimeSec.UTC().Format(time.RFC3339)) } // GenerateBackfillerID generates a unique ID for a Backfiller component. diff --git a/chasm/lib/scheduler/internal/request_id_test.go b/chasm/lib/scheduler/internal/request_id_test.go index 2739ce3fc49..90ea955f1ea 100644 --- a/chasm/lib/scheduler/internal/request_id_test.go +++ b/chasm/lib/scheduler/internal/request_id_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" ) -func TestGenerateWorkflowID(t *testing.T) { - baseWorkflowID := "my-workflow" +func TestGenerateTimestampTargetID(t *testing.T) { + baseTargetID := "my-workflow" nominalTime := time.Date(2024, 6, 15, 10, 30, 45, 123456789, time.UTC) - actual := GenerateWorkflowID(baseWorkflowID, nominalTime) + actual := GenerateTimestampTargetID(baseTargetID, nominalTime) require.Equal(t, "my-workflow-2024-06-15T10:30:45Z", actual) } diff --git a/chasm/lib/scheduler/invoker.go b/chasm/lib/scheduler/invoker.go index da8f5126d84..ac5bcc5e5dc 100644 --- a/chasm/lib/scheduler/invoker.go +++ b/chasm/lib/scheduler/invoker.go @@ -73,6 +73,12 @@ func newInvokerWithState(ctx chasm.MutableContext, state *schedulerpb.InvokerSta // EnqueueBufferedStarts adds new BufferedStarts to the invocation queue, // immediately kicking off a processing task. func (i *Invoker) EnqueueBufferedStarts(ctx chasm.MutableContext, starts []*schedulespb.BufferedStart) { + for _, start := range starts { + if start.GetOccurrenceId() == "" { + i.NextOccurrence++ + start.OccurrenceId = fmt.Sprintf("%d", i.NextOccurrence) + } + } i.BufferedStarts = append(i.BufferedStarts, starts...) if len(starts) > 0 { i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("enqueued %d buffered start(s)", len(starts))) @@ -81,16 +87,17 @@ func (i *Invoker) EnqueueBufferedStarts(ctx chasm.MutableContext, starts []*sche } type processBufferResult struct { - startWorkflows []*schedulespb.BufferedStart - cancelWorkflows []*commonpb.WorkflowExecution - terminateWorkflows []*commonpb.WorkflowExecution + startActions []*schedulespb.BufferedStart + cancelExecutions []*commonpb.Execution + terminateExecutions []*commonpb.Execution // discardStarts will be dropped from the Invoker's BufferedStarts without execution. discardStarts []*schedulespb.BufferedStart // Number of buffered starts dropped due to overlap policy during processing. - overlapSkipped int64 - overlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + overlapSkipped int64 + overlapSkippedByPolicy map[enumspb.ScheduleOverlapPolicy]int64 + overlapSkippedByCustomPolicy map[string]int64 // Number of buffered starts dropped from missing the catchup window, // bucketed by whether a running action contributed to the miss. @@ -110,7 +117,7 @@ func (i *Invoker) recordProcessBufferResult(ctx chasm.MutableContext, result *pr for _, start := range result.discardStarts { discards[start] = true } - for _, start := range result.startWorkflows { + for _, start := range result.startActions { ready[start] = true } @@ -146,8 +153,8 @@ func (i *Invoker) recordProcessBufferResult(ctx chasm.MutableContext, result *pr // Update internal state. i.BufferedStarts = starts - i.CancelWorkflows = append(i.GetCancelWorkflows(), result.cancelWorkflows...) - i.TerminateWorkflows = append(i.GetTerminateWorkflows(), result.terminateWorkflows...) + i.setCancellationExecutions(append(i.cancellationExecutions(), result.cancelExecutions...)) + i.setTerminationExecutions(append(i.terminationExecutions(), result.terminateExecutions...)) i.LastProcessedTime = timestamppb.New(ctx.Now(i)) // Re-arm tasks if this call changed state, or if the LastProcessedTime advance @@ -159,7 +166,7 @@ func (i *Invoker) recordProcessBufferResult(ctx chasm.MutableContext, result *pr // outstanding request. func (i *Invoker) runningWorkflowID(requestID string) string { for _, start := range i.GetBufferedStarts() { - if start.GetRequestId() == requestID && start.GetCompleted() == nil { + if start.GetRequestId() == requestID && !schedulerinternal.IsCompleted(start) { return start.GetWorkflowId() } } @@ -183,7 +190,7 @@ func (i *Invoker) recordCompletedAction( for _, start := range i.BufferedStarts { if start.GetRequestId() == requestID { scheduleTime = start.DesiredTime.AsTime() - if !schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { + if !schedulerinternal.TracksExecution(start) && schedulerinternal.RunID(start) != "" { i.Scheduler.Get(ctx).recordRecentAction(start, completed.GetStatus()) completedUntracked = requestID } else { @@ -198,14 +205,15 @@ func (i *Invoker) recordCompletedAction( }) } + i.releaseWaiting(ctx, completed.GetCloseTime()) + return +} + +func (i *Invoker) releaseWaiting(ctx chasm.MutableContext, closeTime *timestamppb.Timestamp) { // Re-enable deferred starts (Attempt == -1) so they can be re-processed by // ProcessBuffer now that a workflow has completed. This allows the overlap // policy to be re-evaluated. - for _, start := range i.BufferedStarts { - if start.Attempt == -1 { - schedulerinternal.MarkStartUnprocessed(start) - } - } + i.reconsiderWaiting() // Update DesiredTime on the first pending start for metrics. DesiredTime is used // to drive action latency between buffered starts (the time it takes between @@ -217,7 +225,7 @@ func (i *Invoker) recordCompletedAction( return start.Attempt == 0 }) if idx >= 0 { - i.BufferedStarts[idx].DesiredTime = timestamppb.New(completed.GetCloseTime().AsTime()) + i.BufferedStarts[idx].DesiredTime = timestamppb.New(closeTime.AsTime()) } // Apply retention to keep only the last N completed actions. @@ -227,7 +235,14 @@ func (i *Invoker) recordCompletedAction( // kick-off. i.addTasks(ctx) - return +} + +func (i *Invoker) reconsiderWaiting() { + for _, start := range i.BufferedStarts { + if start.Attempt == -1 { + schedulerinternal.MarkStartUnprocessed(start) + } + } } // addTasks adds both ProcessBuffer and Execute tasks as needed. It should be @@ -257,8 +272,8 @@ func (i *Invoker) addTasks(ctx chasm.MutableContext) { } func (i *Invoker) hasExecutableWork() bool { - return len(i.GetCancelWorkflows()) > 0 || - len(i.GetTerminateWorkflows()) > 0 || + return len(i.cancellationExecutions()) > 0 || + len(i.terminationExecutions()) > 0 || len(i.getEligibleBufferedStarts()) > 0 } @@ -282,8 +297,8 @@ func (i *Invoker) nextBackoffDeadline() time.Time { backoff := start.GetBackoffTime().AsTime() // We only care about starts that are retrying. if start.GetAttempt() <= 0 || - start.GetRunId() != "" || - start.GetCompleted() != nil || + schedulerinternal.RunID(start) != "" || + schedulerinternal.IsCompleted(start) || // Backed-off starts will be selected by getEligibleBufferedStarts and kick off // an Execute task, instead. start.BackoffTime.AsTime().Before(lastProcessedTime) { @@ -304,7 +319,7 @@ func (i *Invoker) getEligibleBufferedStarts() []*schedulespb.BufferedStart { lastProcessed := i.GetLastProcessedTime().AsTime() return util.FilterSlice(i.GetBufferedStarts(), func(start *schedulespb.BufferedStart) bool { return start.Attempt > 0 && - start.GetRunId() == "" && + schedulerinternal.RunID(start) == "" && !start.GetBackoffTime().AsTime().After(lastProcessed) }) } @@ -312,17 +327,7 @@ func (i *Invoker) getEligibleBufferedStarts() []*schedulespb.BufferedStart { // runningWorkflowExecutions returns the list of workflow executions that // have been started but not yet completed. func (i *Invoker) runningWorkflowExecutions() []*commonpb.WorkflowExecution { - var running []*commonpb.WorkflowExecution - for _, start := range i.GetBufferedStarts() { - if start.GetRunId() != "" && start.GetCompleted() == nil && - schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { - running = append(running, &commonpb.WorkflowExecution{ - WorkflowId: start.GetWorkflowId(), - RunId: start.GetRunId(), - }) - } - } - return running + return workflowProjection(i.runningExecutions()) } // recentActions combines stored start-only actions with completion-tracked actions @@ -330,26 +335,18 @@ func (i *Invoker) runningWorkflowExecutions() []*commonpb.WorkflowExecution { func (i *Invoker) recentActions(storedActions []*schedulepb.ScheduleActionResult) []*schedulepb.ScheduleActionResult { results := make([]*schedulepb.ScheduleActionResult, 0, len(storedActions)+len(i.GetBufferedStarts())) for _, action := range storedActions { - results = append(results, common.CloneProto(action)) + projection := common.CloneProto(action) + if projection.GetActionExecutionResult() == nil && projection.GetStartWorkflowResult() != nil { + projection.ActionExecutionResult = &commonpb.ActionExecutionResult{Execution: &commonpb.Execution{Type: enumspb.EXECUTION_TYPE_WORKFLOW, BusinessId: projection.StartWorkflowResult.WorkflowId, RunId: projection.StartWorkflowResult.RunId}, Status: &commonpb.ActionExecutionResult_WorkflowStatus{WorkflowStatus: projection.StartWorkflowStatus}} + } + results = append(results, projection) } for _, start := range i.GetBufferedStarts() { // Only include workflows that have been started (have a RunId). - if start.GetRunId() == "" { + if schedulerinternal.RunID(start) == "" { continue } - status := enumspb.WORKFLOW_EXECUTION_STATUS_RUNNING - if start.GetCompleted() != nil { - status = start.GetCompleted().GetStatus() - } - results = append(results, &schedulepb.ScheduleActionResult{ - ScheduleTime: start.GetActualTime(), - ActualTime: start.GetStartTime(), - StartWorkflowResult: &commonpb.WorkflowExecution{ - WorkflowId: start.GetWorkflowId(), - RunId: start.GetRunId(), - }, - StartWorkflowStatus: status, - }) + results = append(results, actionResult(start)) } slices.SortFunc(results, func(a, b *schedulepb.ScheduleActionResult) int { return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) @@ -357,12 +354,18 @@ func (i *Invoker) recentActions(storedActions []*schedulepb.ScheduleActionResult return util.SliceTail(results, recentActionCount) } -// bufferedStartsCount returns starts whose successful StartWorkflowExecution -// result has not been recorded. Started and retained starts are not buffered. +// bufferedStartsCount returns the actions whose successful StartWorkflowExecution +// result has not yet been recorded. BufferedStarts also retains running and completed +// actions for lifecycle tracking and history, so its length is not the API buffer size. +// This preserves V1's distinction: V1 removes selected starts from BufferedStarts +// before recording them as running or recent, while CHASM uses a recorded RunId as +// the durable boundary between those states. +// Count starts without a RunId directly because recent actions include start-only +// ALLOW_ALL records stored outside BufferedStarts and are capped independently. func (i *Invoker) bufferedStartsCount() int { count := 0 for _, start := range i.GetBufferedStarts() { - if start.GetRunId() == "" { + if schedulerinternal.RunID(start) == "" { count++ } } @@ -376,7 +379,7 @@ func (i *Invoker) applyCompletedRetention() { var nonCompleted []*schedulespb.BufferedStart for _, start := range i.BufferedStarts { - if start.GetCompleted() != nil { + if schedulerinternal.IsCompleted(start) && (schedulerinternal.RunID(start) != "" || start.GetAttempt() <= 0) { completed = append(completed, start) } else { nonCompleted = append(nonCompleted, start) @@ -385,7 +388,7 @@ func (i *Invoker) applyCompletedRetention() { // Sort by oldest first. slices.SortFunc(completed, func(a, b *schedulespb.BufferedStart) int { - return a.GetCompleted().GetCloseTime().AsTime().Compare(b.GetCompleted().GetCloseTime().AsTime()) + return schedulerinternal.CompletionTime(a).AsTime().Compare(schedulerinternal.CompletionTime(b).AsTime()) }) if len(completed) > recentActionCount { @@ -394,3 +397,13 @@ func (i *Invoker) applyCompletedRetention() { i.BufferedStarts = append(nonCompleted, completed...) } + +func (i *Invoker) runningExecutions() []*commonpb.Execution { + var running []*commonpb.Execution + for _, start := range i.GetBufferedStarts() { + if schedulerinternal.RunID(start) != "" && !schedulerinternal.IsCompleted(start) && schedulerinternal.TracksExecution(start) { + running = append(running, common.CloneProto(schedulerinternal.Execution(start))) + } + } + return running +} diff --git a/chasm/lib/scheduler/invoker_execute_phase_test.go b/chasm/lib/scheduler/invoker_execute_phase_test.go index 0ec4d29d315..24755deb45b 100644 --- a/chasm/lib/scheduler/invoker_execute_phase_test.go +++ b/chasm/lib/scheduler/invoker_execute_phase_test.go @@ -101,7 +101,7 @@ func TestExecuteTask_SharedBudgetStopsBeforeStarts(t *testing.T) { require.Equal(t, "none", reason) } -func TestExecuteTask_CommitInvalidatesChangedLoadedStart(t *testing.T) { +func TestExecuteTask_CommitMergesCompletionBeforeStartAcknowledgment(t *testing.T) { env := newInvokerExecuteTestEnv(t) ctx := env.MutableContext() invoker := env.Scheduler.Invoker.Get(ctx) @@ -134,10 +134,10 @@ func TestExecuteTask_CommitInvalidatesChangedLoadedStart(t *testing.T) { require.NoError(t, err) require.NoError(t, env.CloseTransaction()) - require.Empty(t, invoker.BufferedStarts[0].GetRunId()) + require.Equal(t, "loaded-run", invoker.BufferedStarts[0].GetRunId()) require.Equal(t, completed, invoker.BufferedStarts[0].GetCompleted()) - require.Zero(t, env.Scheduler.Info.ActionCount) - require.True(t, executeTaskScheduled) + require.EqualValues(t, 1, env.Scheduler.Info.ActionCount) + require.False(t, executeTaskScheduled) } func TestExecuteTask_CommitInvalidatesChangedWorkflowTarget(t *testing.T) { diff --git a/chasm/lib/scheduler/invoker_execute_task.go b/chasm/lib/scheduler/invoker_execute_task.go index 6de3e47be1d..3a6f64f80d3 100644 --- a/chasm/lib/scheduler/invoker_execute_task.go +++ b/chasm/lib/scheduler/invoker_execute_task.go @@ -10,10 +10,8 @@ import ( "time" commonpb "go.temporal.io/api/common/v1" - enumspb "go.temporal.io/api/enums/v1" "go.temporal.io/api/serviceerror" "go.temporal.io/api/workflowservice/v1" - "go.temporal.io/server/api/historyservice/v1" schedulespb "go.temporal.io/server/api/schedule/v1" "go.temporal.io/server/chasm" "go.temporal.io/server/chasm/lib/scheduler/gen/schedulerpb/v1" @@ -23,7 +21,6 @@ import ( "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" "go.temporal.io/server/common/resource" - "go.temporal.io/server/common/util" queueerrors "go.temporal.io/server/service/history/queues/errors" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" @@ -46,7 +43,7 @@ type ( ) const ( - // InvokerMaxStartAttempts is the maximum number of StartWorkflowExecution + // InvokerMaxStartAttempts is the maximum number of start-request // RPCs issued for an individual buffered action, counting the first call. // Attempt numbers are 1-based: recordProcessBufferResult readies a start at // Attempt 1, and each retryable failure increments it. The bound is @@ -79,7 +76,7 @@ func evaluateInvokerExecuteTaskValidity(invoker *Invoker, scheduler *Scheduler) if scheduler.WorkflowMigration != nil { return invokerExecuteTaskValidity{reason: invokerExecuteInvalidatedMigrationPending} } - if len(invoker.GetTerminateWorkflows())+len(invoker.GetCancelWorkflows())+len(invoker.getEligibleBufferedStarts()) == 0 { + if len(invoker.terminationExecutions())+len(invoker.cancellationExecutions())+len(invoker.getEligibleBufferedStarts()) == 0 { return invokerExecuteTaskValidity{reason: invokerExecuteInvalidatedNoWork} } return invokerExecuteTaskValidity{valid: true, reason: reasonNone} @@ -136,9 +133,9 @@ func (h *InvokerExecuteTaskHandler) Validate( return false, nil } -type loadedWorkflowExecution struct { +type loadedExecution struct { index int - expected *commonpb.WorkflowExecution + expected *commonpb.Execution } type loadedBufferedStart struct { @@ -152,8 +149,8 @@ type executionBatch struct { schedulerRef []byte now time.Time maxActions int - terminations []loadedWorkflowExecution - cancellations []loadedWorkflowExecution + terminations []loadedExecution + cancellations []loadedExecution starts []loadedBufferedStart } @@ -174,67 +171,57 @@ type startExecutionResult struct { } type executionBatchResult struct { - terminations []loadedWorkflowExecution - cancellations []loadedWorkflowExecution + terminations []loadedExecution + cancellations []loadedExecution starts []startExecutionResult } type executionCommitOutcome struct { appliedStarts int - latestStartTime time.Time - startOnlyActions []*schedulespb.BufferedStart removedActions int committedRetries int duplicateInvalidations int stateChangedInvalidations int executeTaskScheduled bool + latestStartTime time.Time + startOnlyActions []*schedulespb.BufferedStart } func (i *Invoker) commitExecutionResult( ctx chasm.MutableContext, result executionBatchResult, ) (outcome executionCommitOutcome) { - removeTerminations := revalidateWorkflowResults(i.TerminateWorkflows, result.terminations, &outcome) - i.TerminateWorkflows = deleteIndexed(i.TerminateWorkflows, removeTerminations) - removeCancels := revalidateWorkflowResults(i.CancelWorkflows, result.cancellations, &outcome) - i.CancelWorkflows = deleteIndexed(i.CancelWorkflows, removeCancels) + removeTerminations := revalidateExecutionResults(i.terminationExecutions(), result.terminations, &outcome) + i.setTerminationExecutions(deleteIndexed(i.terminationExecutions(), removeTerminations)) + removeCancels := revalidateExecutionResults(i.cancellationExecutions(), result.cancellations, &outcome) + i.setCancellationExecutions(deleteIndexed(i.cancellationExecutions(), removeCancels)) removeStarts := make(map[int]bool) + matched := make(map[int]bool) for _, startResult := range result.starts { - index := startResult.loaded.index - if index < 0 || index >= len(i.BufferedStarts) || - !proto.Equal(i.BufferedStarts[index], startResult.loaded.expected) { - if startResult.outcome == startExecutionCompleted && index >= 0 && index < len(i.BufferedStarts) && - isAlreadyRecordedStart(i.BufferedStarts[index], startResult.loaded.expected) { - outcome.duplicateInvalidations++ - } else { - outcome.stateChangedInvalidations++ + expected := startResult.loaded.expected + index := -1 + for candidate, live := range i.BufferedStarts { + if !matched[candidate] && sameOccurrence(live, expected) { + index = candidate + break } + } + if index < 0 { + outcome.stateChangedInvalidations++ continue } - + matched[index] = true start := i.BufferedStarts[index] - switch startResult.outcome { - case startExecutionCompleted: - schedulerinternal.MarkStartStarted(start, startResult.runID, startResult.startTime) - start.HasCallback = true - outcome.appliedStarts++ - outcome.latestStartTime = util.MaxTime(outcome.latestStartTime, startResult.startTime.AsTime()) - if !schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) { - outcome.startOnlyActions = append(outcome.startOnlyActions, start) - removeStarts[index] = true - outcome.removedActions++ - } - case startExecutionRetryable: - schedulerinternal.MarkStartRetrying(start, start.GetAttempt()+1, startResult.backoffTime) - outcome.committedRetries++ - case startExecutionFailed: + if outcome.applyStartResult(start, startResult) { removeStarts[index] = true - outcome.removedActions++ - default: } } + i.BufferedStarts = deleteIndexed(i.BufferedStarts, removeStarts) + if outcome.appliedStarts > 0 || len(removeStarts) > 0 { + i.reconsiderWaiting() + } i.getOrCreateEventLog(ctx).LogEvent(ctx, fmt.Sprintf("recordExecuteResult kicked off %d starts, removed %d starts, retried %d starts", @@ -243,34 +230,90 @@ func (i *Invoker) commitExecutionResult( outcome.committedRetries)) i.addTasks(ctx) outcome.executeTaskScheduled = i.hasExecutableWork() + if outcome.appliedStarts > 0 { + i.Scheduler.Get(ctx).Generator.Get(ctx).Generate(ctx) + } return outcome } -func isAlreadyRecordedStart(live, expected *schedulespb.BufferedStart) bool { - if live.GetRunId() == "" { +func (outcome *executionCommitOutcome) applyStartResult(start *schedulespb.BufferedStart, startResult startExecutionResult) bool { + if start.GetStartAccepted() || schedulerinternal.RunID(start) != "" && !schedulerinternal.IsCompleted(start) { + if startResult.outcome == startExecutionCompleted { + outcome.duplicateInvalidations++ + } else { + outcome.stateChangedInvalidations++ + } return false } - liveWithoutResult := common.CloneProto(live) - liveWithoutResult.RunId = "" - liveWithoutResult.StartTime = nil - liveWithoutResult.HasCallback = false - return proto.Equal(liveWithoutResult, expected) + if !compatibleAttempt(start, startResult.loaded.expected) { + outcome.stateChangedInvalidations++ + return false + } + switch startResult.outcome { + case startExecutionCompleted: + if runID := schedulerinternal.RunID(start); runID != "" && runID != startResult.runID { + outcome.stateChangedInvalidations++ + return false + } + if runID := start.GetCompletion().GetExecution().GetRunId(); runID != "" && runID != startResult.runID { + outcome.stateChangedInvalidations++ + return false + } + schedulerinternal.MarkStartStarted(start, startResult.runID, startResult.startTime) + start.HasCallback = true + start.StartAccepted = true + outcome.appliedStarts++ + if start.GetStartTime().AsTime().After(outcome.latestStartTime) { + outcome.latestStartTime = start.GetStartTime().AsTime() + } + if !schedulerinternal.TracksExecution(start) { + outcome.startOnlyActions = append(outcome.startOnlyActions, start) + outcome.removedActions++ + return true + } + case startExecutionRetryable: + schedulerinternal.MarkStartRetrying(start, start.GetAttempt()+1, startResult.backoffTime) + outcome.committedRetries++ + case startExecutionFailed: + if schedulerinternal.IsCompleted(start) { + return false + } + outcome.removedActions++ + return true + default: + outcome.stateChangedInvalidations++ + } + return false +} + +func sameOccurrence(live, expected *schedulespb.BufferedStart) bool { + if live.GetOccurrenceId() != "" || expected.GetOccurrenceId() != "" { + return live.GetOccurrenceId() == expected.GetOccurrenceId() + } + return live.GetRequestId() == expected.GetRequestId() && schedulerinternal.TargetID(live) == schedulerinternal.TargetID(expected) && + proto.Equal(live.GetNominalTime(), expected.GetNominalTime()) && proto.Equal(live.GetActualTime(), expected.GetActualTime()) && live.GetManual() == expected.GetManual() } -func revalidateWorkflowResults( - live []*commonpb.WorkflowExecution, - results []loadedWorkflowExecution, +func compatibleAttempt(live, expected *schedulespb.BufferedStart) bool { + return live.GetAttempt() == expected.GetAttempt() && proto.Equal(live.GetBackoffTime(), expected.GetBackoffTime()) +} + +func revalidateExecutionResults( + live []*commonpb.Execution, + results []loadedExecution, outcome *executionCommitOutcome, ) map[int]bool { remove := make(map[int]bool, len(results)) for _, result := range results { - if result.index < 0 || result.index >= len(live) || !proto.Equal(live[result.index], result.expected) { + index := slices.IndexFunc(live, func(target *commonpb.Execution) bool { return proto.Equal(target, result.expected) }) + if index < 0 || remove[index] { outcome.stateChangedInvalidations++ continue } - remove[result.index] = true + remove[index] = true outcome.removedActions++ } + return remove } @@ -311,7 +354,10 @@ func (h *InvokerExecuteTaskHandler) loadExecutionBatch( batch.scheduler = &Scheduler{ SchedulerState: common.CloneProto(s.SchedulerState), } - batch.lastCompletionState = common.CloneProto(s.LastCompletionResult.Get(ctx)) + batch.lastCompletionState = &schedulerpb.LastCompletionResult{} + if implementation(s.Schedule.GetAction()).ParticipatesInCompletionHistory() { + batch.lastCompletionState = common.CloneProto(s.LastCompletionResult.Get(ctx)) + } // Capture the scheduler's component ref so a per-start completion callback (carrying that // start's request ID in its token) can be built outside the MS lock. @@ -323,17 +369,16 @@ func (h *InvokerExecuteTaskHandler) loadExecutionBatch( batch.now = ctx.Now(i) batch.maxActions = h.maxActionsPerExecution(batch.scheduler) - // Position plus exact state is the commit identity. Request IDs are - // not unique for compatibility with existing schedules. - for index, target := range i.GetTerminateWorkflows() { - batch.terminations = append(batch.terminations, loadedWorkflowExecution{index: index, expected: common.CloneProto(target)}) + // Occurrence identity survives buffer movement while external calls are in flight. + for index, target := range i.terminationExecutions() { + batch.terminations = append(batch.terminations, loadedExecution{index: index, expected: common.CloneProto(target)}) } - for index, target := range i.GetCancelWorkflows() { - batch.cancellations = append(batch.cancellations, loadedWorkflowExecution{index: index, expected: common.CloneProto(target)}) + for index, target := range i.cancellationExecutions() { + batch.cancellations = append(batch.cancellations, loadedExecution{index: index, expected: common.CloneProto(target)}) } lastProcessed := i.GetLastProcessedTime().AsTime() for index, start := range i.GetBufferedStarts() { - if start.GetAttempt() > 0 && start.GetRunId() == "" && !start.GetBackoffTime().AsTime().After(lastProcessed) { + if start.GetAttempt() > 0 && !start.GetStartAccepted() && schedulerinternal.RunID(start) == "" && !start.GetBackoffTime().AsTime().After(lastProcessed) { batch.starts = append(batch.starts, loadedBufferedStart{index: index, expected: common.CloneProto(start)}) } } @@ -358,9 +403,9 @@ func (h *InvokerExecuteTaskHandler) executeBatch(ctx context.Context, batch exec actionsTaken := 0 result := executionBatchResult{} - result.terminations = h.terminateWorkflows(ctx, logger, metricsHandler, batch.scheduler, batch.terminations, &actionsTaken, batch.maxActions) - result.cancellations = h.cancelWorkflows(ctx, logger, metricsHandler, batch.scheduler, batch.cancellations, &actionsTaken, batch.maxActions) - result.starts = h.startWorkflows(ctx, logger, metricsHandler, batch, &actionsTaken) + result.terminations = h.terminateExecutions(ctx, logger, metricsHandler, batch.scheduler, batch.terminations, &actionsTaken, batch.maxActions) + result.cancellations = h.cancelExecutions(ctx, logger, metricsHandler, batch.scheduler, batch.cancellations, &actionsTaken, batch.maxActions) + result.starts = h.startActions(ctx, logger, metricsHandler, batch, &actionsTaken) return result } @@ -379,9 +424,6 @@ func (h *InvokerExecuteTaskHandler) commitExecutionResult( s.advanceLastEventTimeTo(outcome.latestStartTime) s.recordStartOnlyActions(ctx, outcome.startOnlyActions) s.recordActionResult(&schedulerActionResult{actionCount: int64(outcome.appliedStarts)}) - if outcome.appliedStarts > 0 { - s.Generator.Get(ctx).Generate(ctx) - } h.recordExecutionInvalidations(s, outcome) return nil, nil }, @@ -393,16 +435,16 @@ func (h *InvokerExecuteTaskHandler) commitExecutionResult( return outcome, nil } -// cancelWorkflows does a best-effort attempt to cancel all workflow executions provided in targets. -func (h *InvokerExecuteTaskHandler) cancelWorkflows( +// cancelExecutions does a best-effort attempt to cancel all executions provided in targets. +func (h *InvokerExecuteTaskHandler) cancelExecutions( ctx context.Context, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, - targets []loadedWorkflowExecution, + targets []loadedExecution, actionsTaken *int, maxActions int, -) (completed []loadedWorkflowExecution) { +) (completed []loadedExecution) { var wg sync.WaitGroup var resultMutex sync.Mutex @@ -413,13 +455,13 @@ func (h *InvokerExecuteTaskHandler) cancelWorkflows( // Run all cancels concurrently. wg.Go(func() { - err := h.cancelWorkflow(ctx, scheduler, target.expected) + err := implementation(scheduler.Schedule.GetAction()).Cancel(ctx, actionClients{Frontend: h.frontendClient, History: h.historyClient}, scheduler, target.expected) resultMutex.Lock() defer resultMutex.Unlock() if err != nil { - logger.Info("failed to cancel workflow", tag.Error(err), tag.WorkflowID(target.expected.WorkflowId)) + logger.Info("failed to cancel execution", tag.Error(err), tag.NewStringTag("target-id", target.expected.BusinessId)) metricsHandler.Counter(metrics.ScheduleCancelWorkflowErrors.Name()).Record(1) } @@ -436,16 +478,16 @@ func (h *InvokerExecuteTaskHandler) cancelWorkflows( return completed } -// terminateWorkflows does a best-effort attempt to terminate all workflow executions provided in targets. -func (h *InvokerExecuteTaskHandler) terminateWorkflows( +// terminateExecutions does a best-effort attempt to terminate all executions provided in targets. +func (h *InvokerExecuteTaskHandler) terminateExecutions( ctx context.Context, logger log.Logger, metricsHandler metrics.Handler, scheduler *Scheduler, - targets []loadedWorkflowExecution, + targets []loadedExecution, actionsTaken *int, maxActions int, -) (completed []loadedWorkflowExecution) { +) (completed []loadedExecution) { var wg sync.WaitGroup var resultMutex sync.Mutex @@ -456,13 +498,13 @@ func (h *InvokerExecuteTaskHandler) terminateWorkflows( // Run all terminates concurrently. wg.Go(func() { - err := h.terminateWorkflow(ctx, scheduler, target.expected) + err := implementation(scheduler.Schedule.GetAction()).Terminate(ctx, actionClients{Frontend: h.frontendClient, History: h.historyClient}, scheduler, target.expected) resultMutex.Lock() defer resultMutex.Unlock() if err != nil { - logger.Info("failed to terminate workflow", tag.Error(err), tag.WorkflowID(target.expected.WorkflowId)) + logger.Info("failed to terminate execution", tag.Error(err), tag.NewStringTag("target-id", target.expected.BusinessId)) metricsHandler.Counter(metrics.ScheduleTerminateWorkflowErrors.Name()).Record(1) } @@ -479,8 +521,8 @@ func (h *InvokerExecuteTaskHandler) terminateWorkflows( return completed } -// startWorkflows executes the provided list of starts, returning a result with their outcomes. -func (h *InvokerExecuteTaskHandler) startWorkflows( +// startActions executes the provided list of starts, returning a result with their outcomes. +func (h *InvokerExecuteTaskHandler) startActions( ctx context.Context, logger log.Logger, metricsHandler metrics.Handler, @@ -488,7 +530,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflows( actionsTaken *int, ) (results []startExecutionResult) { metricsWithTag := metricsHandler.WithTags( - metrics.StringTag(metrics.ScheduleActionTypeTag, metrics.ScheduleActionStartWorkflow)) + metrics.StringTag(metrics.ScheduleActionTypeTag, batch.scheduler.actionMetadata().Kind.String())) var wg sync.WaitGroup var resultMutex sync.Mutex @@ -503,7 +545,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflows( // Run all starts concurrently. wg.Go(func() { - started, err := h.startWorkflow( + started, err := h.startAction( ctx, metricsHandler, batch.scheduler, @@ -516,7 +558,7 @@ func (h *InvokerExecuteTaskHandler) startWorkflows( defer resultMutex.Unlock() if err != nil { - logger.Info("failed to start workflow", tag.Error(err)) + logger.Info("failed to start action", tag.Error(err)) // Don't count "already started" for the error metric or retry, as it is most likely // due to misconfiguration. @@ -581,86 +623,41 @@ func (h *InvokerExecuteTaskHandler) nextBackoffTime( return timestamppb.New(now.Add(delay)) } -func (h *InvokerExecuteTaskHandler) startWorkflow( +func (h *InvokerExecuteTaskHandler) startAction( ctx context.Context, metricsHandler metrics.Handler, scheduler *Scheduler, start *schedulespb.BufferedStart, lastCompletionState *schedulerpb.LastCompletionResult, schedulerRef []byte, -) (startedWorkflow, error) { - requestSpec := scheduler.GetSchedule().GetAction().GetStartWorkflow() +) (startedExecution, error) { // Inclusive bound: Attempt is 1-based, so the attempt numbered // InvokerMaxStartAttempts is the last one that gets an RPC. if start.Attempt > InvokerMaxStartAttempts { - return startedWorkflow{}, errRetryLimitExceeded + return startedExecution{}, errRetryLimitExceeded } // Get rate limiter permission once per buffered start, on the first attempt only. if start.Attempt == 1 { delay, err := h.getRateLimiterPermission() if err != nil { - return startedWorkflow{}, err + return startedExecution{}, err } if delay > 0 { - return startedWorkflow{}, newRateLimitedError(delay) + return startedExecution{}, newRateLimitedError(delay) } } - reusePolicy := enumspb.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE - if start.Manual { - reusePolicy = enumspb.WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE - } - - tracksCompletionResult := schedulerinternal.TracksCompletionResult(start.GetOverlapPolicy()) - var lcr []*commonpb.Payload - continuedFailure := lastCompletionState.Failure - if tracksCompletionResult && lastCompletionState.Success != nil { - lcr = append(lcr, lastCompletionState.Success) - } - if !tracksCompletionResult { - continuedFailure = nil - } - // Build the completion callback with this start's request ID packed into its token, so the - // completion is matched by a request ID that rides in the callback header and survives - // continue-as-new, rather than the started workflow's callback state which is re-stamped on each - // new run. callback, err := chasm.GenerateNexusCallback(schedulerRef, start.RequestId, h.config.EncodeInternalTokenWithEnvelope(scheduler.Namespace)) if err != nil { - return startedWorkflow{}, err - } - request := &workflowservice.StartWorkflowExecutionRequest{ - CompletionCallbacks: []*commonpb.Callback{callback}, - Header: requestSpec.Header, - Identity: scheduler.identity(), - Input: requestSpec.Input, - Memo: requestSpec.Memo, - Namespace: scheduler.Namespace, - RequestId: start.RequestId, - RetryPolicy: requestSpec.RetryPolicy, - SearchAttributes: scheduler.startWorkflowSearchAttributes(start.NominalTime.AsTime()), - TaskQueue: requestSpec.TaskQueue, - UserMetadata: requestSpec.UserMetadata, - WorkflowExecutionTimeout: requestSpec.WorkflowExecutionTimeout, - WorkflowId: start.WorkflowId, - WorkflowIdReusePolicy: reusePolicy, - WorkflowRunTimeout: requestSpec.WorkflowRunTimeout, - WorkflowTaskTimeout: requestSpec.WorkflowTaskTimeout, - WorkflowType: requestSpec.WorkflowType, - Priority: requestSpec.Priority, - ContinuedFailure: continuedFailure, - LastCompletionResult: &commonpb.Payloads{ - Payloads: lcr, - }, + return startedExecution{}, err } - if h.config.Tweakables(scheduler.Namespace).EnableVersioningOverride { - request.VersioningOverride = requestSpec.VersioningOverride - } - - result, err := h.frontendClient.StartWorkflowExecution(ctx, request) + runID, err := implementation(scheduler.Schedule.GetAction()).Start(ctx, actionClients{Frontend: h.frontendClient, History: h.historyClient}, actionStartInput{ + Scheduler: scheduler, Occurrence: start, Callback: callback, Previous: lastCompletionState, EnableVersioningOverride: h.config.Tweakables(scheduler.Namespace).EnableVersioningOverride, + }) if err != nil { - return startedWorkflow{}, err + return startedExecution{}, err } // This timestamp measures the external RPC, so it must use wall-clock time rather than CHASM transaction time. actualStartTime := time.Now() //nolint:forbidigo @@ -676,52 +673,14 @@ func (h *InvokerExecuteTaskHandler) startWorkflow( Timer(metrics.ScheduleActionE2EDelay.Name()). Record(actualStartTime.Sub(start.ActualTime.AsTime())) } - return startedWorkflow{runID: result.RunId, startTime: actualStartTime}, nil + return startedExecution{runID: runID, startTime: actualStartTime}, nil } -type startedWorkflow struct { +type startedExecution struct { runID string startTime time.Time } -func (h *InvokerExecuteTaskHandler) terminateWorkflow( - ctx context.Context, - scheduler *Scheduler, - target *commonpb.WorkflowExecution, -) error { - request := &historyservice.TerminateWorkflowExecutionRequest{ - NamespaceId: scheduler.NamespaceId, - TerminateRequest: &workflowservice.TerminateWorkflowExecutionRequest{ - Namespace: scheduler.Namespace, - WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: target.WorkflowId}, - Reason: "terminated by schedule overlap policy", - Identity: scheduler.identity(), - FirstExecutionRunId: target.RunId, - }, - } - _, err := h.historyClient.TerminateWorkflowExecution(ctx, request) - return err -} - -func (h *InvokerExecuteTaskHandler) cancelWorkflow( - ctx context.Context, - scheduler *Scheduler, - target *commonpb.WorkflowExecution, -) error { - request := &historyservice.RequestCancelWorkflowExecutionRequest{ - NamespaceId: scheduler.NamespaceId, - CancelRequest: &workflowservice.RequestCancelWorkflowExecutionRequest{ - Namespace: scheduler.Namespace, - WorkflowExecution: &commonpb.WorkflowExecution{WorkflowId: target.WorkflowId}, - Reason: "cancelled by schedule overlap policy", - Identity: scheduler.identity(), - FirstExecutionRunId: target.RunId, - }, - } - _, err := h.historyClient.RequestCancelWorkflowExecution(ctx, request) - return err -} - // getRateLimiterPermission returns a delay for which the caller should wait // before proceeding. If an error is returned, execution should not proceed, and // reservation should be retried. @@ -732,7 +691,8 @@ func (h *InvokerExecuteTaskHandler) getRateLimiterPermission() (delay time.Durat func isAlreadyStartedError(err error) bool { var expectedErr *serviceerror.WorkflowExecutionAlreadyStarted - return errors.As(err, &expectedErr) + var activityErr *serviceerror.ActivityExecutionAlreadyStarted + return errors.As(err, &expectedErr) || errors.As(err, &activityErr) } func isRateLimitedError(err error) (time.Duration, bool) { diff --git a/chasm/lib/scheduler/invoker_process_buffer_task_test.go b/chasm/lib/scheduler/invoker_process_buffer_task_test.go index 852a2a4284c..6ab5cc8bcea 100644 --- a/chasm/lib/scheduler/invoker_process_buffer_task_test.go +++ b/chasm/lib/scheduler/invoker_process_buffer_task_test.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "slices" - "sort" "strings" "testing" "time" @@ -1268,7 +1267,7 @@ func runBufferProcessing(t *testing.T, input bufferProcessingComparisonInput, pl ctx := env.MutableContext() env.Scheduler.Schedule = proto.Clone(input.schedule).(*schedulepb.Schedule) env.Scheduler.Info.CreateTime = timestamppb.New(input.lastProcessedTime) - env.Scheduler.LastEventTime = timestamppb.New(env.TimeSource.Now()) + env.Scheduler.LastEventTime = timestamppb.New(input.lastProcessedTime) initialConflictToken := input.initialConflictToken if initialConflictToken == 0 { initialConflictToken = env.Scheduler.ConflictToken @@ -1334,7 +1333,7 @@ func normalizeTasks(tasksByCategory map[tasks.Category][]tasks.Task) []string { for category := range tasksByCategory { categories = append(categories, category) } - sort.Slice(categories, func(i, j int) bool { return categories[i].ID() < categories[j].ID() }) + slices.SortFunc(categories, func(a, b tasks.Category) int { return a.ID() - b.ID() }) var normalized []string for _, category := range categories { for index, task := range tasksByCategory[category] { @@ -1367,11 +1366,11 @@ func normalizeMetrics(snapshot metricstest.CaptureSnapshot) []string { for key, value := range recording.Tags { tags = append(tags, key+"="+value) } - sort.Strings(tags) + slices.Sort(tags) normalized = append(normalized, fmt.Sprintf("%s:%v:%s", name, recording.Value, strings.Join(tags, ","))) } } - sort.Strings(normalized) + slices.Sort(normalized) return normalized } diff --git a/chasm/lib/scheduler/invoker_tasks.go b/chasm/lib/scheduler/invoker_tasks.go index 8031a99220d..715f8c9cbac 100644 --- a/chasm/lib/scheduler/invoker_tasks.go +++ b/chasm/lib/scheduler/invoker_tasks.go @@ -45,7 +45,7 @@ type ( const ( // Lower bound for the deadline in which buffered actions are dropped. - startWorkflowMinDeadline = 5 * time.Second + startActionMinDeadline = 5 * time.Second ) // Invoker task invalidation and buffered-start drop reasons. Limited cardinality @@ -102,7 +102,7 @@ func (h *InvokerProcessBufferTaskHandler) Execute( // Make sure we have something to start. executionInfo := scheduler.Schedule.GetAction().GetStartWorkflow() - if executionInfo == nil { + if executionInfo == nil && scheduler.Schedule.GetAction().GetStartActivity() == nil { return queueerrors.NewUnprocessableTaskError("schedules must have an Action set") } @@ -129,6 +129,9 @@ func (h *InvokerProcessBufferTaskHandler) recordBufferProcessingMetrics( metrics.StringTag(metrics.ScheduleOverlapPolicyTag, overlapPolicy.String()), ).Counter(metrics.ScheduleOverlapSkipped.Name()).Record(count) } + for name, count := range result.overlapSkippedByCustomPolicy { + metricsHandler.WithTags(metrics.StringTag(metrics.ScheduleOverlapPolicyTag, name)).Counter(metrics.ScheduleOverlapSkipped.Name()).Record(count) + } for actionRunning, count := range result.missedCatchupByActionRunning { newTaggedMetricsHandler(h.metricsHandler, scheduler).WithTags( metrics.StringTag(metrics.ScheduleMissedReasonTag, metrics.ScheduleMissedReasonBufferExpired), @@ -137,10 +140,10 @@ func (h *InvokerProcessBufferTaskHandler) recordBufferProcessingMetrics( } } -// startWorkflowDeadline returns the latest time at which a buffered workflow +// startActionDeadline returns the latest time at which a buffered workflow // should be started, instead of dropped. The deadline puts an upper bound on // the number of retry attempts per buffered start. -func (h *InvokerProcessBufferTaskHandler) startWorkflowDeadline( +func (h *InvokerProcessBufferTaskHandler) startActionDeadline( ctx chasm.Context, scheduler *Scheduler, start *schedulespb.BufferedStart, @@ -159,7 +162,7 @@ func (h *InvokerProcessBufferTaskHandler) startWorkflowDeadline( tweakables := h.config.Tweakables(scheduler.Namespace) timeout = catchupWindow(scheduler, tweakables) - timeout = max(timeout, startWorkflowMinDeadline) + timeout = max(timeout, startActionMinDeadline) return start.ActualTime.AsTime().Add(timeout) } diff --git a/chasm/lib/scheduler/migration/identity_test.go b/chasm/lib/scheduler/migration/identity_test.go index a828b774ae0..ecd9aa8cd1f 100644 --- a/chasm/lib/scheduler/migration/identity_test.go +++ b/chasm/lib/scheduler/migration/identity_test.go @@ -36,7 +36,7 @@ func TestSameTimePendingStartsReceiveUniqueIdentities(t *testing.T) { // The first start keeps the undecorated workflow ID, so a migrated action // still dedups against one the V1 scheduler had already started. require.Equal(t, - schedulerinternal.GenerateWorkflowID("workflow-id", when.AsTime()), + schedulerinternal.GenerateTimestampTargetID("workflow-id", when.AsTime()), converted[0].GetWorkflowId()) } diff --git a/chasm/lib/scheduler/migration/migration.go b/chasm/lib/scheduler/migration/migration.go index 6671d7771d9..51d193aaa09 100644 --- a/chasm/lib/scheduler/migration/migration.go +++ b/chasm/lib/scheduler/migration/migration.go @@ -289,7 +289,7 @@ func convertBufferedStartsLegacyToCHASM( } if v2Start.WorkflowId == "" { - v2Start.WorkflowId = schedulerinternal.GenerateWorkflowID( + v2Start.WorkflowId = schedulerinternal.GenerateTimestampTargetID( baseWorkflowID, v1Start.GetNominalTime().AsTime(), ) @@ -331,12 +331,10 @@ func convertRunningWorkflowsToBufferedStarts( bufferedStarts := make([]*schedulespb.BufferedStart, len(runningWorkflows)) for i, wf := range runningWorkflows { - bufferedStarts[i] = &schedulespb.BufferedStart{ + start := &schedulespb.BufferedStart{ NominalTime: timestamppb.New(migrationTime), ActualTime: timestamppb.New(migrationTime), - StartTime: timestamppb.New(migrationTime), WorkflowId: wf.WorkflowId, - RunId: wf.RunId, // RequestId will be used with AttachRequestID to register Nexus // callbacks for tracking workflow completion after migration. // Include the RunId in the tag to ensure each running workflow @@ -350,12 +348,13 @@ func convertRunningWorkflowsToBufferedStarts( migrationTime, migrationTime, ), - Attempt: 1, - Completed: nil, // Migrated running workflows must have a Nexus callback attached once the // migrated schedule target has been created. HasCallback: false, } + schedulerinternal.MarkStartReady(start) + schedulerinternal.MarkStartStarted(start, wf.RunId, timestamppb.New(migrationTime)) + bufferedStarts[i] = start } return bufferedStarts @@ -400,12 +399,10 @@ func convertRecentActionsToBufferedStarts( continue } - bufferedStarts = append(bufferedStarts, &schedulespb.BufferedStart{ + start := &schedulespb.BufferedStart{ NominalTime: action.ScheduleTime, ActualTime: action.ActualTime, - StartTime: action.ActualTime, WorkflowId: action.StartWorkflowResult.WorkflowId, - RunId: action.StartWorkflowResult.RunId, RequestId: schedulerinternal.GenerateRequestID( namespaceID, scheduleID, @@ -414,12 +411,14 @@ func convertRecentActionsToBufferedStarts( action.ScheduleTime.AsTime(), action.ActualTime.AsTime(), ), - Attempt: 1, - Completed: &schedulespb.CompletedResult{ - Status: action.StartWorkflowStatus, - CloseTime: timestamppb.New(migrationTime), - }, + } + schedulerinternal.MarkStartReady(start) + schedulerinternal.MarkStartStarted(start, action.StartWorkflowResult.RunId, action.ActualTime) + schedulerinternal.MarkStartCompleted(start, &schedulespb.CompletedResult{ + Status: action.StartWorkflowStatus, + CloseTime: timestamppb.New(migrationTime), }) + bufferedStarts = append(bufferedStarts, start) } return bufferedStarts diff --git a/chasm/lib/scheduler/proto/v1/message.proto b/chasm/lib/scheduler/proto/v1/message.proto index d52771e20bd..e8e93f78d75 100644 --- a/chasm/lib/scheduler/proto/v1/message.proto +++ b/chasm/lib/scheduler/proto/v1/message.proto @@ -98,6 +98,9 @@ message InvokerState { google.protobuf.Timestamp last_processed_time = 5; reserved 6; + repeated temporal.api.common.v1.Execution cancel_executions = 7; + repeated temporal.api.common.v1.Execution terminate_executions = 8; + int64 next_occurrence = 9; } // CHASM scheduler's Backfiller internal state. Backfill requests are 1:1 diff --git a/chasm/lib/scheduler/scheduler.go b/chasm/lib/scheduler/scheduler.go index 3490cb47260..9b61ee22a68 100644 --- a/chasm/lib/scheduler/scheduler.go +++ b/chasm/lib/scheduler/scheduler.go @@ -133,6 +133,14 @@ func NewScheduler( input *schedulepb.Schedule, patch *schedulepb.SchedulePatch, ) (*Scheduler, error) { + if err := ValidateScheduleActionPolicies(input, patch); err != nil { + return nil, err + } + if input.GetAction().GetStartActivity() != nil { + if err := implementation(input.GetAction()).Validate(input.GetAction()); err != nil { + return nil, err + } + } var zero time.Time sched := &Scheduler{ @@ -245,6 +253,9 @@ func (s *Scheduler) applyPausePatch(ctx chasm.MutableContext, patch *schedulepb. // handlePatch creates backfillers to fulfill the given patch request. func (s *Scheduler) handlePatch(ctx chasm.MutableContext, patch *schedulepb.SchedulePatch) error { + if err := ValidateScheduleActionPolicies(s.Schedule, patch); err != nil { + return err + } if patch == nil { return nil } @@ -411,7 +422,11 @@ func (s *Scheduler) NewImmediateBackfiller( } // Trigger backfills fire a single action at creation time; processTrigger uses // LastProcessedTime as that action's deterministic time, so it must be set. - backfiller.LastProcessedTime = timestamppb.New(ctx.Now(s)) + triggerTime := ctx.Now(s) + if request.GetScheduledTime() != nil && !request.GetScheduledTime().AsTime().IsZero() { + triggerTime = request.GetScheduledTime().AsTime() + } + backfiller.LastProcessedTime = timestamppb.New(triggerTime) return backfiller } @@ -619,15 +634,12 @@ func (s *Scheduler) recordRecentAction( start *schedulespb.BufferedStart, status enumspb.WorkflowExecutionStatus, ) { - s.Info.RecentActions = append(s.Info.RecentActions, &schedulepb.ScheduleActionResult{ - ScheduleTime: start.GetActualTime(), - ActualTime: start.GetStartTime(), - StartWorkflowResult: &commonpb.WorkflowExecution{ - WorkflowId: start.GetWorkflowId(), - RunId: start.GetRunId(), - }, - StartWorkflowStatus: status, - }) + result := actionResult(start) + if result.GetActionExecutionResult().GetExecution().GetType() == enumspb.EXECUTION_TYPE_WORKFLOW { + result.StartWorkflowStatus = status + result.ActionExecutionResult.Status = &commonpb.ActionExecutionResult_WorkflowStatus{WorkflowStatus: status} + } + s.Info.RecentActions = append(s.Info.RecentActions, result) slices.SortFunc(s.Info.RecentActions, func(a, b *schedulepb.ScheduleActionResult) int { return a.GetActualTime().AsTime().Compare(b.GetActualTime().AsTime()) }) @@ -715,7 +727,7 @@ func (s *Scheduler) HandleNexusCompletion( ) return nil } - if start.GetCompleted() != nil { + if internal.IsCompleted(start) { // Completion callbacks may be validly redelivered, for example after a workflow reset. // Preserve state but keep the duplicate observable through the log and metric. s.recordIgnoredCallback( @@ -728,7 +740,7 @@ func (s *Scheduler) HandleNexusCompletion( return nil } workflowID := start.GetWorkflowId() - tracksCompletionResult := internal.TracksCompletionResult(start.GetOverlapPolicy()) + tracksCompletionResult := implementation(s.Schedule.GetAction()).ParticipatesInCompletionHistory() && internal.TracksExecution(start) // Record how long it took for the callback to arrive after the action completed. // Use ctx.Now instead of time.Since to use a consistent time source across nodes, @@ -741,10 +753,9 @@ func (s *Scheduler) HandleNexusCompletion( // Handle last completed/failed status and payloads. // // TODO - also record payload sizes once we have metrics wired into CHASM context. - var wfStatus enumspb.WorkflowExecutionStatus + wfStatus := implementation(s.Schedule.GetAction()).Completion(info, internal.Execution(start)).Result.GetWorkflowStatus() switch outcome := info.Outcome.(type) { case *persistencespb.ChasmNexusCompletion_Failure: - wfStatus = executionStatusFromFailure(outcome.Failure) if tracksCompletionResult { previousResult := s.LastCompletionResult.Get(ctx) // Most-recent success is kept after failure. s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ @@ -753,14 +764,12 @@ func (s *Scheduler) HandleNexusCompletion( }) } case *persistencespb.ChasmNexusCompletion_Success: - wfStatus = enumspb.WORKFLOW_EXECUTION_STATUS_COMPLETED if tracksCompletionResult { s.LastCompletionResult = chasm.NewDataField(ctx, &schedulerpb.LastCompletionResult{ Success: outcome.Success, }) } default: - wfStatus = enumspb.WORKFLOW_EXECUTION_STATUS_FAILED } // Handle pause-on-failure. @@ -780,6 +789,8 @@ func (s *Scheduler) HandleNexusCompletion( Status: wfStatus, CloseTime: info.CloseTime, } + start.Completion = &commonpb.ActionExecutionResult{Execution: common.CloneProto(internal.Execution(start)), Status: &commonpb.ActionExecutionResult_WorkflowStatus{WorkflowStatus: wfStatus}} + start.CompletionTime = info.CloseTime invoker.recordCompletedAction(ctx, completed, info.RequestId) // Generate immediately after recording completions, so that an idle task @@ -810,7 +821,7 @@ func (s *Scheduler) Describe( // Describe must not mutate the cached component: default on a clone, not s.Schedule. schedule := common.CloneProto(s.Schedule) - if schedule.GetPolicies().GetOverlapPolicy() == enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED { + if schedule.GetPolicies().GetOverlapPolicy() == enumspb.SCHEDULE_OVERLAP_POLICY_UNSPECIFIED && schedule.GetPolicies().GetCustomOverlapPolicy() == nil { schedule.Policies.OverlapPolicy = s.overlapPolicy() } schedule.Policies.CatchupWindow = durationpb.New( @@ -833,6 +844,9 @@ func (s *Scheduler) Describe( invoker := s.Invoker.Get(ctx) info := common.CloneProto(s.Info) info.RunningWorkflows = invoker.runningWorkflowExecutions() + info.RunningExecutions = invoker.runningExecutions() + info.ActionKind = s.actionMetadata().Kind + info.ActionType = s.actionMetadata().Type info.RecentActions = s.recentActions(ctx) info.FutureActionTimes = futureActionTimes // Only starts that have not reached StartWorkflowExecution count as buffered. @@ -956,6 +970,10 @@ func (s *Scheduler) MigrateToWorkflow( ctx chasm.MutableContext, req *schedulerpb.MigrateToWorkflowRequest, ) (*schedulerpb.MigrateToWorkflowResponse, error) { + if s.Schedule.GetAction().GetStartActivity() != nil { + return nil, serviceerror.NewInvalidArgument("activity schedules cannot migrate to the workflow-backed scheduler") + } + if s.Sentinel { return nil, ErrSentinel } @@ -1002,6 +1020,18 @@ func (s *Scheduler) Update( return nil, ErrConflictTokenMismatch } + if err := validateActionKind(s.Schedule.GetAction(), req.FrontendRequest.GetSchedule().GetAction()); err != nil { + return nil, err + } + if err := ValidateScheduleActionPolicies(req.FrontendRequest.GetSchedule(), nil); err != nil { + return nil, err + } + if req.FrontendRequest.GetSchedule().GetAction().GetStartActivity() != nil { + if err := implementation(req.FrontendRequest.GetSchedule().GetAction()).Validate(req.FrontendRequest.GetSchedule().GetAction()); err != nil { + return nil, err + } + } + // Update custom search attributes. if req.FrontendRequest.GetSearchAttributes() != nil { // To preserve compatibility with V1 scheduler, we do a full replacement @@ -1170,15 +1200,15 @@ func (s *Scheduler) ListInfo( } } -// startWorkflowSearchAttributes returns the search attributes to be applied to -// workflows kicked off. Includes custom search attributes and Temporal-managed. -func (s *Scheduler) startWorkflowSearchAttributes( +// startActionSearchAttributes returns the search attributes to be applied to +// started executions, including user and Temporal-managed attributes. +func (s *Scheduler) startActionSearchAttributes( nominal time.Time, ) *commonpb.SearchAttributes { scheduledStartTime := chasm.SearchAttributeTemporalScheduledStartTime.Value(nominal) scheduledByID := chasm.SearchAttributeTemporalScheduledByID.Value(s.ScheduleId) fields := payload.MergeMapOfPayload( - s.Schedule.GetAction().GetStartWorkflow().GetSearchAttributes().GetIndexedFields(), + s.actionMetadata().SearchAttributes.GetIndexedFields(), map[string]*commonpb.Payload{ scheduledStartTime.Field: scheduledStartTime.Value.MustEncode(), scheduledByID.Field: scheduledByID.Value.MustEncode(), diff --git a/chasm/lib/scheduler/spec_processor.go b/chasm/lib/scheduler/spec_processor.go index 383f1113640..547e6de55a8 100644 --- a/chasm/lib/scheduler/spec_processor.go +++ b/chasm/lib/scheduler/spec_processor.go @@ -5,8 +5,8 @@ import ( "time" enumspb "go.temporal.io/api/enums/v1" + schedulepb "go.temporal.io/api/schedule/v1" schedulespb "go.temporal.io/server/api/schedule/v1" - schedulerinternal "go.temporal.io/server/chasm/lib/scheduler/internal" "go.temporal.io/server/common/log" "go.temporal.io/server/common/log/tag" "go.temporal.io/server/common/metrics" @@ -32,7 +32,7 @@ type ( scheduler *Scheduler, start, end time.Time, overlapPolicy enumspb.ScheduleOverlapPolicy, - workflowID string, + targetIDBase string, backfillID string, manual bool, limit *int, @@ -78,19 +78,23 @@ func (s *SpecProcessorImpl) ProcessTimeRange( scheduler *Scheduler, start, end time.Time, overlapPolicy enumspb.ScheduleOverlapPolicy, - workflowID string, + targetIDBase string, backfillID string, manual bool, limit *int, ) (*ProcessedTimeRange, error) { tweakables := s.config.Tweakables(scheduler.Namespace) metricsHandler := newTaggedMetricsHandler(s.metricsHandler, scheduler) - overlapPolicy = scheduler.resolveOverlapPolicy(overlapPolicy) + resolved, resolveErr := scheduler.resolvedPolicy(overlapPolicy, nil) + if resolveErr != nil { + return nil, resolveErr + } + overlapPolicy = resolved.Builtin s.logger.Debug("ProcessTimeRange", tag.Time("start", start), tag.Time("end", end), - tag.Any("overlap-policy", overlapPolicy), + tag.String("overlap-policy", resolved.String()), tag.Bool("manual", manual)) // Peek at paused/remaining actions state and don't bother if we're not going to @@ -173,14 +177,18 @@ func (s *SpecProcessorImpl) ProcessTimeRange( droppedCount++ continue } - bufferedStarts = append(bufferedStarts, &schedulespb.BufferedStart{ + bufferedStart := &schedulespb.BufferedStart{ NominalTime: timestamppb.New(next.Nominal), ActualTime: timestamppb.New(next.Next), OverlapPolicy: overlapPolicy, Manual: manual, RequestId: generateRequestID(scheduler, backfillID, next.Nominal, next.Next), - WorkflowId: schedulerinternal.GenerateWorkflowID(workflowID, next.Nominal), - }) + } + if resolved.Custom != "" { + bufferedStart.CustomOverlapPolicy = &schedulepb.CustomOverlapPolicy{Name: resolved.Custom} + } + scheduler.newBufferedExecution(bufferedStart, targetIDBase) + bufferedStarts = append(bufferedStarts, bufferedStart) if limit != nil { if (*limit)--; *limit <= 0 { diff --git a/chasm/lib/scheduler/spec_processor_mock.go b/chasm/lib/scheduler/spec_processor_mock.go index 8a3ea91f345..dcd0b17f4ab 100644 --- a/chasm/lib/scheduler/spec_processor_mock.go +++ b/chasm/lib/scheduler/spec_processor_mock.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: spec_processor.go +// Source: chasm/lib/scheduler/spec_processor.go // // Generated by this command: // -// mockgen -package scheduler -source spec_processor.go -destination spec_processor_mock.go +// mockgen -package scheduler -source chasm/lib/scheduler/spec_processor.go -destination chasm/lib/scheduler/spec_processor_mock.go // // Package scheduler is a generated GoMock package. @@ -58,16 +58,16 @@ func (mr *MockSpecProcessorMockRecorder) NextTime(arg0, after any) *gomock.Call } // ProcessTimeRange mocks base method. -func (m *MockSpecProcessor) ProcessTimeRange(arg0 *Scheduler, start, end time.Time, overlapPolicy enums.ScheduleOverlapPolicy, workflowID, backfillID string, manual bool, limit *int) (*ProcessedTimeRange, error) { +func (m *MockSpecProcessor) ProcessTimeRange(arg0 *Scheduler, start, end time.Time, overlapPolicy enums.ScheduleOverlapPolicy, targetIDBase, backfillID string, manual bool, limit *int) (*ProcessedTimeRange, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProcessTimeRange", arg0, start, end, overlapPolicy, workflowID, backfillID, manual, limit) + ret := m.ctrl.Call(m, "ProcessTimeRange", arg0, start, end, overlapPolicy, targetIDBase, backfillID, manual, limit) ret0, _ := ret[0].(*ProcessedTimeRange) ret1, _ := ret[1].(error) return ret0, ret1 } // ProcessTimeRange indicates an expected call of ProcessTimeRange. -func (mr *MockSpecProcessorMockRecorder) ProcessTimeRange(arg0, start, end, overlapPolicy, workflowID, backfillID, manual, limit any) *gomock.Call { +func (mr *MockSpecProcessorMockRecorder) ProcessTimeRange(arg0, start, end, overlapPolicy, targetIDBase, backfillID, manual, limit any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessTimeRange", reflect.TypeOf((*MockSpecProcessor)(nil).ProcessTimeRange), arg0, start, end, overlapPolicy, workflowID, backfillID, manual, limit) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProcessTimeRange", reflect.TypeOf((*MockSpecProcessor)(nil).ProcessTimeRange), arg0, start, end, overlapPolicy, targetIDBase, backfillID, manual, limit) } diff --git a/proto/internal/temporal/server/api/schedule/v1/message.proto b/proto/internal/temporal/server/api/schedule/v1/message.proto index 71b97922939..11fe5468004 100644 --- a/proto/internal/temporal/server/api/schedule/v1/message.proto +++ b/proto/internal/temporal/server/api/schedule/v1/message.proto @@ -53,6 +53,14 @@ message BufferedStart { // attached. False when a BufferedStart originated from a migrated V1 // workflow. Only used by CHASM scheduler, for migration from V1. bool has_callback = 13; + // Generic execution identity. Workflow fields remain a compatibility projection. + temporal.api.common.v1.Execution execution = 14; + temporal.api.common.v1.ActionExecutionResult completion = 15; + google.protobuf.Timestamp completion_time = 16; + temporal.api.schedule.v1.CustomOverlapPolicy custom_overlap_policy = 17; + // Assigned on enqueue, independent of request IDs and buffer position. + string occurrence_id = 18; + bool start_accepted = 19; } // Result when a workflow execution has completed.