diff --git a/CHANGELOG.md b/CHANGELOG.md index 0092fd4..3b049c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,12 @@ Versioning. ### Added +- Added autonomous vanished-runner recovery with exact GitHub run + force-cancellation, one full rerun, crash reconstruction from `run_attempt`, + bounded command observation and fsync-backed transaction state. +- Added short-lived GitHub App installation-token minting from an existing + encrypted GARM credential envelope, without persisting private keys or + installation tokens outside their existing authority. - Mirrored verified `nddev_tool_cache_event` records into the GitHub runner diagnostic directory so teardown bundles retain job-level cache evidence. - Added bounded extraction of `nddev_tool_cache_event` records from verified diff --git a/cmd/gha-garm-app-token/main.go b/cmd/gha-garm-app-token/main.go new file mode 100644 index 0000000..abf3c4a --- /dev/null +++ b/cmd/gha-garm-app-token/main.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "time" + + "github.com/BurntSushi/toml" + "github.com/NDDev-OpenNetwork/github-actions/internal/garmapptoken" +) + +type garmConfig struct { + Database struct { + Passphrase string `toml:"passphrase"` + } `toml:"database"` +} + +func main() { os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) } + +func run(arguments []string, stdin io.Reader, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("gha-garm-app-token", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("garm-config", "", "absolute GARM config path") + endpointValue := flags.String("github-endpoint", "https://api.github.com", "GitHub API endpoint") + if err := flags.Parse(arguments); err != nil || *configPath == "" || flags.NArg() != 0 { + fmt.Fprintln(stderr, "usage: gha-garm-app-token --garm-config PATH [--github-endpoint URL]") + return 2 + } + var configuration garmConfig + if _, err := toml.DecodeFile(*configPath, &configuration); err != nil || len(configuration.Database.Passphrase) != 32 { + fmt.Fprintln(stderr, "GARM database passphrase configuration is invalid") + return 1 + } + endpoint, err := url.Parse(*endpointValue) + if err != nil { + fmt.Fprintln(stderr, "GitHub endpoint is invalid") + return 1 + } + token, err := (garmapptoken.Minter{ + Endpoint: endpoint, HTTP: &http.Client{Timeout: 20 * time.Second}, Now: time.Now, + Passphrase: []byte(configuration.Database.Passphrase), + }).Mint(context.Background(), stdin) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + fmt.Fprintln(stdout, token.Value) + return 0 +} diff --git a/cmd/gha-vanished-job-recovery/main.go b/cmd/gha-vanished-job-recovery/main.go new file mode 100644 index 0000000..7fcf53a --- /dev/null +++ b/cmd/gha-vanished-job-recovery/main.go @@ -0,0 +1,234 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/NDDev-OpenNetwork/github-actions/internal/vanishedjob" +) + +type config struct { + StateFile string `json:"state_file"` + LockFile string `json:"lock_file"` + TokenFile string `json:"token_file"` + TokenCommand []string `json:"token_command"` + GitHubEndpoint string `json:"github_endpoint"` + Policy vanishedjob.Policy `json:"policy"` + Observation struct { + Argv []string `json:"argv"` + TimeoutSeconds int `json:"timeout_seconds"` + } `json:"observation"` +} + +type jsonEvents struct{ encoder *json.Encoder } + +func (sink jsonEvents) Emit(_ context.Context, event vanishedjob.Event) error { + return sink.encoder.Encode(event) +} + +func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } + +func run(arguments []string, stdout, stderr io.Writer) int { + flags := flag.NewFlagSet("gha-vanished-job-recovery", flag.ContinueOnError) + flags.SetOutput(stderr) + configPath := flags.String("config", "", "absolute recovery config path") + jobPath := flags.String("job", "", "absolute observed job JSON path") + if err := flags.Parse(arguments); err != nil { + return 2 + } + if *configPath == "" || flags.NArg() != 1 || flags.Arg(0) != "plan" && flags.Arg(0) != "apply" { + fmt.Fprintln(stderr, "usage: gha-vanished-job-recovery --config PATH [--job PATH] ") + return 2 + } + configuration, err := loadConfig(*configPath) + if err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + var jobs []vanishedjob.Job + if *jobPath != "" { + job, loadErr := loadJob(*jobPath) + if loadErr != nil { + fmt.Fprintln(stderr, loadErr) + return 1 + } + jobs = []vanishedjob.Job{job} + } else { + observation, observeErr := (vanishedjob.CommandObserver{ + Argv: configuration.Observation.Argv, + Timeout: time.Duration(configuration.Observation.TimeoutSeconds) * time.Second, + }).Observe(context.Background()) + if observeErr != nil { + fmt.Fprintln(stderr, observeErr) + return 1 + } + jobs = observation.Jobs + } + store := vanishedjob.FileStore{Path: configuration.StateFile, LockPath: configuration.LockFile} + if flags.Arg(0) == "plan" { + decisions := make([]vanishedjob.Decision, 0, len(jobs)) + for _, job := range jobs { + existing, getErr := store.Get(vanishedjob.RecordKey(job.Repository, job.RunID, job.RunAttempt)) + if getErr != nil { + fmt.Fprintln(stderr, getErr) + return 1 + } + if existing == nil { + existing, _, getErr = store.ForRun(job.Repository, job.RunID) + } + if getErr != nil { + fmt.Fprintln(stderr, getErr) + return 1 + } + decision, evaluateErr := vanishedjob.Evaluate(configuration.Policy, job, existing, time.Now().UTC()) + if evaluateErr != nil { + fmt.Fprintln(stderr, evaluateErr) + return 1 + } + decisions = append(decisions, decision) + } + if err := json.NewEncoder(stdout).Encode(decisions); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + return 0 + } + endpoint, _ := url.Parse(configuration.GitHubEndpoint) + tokens := make(map[string]string) + for _, job := range jobs { + token, present := tokens[job.Repository] + if !present { + var tokenErr error + token, tokenErr = resolveToken(context.Background(), configuration, job.Repository) + if tokenErr != nil { + fmt.Fprintln(stderr, tokenErr) + return 1 + } + tokens[job.Repository] = token + } + controller := vanishedjob.Controller{ + Policy: configuration.Policy, Store: store, + Client: vanishedjob.GitHubClient{Endpoint: endpoint, Token: token, HTTP: &http.Client{Timeout: 20 * time.Second}}, + Events: jsonEvents{json.NewEncoder(stdout)}, Now: time.Now, + } + if _, err := controller.Reconcile(context.Background(), job); err != nil { + fmt.Fprintln(stderr, err) + return 1 + } + } + return 0 +} + +func loadConfig(path string) (config, error) { + if !boundedAbsolute(path) { + return config{}, fmt.Errorf("recovery config path must be absolute and bounded") + } + data, err := os.ReadFile(path) + if err != nil { + return config{}, err + } + var configuration config + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&configuration); err != nil { + return config{}, err + } + endpoint, endpointErr := url.Parse(configuration.GitHubEndpoint) + validTokenSource := boundedAbsolute(configuration.TokenFile) != (len(configuration.TokenCommand) > 0 && filepath.IsAbs(configuration.TokenCommand[0])) + if !boundedAbsolute(configuration.StateFile) || !boundedAbsolute(configuration.LockFile) || !validTokenSource || endpointErr != nil || endpoint.Scheme != "https" && endpoint.Scheme != "http" || endpoint.Host == "" { + return config{}, fmt.Errorf("recovery config paths or endpoint are invalid") + } + if err := configuration.Policy.Validate(); err != nil { + return config{}, err + } + if len(configuration.Observation.Argv) > 0 && (configuration.Observation.TimeoutSeconds <= 0 || !filepath.IsAbs(configuration.Observation.Argv[0])) { + return config{}, fmt.Errorf("recovery observation command is invalid") + } + return configuration, nil +} + +func resolveToken(ctx context.Context, configuration config, repository string) (string, error) { + if configuration.TokenFile != "" { + return readSecret(configuration.TokenFile) + } + commandCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + command := exec.CommandContext(commandCtx, configuration.TokenCommand[0], configuration.TokenCommand[1:]...) + command.Env = append(os.Environ(), "GHA_VANISHED_REPOSITORY="+repository) + var stdout, stderr bytes.Buffer + command.Stdout = &limitedWriter{buffer: &stdout, remaining: 64*1024 + 1} + command.Stderr = &limitedWriter{buffer: &stderr, remaining: 64*1024 + 1} + if err := command.Run(); err != nil { + return "", fmt.Errorf("resolve GitHub token: %w: %s", err, strings.TrimSpace(stderr.String())) + } + if stdout.Len() > 64*1024 { + return "", fmt.Errorf("resolved GitHub token exceeds bounded output") + } + token := strings.TrimSpace(stdout.String()) + if token == "" || strings.ContainsAny(token, "\r\n") { + return "", fmt.Errorf("token command must return one non-empty line") + } + return token, nil +} + +type limitedWriter struct { + buffer *bytes.Buffer + remaining int +} + +func (writer *limitedWriter) Write(value []byte) (int, error) { + if len(value) > writer.remaining { + return 0, fmt.Errorf("command output exceeds bounded size") + } + writer.remaining -= len(value) + return writer.buffer.Write(value) +} + +func loadJob(path string) (vanishedjob.Job, error) { + if !boundedAbsolute(path) { + return vanishedjob.Job{}, fmt.Errorf("job observation path must be absolute and bounded") + } + file, err := os.Open(path) + if err != nil { + return vanishedjob.Job{}, err + } + defer file.Close() + var job vanishedjob.Job + decoder := json.NewDecoder(io.LimitReader(file, 64*1024+1)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&job); err != nil { + return vanishedjob.Job{}, err + } + return job, nil +} + +func readSecret(path string) (string, error) { + info, err := os.Stat(path) + if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 { + return "", fmt.Errorf("token file must be a private regular file") + } + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + token := strings.TrimSpace(string(data)) + if token == "" || strings.ContainsAny(token, "\r\n") { + return "", fmt.Errorf("token file must contain one non-empty line") + } + return token, nil +} + +func boundedAbsolute(path string) bool { + return filepath.IsAbs(path) && filepath.Clean(path) != string(filepath.Separator) +} diff --git a/cmd/gha-vanished-job-recovery/main_test.go b/cmd/gha-vanished-job-recovery/main_test.go new file mode 100644 index 0000000..f1a1267 --- /dev/null +++ b/cmd/gha-vanished-job-recovery/main_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "bytes" + "context" + "os" + "path/filepath" + "testing" +) + +func TestPlanIsReadOnlyAndClassifiesVanishedCIJob(t *testing.T) { + directory := t.TempDir() + state, lock, token := filepath.Join(directory, "state.json"), filepath.Join(directory, "state.lock"), filepath.Join(directory, "token") + if err := os.WriteFile(token, []byte("secret-token"), 0o600); err != nil { + t.Fatal(err) + } + configPath := filepath.Join(directory, "config.json") + jobPath := filepath.Join(directory, "job.json") + configuration := `{"state_file":"` + state + `","lock_file":"` + lock + `","token_file":"` + token + `","github_endpoint":"https://api.github.com","policy":{"schema_version":1,"missing_runner_grace_seconds":120,"scale_sets":{"example-ci":"force-cancel-full-rerun"}}}` + job := `{"repository":"example-org/example-repo","scale_set":"example-ci","run_id":42,"job_id":84,"runner_id":21,"runner_name":"example-runner","job_status":"in_progress","started_at":"2026-08-26T13:00:00Z","runner_present":false,"run_status":"in_progress","run_attempt":1}` + if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(jobPath, []byte(job), 0o600); err != nil { + t.Fatal(err) + } + var stdout, stderr bytes.Buffer + if code := run([]string{"--config", configPath, "--job", jobPath, "plan"}, &stdout, &stderr); code != 0 { + t.Fatalf("code=%d stderr=%s", code, stderr.String()) + } + if !bytes.Contains(stdout.Bytes(), []byte(`"action":"force-cancel"`)) { + t.Fatalf("plan=%s", stdout.String()) + } + if _, err := os.Stat(state); !os.IsNotExist(err) { + t.Fatal("plan mutated recovery state") + } +} + +func TestResolveTokenCommandReceivesOnlyRepositoryIdentity(t *testing.T) { + directory := t.TempDir() + command := filepath.Join(directory, "token") + script := "#!/bin/sh\n[ \"$GHA_VANISHED_REPOSITORY\" = 'example-org/example-repo' ]\nprintf '%s\\n' 'installation-token'\n" + if err := os.WriteFile(command, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + token, err := resolveToken(context.Background(), config{TokenCommand: []string{command}}, "example-org/example-repo") + if err != nil { + t.Fatal(err) + } + if token != "installation-token" { + t.Fatalf("token=%q", token) + } +} diff --git a/internal/garmapptoken/token.go b/internal/garmapptoken/token.go new file mode 100644 index 0000000..a7c1e46 --- /dev/null +++ b/internal/garmapptoken/token.go @@ -0,0 +1,123 @@ +package garmapptoken + +import ( + "bytes" + "context" + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + providerutil "github.com/cloudbase/garm-provider-common/util" +) + +const maxCredentialEnvelopeBytes = 64 << 10 + +type appCredential struct { + AppID int64 `json:"app_id"` + InstallationID int64 `json:"installation_id"` + PrivateKeyBytes []byte `json:"private_key_bytes"` +} + +type Token struct { + Value string + ExpiresAt time.Time +} + +type Minter struct { + Endpoint *url.URL + HTTP *http.Client + Now func() time.Time + Passphrase []byte +} + +func (minter Minter) Mint(ctx context.Context, envelope io.Reader) (Token, error) { + if minter.Endpoint == nil || minter.HTTP == nil || minter.Now == nil || len(minter.Passphrase) != 32 || minter.Endpoint.Scheme != "https" && minter.Endpoint.Scheme != "http" || minter.Endpoint.Host == "" { + return Token{}, fmt.Errorf("GARM GitHub App token minter is incomplete") + } + sealed, err := io.ReadAll(io.LimitReader(envelope, maxCredentialEnvelopeBytes+1)) + if err != nil || len(sealed) == 0 || len(sealed) > maxCredentialEnvelopeBytes { + return Token{}, fmt.Errorf("GARM credential envelope is invalid") + } + plaintext, err := providerutil.Unseal(sealed, minter.Passphrase) + if err != nil { + return Token{}, fmt.Errorf("unseal GARM credential: %w", err) + } + defer clear(plaintext) + var credential appCredential + decoder := json.NewDecoder(bytes.NewReader(plaintext)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&credential); err != nil || credential.AppID <= 0 || credential.InstallationID <= 0 || len(credential.PrivateKeyBytes) == 0 { + return Token{}, fmt.Errorf("GARM GitHub App credential is invalid") + } + defer clear(credential.PrivateKeyBytes) + block, _ := pem.Decode(credential.PrivateKeyBytes) + if block == nil { + return Token{}, fmt.Errorf("GARM GitHub App private key is invalid") + } + privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes) + if err != nil { + return Token{}, fmt.Errorf("parse GARM GitHub App private key: %w", err) + } + now := minter.Now().UTC() + jwt, err := signJWT(privateKey, credential.AppID, now) + if err != nil { + return Token{}, err + } + endpoint := *minter.Endpoint + endpoint.Path = strings.TrimSuffix(endpoint.Path, "/") + "/app/installations/" + strconv.FormatInt(credential.InstallationID, 10) + "/access_tokens" + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), nil) + if err != nil { + return Token{}, err + } + request.Header.Set("Authorization", "Bearer "+jwt) + request.Header.Set("Accept", "application/vnd.github+json") + request.Header.Set("X-GitHub-Api-Version", "2022-11-28") + response, err := minter.HTTP.Do(request) + if err != nil { + return Token{}, fmt.Errorf("mint GitHub App installation token: %w", err) + } + defer response.Body.Close() + body, err := io.ReadAll(io.LimitReader(response.Body, 64*1024+1)) + if err != nil || len(body) > 64*1024 { + return Token{}, fmt.Errorf("read GitHub App installation response") + } + if response.StatusCode != http.StatusCreated { + return Token{}, fmt.Errorf("mint GitHub App installation token: HTTP %d", response.StatusCode) + } + var decoded struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + if err := json.Unmarshal(body, &decoded); err != nil || decoded.Token == "" || decoded.ExpiresAt.Before(now.Add(time.Minute)) { + return Token{}, fmt.Errorf("GitHub App installation token response is invalid") + } + return Token{Value: decoded.Token, ExpiresAt: decoded.ExpiresAt.UTC()}, nil +} + +func signJWT(privateKey *rsa.PrivateKey, appID int64, now time.Time) (string, error) { + header := base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"RS256","typ":"JWT"}`)) + claims, err := json.Marshal(map[string]int64{"iat": now.Add(-time.Minute).Unix(), "exp": now.Add(9 * time.Minute).Unix(), "iss": appID}) + if err != nil { + return "", err + } + payload := base64.RawURLEncoding.EncodeToString(claims) + unsigned := header + "." + payload + digest := sha256.Sum256([]byte(unsigned)) + signature, err := rsa.SignPKCS1v15(rand.Reader, privateKey, crypto.SHA256, digest[:]) + if err != nil { + return "", fmt.Errorf("sign GitHub App JWT: %w", err) + } + return unsigned + "." + base64.RawURLEncoding.EncodeToString(signature), nil +} diff --git a/internal/garmapptoken/token_test.go b/internal/garmapptoken/token_test.go new file mode 100644 index 0000000..082c7cc --- /dev/null +++ b/internal/garmapptoken/token_test.go @@ -0,0 +1,43 @@ +package garmapptoken + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + providerutil "github.com/cloudbase/garm-provider-common/util" + "github.com/stretchr/testify/require" +) + +func TestMinterUnsealsCredentialAndMintsInstallationToken(t *testing.T) { + now := time.Date(2026, 8, 26, 16, 0, 0, 0, time.UTC) + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}) + credential, err := json.Marshal(appCredential{AppID: 123, InstallationID: 456, PrivateKeyBytes: keyPEM}) + require.NoError(t, err) + passphrase := []byte("0123456789abcdefghijklmnopqrstuv") + sealed, err := providerutil.Seal(credential, passphrase) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + require.Equal(t, "/app/installations/456/access_tokens", request.URL.Path) + require.Contains(t, request.Header.Get("Authorization"), "Bearer eyJ") + writer.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(writer).Encode(map[string]any{"token": "ghs_short_lived", "expires_at": now.Add(time.Hour)}) + })) + defer server.Close() + endpoint, err := url.Parse(server.URL) + require.NoError(t, err) + token, err := (Minter{Endpoint: endpoint, HTTP: server.Client(), Now: func() time.Time { return now }, Passphrase: passphrase}).Mint(context.Background(), bytes.NewReader(sealed)) + require.NoError(t, err) + require.Equal(t, "ghs_short_lived", token.Value) +} diff --git a/internal/vanishedjob/controller.go b/internal/vanishedjob/controller.go new file mode 100644 index 0000000..3ca85c5 --- /dev/null +++ b/internal/vanishedjob/controller.go @@ -0,0 +1,113 @@ +package vanishedjob + +import ( + "context" + "fmt" + "time" +) + +type RunClient interface { + ForceCancel(context.Context, string, int64) error + FullRerun(context.Context, string, int64) error +} + +type Event struct { + At time.Time `json:"at"` + Action Action `json:"action"` + Reason string `json:"reason"` + Repository string `json:"repository,omitempty"` + RunID int64 `json:"run_id,omitempty"` + JobID int64 `json:"job_id,omitempty"` + RunnerID int64 `json:"runner_id,omitempty"` + ScaleSet string `json:"scale_set,omitempty"` + Error string `json:"error,omitempty"` +} + +type EventSink interface { + Emit(context.Context, Event) error +} + +type Controller struct { + Policy Policy + Store FileStore + Client RunClient + Events EventSink + Now func() time.Time +} + +func (controller Controller) Reconcile(ctx context.Context, job Job) (Decision, error) { + if controller.Client == nil || controller.Events == nil || controller.Now == nil { + return Decision{}, fmt.Errorf("vanished-runner recovery controller is incomplete") + } + key := RecordKey(job.Repository, job.RunID, job.RunAttempt) + existing, err := controller.Store.Get(key) + if err != nil { + return Decision{}, fmt.Errorf("read vanished-runner recovery: %w", err) + } + // A replacement attempt is indexed by the original record, not by its new + // attempt number. Find that exact active transaction when the direct key is + // absent; there can be only one record for a repository/run at a time. + if existing == nil { + existing, key, err = controller.Store.ForRun(job.Repository, job.RunID) + if err != nil { + return Decision{}, err + } + } + decision, err := Evaluate(controller.Policy, job, existing, controller.Now().UTC()) + if err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + if existing == nil && decision.Action == ActionForceCancel { + created, beginErr := controller.Store.Begin(decision.Record) + if beginErr != nil { + return decision, controller.emit(ctx, job, decision, beginErr) + } + if !created { + return decision, nil + } + key = RecordKey(decision.Record.Repository, decision.Record.RunID, decision.Record.OriginalAttempt) + existing = &decision.Record + } + switch decision.Action { + case ActionForceCancel: + if err := controller.Client.ForceCancel(ctx, job.Repository, job.RunID); err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + if existing.Stage == StageDetected { + if err := controller.Store.Advance(key, StageDetected, StageCancelRequested, controller.Now()); err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + } + case ActionFullRerun: + if err := controller.Client.FullRerun(ctx, job.Repository, job.RunID); err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + if err := controller.Store.Advance(key, existing.Stage, StageRerunRequested, controller.Now()); err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + case ActionComplete: + result := Result{ + Key: key, OriginalAttempt: existing.OriginalAttempt, ReplacementAttempt: job.RunAttempt, + Conclusion: job.RunConclusion, FinishedAt: controller.Now().UTC(), + } + if err := controller.Store.Finish(key, existing.Stage, result); err != nil { + return decision, controller.emit(ctx, job, decision, err) + } + } + return decision, controller.emit(ctx, job, decision, nil) +} + +func (controller Controller) emit(ctx context.Context, job Job, decision Decision, operationErr error) error { + event := Event{ + At: controller.Now().UTC(), Action: decision.Action, Reason: decision.Reason, + Repository: job.Repository, RunID: job.RunID, JobID: job.JobID, + RunnerID: job.RunnerID, ScaleSet: job.ScaleSet, + } + if operationErr != nil { + event.Error = operationErr.Error() + } + if err := controller.Events.Emit(ctx, event); err != nil { + return fmt.Errorf("emit vanished-runner recovery event: %w", err) + } + return operationErr +} diff --git a/internal/vanishedjob/controller_test.go b/internal/vanishedjob/controller_test.go new file mode 100644 index 0000000..0efb059 --- /dev/null +++ b/internal/vanishedjob/controller_test.go @@ -0,0 +1,58 @@ +package vanishedjob + +import ( + "context" + "path/filepath" + "testing" + "time" +) + +type fakeRunClient struct{ cancels, reruns int } + +func (client *fakeRunClient) ForceCancel(context.Context, string, int64) error { + client.cancels++ + return nil +} +func (client *fakeRunClient) FullRerun(context.Context, string, int64) error { + client.reruns++ + return nil +} + +type eventLog struct{ events []Event } + +func (log *eventLog) Emit(_ context.Context, event Event) error { + log.events = append(log.events, event) + return nil +} + +func TestControllerExecutesAndPersistsOneFullRecovery(t *testing.T) { + now := time.Date(2026, 8, 26, 14, 0, 0, 0, time.UTC) + directory := t.TempDir() + client, events := &fakeRunClient{}, &eventLog{} + controller := Controller{ + Policy: testPolicy(t), Store: FileStore{Path: filepath.Join(directory, "state.json"), LockPath: filepath.Join(directory, "state.lock")}, + Client: client, Events: events, Now: func() time.Time { return now }, + } + job := testJob(now) + decision, err := controller.Reconcile(context.Background(), job) + if err != nil || decision.Action != ActionForceCancel || client.cancels != 1 { + t.Fatalf("cancel decision=%#v client=%#v err=%v", decision, client, err) + } + job.RunStatus, job.RunConclusion = "completed", "cancelled" + decision, err = controller.Reconcile(context.Background(), job) + if err != nil || decision.Action != ActionFullRerun || client.reruns != 1 { + t.Fatalf("rerun decision=%#v client=%#v err=%v", decision, client, err) + } + job.RunAttempt, job.RunStatus, job.RunConclusion = 2, "completed", "success" + decision, err = controller.Reconcile(context.Background(), job) + if err != nil || decision.Action != ActionComplete || client.reruns != 1 { + t.Fatalf("complete decision=%#v client=%#v err=%v", decision, client, err) + } + key := RecordKey(job.Repository, job.RunID, 1) + if record, err := controller.Store.Get(key); err != nil || record != nil { + t.Fatalf("terminal record=%#v err=%v", record, err) + } + if len(events.events) != 3 { + t.Fatalf("events=%#v", events.events) + } +} diff --git a/internal/vanishedjob/github.go b/internal/vanishedjob/github.go new file mode 100644 index 0000000..9928137 --- /dev/null +++ b/internal/vanishedjob/github.go @@ -0,0 +1,65 @@ +package vanishedjob + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" +) + +type GitHubClient struct { + Endpoint *url.URL + Token string + HTTP *http.Client +} + +func (client GitHubClient) ForceCancel(ctx context.Context, repository string, runID int64) error { + status, err := client.post(ctx, repository, runID, "force-cancel") + if err != nil { + return err + } + // A 409 means the run crossed terminal concurrently with the request. The + // next authoritative observation advances to rerun; repeating cancellation + // cannot improve that state. + if status != http.StatusAccepted && status != http.StatusConflict { + return fmt.Errorf("force-cancel workflow run returned HTTP %d", status) + } + return nil +} + +func (client GitHubClient) FullRerun(ctx context.Context, repository string, runID int64) error { + status, err := client.post(ctx, repository, runID, "rerun") + if err != nil { + return err + } + if status != http.StatusCreated { + return fmt.Errorf("rerun workflow returned HTTP %d", status) + } + return nil +} + +func (client GitHubClient) post(ctx context.Context, repository string, runID int64, operation string) (int, error) { + parts := strings.Split(repository, "/") + if client.Endpoint == nil || client.Endpoint.Scheme != "https" && client.Endpoint.Scheme != "http" || len(parts) != 2 || parts[0] == "" || parts[1] == "" || runID <= 0 || client.Token == "" || client.HTTP == nil { + return 0, fmt.Errorf("GitHub vanished-runner client is incomplete") + } + endpoint := *client.Endpoint + endpoint.Path = strings.TrimSuffix(endpoint.Path, "/") + "/repos/" + url.PathEscape(parts[0]) + "/" + url.PathEscape(parts[1]) + "/actions/runs/" + strconv.FormatInt(runID, 10) + "/" + operation + request, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint.String(), nil) + if err != nil { + return 0, err + } + request.Header.Set("Accept", "application/vnd.github+json") + request.Header.Set("Authorization", "Bearer "+client.Token) + request.Header.Set("X-GitHub-Api-Version", "2026-03-10") + response, err := client.HTTP.Do(request) + if err != nil { + return 0, err + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64*1024)) + return response.StatusCode, nil +} diff --git a/internal/vanishedjob/github_test.go b/internal/vanishedjob/github_test.go new file mode 100644 index 0000000..286a528 --- /dev/null +++ b/internal/vanishedjob/github_test.go @@ -0,0 +1,55 @@ +package vanishedjob + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestGitHubClientUsesExactRunOperations(t *testing.T) { + requests := []string{} + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Header.Get("Authorization") != "Bearer secret-token" || request.Header.Get("X-GitHub-Api-Version") != "2026-03-10" { + t.Fatal("request omitted bounded authentication or API version") + } + requests = append(requests, request.URL.Path) + if request.URL.Path == "/repos/example-org/example-repo/actions/runs/42/force-cancel" { + writer.WriteHeader(http.StatusAccepted) + return + } + if request.URL.Path == "/repos/example-org/example-repo/actions/runs/42/rerun" { + writer.WriteHeader(http.StatusCreated) + return + } + writer.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + endpoint, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + client := GitHubClient{Endpoint: endpoint, Token: "secret-token", HTTP: server.Client()} + if err := client.ForceCancel(context.Background(), "example-org/example-repo", 42); err != nil { + t.Fatal(err) + } + if err := client.FullRerun(context.Background(), "example-org/example-repo", 42); err != nil { + t.Fatal(err) + } + if len(requests) != 2 { + t.Fatalf("requests=%v", requests) + } +} + +func TestGitHubClientTreatsTerminalCancelRaceAsProgress(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + writer.WriteHeader(http.StatusConflict) + })) + defer server.Close() + endpoint, _ := url.Parse(server.URL) + client := GitHubClient{Endpoint: endpoint, Token: "secret-token", HTTP: server.Client()} + if err := client.ForceCancel(context.Background(), "example-org/example-repo", 42); err != nil { + t.Fatal(err) + } +} diff --git a/internal/vanishedjob/observe.go b/internal/vanishedjob/observe.go new file mode 100644 index 0000000..3a003c3 --- /dev/null +++ b/internal/vanishedjob/observe.go @@ -0,0 +1,74 @@ +package vanishedjob + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os/exec" + "path/filepath" + "time" +) + +const maxObservationBytes = 4 << 20 + +type Observation struct { + Jobs []Job `json:"jobs"` +} + +type Observer interface { + Observe(context.Context) (Observation, error) +} + +type CommandObserver struct { + Argv []string + Timeout time.Duration +} + +func (observer CommandObserver) Observe(ctx context.Context) (Observation, error) { + if len(observer.Argv) == 0 || !filepath.IsAbs(observer.Argv[0]) || observer.Timeout <= 0 || observer.Timeout > 5*time.Minute { + return Observation{}, fmt.Errorf("vanished-runner observer command is invalid") + } + commandCtx, cancel := context.WithTimeout(ctx, observer.Timeout) + defer cancel() + command := exec.CommandContext(commandCtx, observer.Argv[0], observer.Argv[1:]...) + var stdout, stderr bytes.Buffer + command.Stdout = &boundedBuffer{buffer: &stdout, limit: maxObservationBytes} + command.Stderr = &boundedBuffer{buffer: &stderr, limit: 64 << 10} + if err := command.Run(); err != nil { + return Observation{}, fmt.Errorf("observe vanished-runner jobs: %w: %s", err, stderr.String()) + } + decoder := json.NewDecoder(&stdout) + decoder.DisallowUnknownFields() + var observation Observation + if err := decoder.Decode(&observation); err != nil { + return Observation{}, fmt.Errorf("decode vanished-runner observation: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Observation{}, fmt.Errorf("vanished-runner observation has trailing content") + } + seen := make(map[string]struct{}, len(observation.Jobs)) + for _, job := range observation.Jobs { + key := fmt.Sprintf("%s/%d/%d", job.Repository, job.RunID, job.JobID) + if _, exists := seen[key]; exists { + return Observation{}, fmt.Errorf("vanished-runner observation contains duplicate job %s", key) + } + seen[key] = struct{}{} + } + return observation, nil +} + +type boundedBuffer struct { + buffer *bytes.Buffer + limit int +} + +func (writer *boundedBuffer) Write(value []byte) (int, error) { + if writer.buffer.Len()+len(value) > writer.limit { + return 0, fmt.Errorf("command output exceeds %d bytes", writer.limit) + } + return writer.buffer.Write(value) +} diff --git a/internal/vanishedjob/observe_test.go b/internal/vanishedjob/observe_test.go new file mode 100644 index 0000000..f57543b --- /dev/null +++ b/internal/vanishedjob/observe_test.go @@ -0,0 +1,30 @@ +package vanishedjob + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestCommandObserverReadsBatch(t *testing.T) { + directory := t.TempDir() + command := filepath.Join(directory, "observe") + require.NoError(t, os.WriteFile(command, []byte("#!/bin/sh\nprintf '%s\\n' '{\"jobs\":[{\"repository\":\"org/repo\",\"run_id\":42,\"job_id\":7}]}'\n"), 0o700)) + observation, err := (CommandObserver{Argv: []string{command}, Timeout: time.Second}).Observe(context.Background()) + require.NoError(t, err) + require.Len(t, observation.Jobs, 1) + require.Equal(t, int64(42), observation.Jobs[0].RunID) +} + +func TestCommandObserverRejectsDuplicates(t *testing.T) { + directory := t.TempDir() + command := filepath.Join(directory, "observe") + payload := "{\"jobs\":[{\"repository\":\"org/repo\",\"run_id\":42,\"job_id\":7},{\"repository\":\"org/repo\",\"run_id\":42,\"job_id\":7}]}" + require.NoError(t, os.WriteFile(command, []byte("#!/bin/sh\nprintf '%s\\n' '"+payload+"'\n"), 0o700)) + _, err := (CommandObserver{Argv: []string{command}, Timeout: time.Second}).Observe(context.Background()) + require.ErrorContains(t, err, "duplicate job") +} diff --git a/internal/vanishedjob/policy.go b/internal/vanishedjob/policy.go new file mode 100644 index 0000000..d9609b7 --- /dev/null +++ b/internal/vanishedjob/policy.go @@ -0,0 +1,60 @@ +package vanishedjob + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "time" +) + +type Mode string + +const ( + ModeObserve Mode = "observe" + ModeFullRerun Mode = "force-cancel-full-rerun" +) + +type Policy struct { + SchemaVersion int `json:"schema_version"` + MissingRunnerGraceSeconds int `json:"missing_runner_grace_seconds"` + ScaleSets map[string]Mode `json:"scale_sets"` +} + +func DecodePolicy(reader io.Reader) (Policy, error) { + data, err := io.ReadAll(io.LimitReader(reader, 64*1024+1)) + if err != nil || len(data) > 64*1024 { + return Policy{}, fmt.Errorf("read vanished-runner recovery policy: invalid bounded content") + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + var policy Policy + if err := decoder.Decode(&policy); err != nil { + return Policy{}, fmt.Errorf("decode vanished-runner recovery policy: %w", err) + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return Policy{}, fmt.Errorf("vanished-runner recovery policy has trailing content") + } + if err := policy.Validate(); err != nil { + return Policy{}, err + } + return policy, nil +} + +func (policy Policy) Validate() error { + if policy.SchemaVersion != 1 || policy.MissingRunnerGraceSeconds < 60 || policy.MissingRunnerGraceSeconds > 3600 || len(policy.ScaleSets) == 0 { + return fmt.Errorf("vanished-runner recovery policy identity or grace is invalid") + } + for scaleSet, mode := range policy.ScaleSets { + if scaleSet == "" || (mode != ModeObserve && mode != ModeFullRerun) { + return fmt.Errorf("vanished-runner recovery policy for scale set %q is invalid", scaleSet) + } + } + return nil +} + +func (policy Policy) Grace() time.Duration { + return time.Duration(policy.MissingRunnerGraceSeconds) * time.Second +} diff --git a/internal/vanishedjob/state.go b/internal/vanishedjob/state.go new file mode 100644 index 0000000..835e66b --- /dev/null +++ b/internal/vanishedjob/state.go @@ -0,0 +1,113 @@ +package vanishedjob + +import ( + "fmt" + "time" +) + +type Stage string + +const ( + StageDetected Stage = "detected" + StageCancelRequested Stage = "cancel-requested" + StageRerunRequested Stage = "rerun-requested" +) + +type Action string + +const ( + ActionNone Action = "none" + ActionIncident Action = "incident" + ActionForceCancel Action = "force-cancel" + ActionAwaitCancel Action = "await-cancel" + ActionFullRerun Action = "full-rerun" + ActionAwaitRerun Action = "await-rerun" + ActionComplete Action = "complete" +) + +type Job struct { + Repository string `json:"repository"` + ScaleSet string `json:"scale_set"` + RunID int64 `json:"run_id"` + JobID int64 `json:"job_id"` + RunnerID int64 `json:"runner_id"` + RunnerName string `json:"runner_name"` + JobStatus string `json:"job_status"` + StartedAt time.Time `json:"started_at"` + RunnerPresent bool `json:"runner_present"` + RunStatus string `json:"run_status"` + RunConclusion string `json:"run_conclusion,omitempty"` + RunAttempt int `json:"run_attempt"` +} + +type Record struct { + Repository string `json:"repository"` + RunID int64 `json:"run_id"` + JobID int64 `json:"job_id"` + RunnerID int64 `json:"runner_id"` + RunnerName string `json:"runner_name"` + ScaleSet string `json:"scale_set"` + OriginalAttempt int `json:"original_attempt"` + Stage Stage `json:"stage"` + UpdatedAt time.Time `json:"updated_at"` +} + +type Decision struct { + Action Action `json:"action"` + Reason string `json:"reason"` + Record Record `json:"record,omitempty"` +} + +func Evaluate(policy Policy, job Job, existing *Record, now time.Time) (Decision, error) { + if err := policy.Validate(); err != nil { + return Decision{}, err + } + mode, configured := policy.ScaleSets[job.ScaleSet] + if !configured { + return Decision{Action: ActionNone, Reason: "scale-set-unconfigured"}, nil + } + if existing == nil { + if job.Repository == "" || job.RunID <= 0 || job.JobID <= 0 || job.RunnerID <= 0 || job.RunnerName == "" || job.RunAttempt <= 0 || job.StartedAt.IsZero() { + return Decision{}, fmt.Errorf("vanished-runner job identity is incomplete") + } + if job.JobStatus != "in_progress" || job.RunnerPresent || now.Sub(job.StartedAt) < policy.Grace() { + return Decision{Action: ActionNone, Reason: "no-aged-vanished-runner"}, nil + } + record := Record{ + Repository: job.Repository, RunID: job.RunID, JobID: job.JobID, + RunnerID: job.RunnerID, RunnerName: job.RunnerName, ScaleSet: job.ScaleSet, + OriginalAttempt: job.RunAttempt, Stage: StageDetected, UpdatedAt: now.UTC(), + } + if mode == ModeObserve { + return Decision{Action: ActionIncident, Reason: "side-effecting-workflow-requires-policy", Record: record}, nil + } + return Decision{Action: ActionForceCancel, Reason: "aged-job-runner-absent", Record: record}, nil + } + if existing.Repository != job.Repository || existing.RunID != job.RunID || existing.JobID != job.JobID || existing.RunnerID != job.RunnerID || existing.RunnerName != job.RunnerName || existing.ScaleSet != job.ScaleSet || existing.OriginalAttempt < 1 { + return Decision{}, fmt.Errorf("vanished-runner recovery identity changed") + } + // A crash after rerun but before journal update is reconstructed from the + // authoritative attempt number, so the same run can never be rerun twice. + if job.RunAttempt > existing.OriginalAttempt { + if job.RunStatus == "completed" { + return Decision{Action: ActionComplete, Reason: "replacement-attempt-terminal", Record: *existing}, nil + } + return Decision{Action: ActionAwaitRerun, Reason: "replacement-attempt-active", Record: *existing}, nil + } + switch existing.Stage { + case StageDetected: + if job.RunStatus == "completed" { + return Decision{Action: ActionFullRerun, Reason: "force-cancel-became-terminal", Record: *existing}, nil + } + return Decision{Action: ActionForceCancel, Reason: "resume-force-cancel", Record: *existing}, nil + case StageCancelRequested: + if job.RunStatus == "completed" { + return Decision{Action: ActionFullRerun, Reason: "cancel-terminal", Record: *existing}, nil + } + return Decision{Action: ActionAwaitCancel, Reason: "cancel-in-progress", Record: *existing}, nil + case StageRerunRequested: + return Decision{Action: ActionAwaitRerun, Reason: "rerun-attempt-not-visible", Record: *existing}, nil + default: + return Decision{}, fmt.Errorf("vanished-runner recovery stage is invalid") + } +} diff --git a/internal/vanishedjob/state_test.go b/internal/vanishedjob/state_test.go new file mode 100644 index 0000000..24b4213 --- /dev/null +++ b/internal/vanishedjob/state_test.go @@ -0,0 +1,75 @@ +package vanishedjob + +import ( + "strings" + "testing" + "time" +) + +func testPolicy(t *testing.T) Policy { + t.Helper() + policy, err := DecodePolicy(strings.NewReader(`{ + "schema_version":1, + "missing_runner_grace_seconds":120, + "scale_sets":{"example-ci":"force-cancel-full-rerun","example-release":"observe"} +}`)) + if err != nil { + t.Fatal(err) + } + return policy +} + +func testJob(now time.Time) Job { + return Job{ + Repository: "example-org/example-repo", ScaleSet: "example-ci", RunID: 42, JobID: 84, + RunnerID: 21, RunnerName: "example-runner", JobStatus: "in_progress", + StartedAt: now.Add(-3 * time.Minute), RunStatus: "in_progress", RunAttempt: 1, + } +} + +func TestEvaluateFullRecoveryLifecycleAndCrashReconstruction(t *testing.T) { + now := time.Date(2026, 8, 26, 14, 0, 0, 0, time.UTC) + policy, job := testPolicy(t), testJob(now) + detected, err := Evaluate(policy, job, nil, now) + if err != nil || detected.Action != ActionForceCancel || detected.Record.Stage != StageDetected { + t.Fatalf("detection = %#v, %v", detected, err) + } + cancelled := detected.Record + cancelled.Stage = StageCancelRequested + waiting, err := Evaluate(policy, job, &cancelled, now.Add(time.Minute)) + if err != nil || waiting.Action != ActionAwaitCancel { + t.Fatalf("cancel wait = %#v, %v", waiting, err) + } + job.RunStatus, job.RunConclusion = "completed", "cancelled" + rerun, err := Evaluate(policy, job, &cancelled, now.Add(2*time.Minute)) + if err != nil || rerun.Action != ActionFullRerun { + t.Fatalf("rerun = %#v, %v", rerun, err) + } + // Simulate a crash after GitHub accepted the rerun but before Stage was + // persisted. The incremented attempt is authoritative and suppresses replay. + job.RunAttempt, job.RunStatus = 2, "in_progress" + reconstructed, err := Evaluate(policy, job, &cancelled, now.Add(3*time.Minute)) + if err != nil || reconstructed.Action != ActionAwaitRerun { + t.Fatalf("reconstructed = %#v, %v", reconstructed, err) + } + job.RunStatus, job.RunConclusion = "completed", "success" + complete, err := Evaluate(policy, job, &cancelled, now.Add(4*time.Minute)) + if err != nil || complete.Action != ActionComplete { + t.Fatalf("complete = %#v, %v", complete, err) + } +} + +func TestEvaluateSeparatesReleaseAndTransientAbsence(t *testing.T) { + now := time.Date(2026, 8, 26, 14, 0, 0, 0, time.UTC) + policy, job := testPolicy(t), testJob(now) + job.ScaleSet = "example-release" + decision, err := Evaluate(policy, job, nil, now) + if err != nil || decision.Action != ActionIncident { + t.Fatalf("release = %#v, %v", decision, err) + } + job.ScaleSet, job.RunnerPresent = "example-ci", true + decision, err = Evaluate(policy, job, nil, now) + if err != nil || decision.Action != ActionNone { + t.Fatalf("present runner = %#v, %v", decision, err) + } +} diff --git a/internal/vanishedjob/store.go b/internal/vanishedjob/store.go new file mode 100644 index 0000000..42b8218 --- /dev/null +++ b/internal/vanishedjob/store.go @@ -0,0 +1,263 @@ +package vanishedjob + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "golang.org/x/sys/unix" +) + +const recoveryHistoryLimit = 128 +const maximumStateBytes = 1024 * 1024 + +type Result struct { + Key string `json:"key"` + OriginalAttempt int `json:"original_attempt"` + ReplacementAttempt int `json:"replacement_attempt"` + Conclusion string `json:"conclusion"` + FinishedAt time.Time `json:"finished_at"` +} + +type fileState struct { + SchemaVersion int `json:"schema_version"` + Generation uint64 `json:"generation"` + Records map[string]Record `json:"records"` + Finished []Result `json:"finished"` +} + +type FileStore struct { + Path string + LockPath string +} + +func RecordKey(repository string, runID int64, attempt int) string { + return repository + "/runs/" + strconv.FormatInt(runID, 10) + "/attempts/" + strconv.Itoa(attempt) +} + +func (store FileStore) Get(key string) (*Record, error) { + var found *Record + err := store.readLocked(func(state fileState) error { + if record, exists := state.Records[key]; exists { + copy := record + found = © + } + return nil + }) + return found, err +} + +func (store FileStore) ForRun(repository string, runID int64) (*Record, string, error) { + var found *Record + var key string + err := store.readLocked(func(state fileState) error { + for candidateKey, record := range state.Records { + if record.Repository != repository || record.RunID != runID { + continue + } + if found != nil { + return fmt.Errorf("multiple vanished-runner recoveries exist for one run") + } + copy := record + found, key = ©, candidateKey + } + return nil + }) + return found, key, err +} + +func (store FileStore) Begin(record Record) (bool, error) { + if err := record.Validate(); err != nil { + return false, err + } + key := RecordKey(record.Repository, record.RunID, record.OriginalAttempt) + created := false + err := store.locked(func(state *fileState) (bool, error) { + if _, exists := state.Records[key]; exists { + return false, nil + } + for _, result := range state.Finished { + if result.Key == key { + return false, nil + } + } + state.Records[key] = record + created = true + return true, nil + }) + return created, err +} + +func (store FileStore) Advance(key string, expected, next Stage, at time.Time) error { + if at.IsZero() || expected == StageDetected && next != StageCancelRequested || expected == StageCancelRequested && next != StageRerunRequested || expected == StageRerunRequested { + return fmt.Errorf("vanished-runner recovery transition is invalid") + } + return store.locked(func(state *fileState) (bool, error) { + record, exists := state.Records[key] + if !exists || record.Stage != expected { + return false, fmt.Errorf("vanished-runner recovery stage changed") + } + record.Stage, record.UpdatedAt = next, at.UTC() + state.Records[key] = record + return true, nil + }) +} + +func (store FileStore) Finish(key string, expected Stage, result Result) error { + if result.ReplacementAttempt <= result.OriginalAttempt || result.Conclusion == "" { + return fmt.Errorf("vanished-runner recovery result is invalid") + } + return store.locked(func(state *fileState) (bool, error) { + record, exists := state.Records[key] + if !exists || record.Stage != expected || result.Key != key || result.OriginalAttempt != record.OriginalAttempt || result.FinishedAt.IsZero() { + return false, fmt.Errorf("vanished-runner recovery finish identity changed") + } + delete(state.Records, key) + state.Finished = append(state.Finished, result) + if len(state.Finished) > recoveryHistoryLimit { + state.Finished = slices.Clone(state.Finished[len(state.Finished)-recoveryHistoryLimit:]) + } + return true, nil + }) +} + +func (store FileStore) locked(update func(*fileState) (bool, error)) error { + if !filepath.IsAbs(store.Path) || !filepath.IsAbs(store.LockPath) || filepath.Clean(store.Path) == string(filepath.Separator) || filepath.Clean(store.LockPath) == string(filepath.Separator) || filepath.Clean(store.Path) == filepath.Clean(store.LockPath) { + return fmt.Errorf("vanished-runner recovery paths are unsafe") + } + if err := os.MkdirAll(filepath.Dir(store.Path), 0o700); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(store.LockPath), 0o700); err != nil { + return err + } + lock, err := os.OpenFile(store.LockPath, os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return err + } + defer lock.Close() + if err := unix.Flock(int(lock.Fd()), unix.LOCK_EX); err != nil { + return err + } + defer unix.Flock(int(lock.Fd()), unix.LOCK_UN) //nolint:errcheck + state, err := readState(store.Path) + if err != nil { + return err + } + changed, err := update(&state) + if err != nil { + return err + } + if !changed { + return nil + } + state.Generation++ + return writeState(store.Path, state) +} + +func (store FileStore) readLocked(read func(fileState) error) error { + if !filepath.IsAbs(store.Path) || !filepath.IsAbs(store.LockPath) || filepath.Clean(store.Path) == filepath.Clean(store.LockPath) { + return fmt.Errorf("vanished-runner recovery paths are unsafe") + } + if _, err := os.Stat(store.Path); os.IsNotExist(err) { + return read(fileState{SchemaVersion: 1, Records: map[string]Record{}}) + } else if err != nil { + return err + } + lock, err := os.OpenFile(store.LockPath, os.O_RDWR, 0) + if err != nil { + return err + } + defer lock.Close() + if err := unix.Flock(int(lock.Fd()), unix.LOCK_SH); err != nil { + return err + } + defer unix.Flock(int(lock.Fd()), unix.LOCK_UN) //nolint:errcheck + state, err := readState(store.Path) + if err != nil { + return err + } + return read(state) +} + +func readState(path string) (fileState, error) { + file, err := os.Open(path) + if os.IsNotExist(err) { + return fileState{SchemaVersion: 1, Records: map[string]Record{}}, nil + } + if err != nil { + return fileState{}, err + } + defer file.Close() + data, err := io.ReadAll(io.LimitReader(file, maximumStateBytes+1)) + if err != nil || len(data) > maximumStateBytes { + return fileState{}, fmt.Errorf("vanished-runner recovery state exceeds its bounded size") + } + var state fileState + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&state); err != nil || state.SchemaVersion != 1 || state.Records == nil { + return fileState{}, fmt.Errorf("vanished-runner recovery state is invalid") + } + var trailing any + if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { + return fileState{}, fmt.Errorf("vanished-runner recovery state has trailing content") + } + for key, record := range state.Records { + if err := record.Validate(); err != nil || key != RecordKey(record.Repository, record.RunID, record.OriginalAttempt) { + return fileState{}, fmt.Errorf("vanished-runner recovery record %q is invalid", key) + } + } + return state, nil +} + +func (record Record) Validate() error { + parts := strings.Split(record.Repository, "/") + if len(parts) != 2 || parts[0] == "" || parts[1] == "" || record.RunID <= 0 || record.JobID <= 0 || record.RunnerID <= 0 || strings.TrimSpace(record.RunnerName) == "" || strings.TrimSpace(record.ScaleSet) == "" || record.OriginalAttempt <= 0 || record.UpdatedAt.IsZero() || record.Stage != StageDetected && record.Stage != StageCancelRequested && record.Stage != StageRerunRequested { + return fmt.Errorf("vanished-runner recovery record identity is invalid") + } + return nil +} + +func writeState(path string, state fileState) error { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + temporary, err := os.CreateTemp(filepath.Dir(path), ".vanished-job-") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + if err := temporary.Chmod(0o600); err != nil { + return err + } + if _, err := temporary.Write(data); err != nil { + return err + } + if err := temporary.Sync(); err != nil { + return err + } + if err := temporary.Close(); err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + directory, err := os.Open(filepath.Dir(path)) + if err != nil { + return err + } + defer directory.Close() + return directory.Sync() +} diff --git a/internal/vanishedjob/store_test.go b/internal/vanishedjob/store_test.go new file mode 100644 index 0000000..47d3ee5 --- /dev/null +++ b/internal/vanishedjob/store_test.go @@ -0,0 +1,52 @@ +package vanishedjob + +import ( + "path/filepath" + "testing" + "time" +) + +func TestFileStorePersistsCASLifecycleAndSuppressesReplay(t *testing.T) { + directory := t.TempDir() + store := FileStore{Path: filepath.Join(directory, "state.json"), LockPath: filepath.Join(directory, "state.lock")} + at := time.Date(2026, 8, 26, 14, 0, 0, 0, time.UTC) + record := Record{ + Repository: "example-org/example-repo", RunID: 42, JobID: 84, RunnerID: 21, + RunnerName: "example-runner", ScaleSet: "example-ci", OriginalAttempt: 1, + Stage: StageDetected, UpdatedAt: at, + } + created, err := store.Begin(record) + if err != nil || !created { + t.Fatalf("begin=%t err=%v", created, err) + } + reopened := FileStore{Path: store.Path, LockPath: store.LockPath} + created, err = reopened.Begin(record) + if err != nil || created { + t.Fatalf("duplicate begin=%t err=%v", created, err) + } + key := RecordKey(record.Repository, record.RunID, record.OriginalAttempt) + if err := reopened.Advance(key, StageDetected, StageCancelRequested, at.Add(time.Minute)); err != nil { + t.Fatal(err) + } + if err := reopened.Advance(key, StageDetected, StageRerunRequested, at.Add(2*time.Minute)); err == nil { + t.Fatal("stale stage compare-and-swap succeeded") + } + if err := reopened.Advance(key, StageCancelRequested, StageRerunRequested, at.Add(2*time.Minute)); err != nil { + t.Fatal(err) + } + result := Result{Key: key, OriginalAttempt: 1, ReplacementAttempt: 2, Conclusion: "success", FinishedAt: at.Add(3 * time.Minute)} + if err := reopened.Finish(key, StageRerunRequested, result); err != nil { + t.Fatal(err) + } + if current, err := store.Get(key); err != nil || current != nil { + t.Fatalf("finished record=%#v err=%v", current, err) + } + created, err = store.Begin(record) + if err != nil || created { + t.Fatalf("finished replay begin=%t err=%v", created, err) + } + state, err := readState(store.Path) + if err != nil || len(state.Finished) != 1 || state.Generation != 4 { + t.Fatalf("state=%#v err=%v", state, err) + } +}