feat(recovery): model vanished runner job recovery

CHANGELOG.md

@@ -46,6 +46,12 @@ Versioning.
 
 ### Added
 
+- Added autonomous vanished-runner recovery with exact GitHub run
+  force-cancellation, one full rerun, crash reconstruction from `run_attempt`,
+  bounded command observation and fsync-backed transaction state.
+- Added short-lived GitHub App installation-token minting from an existing
+  encrypted GARM credential envelope, without persisting private keys or
+  installation tokens outside their existing authority.
 - Mirrored verified `nddev_tool_cache_event` records into the GitHub runner
   diagnostic directory so teardown bundles retain job-level cache evidence.
 - Added bounded extraction of `nddev_tool_cache_event` records from verified

cmd/gha-garm-app-token/main.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/NDDev-OpenNetwork/github-actions/internal/garmapptoken"
+)
+
+type garmConfig struct {
+	Database struct {
+		Passphrase string `toml:"passphrase"`
+	} `toml:"database"`
+}
+
+func main() { os.Exit(run(os.Args[1:], os.Stdin, os.Stdout, os.Stderr)) }
+
+func run(arguments []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	flags := flag.NewFlagSet("gha-garm-app-token", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	configPath := flags.String("garm-config", "", "absolute GARM config path")
+	endpointValue := flags.String("github-endpoint", "https://api.github.com", "GitHub API endpoint")
+	if err := flags.Parse(arguments); err != nil || *configPath == "" || flags.NArg() != 0 {
+		fmt.Fprintln(stderr, "usage: gha-garm-app-token --garm-config PATH [--github-endpoint URL]")
+		return 2
+	}
+	var configuration garmConfig
+	if _, err := toml.DecodeFile(*configPath, &configuration); err != nil || len(configuration.Database.Passphrase) != 32 {
+		fmt.Fprintln(stderr, "GARM database passphrase configuration is invalid")
+		return 1
+	}
+	endpoint, err := url.Parse(*endpointValue)
+	if err != nil {
+		fmt.Fprintln(stderr, "GitHub endpoint is invalid")
+		return 1
+	}
+	token, err := (garmapptoken.Minter{
+		Endpoint: endpoint, HTTP: &http.Client{Timeout: 20 * time.Second}, Now: time.Now,
+		Passphrase: []byte(configuration.Database.Passphrase),
+	}).Mint(context.Background(), stdin)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	fmt.Fprintln(stdout, token.Value)
+	return 0
+}

cmd/gha-vanished-job-recovery/main.go

@@ -0,0 +1,234 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/NDDev-OpenNetwork/github-actions/internal/vanishedjob"
+)
+
+type config struct {
+	StateFile      string             `json:"state_file"`
+	LockFile       string             `json:"lock_file"`
+	TokenFile      string             `json:"token_file"`
+	TokenCommand   []string           `json:"token_command"`
+	GitHubEndpoint string             `json:"github_endpoint"`
+	Policy         vanishedjob.Policy `json:"policy"`
+	Observation    struct {
+		Argv           []string `json:"argv"`
+		TimeoutSeconds int      `json:"timeout_seconds"`
+	} `json:"observation"`
+}
+
+type jsonEvents struct{ encoder *json.Encoder }
+
+func (sink jsonEvents) Emit(_ context.Context, event vanishedjob.Event) error {
+	return sink.encoder.Encode(event)
+}
+
+func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) }
+
+func run(arguments []string, stdout, stderr io.Writer) int {
+	flags := flag.NewFlagSet("gha-vanished-job-recovery", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	configPath := flags.String("config", "", "absolute recovery config path")
+	jobPath := flags.String("job", "", "absolute observed job JSON path")
+	if err := flags.Parse(arguments); err != nil {
+		return 2
+	}
+	if *configPath == "" || flags.NArg() != 1 || flags.Arg(0) != "plan" && flags.Arg(0) != "apply" {
+		fmt.Fprintln(stderr, "usage: gha-vanished-job-recovery --config PATH [--job PATH] <plan|apply>")
+		return 2
+	}
+	configuration, err := loadConfig(*configPath)
+	if err != nil {
+		fmt.Fprintln(stderr, err)
+		return 1
+	}
+	var jobs []vanishedjob.Job
+	if *jobPath != "" {
+		job, loadErr := loadJob(*jobPath)
+		if loadErr != nil {
+			fmt.Fprintln(stderr, loadErr)
+			return 1
+		}
+		jobs = []vanishedjob.Job{job}
+	} else {
+		observation, observeErr := (vanishedjob.CommandObserver{
+			Argv:    configuration.Observation.Argv,
+			Timeout: time.Duration(configuration.Observation.TimeoutSeconds) * time.Second,
+		}).Observe(context.Background())
+		if observeErr != nil {
+			fmt.Fprintln(stderr, observeErr)
+			return 1
+		}
+		jobs = observation.Jobs
+	}
+	store := vanishedjob.FileStore{Path: configuration.StateFile, LockPath: configuration.LockFile}
+	if flags.Arg(0) == "plan" {
+		decisions := make([]vanishedjob.Decision, 0, len(jobs))
+		for _, job := range jobs {
+			existing, getErr := store.Get(vanishedjob.RecordKey(job.Repository, job.RunID, job.RunAttempt))
+			if getErr != nil {
+				fmt.Fprintln(stderr, getErr)
+				return 1
+			}
+			if existing == nil {
+				existing, _, getErr = store.ForRun(job.Repository, job.RunID)
+			}
+			if getErr != nil {
+				fmt.Fprintln(stderr, getErr)
+				return 1
+			}
+			decision, evaluateErr := vanishedjob.Evaluate(configuration.Policy, job, existing, time.Now().UTC())
+			if evaluateErr != nil {
+				fmt.Fprintln(stderr, evaluateErr)
+				return 1
+			}
+			decisions = append(decisions, decision)
+		}
+		if err := json.NewEncoder(stdout).Encode(decisions); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+		return 0
+	}
+	endpoint, _ := url.Parse(configuration.GitHubEndpoint)
+	tokens := make(map[string]string)
+	for _, job := range jobs {
+		token, present := tokens[job.Repository]
+		if !present {
+			var tokenErr error
+			token, tokenErr = resolveToken(context.Background(), configuration, job.Repository)
+			if tokenErr != nil {
+				fmt.Fprintln(stderr, tokenErr)
+				return 1
+			}
+			tokens[job.Repository] = token
+		}
+		controller := vanishedjob.Controller{
+			Policy: configuration.Policy, Store: store,
+			Client: vanishedjob.GitHubClient{Endpoint: endpoint, Token: token, HTTP: &http.Client{Timeout: 20 * time.Second}},
+			Events: jsonEvents{json.NewEncoder(stdout)}, Now: time.Now,
+		}
+		if _, err := controller.Reconcile(context.Background(), job); err != nil {
+			fmt.Fprintln(stderr, err)
+			return 1
+		}
+	}
+	return 0
+}
+
+func loadConfig(path string) (config, error) {
+	if !boundedAbsolute(path) {
+		return config{}, fmt.Errorf("recovery config path must be absolute and bounded")
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return config{}, err
+	}
+	var configuration config
+	decoder := json.NewDecoder(bytes.NewReader(data))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&configuration); err != nil {
+		return config{}, err
+	}
+	endpoint, endpointErr := url.Parse(configuration.GitHubEndpoint)
+	validTokenSource := boundedAbsolute(configuration.TokenFile) != (len(configuration.TokenCommand) > 0 && filepath.IsAbs(configuration.TokenCommand[0]))
+	if !boundedAbsolute(configuration.StateFile) || !boundedAbsolute(configuration.LockFile) || !validTokenSource || endpointErr != nil || endpoint.Scheme != "https" && endpoint.Scheme != "http" || endpoint.Host == "" {
+		return config{}, fmt.Errorf("recovery config paths or endpoint are invalid")
+	}
+	if err := configuration.Policy.Validate(); err != nil {
+		return config{}, err
+	}
+	if len(configuration.Observation.Argv) > 0 && (configuration.Observation.TimeoutSeconds <= 0 || !filepath.IsAbs(configuration.Observation.Argv[0])) {
+		return config{}, fmt.Errorf("recovery observation command is invalid")
+	}
+	return configuration, nil
+}
+
+func resolveToken(ctx context.Context, configuration config, repository string) (string, error) {
+	if configuration.TokenFile != "" {
+		return readSecret(configuration.TokenFile)
+	}
+	commandCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
+	defer cancel()
+	command := exec.CommandContext(commandCtx, configuration.TokenCommand[0], configuration.TokenCommand[1:]...)
+	command.Env = append(os.Environ(), "GHA_VANISHED_REPOSITORY="+repository)
+	var stdout, stderr bytes.Buffer
+	command.Stdout = &limitedWriter{buffer: &stdout, remaining: 64*1024 + 1}
+	command.Stderr = &limitedWriter{buffer: &stderr, remaining: 64*1024 + 1}
+	if err := command.Run(); err != nil {
+		return "", fmt.Errorf("resolve GitHub token: %w: %s", err, strings.TrimSpace(stderr.String()))
+	}
+	if stdout.Len() > 64*1024 {
+		return "", fmt.Errorf("resolved GitHub token exceeds bounded output")
+	}
+	token := strings.TrimSpace(stdout.String())
+	if token == "" || strings.ContainsAny(token, "\r\n") {
+		return "", fmt.Errorf("token command must return one non-empty line")
+	}
+	return token, nil
+}
+
+type limitedWriter struct {
+	buffer    *bytes.Buffer
+	remaining int
+}
+
+func (writer *limitedWriter) Write(value []byte) (int, error) {
+	if len(value) > writer.remaining {
+		return 0, fmt.Errorf("command output exceeds bounded size")
+	}
+	writer.remaining -= len(value)
+	return writer.buffer.Write(value)
+}
+
+func loadJob(path string) (vanishedjob.Job, error) {
+	if !boundedAbsolute(path) {
+		return vanishedjob.Job{}, fmt.Errorf("job observation path must be absolute and bounded")
+	}
+	file, err := os.Open(path)
+	if err != nil {
+		return vanishedjob.Job{}, err
+	}
+	defer file.Close()
+	var job vanishedjob.Job
+	decoder := json.NewDecoder(io.LimitReader(file, 64*1024+1))
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&job); err != nil {
+		return vanishedjob.Job{}, err
+	}
+	return job, nil
+}
+
+func readSecret(path string) (string, error) {
+	info, err := os.Stat(path)
+	if err != nil || !info.Mode().IsRegular() || info.Mode().Perm()&0o077 != 0 {
+		return "", fmt.Errorf("token file must be a private regular file")
+	}
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	token := strings.TrimSpace(string(data))
+	if token == "" || strings.ContainsAny(token, "\r\n") {
+		return "", fmt.Errorf("token file must contain one non-empty line")
+	}
+	return token, nil
+}
+
+func boundedAbsolute(path string) bool {
+	return filepath.IsAbs(path) && filepath.Clean(path) != string(filepath.Separator)
+}

cmd/gha-vanished-job-recovery/main_test.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestPlanIsReadOnlyAndClassifiesVanishedCIJob(t *testing.T) {
+	directory := t.TempDir()
+	state, lock, token := filepath.Join(directory, "state.json"), filepath.Join(directory, "state.lock"), filepath.Join(directory, "token")
+	if err := os.WriteFile(token, []byte("secret-token"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	configPath := filepath.Join(directory, "config.json")
+	jobPath := filepath.Join(directory, "job.json")
+	configuration := `{"state_file":"` + state + `","lock_file":"` + lock + `","token_file":"` + token + `","github_endpoint":"https://api.github.com","policy":{"schema_version":1,"missing_runner_grace_seconds":120,"scale_sets":{"example-ci":"force-cancel-full-rerun"}}}`
+	job := `{"repository":"example-org/example-repo","scale_set":"example-ci","run_id":42,"job_id":84,"runner_id":21,"runner_name":"example-runner","job_status":"in_progress","started_at":"2026-08-26T13:00:00Z","runner_present":false,"run_status":"in_progress","run_attempt":1}`
+	if err := os.WriteFile(configPath, []byte(configuration), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(jobPath, []byte(job), 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var stdout, stderr bytes.Buffer
+	if code := run([]string{"--config", configPath, "--job", jobPath, "plan"}, &stdout, &stderr); code != 0 {
+		t.Fatalf("code=%d stderr=%s", code, stderr.String())
+	}
+	if !bytes.Contains(stdout.Bytes(), []byte(`"action":"force-cancel"`)) {
+		t.Fatalf("plan=%s", stdout.String())
+	}
+	if _, err := os.Stat(state); !os.IsNotExist(err) {
+		t.Fatal("plan mutated recovery state")
+	}
+}
+
+func TestResolveTokenCommandReceivesOnlyRepositoryIdentity(t *testing.T) {
+	directory := t.TempDir()
+	command := filepath.Join(directory, "token")
+	script := "#!/bin/sh\n[ \"$GHA_VANISHED_REPOSITORY\" = 'example-org/example-repo' ]\nprintf '%s\\n' 'installation-token'\n"
+	if err := os.WriteFile(command, []byte(script), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	token, err := resolveToken(context.Background(), config{TokenCommand: []string{command}}, "example-org/example-repo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if token != "installation-token" {
+		t.Fatalf("token=%q", token)
+	}
+}