diff --git a/application/README.md b/application/README.md index e86bd0c..98a820a 100644 --- a/application/README.md +++ b/application/README.md @@ -269,6 +269,38 @@ Usage: neuron remove example:echo@1.0.0 ``` +### `neuron builtins` + +Report the capability runtimes N.O.R.E. provides without installation. + +``` +Usage: + neuron builtins [name] [flags] + +Flags: + -v, --verbose Include the full descriptions +``` + +A built-in capability runtime is part of the runtime: it is always available, it is never installed from a registry, and it is the capability runtime an assembly uses when it names none. The command reports the contract of every built-in, or the contract of one built-in when it is named. It needs neither a running runtime nor a project. + +```bash +# List the built-in capability runtimes +neuron builtins + +# The contract of one built-in +neuron builtins http + +# The contract with the full descriptions +neuron builtins http --verbose +``` + +``` +neuron:core:command Runs a local program +neuron:core:filesystem Reads files and directories +neuron:core:http Makes an HTTP request +neuron:core:set Materializes a capability's static value set +``` + ### `neuron capability runtime` Inspect capability runtime packages installed in the local store. (Installing or removing a package is done with the top-level `neuron add` and `neuron remove` commands, since those change what a project pins.) diff --git a/application/compiler/compiler.go b/application/compiler/compiler.go index 4dcc5c0..6029ef5 100644 --- a/application/compiler/compiler.go +++ b/application/compiler/compiler.go @@ -250,6 +250,9 @@ func convertBinding(conn manifest.Binding, capabilityMap map[string]core.Capabil if !ok { return core.Binding{}, fmt.Errorf("binding from %q to %q: to capability %q not found", conn.From, conn.To, conn.To) } + if err := conn.JoinPolicy.Validate(); err != nil { + return core.Binding{}, fmt.Errorf("binding %s -> %s: %w", conn.From, conn.To, err) + } var mappings []core.MappingRule for _, m := range conn.Mappings { @@ -285,6 +288,7 @@ func convertBinding(conn manifest.Binding, capabilityMap map[string]core.Capabil To: core.Endpoint{ CapabilityID: toSvc.Metadata.ID, }, + JoinPolicy: conn.JoinPolicy, Mappings: mappings, Validations: validations, }, nil diff --git a/application/compiler/join_test.go b/application/compiler/join_test.go new file mode 100644 index 0000000..5f0c087 --- /dev/null +++ b/application/compiler/join_test.go @@ -0,0 +1,29 @@ +package compiler + +import ( + "testing" + + "github.com/neuron-runtime/neuron/shared/types/core" +) + +func TestCompileCarriesJoinPolicy(t *testing.T) { + m := testManifest() + m.Bindings[0].JoinPolicy = core.JoinPolicyAll + + sys, err := New().Compile(m) + if err != nil { + t.Fatalf("compile: %v", err) + } + if got := sys.Specification.Bindings[0].JoinPolicy; got != core.JoinPolicyAll { + t.Fatalf("binding join policy = %q, want %q", got, core.JoinPolicyAll) + } +} + +func TestCompileRejectsUnsupportedJoinPolicy(t *testing.T) { + m := testManifest() + m.Bindings[0].JoinPolicy = core.JoinPolicy("any") + + if _, err := New().Compile(m); err == nil { + t.Fatal("an unsupported join policy must be rejected at build time") + } +} diff --git a/application/compiler/manifest/manifest.go b/application/compiler/manifest/manifest.go index bf7b277..96e86d3 100644 --- a/application/compiler/manifest/manifest.go +++ b/application/compiler/manifest/manifest.go @@ -108,8 +108,15 @@ type CapabilityRuntimeSpec struct { // Binding describes a directed edge between two capabilities. type Binding struct { - From string `json:"from"` - To string `json:"to"` + From string `json:"from"` + To string `json:"to"` + + // JoinPolicy selects when To becomes ready when several bindings converge + // on it. It is omitted when the author declares nothing, which means + // core.JoinPolicyAll. Every incoming binding for one target must declare + // the same effective policy. + JoinPolicy core.JoinPolicy `json:"joinPolicy,omitempty"` + Mappings []BindingMapping `json:"mappings"` Validations []BindingValidation `json:"validations"` } diff --git a/application/compiler/manifest/yaml.go b/application/compiler/manifest/yaml.go index 323ad77..dcbd35c 100644 --- a/application/compiler/manifest/yaml.go +++ b/application/compiler/manifest/yaml.go @@ -149,8 +149,9 @@ func runtimeConfigFrom(spec project.CapabilitySpec) *core.RuntimeConfig { func bindingFrom(rc project.ResolvedBinding) Binding { conn := Binding{ - From: rc.Definition.From, - To: rc.Definition.To, + From: rc.Definition.From, + To: rc.Definition.To, + JoinPolicy: rc.Definition.JoinPolicy, } for _, m := range rc.Definition.Mappings { diff --git a/application/compiler/manifest/yaml_test.go b/application/compiler/manifest/yaml_test.go index 0abc102..d018c45 100644 --- a/application/compiler/manifest/yaml_test.go +++ b/application/compiler/manifest/yaml_test.go @@ -81,3 +81,28 @@ func TestFromResolvedProjectWithoutAResultContractStaysNil(t *testing.T) { t.Fatalf("result = %+v, want nil when the YAML assembly declares none", got.Result) } } + +func TestFromResolvedProjectCarriesBindingJoinPolicy(t *testing.T) { + rp := &project.ResolvedProject{ + Assembly: project.ResolvedAssembly{ + Bindings: []project.ResolvedBinding{{ + Definition: project.BindingFile{ + From: "a", + To: "b", + JoinPolicy: core.JoinPolicyAll, + }, + }}, + }, + } + + got := FromResolvedProject(rp) + if got == nil { + t.Fatal("FromResolvedProject returned nil") + } + if len(got.Bindings) != 1 { + t.Fatalf("bindings = %d, want 1", len(got.Bindings)) + } + if got.Bindings[0].JoinPolicy != core.JoinPolicyAll { + t.Fatalf("join policy = %q, want %q", got.Bindings[0].JoinPolicy, core.JoinPolicyAll) + } +} diff --git a/application/config/builtins.go b/application/config/builtins.go new file mode 100644 index 0000000..1f07b43 --- /dev/null +++ b/application/config/builtins.go @@ -0,0 +1,148 @@ +package config + +import ( + "encoding/json" + "fmt" + + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// BuiltinsConfig bounds the built-in capability runtimes N.O.R.E. provides. +// +// The built-ins are the only capability runtimes that are not installed +// artifacts: they are compiled into the runtime and they reach outside it — +// they open sockets, run programs, and touch the filesystem. The reach they +// are allowed is configured here, per built-in, because a capability that can +// reach the whole machine is not a capability the author controls. +// +// This configuration can only narrow the limits compiled into the runtime. The +// ceilings live in the runtime, not here, so a project configuration file — +// which is part of the assembly's own input — cannot widen what a capability +// can reach. +type BuiltinsConfig struct { + // Filesystem bounds the filesystem built-in. + Filesystem BuiltinFilesystemConfig `yaml:"filesystem,omitempty" mapstructure:"filesystem"` + + // Command bounds the command built-in. + Command BuiltinCommandConfig `yaml:"command,omitempty" mapstructure:"command"` + + // HTTP bounds the http built-in. + HTTP BuiltinHTTPConfig `yaml:"http,omitempty" mapstructure:"http"` + + // Limits bounds the size of what the built-ins read, write, and return. + Limits BuiltinLimitsConfig `yaml:"limits,omitempty" mapstructure:"limits"` +} + +// BuiltinFilesystemConfig bounds the filesystem built-in's reach. +type BuiltinFilesystemConfig struct { + // Roots are additional directories the filesystem and command built-ins + // may reach, resolved against the project root. + // + // The project root is always permitted: it is where the assembly lives. + // These roots extend that reach to the directories the assembly is + // expected to work with. A path that resolves outside every permitted + // root is refused. + Roots []string `yaml:"roots,omitempty" mapstructure:"roots"` +} + +// BuiltinCommandConfig bounds the command built-in's reach. +type BuiltinCommandConfig struct { + // AllowedExecutables restricts the programs the command built-in may run. + // + // When it is empty, any program resolvable on the program path may run. + // When it is not empty, only the listed programs may run, matched by + // their resolved absolute path. + AllowedExecutables []string `yaml:"allowedExecutables,omitempty" mapstructure:"allowedExecutables"` +} + +// BuiltinHTTPConfig bounds the http built-in's reach. +type BuiltinHTTPConfig struct { + // AllowedHosts restricts the http built-in to a set of hosts. + // + // When it is empty, the built-in may contact any public host and refuses + // loopback, private, link-local, multicast, unspecified, and cloud + // metadata addresses. When it is not empty, only the listed hosts may be + // contacted — including ones that resolve to private addresses, because + // listing a host is the explicit decision to reach it. + AllowedHosts []string `yaml:"allowedHosts,omitempty" mapstructure:"allowedHosts"` +} + +// BuiltinLimitsConfig bounds the size of what a built-in may read, write, and +// return. +// +// Every limit here is also compiled into the runtime as a ceiling. A value +// above the ceiling is refused rather than clamped, so a configuration that +// asks for more than the runtime allows fails loudly instead of silently +// running under a different limit than the one that was configured. +type BuiltinLimitsConfig struct { + // MaxCommandOutputBytes bounds the output retained from one stream of a + // single command. + MaxCommandOutputBytes int64 `yaml:"maxCommandOutputBytes,omitempty" mapstructure:"maxCommandOutputBytes"` + + // MaxHTTPRequestBodyBytes bounds the request body a single http request + // may send. + MaxHTTPRequestBodyBytes int64 `yaml:"maxHTTPRequestBodyBytes,omitempty" mapstructure:"maxHTTPRequestBodyBytes"` + + // MaxHTTPResponseBytes bounds the response body a single http request + // may read. + MaxHTTPResponseBytes int64 `yaml:"maxHTTPResponseBytes,omitempty" mapstructure:"maxHTTPResponseBytes"` + + // MaxFileReadBytes bounds the file content a single filesystem read may + // return. + MaxFileReadBytes int64 `yaml:"maxFileReadBytes,omitempty" mapstructure:"maxFileReadBytes"` + + // MaxDirectoryEntries bounds the entries a single filesystem listing may + // return. + MaxDirectoryEntries int `yaml:"maxDirectoryEntries,omitempty" mapstructure:"maxDirectoryEntries"` + + // DefaultTimeout bounds a single built-in operation when the capability + // does not set a timeout of its own, as a duration string such as "60s". + DefaultTimeout string `yaml:"defaultTimeout,omitempty" mapstructure:"defaultTimeout"` +} + +// BuiltinPolicy resolves the policy the built-in capability runtimes run +// under from the effective configuration. +// +// The resolution is the shared one, so the configuration can only narrow the +// compiled-in ceilings. A configuration that asks for more than a ceiling +// allows is refused here rather than quietly clamped, because a limit that +// silently differs from the configured one is a limit nobody is looking at. +func (c Config) BuiltinPolicy() (builtins.Policy, error) { + policy, err := builtins.Effective(builtins.Policy{ + FilesystemRoots: c.Builtins.Filesystem.Roots, + AllowedExecutables: c.Builtins.Command.AllowedExecutables, + AllowedHTTPHosts: c.Builtins.HTTP.AllowedHosts, + MaxCommandOutputBytes: c.Builtins.Limits.MaxCommandOutputBytes, + MaxHTTPRequestBodyBytes: c.Builtins.Limits.MaxHTTPRequestBodyBytes, + MaxHTTPResponseBytes: c.Builtins.Limits.MaxHTTPResponseBytes, + MaxFileReadBytes: c.Builtins.Limits.MaxFileReadBytes, + MaxDirectoryEntries: c.Builtins.Limits.MaxDirectoryEntries, + DefaultTimeout: c.Builtins.Limits.DefaultTimeout, + }) + if err != nil { + return builtins.Policy{}, fmt.Errorf("invalid built-in capability policy: %w", err) + } + return policy, nil +} + +// resolveBuiltinPaths expands the configured built-in filesystem roots against +// the project root so the runtime compares absolute paths. +func resolveBuiltinPaths(cfg *Config, projectDir string) { + for i := range cfg.Builtins.Filesystem.Roots { + cfg.Builtins.Filesystem.Roots[i] = Expand(cfg.Builtins.Filesystem.Roots[i], projectDir) + } +} + +// BuiltinPolicyJSON encodes the effective built-in policy for the runtime's +// command line. +// +// The policy travels as JSON because the runtime is a separate process (and a +// separate Go module): the command line is the transport that already carries +// the project's runtime settings to it. +func BuiltinPolicyJSON(policy builtins.Policy) (string, error) { + data, err := json.Marshal(policy) + if err != nil { + return "", fmt.Errorf("encode built-in capability policy: %w", err) + } + return string(data), nil +} diff --git a/application/config/builtins_test.go b/application/config/builtins_test.go new file mode 100644 index 0000000..0541430 --- /dev/null +++ b/application/config/builtins_test.go @@ -0,0 +1,224 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +func TestBuiltinPolicyDefaultsToTheCeilings(t *testing.T) { + cfg, err := Load(Options{GlobalPath: "/nonexistent/global.yaml", ProjectDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + + policy, err := cfg.BuiltinPolicy() + if err != nil { + t.Fatal(err) + } + + ceilings := builtins.DefaultPolicy() + if policy.MaxCommandOutputBytes != ceilings.MaxCommandOutputBytes { + t.Errorf("maxCommandOutputBytes = %d, want %d", policy.MaxCommandOutputBytes, ceilings.MaxCommandOutputBytes) + } + if policy.MaxHTTPRequestBodyBytes != ceilings.MaxHTTPRequestBodyBytes { + t.Errorf("maxHTTPRequestBodyBytes = %d, want %d", policy.MaxHTTPRequestBodyBytes, ceilings.MaxHTTPRequestBodyBytes) + } + if policy.MaxHTTPResponseBytes != ceilings.MaxHTTPResponseBytes { + t.Errorf("maxHTTPResponseBytes = %d, want %d", policy.MaxHTTPResponseBytes, ceilings.MaxHTTPResponseBytes) + } + if policy.MaxFileReadBytes != ceilings.MaxFileReadBytes { + t.Errorf("maxFileReadBytes = %d, want %d", policy.MaxFileReadBytes, ceilings.MaxFileReadBytes) + } + if policy.MaxDirectoryEntries != ceilings.MaxDirectoryEntries { + t.Errorf("maxDirectoryEntries = %d, want %d", policy.MaxDirectoryEntries, ceilings.MaxDirectoryEntries) + } + if policy.DefaultTimeout != ceilings.DefaultTimeout { + t.Errorf("defaultTimeout = %q, want %q", policy.DefaultTimeout, ceilings.DefaultTimeout) + } + if len(policy.AllowedHTTPHosts) != 0 { + t.Errorf("allowedHTTPHosts = %v, want none", policy.AllowedHTTPHosts) + } + if len(policy.AllowedExecutables) != 0 { + t.Errorf("allowedExecutables = %v, want none", policy.AllowedExecutables) + } +} + +func TestBuiltinPolicyReadsTheProjectConfiguration(t *testing.T) { + dir := t.TempDir() + write(t, dir, "neuron.config.yaml", ` +builtins: + filesystem: + roots: + - data + - /srv/shared + command: + allowedExecutables: + - git + - ls + http: + allowedHosts: + - api.example.com + limits: + maxCommandOutputBytes: 4096 + maxHTTPResponseBytes: 65536 + defaultTimeout: 15s +`) + + cfg, err := Load(Options{ + GlobalPath: "/nonexistent/global.yaml", + ProjectPath: filepath.Join(dir, "neuron.config.yaml"), + }) + if err != nil { + t.Fatal(err) + } + + policy, err := cfg.BuiltinPolicy() + if err != nil { + t.Fatal(err) + } + + if len(policy.FilesystemRoots) != 2 { + t.Fatalf("filesystemRoots = %v, want 2 roots", policy.FilesystemRoots) + } + // A relative root resolves against the project root. + if want := filepath.Join(dir, "data"); policy.FilesystemRoots[0] != want { + t.Errorf("filesystemRoots[0] = %q, want %q", policy.FilesystemRoots[0], want) + } + if policy.FilesystemRoots[1] != "/srv/shared" { + t.Errorf("filesystemRoots[1] = %q, want /srv/shared", policy.FilesystemRoots[1]) + } + if len(policy.AllowedExecutables) != 2 { + t.Errorf("allowedExecutables = %v, want 2 entries", policy.AllowedExecutables) + } + if len(policy.AllowedHTTPHosts) != 1 || policy.AllowedHTTPHosts[0] != "api.example.com" { + t.Errorf("allowedHTTPHosts = %v, want [api.example.com]", policy.AllowedHTTPHosts) + } + if policy.MaxCommandOutputBytes != 4096 { + t.Errorf("maxCommandOutputBytes = %d, want 4096", policy.MaxCommandOutputBytes) + } + if policy.MaxHTTPResponseBytes != 65536 { + t.Errorf("maxHTTPResponseBytes = %d, want 65536", policy.MaxHTTPResponseBytes) + } + if policy.DefaultTimeout != "15s" { + t.Errorf("defaultTimeout = %q, want 15s", policy.DefaultTimeout) + } + // Limits the configuration did not mention keep the compiled-in ceiling. + ceilings := builtins.DefaultPolicy() + if policy.MaxFileReadBytes != ceilings.MaxFileReadBytes { + t.Errorf("maxFileReadBytes = %d, want %d", policy.MaxFileReadBytes, ceilings.MaxFileReadBytes) + } +} + +func TestBuiltinPolicyRefusesToExceedTheCeilings(t *testing.T) { + dir := t.TempDir() + write(t, dir, "neuron.config.yaml", ` +builtins: + limits: + maxCommandOutputBytes: 67108864 +`) + + _, err := Load(Options{ + GlobalPath: "/nonexistent/global.yaml", + ProjectPath: filepath.Join(dir, "neuron.config.yaml"), + }) + if err == nil { + t.Fatal("expected the configuration to be refused") + } + if !strings.Contains(err.Error(), "maxCommandOutputBytes") { + t.Errorf("error = %v, want it to name the limit that exceeded the ceiling", err) + } +} + +func TestBuiltinPolicyRefusesAnUnusableTimeout(t *testing.T) { + dir := t.TempDir() + write(t, dir, "neuron.config.yaml", ` +builtins: + limits: + defaultTimeout: never +`) + + if _, err := Load(Options{ + GlobalPath: "/nonexistent/global.yaml", + ProjectPath: filepath.Join(dir, "neuron.config.yaml"), + }); err == nil { + t.Fatal("expected the configuration to be refused") + } +} + +func TestBuiltinPolicyRejectsAnUnknownKey(t *testing.T) { + dir := t.TempDir() + write(t, dir, "neuron.config.yaml", ` +builtins: + http: + allowedHosts: + - api.example.com + followRedirects: true +`) + + _, err := Load(Options{ + GlobalPath: "/nonexistent/global.yaml", + ProjectPath: filepath.Join(dir, "neuron.config.yaml"), + }) + if err == nil { + t.Fatal("expected the configuration to be refused") + } + if !strings.Contains(err.Error(), "followRedirects") { + t.Errorf("error = %v, want it to name the unknown key", err) + } +} + +func TestBuiltinPolicyJSONRoundTrips(t *testing.T) { + policy, err := builtins.Effective(builtins.Policy{ + AllowedHTTPHosts: []string{"api.example.com"}, + DefaultTimeout: "30s", + }) + if err != nil { + t.Fatal(err) + } + + encoded, err := BuiltinPolicyJSON(policy) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(encoded, "api.example.com") { + t.Errorf("encoded policy = %s, want it to carry the allowed host", encoded) + } +} + +func TestBuiltinRootsResolveAgainstTheProjectRoot(t *testing.T) { + dir := t.TempDir() + write(t, dir, "neuron.config.yaml", ` +builtins: + filesystem: + roots: + - ./data + - ~/shared +`) + + cfg, err := Load(Options{ + GlobalPath: "/nonexistent/global.yaml", + ProjectPath: filepath.Join(dir, "neuron.config.yaml"), + }) + if err != nil { + t.Fatal(err) + } + + roots := cfg.Builtins.Filesystem.Roots + if len(roots) != 2 { + t.Fatalf("roots = %v, want 2 roots", roots) + } + if !filepath.IsAbs(roots[0]) { + t.Errorf("roots[0] = %q, want an absolute path", roots[0]) + } + home, err := os.UserHomeDir() + if err != nil { + t.Skip("no home directory in this environment") + } + if want := filepath.Join(home, "shared"); roots[1] != want { + t.Errorf("roots[1] = %q, want %q", roots[1], want) + } +} diff --git a/application/config/config.go b/application/config/config.go index ebeb2a1..b1759a3 100644 --- a/application/config/config.go +++ b/application/config/config.go @@ -58,6 +58,10 @@ type Config struct { Providers ProvidersConfig `yaml:"providers" mapstructure:"providers"` Inspector InspectorConfig `yaml:"inspector" mapstructure:"inspector"` + // Builtins bounds the built-in capability runtimes N.O.R.E. provides. + // It can only narrow the limits compiled into the runtime. + Builtins BuiltinsConfig `yaml:"builtins,omitempty" mapstructure:"builtins"` + // Dev controls developer-experience options. Dev DevConfig `yaml:"dev,omitempty" mapstructure:"dev"` @@ -169,17 +173,23 @@ type DaemonConfig struct { PIDFile string `yaml:"pidFile,omitempty" mapstructure:"pidFile"` } -// StorageConfig selects the storage provider and its root directory. +// StorageConfig selects the storage driver and its root directory. // // Storage is internal: Neuron manages it and it cannot be set from a // configuration file. It remains on Config only so the daemon bootstrap can -// derive its data directory and the CLI can test programmatically. +// derive its data directory and forward the selected driver to the runtime. type StorageConfig struct { + // Provider/Driver names a registered storage driver; "sqlite" is the default. Provider string `yaml:"provider" mapstructure:"provider"` + Driver string `yaml:"driver" mapstructure:"driver"` - // Directory is the provider's data directory. The storage implementation + // Directory/Path is the driver's data directory. The storage implementation // decides the individual subdirectories (assemblies, instances, ...). Directory string `yaml:"directory" mapstructure:"directory"` + Path string `yaml:"path" mapstructure:"path"` + + // DSN is the PostgreSQL connection string. Must not be committed. + DSN string `yaml:"dsn" mapstructure:"dsn"` } // CapabilityRuntimesConfig lists the executable registries Neuron can resolve capabilities diff --git a/application/config/defaults.go b/application/config/defaults.go index 8cb7c3a..5ac7477 100644 --- a/application/config/defaults.go +++ b/application/config/defaults.go @@ -1,11 +1,18 @@ package config +import "github.com/neuron-runtime/neuron/shared/types/builtins" + // Defaults returns the built-in Neuron configuration. // // These values are compiled into Neuron and are the base of every effective // configuration produced by Load. Lower layers (global, project, environment, // CLI) override them. func Defaults() Config { + // The built-in limits default to the ceilings compiled into the runtime, + // so the effective configuration states the limits that actually apply. + // A project may lower them; it can never raise them. + ceilings := builtins.DefaultPolicy() + return Config{ Lang: "typescript", @@ -30,8 +37,10 @@ func Defaults() Config { }, Storage: StorageConfig{ - Provider: "file", + Provider: "sqlite", + Driver: "sqlite", Directory: DefaultDataDir(), + Path: DefaultDataDir(), }, // No registries are compiled in. A project that resolves external @@ -58,5 +67,16 @@ func Defaults() Config { Dev: DevConfig{ MaxWorkers: 1, }, + + Builtins: BuiltinsConfig{ + Limits: BuiltinLimitsConfig{ + MaxCommandOutputBytes: ceilings.MaxCommandOutputBytes, + MaxHTTPRequestBodyBytes: ceilings.MaxHTTPRequestBodyBytes, + MaxHTTPResponseBytes: ceilings.MaxHTTPResponseBytes, + MaxFileReadBytes: ceilings.MaxFileReadBytes, + MaxDirectoryEntries: ceilings.MaxDirectoryEntries, + DefaultTimeout: ceilings.DefaultTimeout, + }, + }, } } diff --git a/application/config/loader.go b/application/config/loader.go index 62a12d7..0e2a05e 100644 --- a/application/config/loader.go +++ b/application/config/loader.go @@ -135,6 +135,13 @@ func Load(opts Options) (Config, error) { resolvePaths(&cfg, cfg.ProjectDir) + // The built-in policy is resolved here so a configuration that asks for + // more than the compiled-in ceilings fails when the configuration is + // loaded, rather than when a capability first reaches for the limit. + if _, err := cfg.BuiltinPolicy(); err != nil { + return Config{}, err + } + if err := validateAssemblyDeclarations(&cfg); err != nil { return Config{}, fmt.Errorf("invalid assemblies: %w", err) } @@ -190,7 +197,10 @@ func registerDefaults(v *viper.Viper, cfg Config) { v.SetDefault("daemon.pidFile", cfg.Daemon.PIDFile) v.SetDefault("storage.provider", cfg.Storage.Provider) + v.SetDefault("storage.driver", cfg.Storage.Driver) v.SetDefault("storage.directory", cfg.Storage.Directory) + v.SetDefault("storage.path", cfg.Storage.Path) + v.SetDefault("storage.dsn", cfg.Storage.DSN) v.SetDefault("inspector.enabled", cfg.Inspector.Enabled) v.SetDefault("inspector.address", cfg.Inspector.Address) @@ -235,12 +245,35 @@ func registerDefaults(v *viper.Viper, cfg Config) { v.SetDefault("dev.maxWorkers", cfg.Dev.MaxWorkers) v.SetDefault("entry", cfg.Entry) v.SetDefault("variables", cfg.Variables) + + v.SetDefault("builtins.filesystem.roots", cfg.Builtins.Filesystem.Roots) + v.SetDefault("builtins.command.allowedExecutables", cfg.Builtins.Command.AllowedExecutables) + v.SetDefault("builtins.http.allowedHosts", cfg.Builtins.HTTP.AllowedHosts) + v.SetDefault("builtins.limits.maxCommandOutputBytes", cfg.Builtins.Limits.MaxCommandOutputBytes) + v.SetDefault("builtins.limits.maxHTTPRequestBodyBytes", cfg.Builtins.Limits.MaxHTTPRequestBodyBytes) + v.SetDefault("builtins.limits.maxHTTPResponseBytes", cfg.Builtins.Limits.MaxHTTPResponseBytes) + v.SetDefault("builtins.limits.maxFileReadBytes", cfg.Builtins.Limits.MaxFileReadBytes) + v.SetDefault("builtins.limits.maxDirectoryEntries", cfg.Builtins.Limits.MaxDirectoryEntries) + v.SetDefault("builtins.limits.defaultTimeout", cfg.Builtins.Limits.DefaultTimeout) } // resolvePaths expands any relative or "~"-prefixed path fields against the // project root so downstream consumers always receive absolute paths. func resolvePaths(cfg *Config, projectDir string) { + if cfg.Storage.Driver == "" { + cfg.Storage.Driver = cfg.Storage.Provider + } + if cfg.Storage.Provider == "" { + cfg.Storage.Provider = cfg.Storage.Driver + } + if cfg.Storage.Path == "" { + cfg.Storage.Path = cfg.Storage.Directory + } + if cfg.Storage.Directory == "" { + cfg.Storage.Directory = cfg.Storage.Path + } cfg.Storage.Directory = Expand(cfg.Storage.Directory, projectDir) + cfg.Storage.Path = Expand(cfg.Storage.Path, projectDir) cfg.Daemon.Socket = Expand(cfg.Daemon.Socket, projectDir) cfg.Daemon.PIDFile = Expand(cfg.Daemon.PIDFile, projectDir) cfg.CapabilityRuntimes.StoreDir = Expand(cfg.CapabilityRuntimes.StoreDir, projectDir) @@ -264,6 +297,8 @@ func resolvePaths(cfg *Config, projectDir string) { cfg.Providers.LocalRoots[i] = Expand(cfg.Providers.LocalRoots[i], projectDir) } + resolveBuiltinPaths(cfg, projectDir) + for i := range cfg.CapabilityRuntimes.Registries { reg := &cfg.CapabilityRuntimes.Registries[i] if reg.Name == "local" && reg.URL != "" && reg.URL != "local://" { diff --git a/application/daemon/config.go b/application/daemon/config.go index 2f3c971..5b27afa 100644 --- a/application/daemon/config.go +++ b/application/daemon/config.go @@ -55,8 +55,24 @@ func ConfigFromEffective(cfg config.Config) Config { c.Args = append(c.Args, "--socket", cfg.Daemon.Socket) } - if cfg.Storage.Directory != "" { - c.Args = append(c.Args, "--data-dir", cfg.Storage.Directory) + driver := cfg.Storage.Driver + if driver == "" { + driver = cfg.Storage.Provider + } + if driver != "" { + c.Args = append(c.Args, "--storage-driver", driver) + } + + dir := cfg.Storage.Directory + if dir == "" { + dir = cfg.Storage.Path + } + if dir != "" { + c.Args = append(c.Args, "--data-dir", dir) + c.Args = append(c.Args, "--storage-path", dir) + } + if cfg.Storage.DSN != "" { + c.Args = append(c.Args, "--storage-dsn", cfg.Storage.DSN) } if cfg.Runtime.Workers.Max > 0 { diff --git a/application/daemon/config_test.go b/application/daemon/config_test.go index 33296b4..d1827c0 100644 --- a/application/daemon/config_test.go +++ b/application/daemon/config_test.go @@ -19,6 +19,9 @@ func noreArgs(t *testing.T, args []string) []string { _ = fs.String("socket", "", "") _ = fs.Int("workers", 8, "") _ = fs.String("data-dir", "", "") + _ = fs.String("storage-driver", "", "") + _ = fs.String("storage-path", "", "") + _ = fs.String("storage-dsn", "", "") _ = fs.String("detached-drain-timeout", "30s", "") if err := fs.Parse(args); err != nil { diff --git a/application/internal/cli/build/build.go b/application/internal/cli/build/build.go index b3699ca..7facf68 100644 --- a/application/internal/cli/build/build.go +++ b/application/internal/cli/build/build.go @@ -488,10 +488,19 @@ func resolveFrozenCapabilityRuntimes(ctx context.Context, cfg config.Config, req capabilityRuntimeReqs := make([]capabilityruntime.Requirement, 0, len(requirements)) for _, req := range requirements { - // Core capability runtimes (neuron:core:*) run in-process inside N.O.R.E. and - // the legacy bare core names too; there is nothing to resolve or - // install for them. - if core.IsCoreRuntimeType(core.CapabilityRuntimeType(req.Name)) { + runtimeType := core.CapabilityRuntimeType(req.Name) + + // A reference to a built-in that does not exist is a mistake in the + // assembly. It is reported here, where the author can see it, rather + // than being sent to the package resolver. + if err := core.UnknownBuiltinError(runtimeType); err != nil { + return nil, err + } + + // Built-in capability runtimes (neuron:core:*) run in-process inside + // N.O.R.E. and the legacy bare core names too; there is nothing to + // resolve or install for them. + if core.IsCoreRuntimeType(runtimeType) { continue } var registries []string diff --git a/application/internal/cli/builtins/builtins.go b/application/internal/cli/builtins/builtins.go new file mode 100644 index 0000000..e8af6f8 --- /dev/null +++ b/application/internal/cli/builtins/builtins.go @@ -0,0 +1,178 @@ +// Package builtins implements the `neuron builtins` command, which reports the +// capability runtimes the runtime provides without installation. +package builtins + +import ( + "fmt" + "io" + "sort" + "strings" + "text/tabwriter" + + "github.com/neuron-runtime/neuron/application/internal/cli/command" + "github.com/neuron-runtime/neuron/shared/types/core" + "github.com/spf13/cobra" +) + +// New returns the `neuron builtins` command. +// +// The command reports a contract that is compiled into the runtime, so it +// needs neither a running runtime nor a project. It is the place an author +// looks up what a built-in accepts and what it guarantees. +func New() *cobra.Command { + cmd := &cobra.Command{ + Use: command.Builtins, + Short: "Report the built-in capability runtimes", + Long: `Report the capability runtimes N.O.R.E. provides without installation. + +A built-in capability runtime is part of the runtime: it is always available, it is +never installed from a registry, and it is the capability runtime an assembly uses +when it names none. + +The command reports the contract of every built-in, or the contract of one built-in +when it is named. Use --verbose for the full descriptions. + +The command needs neither a running runtime nor a project.`, + Args: cobra.MaximumNArgs(1), + RunE: run, + } + // The command declares the flag it reads, so it reports the contract the + // same way whether it runs on its own or under the root command. The root + // command has a verbose flag of its own with a different meaning, and this + // command shows no runtime logs. + cmd.Flags().BoolP("verbose", "v", false, "Include the full descriptions") + return cmd +} + +// run reports the built-ins, or the one that was named. +func run(cmd *cobra.Command, args []string) error { + verbose, _ := cmd.Flags().GetBool("verbose") + out := cmd.OutOrStdout() + + if len(args) == 1 { + return describe(out, args[0], verbose) + } + return list(out, verbose) +} + +// list reports every built-in. +func list(out io.Writer, verbose bool) error { + descriptors := core.BuiltinDescriptors() + // The listing is sorted by name so it reads the same way as the list of + // available built-ins in an error message. + sort.Slice(descriptors, func(i, j int) bool { return descriptors[i].Name < descriptors[j].Name }) + + writer := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) + for _, descriptor := range descriptors { + if _, err := fmt.Fprintf(writer, "%s\t%s\n", descriptor.Type(), descriptor.Summary); err != nil { + return err + } + } + if err := writer.Flush(); err != nil { + return err + } + + if verbose { + for _, descriptor := range descriptors { + if _, err := fmt.Fprintf(out, "\n%s\n %s\n", descriptor.Type(), descriptor.Description); err != nil { + return err + } + } + } + return nil +} + +// describe reports one built-in. +func describe(out io.Writer, name string, verbose bool) error { + descriptor, ok := core.LookupBuiltin(core.CapabilityRuntimeType(canonical(name))) + if !ok { + return unknown(name) + } + + if _, err := fmt.Fprintf(out, "%s\n %s\n", descriptor.Type(), descriptor.Summary); err != nil { + return err + } + + if verbose { + if _, err := fmt.Fprintf(out, "\n%s\n", descriptor.Description); err != nil { + return err + } + } + + if err := writeParameters(out, "Parameters", descriptor.Parameters, verbose); err != nil { + return err + } + if err := writeParameters(out, "Results", descriptor.Results, verbose); err != nil { + return err + } + + if len(descriptor.Constraints) > 0 { + if _, err := fmt.Fprintf(out, "\nConstraints:\n"); err != nil { + return err + } + for _, constraint := range descriptor.Constraints { + if _, err := fmt.Fprintf(out, " - %s\n", constraint); err != nil { + return err + } + } + } + + return nil +} + +// writeParameters writes the parameters or the results of a built-in. +func writeParameters(out io.Writer, title string, parameters []core.BuiltinParameter, verbose bool) error { + if len(parameters) == 0 { + return nil + } + + if _, err := fmt.Fprintf(out, "\n%s:\n", title); err != nil { + return err + } + + for _, parameter := range parameters { + if _, err := fmt.Fprintf(out, " %s (%s%s)\n", parameter.Name, parameter.Type, qualification(parameter)); err != nil { + return err + } + if verbose && parameter.Description != "" { + if _, err := fmt.Fprintf(out, " %s\n", parameter.Description); err != nil { + return err + } + } + } + return nil +} + +// qualification states how a parameter is satisfied: it is required, it has +// a default, or it is optional. +func qualification(parameter core.BuiltinParameter) string { + switch { + case parameter.Required: + return ", required" + case parameter.Default != "": + return fmt.Sprintf(", default %q", parameter.Default) + default: + return "" + } +} + +// canonical turns a name into the canonical form of a core capability runtime, +// so that a name that names a removed built-in is reported as removed rather +// than as unknown. +func canonical(name string) string { + trimmed := strings.TrimSpace(name) + if strings.Contains(trimmed, ":") { + return trimmed + } + return string(core.CoreName(trimmed)) +} + +// unknown reports a name that names no built-in. +func unknown(name string) error { + canonicalName := canonical(name) + if err := core.UnknownBuiltinError(core.CapabilityRuntimeType(canonicalName)); err != nil { + return err + } + return fmt.Errorf("unknown built-in capability runtime %q; the built-in capability runtimes are %s", + name, strings.Join(core.BuiltinTypeNames(), ", ")) +} diff --git a/application/internal/cli/builtins/builtins_test.go b/application/internal/cli/builtins/builtins_test.go new file mode 100644 index 0000000..eb81c8d --- /dev/null +++ b/application/internal/cli/builtins/builtins_test.go @@ -0,0 +1,110 @@ +package builtins + +import ( + "bytes" + "strings" + "testing" + + "github.com/spf13/cobra" +) + +// execute runs the command with the given arguments and returns its output. +func execute(t *testing.T, args ...string) (string, error) { + t.Helper() + + cmd := New() + out := &bytes.Buffer{} + cmd.SetOut(out) + cmd.SetErr(out) + cmd.SetArgs(args) + + err := cmd.Execute() + return out.String(), err +} + +// The listing names every built-in with the type an assembly refers to it by. +func TestListNamesEveryBuiltin(t *testing.T) { + out, err := execute(t) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + for _, name := range []string{"neuron:core:set", "neuron:core:http", "neuron:core:command", "neuron:core:filesystem"} { + if !strings.Contains(out, name) { + t.Errorf("the listing does not name %q:\n%s", name, out) + } + } +} + +// The detail of a built-in states what it accepts and what it produces. +func TestDetailStatesTheContract(t *testing.T) { + out, err := execute(t, "http") + if err != nil { + t.Fatalf("Execute: %v", err) + } + + for _, expected := range []string{"neuron:core:http", "Parameters:", "url (string, required)", "Results:", "status_code (number, required)", "Constraints:"} { + if !strings.Contains(out, expected) { + t.Errorf("the detail does not state %q:\n%s", expected, out) + } + } +} + +// The verbose detail explains what each parameter is for. +func TestVerboseDetailExplainsTheParameters(t *testing.T) { + out, err := execute(t, "http", "--verbose") + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if !strings.Contains(out, "The absolute URL to request.") { + t.Errorf("the verbose detail does not explain the parameters:\n%s", out) + } +} + +// A built-in that was removed is reported as removed, and the replacement is +// named. +func TestRemovedBuiltinIsReportedAsRemoved(t *testing.T) { + _, err := execute(t, "delay") + if err == nil { + t.Fatal("Execute accepted a built-in that was removed") + } + if !strings.Contains(err.Error(), "was removed") { + t.Errorf("error = %v, want it to report the removal", err) + } + if !strings.Contains(err.Error(), "instead") { + t.Errorf("error = %v, want it to name a replacement", err) + } +} + +// A name that names no built-in is reported with the built-ins that exist. +func TestUnknownBuiltinIsReported(t *testing.T) { + _, err := execute(t, "storage") + if err == nil { + t.Fatal("Execute accepted a name that names no built-in") + } + if !strings.Contains(err.Error(), "neuron:core:http") { + t.Errorf("error = %v, want it to list the built-ins", err) + } +} + +// The canonical name of a built-in is accepted as well as the bare one. +func TestCanonicalNameIsAccepted(t *testing.T) { + out, err := execute(t, "neuron:core:filesystem") + if err != nil { + t.Fatalf("Execute: %v", err) + } + if !strings.Contains(out, "neuron:core:filesystem") { + t.Errorf("the detail does not name the built-in:\n%s", out) + } +} + +// The command is not bound to a project, so it reports the contract without a +// configuration. +func TestTheCommandNeedsNoProject(t *testing.T) { + cmd := New() + if cmd.RunE == nil { + t.Fatal("the command has no handler") + } + var _ *cobra.Command = cmd +} diff --git a/application/internal/cli/cli.go b/application/internal/cli/cli.go index c6f8aa9..1a28fdc 100644 --- a/application/internal/cli/cli.go +++ b/application/internal/cli/cli.go @@ -9,6 +9,7 @@ import ( "github.com/neuron-runtime/neuron/application/config" "github.com/neuron-runtime/neuron/application/internal/cli/bootstrap" "github.com/neuron-runtime/neuron/application/internal/cli/build" + "github.com/neuron-runtime/neuron/application/internal/cli/builtins" "github.com/neuron-runtime/neuron/application/internal/cli/capabilityruntime" "github.com/neuron-runtime/neuron/application/internal/cli/command" "github.com/neuron-runtime/neuron/application/internal/cli/daemon" @@ -87,6 +88,7 @@ func init() { versioncmd.New(), installcmd.New(), providercmd.New(), + builtins.New(), ) } @@ -191,10 +193,37 @@ func runProjectRuntime(cmd *cobra.Command, args []string) error { // Add workers noreArgs = append(noreArgs, "--workers", fmt.Sprintf("%d", cfg.Runtime.Workers.Max)) - // Add data dir - if cfg.Storage.Directory != "" { - noreArgs = append(noreArgs, "--data-dir", cfg.Storage.Directory) + // Add storage backend and data dir + driver := cfg.Storage.Driver + if driver == "" { + driver = cfg.Storage.Provider } + if driver != "" { + noreArgs = append(noreArgs, "--storage-driver", driver) + } + + dir := cfg.Storage.Directory + if dir == "" { + dir = cfg.Storage.Path + } + if dir != "" { + noreArgs = append(noreArgs, "--data-dir", dir) + noreArgs = append(noreArgs, "--storage-path", dir) + } + if cfg.Storage.DSN != "" { + noreArgs = append(noreArgs, "--storage-dsn", cfg.Storage.DSN) + } + // The built-in capability policy travels with the runtime: it states what + // the built-in capability runtimes may reach, and the runtime enforces it. + policy, err := cfg.BuiltinPolicy() + if err != nil { + return err + } + policyJSON, err := config.BuiltinPolicyJSON(policy) + if err != nil { + return err + } + noreArgs = append(noreArgs, "--builtin-policy", policyJSON) // Execute nore with the project's stdout/stderr so the user sees the logs noreCmd := exec.CommandContext(cmd.Context(), noreBin, noreArgs...) diff --git a/application/internal/cli/cli_test.go b/application/internal/cli/cli_test.go index 7093447..0bd4342 100644 --- a/application/internal/cli/cli_test.go +++ b/application/internal/cli/cli_test.go @@ -1,8 +1,10 @@ package cli import ( + "encoding/json" "io" "os" + "path/filepath" "strings" "testing" ) @@ -87,3 +89,68 @@ func TestUndocumentedCapabilityListIsNotACommand(t *testing.T) { t.Errorf("capability list resolved to %q with no remaining args; the list command must live under capability runtime", cmd.Use) } } + +// TestProjectRuntimeReceivesTheBuiltinPolicy pins the wiring between the +// project's built-in policy and the runtime that enforces it. The policy +// travels with the runtime, because a configuration the runtime never +// receives is a promise the runtime never keeps. +func TestProjectRuntimeReceivesTheBuiltinPolicy(t *testing.T) { + dir := t.TempDir() + + argsFile := filepath.Join(dir, "nore-args.txt") + fakeNore := filepath.Join(dir, "nore") + script := "#!/bin/sh\nfor arg in \"$@\"; do printf '%s\\n' \"$arg\" >> " + argsFile + "; done\n" + if err := os.WriteFile(fakeNore, []byte(script), 0o755); err != nil { + t.Fatalf("write the fake runtime: %v", err) + } + + config := "builtins:\n http:\n allowedHosts:\n - api.example.com\n" + if err := os.WriteFile(filepath.Join(dir, "neuron.config.yaml"), []byte(config), 0o644); err != nil { + t.Fatalf("write the project configuration: %v", err) + } + + wd, err := os.Getwd() + if err != nil { + t.Fatalf("getwd: %v", err) + } + if err := os.Chdir(dir); err != nil { + t.Fatalf("chdir %s: %v", dir, err) + } + t.Cleanup(func() { + _ = os.Chdir(wd) + }) + + RootCmd.SetArgs([]string{"--nore-path", fakeNore}) + RootCmd.SetOut(io.Discard) + RootCmd.SetErr(io.Discard) + if err := RootCmd.Execute(); err != nil { + t.Fatalf("bare `neuron`: %v", err) + } + + data, err := os.ReadFile(argsFile) + if err != nil { + t.Fatalf("read the runtime arguments: %v", err) + } + args := strings.Split(strings.TrimRight(string(data), "\n"), "\n") + + index := -1 + for i, arg := range args { + if arg == "--builtin-policy" { + index = i + break + } + } + if index < 0 || index+1 >= len(args) { + t.Fatalf("the runtime was not started with a built-in policy: %v", args) + } + + var policy struct { + AllowedHTTPHosts []string `json:"allowedHTTPHosts"` + } + if err := json.Unmarshal([]byte(args[index+1]), &policy); err != nil { + t.Fatalf("the built-in policy is not JSON: %v\n%s", err, args[index+1]) + } + if len(policy.AllowedHTTPHosts) != 1 || policy.AllowedHTTPHosts[0] != "api.example.com" { + t.Errorf("allowedHTTPHosts = %v, want [api.example.com]", policy.AllowedHTTPHosts) + } +} diff --git a/application/internal/cli/command/command.go b/application/internal/cli/command/command.go index cafd22a..965d4d6 100644 --- a/application/internal/cli/command/command.go +++ b/application/internal/cli/command/command.go @@ -27,6 +27,10 @@ var ( CapabilityRuntimeList = "list" CapabilityRuntimeInspect = "inspect [name@version]" + // Builtins reports the capability runtimes the runtime provides without + // installation. + Builtins = "builtins [name]" + Add = "add [name@version]" Remove = "remove [name@version]" diff --git a/application/internal/cli/run/run.go b/application/internal/cli/run/run.go index ace9355..7524b30 100644 --- a/application/internal/cli/run/run.go +++ b/application/internal/cli/run/run.go @@ -21,6 +21,7 @@ import ( "github.com/neuron-runtime/neuron/application/internal/cli/utils" "github.com/neuron-runtime/neuron/application/internal/ref" "github.com/neuron-runtime/neuron/application/project" + "github.com/neuron-runtime/neuron/shared/types/builtins" capabilityrt "github.com/neuron-runtime/neuron/shared/types/capabilityruntime" core "github.com/neuron-runtime/neuron/shared/types/core" "github.com/neuron-runtime/neuron/shared/types/projectruntime" @@ -178,6 +179,12 @@ func runDirect(cmd *cobra.Command, reference string) error { return fmt.Errorf("capability runtime name is required (e.g. neuron:core:set)") } + // A reference to a built-in that does not exist is a mistake in the + // invocation, and the error says which built-ins exist. + if err := core.UnknownBuiltinError(core.CapabilityRuntimeType(typ)); err != nil { + return err + } + execParams, err := loadExecParams() if err != nil { return err @@ -196,8 +203,7 @@ func runDirect(cmd *cobra.Command, reference string) error { typ, version, core.CanonicalCoreType(coreType)) } req.CoreType = string(core.CanonicalCoreType(coreType)) - } else { - // Direct invocation is project-independent, so it resolves against the + } else { // Direct invocation is project-independent, so it resolves against the // default machine-global store — exactly where `neuron add` installs when // no store directory is configured. Requiring the artifact to be installed // keeps direct invocation free of implicit resolution or installation. @@ -231,6 +237,24 @@ func runDirect(cmd *cobra.Command, reference string) error { if storeDir != "" { childArgs = append(childArgs, "--capability-runtime-store", storeDir) } + + // A direct invocation is project-independent, so it runs under the + // compiled-in ceilings. The policy travels with the request so the child + // enforces the same limits the parent would. + // + // Private networks are permitted here because the request is the + // operator's own: someone running a capability runtime by hand is + // reaching a service they chose, not an address that came out of an + // assembly's input. + policy, err := builtins.Effective(builtins.Policy{AllowPrivateNetworks: true}) + if err != nil { + return fmt.Errorf("resolve the built-in capability policy: %w", err) + } + policyJSON, err := config.BuiltinPolicyJSON(policy) + if err != nil { + return err + } + childArgs = append(childArgs, "--builtin-policy", policyJSON) child := exec.CommandContext(ctx, noreBin, childArgs...) child.Stdin = bytes.NewReader(requestJSON) child.Stderr = cmd.ErrOrStderr() diff --git a/application/project/resolver.go b/application/project/resolver.go index e47e7f0..3858e36 100644 --- a/application/project/resolver.go +++ b/application/project/resolver.go @@ -274,6 +274,9 @@ func (r *Resolver) resolveBindings( if connRef.To != "" { connFile.To = connRef.To } + if connRef.JoinPolicy != "" { + connFile.JoinPolicy = connRef.JoinPolicy + } } else { // Inline binding connFile = BindingFile{ @@ -285,6 +288,7 @@ func (r *Resolver) resolveBindings( }, From: connRef.From, To: connRef.To, + JoinPolicy: connRef.JoinPolicy, Mappings: connRef.Mappings, Validations: connRef.Validations, } diff --git a/application/project/types.go b/application/project/types.go index afda6aa..43bd9b7 100644 --- a/application/project/types.go +++ b/application/project/types.go @@ -79,6 +79,7 @@ type BindingReference struct { // Inline definition From string `yaml:"from"` To string `yaml:"to"` + JoinPolicy core.JoinPolicy `yaml:"joinPolicy,omitempty"` Mappings []MappingDefinition `yaml:"mappings,omitempty"` Validations []BindingValidationRule `yaml:"validations,omitempty"` @@ -95,6 +96,7 @@ type BindingFile struct { From string `yaml:"from"` To string `yaml:"to"` + JoinPolicy core.JoinPolicy `yaml:"joinPolicy,omitempty"` Mappings []MappingDefinition `yaml:"mappings,omitempty"` Validations []BindingValidationRule `yaml:"validations,omitempty"` } diff --git a/application/project/validator.go b/application/project/validator.go index 83c1669..1ce17dc 100644 --- a/application/project/validator.go +++ b/application/project/validator.go @@ -68,6 +68,10 @@ func validateBindingBasic( ) error { prefix := fmt.Sprintf("bindings[%d]", index) + if err := conn.JoinPolicy.Validate(); err != nil { + return fmt.Errorf("%s.joinPolicy: %w", prefix, err) + } + if conn.Entry == "" { if strings.TrimSpace(conn.From) == "" { return fmt.Errorf("%s.from is required", prefix) @@ -109,6 +113,9 @@ func validateBindingFile(conn BindingFile) error { if strings.TrimSpace(conn.To) == "" { return fmt.Errorf("binding.to is required") } + if err := conn.JoinPolicy.Validate(); err != nil { + return fmt.Errorf("binding.joinPolicy: %w", err) + } for j, m := range conn.Mappings { if strings.TrimSpace(m.Target) == "" { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5d22571..b2a2d85 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -300,7 +300,7 @@ Built-in modules are the exception that proves the rule: they run in-process ins | **Instances** | Creating, running, pausing, restarting, and removing living realizations of Assemblies | | **Capability runtimes** | Hosting and supervising modules — in-process for built-ins, out-of-process for external modules (process and WASM providers) | | **Scheduling** | Advancing executions, evaluating binding mappings and validations, handling cancellations | -| **Persistence** | Durable storage of registered assemblies, instances, executions, and events through an interchangeable storage provider (SQLite by default) | +| **Persistence** | Durable storage of registered assemblies, instances, executions, and events through the `storage.Store` contract. SQLite is the default driver; `storage.provider` (or `--storage-driver` / `NEURON_STORAGE_PROVIDER`) selects another registered driver | | **API** | The local HTTP/JSON transport over which the CLI talks to the runtime | N.O.R.E. deliberately does **not**: @@ -324,7 +324,7 @@ flowchart TB rt[provider
capability runtime host] pl[planner
assembly → execution plan] rv[resolver
CEL expressions] - st[storage
provider + SQLite] + st[storage
contract + driver registry] sys[assembly
registered assembly repository] plugin[plugin
frozen records → adapters] end @@ -352,7 +352,7 @@ flowchart TB | `mechanism` | Provider registry — one provider contract, multiple first-party providers (process workers, WASM), with health checks, worker pooling, restart, and cancellation | | `planner` | Compiles the registered Assembly + frozen capability runtimes into an executable plan. Source-language agnostic; owns no HTTP clients, registries, or file downloads | | `resolver` | CEL expression resolver — binding mappings and validations | -| `storage` | Provider interface + SQLite implementation — assemblies, instances, executions, events | +| `storage` | Backend-neutral `Store` contract, sentinel errors, key construction, driver registry, and transactions. Concrete drivers live under `storage/` and are imported only by the composition root | | `plugin` | Boundary between frozen capability runtime records and the providers — thin adapters, no process/socket/WASM machinery itself | | `assembly` | Registered assembly repository and indexing | @@ -506,12 +506,10 @@ There are two ways a capability type gets a capability runtime. | Capability type | Behavior | | --- | --- | -| `neuron:core:set` | Sets static values from config | -| `neuron:core:ai` | In-process mock/placeholder capability runtime | -| `neuron:core:log` | Logs its input | -| `neuron:core:http` | Makes an HTTP request | -| `neuron:core:delay` | Waits a configured interval | -| `neuron:core:command` | Runs an OS command | +| `neuron:core:set` | Materializes a capability's value set (configuration with params overlaid) | +| `neuron:core:http` | Performs an HTTP request with SSRF protection, host allowlist, and retry policy | +| `neuron:core:command` | Runs a local program with cwd confinement, env allowlist, and process-group termination | +| `neuron:core:filesystem` | Reads files and lists directories, confined to the project root and extra roots | They are registered by `RegisterCoreRuntimes` in `nore/internal/registry`, under both the canonical namespaced name (`neuron:core:set`, …) and the legacy bare name. @@ -873,13 +871,61 @@ From there the normal evolution holds: the artifact is distributed through a reg ## Persistence -N.O.R.E. persists registered assemblies, instances, executions, and the event log through a small storage provider interface (`storage.Store`), implemented today by SQLite under the configured data directory (`~/.neuron/nore`). +N.O.R.E. persists registered assemblies, instance metadata, execution snapshots, and the event log through one backend-neutral contract, `storage.Store`. Generic runtime packages depend only on that contract. Concrete drivers live under `nore/internal/storage/`; only the composition root (`cmd/nore`) imports one, so the execution, event, instance, assembly, and planner packages cannot acquire a backend dependency. -Executions are kept in an in-memory store that writes through to persistent storage, so live executions are fully in-memory for speed and durable for recovery. On restart, N.O.R.E. restores persisted instances and their in-flight executions as metadata-only records. +SQLite is the default driver. `--storage-driver` selects a registered driver, and its default comes from `NEURON_STORAGE_PROVIDER`; the CLI forwards the effective internal storage provider to the runtime as that flag. Storage configuration itself is engine-owned and is rejected from project files. `storage.Open` performs the whole lifecycle: validate the driver, construct it, run migrations, and `Ping`. An unknown driver or a failed open aborts startup with an actionable error; there is no silent fallback to another backend. A store is passed to runtime components as `storage.Store`; no component constructs its own backend. -Execution history and retention are intended to become a **configurable storage policy** (`none | memory | local`) so retention is an operational choice rather than a fixed behavior — and so execution semantics never depend on persistent storage being enabled. The pluggable store is in place; the policy is not. +The contract has explicit failure semantics. Missing records are `storage.ErrNotFound`, compare-and-swap losses are `storage.ErrConflict`, insert-only collisions are `storage.ErrAlreadyExists`, and nested/re-entrant transaction use is `storage.ErrNestedTransaction` / `storage.ErrTransactionActive`. Callers distinguish absence from backend or decode failures with `errors.Is`; a failed read is never reported as an empty list or a missing record. ---- +### Transactions + +`storage.Store.WithTx(ctx, fn)` runs `fn` against a `storage.Tx` that exposes the same Get/Put/Delete/List operations bound to one transaction. Returning an error or a cancelled context rolls back. A plain `Store` operation attempted while a transaction is active returns `ErrTransactionActive`; a nested `WithTx` returns `ErrNestedTransaction` rather than joining or deadlocking. Store and transaction concurrency rules are part of the contract suite (`nore/internal/storage/storagetest`), not an assumption about SQLite. + +Transactions are required when one logical operation writes more than one key: + +- **Assembly registration** writes the artifact, prunes artifacts replaced for the same version, and updates the version index atomically. A failed registration leaves no partial artifact or index entry. +- **Execution save** reads the persisted revision, compares it with the writer's revision, and writes the next snapshot in one transaction. This is the compare-and-swap guard that rejects a stale writer. +- **Execution transition plus dispatch intent** (Phase 4) and **timer claim** (Phase 5) will use the same boundary. They are not implemented yet. + +Single-key reads and writes use the plain `Store`; they do not need a transaction. + +### Execution guarantees + +Three guarantees are separate and must not be conflated: + +| Guarantee | Meaning | Status | +| --- | --- | --- | +| **Observability** | A run's transitions and output can be inspected through events and the API | Implemented | +| **Persistence** | Registered assemblies, instance metadata, execution history, and events survive a restart and remain queryable | Implemented | +| **Durable execution** | Unfinished work resumes after a restart and reaches a terminal state exactly once | Not implemented | + +Persistence does not imply durable execution. N.O.R.E. keeps live executions in memory and writes snapshots through to storage, so live work is fast and completed history survives a restart. Saving a snapshot is not the same as recording a resumable transition and dispatch intent. + +On restart, N.O.R.E. restores instance metadata and execution records as **metadata-only** records. A restored instance has no event bus, scheduler, capability runtime engine, or worker processes, so it cannot be started or resumed. Executions that were unfinished when the process stopped are marked failed with the reason no runtime could resume them. This is the current non-durable mode, not an execution guarantee N.O.R.E. claims to provide. + +> [!WARNING] +> Creating a new instance is how work is re-run today. Do not treat a restored instance or a persisted `running` execution as live work that will continue. + +### Durability modes + +The intended policy surface names three modes: + +| Mode | Semantics | +| --- | --- | +| `execution.durability: none` | No execution/event history is retained. Execution semantics do not depend on storage. | +| `execution.durability: history` | Execution and event history is retained for inspection, but unfinished work is not resumed. Restore marks it failed. This is the behavior N.O.R.E. implements today. | +| `execution.durability: durable` | Execution transitions and dispatch intent are persisted atomically; unfinished work is restored and resumed from the last committed state, with idempotent dispatch. Planned for Phase 4; not implemented. | + +The configuration key is not wired yet: today the `history` behavior is unconditional. Defining the modes here gives Phase 4 a contract to implement rather than an emergent behavior to reverse-engineer. + +### Retention limitation + +Registered assemblies are stored under `assemblies/{id}/{version}/{hash}/{env}/manifest`. A key that carries a hash resolves exactly that artifact; an unknown or mismatched artifact is an error, never a fallback to the latest content. + +Registration is not content-addressed immutable. Registering the same assembly version again replaces the version's previous artifact(s) and prunes the earlier hashes, so a given `name:version` addresses exactly one artifact, the newest one. A hash that is no longer retained resolves to `ErrNotFound`. Phase 4 removes this retention limitation for artifacts referenced by durable executions. + +> [!NOTE] +> Execution history and events are an observability and persistence concern, not part of execution semantics. The current implementation always retains history; making retention configurable is a separate policy layer, and execution must not depend on it being enabled. ## Security & isolation diff --git a/docs/BUILTINS.md b/docs/BUILTINS.md new file mode 100644 index 0000000..2884be2 --- /dev/null +++ b/docs/BUILTINS.md @@ -0,0 +1,198 @@ +# Built-in Capability Runtimes + +Neuron ships four built-in capability runtimes. They run **in-process** inside the N.O.R.E. runtime engine and require no resolution, installation, or external dependency. Referencing one in a Capability is a plain module reference — the runtime dispatches it directly to the in-process implementation. + +| Built-in | Purpose | +| --- | --- | +| `neuron:core:set` | Materializes a capability's value set: the capability's configuration with its params overlaid | +| `neuron:core:http` | Performs an HTTP request with SSRF protection, host allowlist, and optional retry policy | +| `neuron:core:command` | Runs a local program with working-directory confinement, environment allowlist, and process-group termination | +| `neuron:core:filesystem` | Reads files and lists directories, confined to the project root and any extra roots the project declares | + +The `neuron builtins` command reports the contract of every built-in, or the contract of one built-in when named: + +```bash +neuron builtins # every built-in +neuron builtins http # one built-in +neuron builtins http -v # with the full descriptions +``` + +--- + +## `neuron:core:set` + +**Summary:** Materializes a capability's value set. + +**Description:** +`set` combines the capability's own configuration with the params bound to the invocation and returns the combination, with params overriding configuration. It is the default runtime for a capability that declares none, which makes it the way an assembly supplies a value set to a downstream capability without writing a capability runtime. + +**Parameters:** + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| `format` | string | no | `json` | How the values of the set are represented: `json` or `text`. | + +**Results:** + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `value` | object | yes | The merged value set: the capability's configuration with the params overlaid on it. The result of the capability carries the names of the merged set. | + +**Constraints:** + +- Reads only the capability's own input; it reaches nothing outside the runtime. +- The format is an instruction to the built-in rather than a value of the set, so it is not returned as one. + +--- + +## `neuron:core:http` + +**Summary:** Makes an HTTP request. + +**Description:** +`http` performs a single HTTP request and returns the status, the response headers, and the body. It is the general-purpose integration boundary: an author who needs to talk to an HTTP API should not have to write a capability runtime for it. + +**Parameters:** + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| `url` | string | yes | — | The absolute URL to request. | +| `method` | string | no | `GET` | The HTTP method. | +| `headers` | object of strings | no | — | Request headers. | +| `query` | object of strings | no | — | Query string parameters, appended to the URL. | +| `body` | any | no | — | The request body, encoded as JSON. | +| `contentType` | string | no | `application/json` | The request content type. Only used when a body is present. | +| `auth` | object | no | — | Credentials for the request: `{type: "bearer" \| "basic" \| "apiKey", ...}`. | +| `timeout` | duration string | no | — | How long the request may take, such as `30s`. | +| `format` | string | no | `json` | How the response body is represented: `json` or `text`. | +| `followRedirects` | boolean | no | `false` | Whether redirects are followed. Each hop is checked against the address policy. | +| `retry` | object | no | — | Retry behavior: `{attempts: n, backoff: "200ms", methods: [...]}`. Off by default. | + +**Results:** + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `status_code` | number | yes | The HTTP response status code. | +| `headers` | object of strings | yes | The response headers. | +| `body` | any or string | yes | The response body: parsed JSON in the json format, a plain string in the text format. | +| `content_type` | string | yes | The response content type. | +| `duration_ms` | number | yes | How long the request took, in milliseconds. | + +**Constraints:** + +- Refuses loopback, private, link-local, and cloud-metadata addresses, so a request cannot be aimed at the host or an internal service. +- Does not follow redirects unless asked to; when it does, every hop is checked against the same address policy. +- Does not retry unless asked to, and never retries a method that was not opted in. +- Bounds the request body and the response body. + +--- + +## `neuron:core:command` + +**Summary:** Runs a local program. + +**Description:** +`command` executes a local program with structured arguments and returns its exit code, standard output, and standard error. It is the boundary between an assembly and the programs already installed on the machine. + +**Parameters:** + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| `command` | string | yes | — | The program to run. A single executable, never a shell command line. | +| `args` | array of strings | no | — | The program's arguments. | +| `env` | object of strings | no | — | Environment variables added to the program's environment. | +| `cwd` | string | no | — | The working directory, resolved against the allowed root. | +| `stdin` | string | no | — | Text written to the program's standard input. | +| `timeout` | duration string | no | — | How long the program may run, such as `30s`. | +| `format` | string | no | `text` | How standard output is represented: `json` or `text`. | +| `failOnNonZeroExit` | boolean | no | `true` | Whether a non-zero exit fails the capability. Set it to false to inspect the result instead. | +| `maxOutputBytes` | number | no | — | The most output to retain per stream. The runtime's ceiling applies when this is absent. | + +**Results:** + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `exit_code` | number | yes | The program's exit code. | +| `stdout` | any or string | yes | The program's standard output: parsed JSON in the json format, a plain string in the text format. | +| `stderr` | string | yes | The program's standard error. | +| `success` | boolean | yes | Whether the program exited with code zero. | +| `timed_out` | boolean | yes | Whether the program was terminated by its deadline. | +| `duration_ms` | number | yes | How long the program ran, in milliseconds. | +| `stdout_truncated` | boolean | no | Present when the standard output was shortened to the output limit. | +| `stderr_truncated` | boolean | no | Present when the standard error was shortened to the output limit. | + +**Constraints:** + +- Never runs a shell: the program and its arguments are passed to the operating system separately, so an argument can never become a command. +- Runs only the programs the policy permits, when the policy names any. +- Runs the program with an allowlisted environment, never the runtime's own environment. +- Confines the working directory to an allowed root. +- Bounds the retained output and reports truncation rather than hiding it. +- Terminates the program's whole process group on cancellation or deadline, so no child survives the capability. + +--- + +## `neuron:core:filesystem` + +**Summary:** Reads files and directories. + +**Description:** +`filesystem` reads a file, lists a directory, and reports what a path is. It is the read-only view of the machine the assembly runs on: the files the assembly was written to work with, and nothing else. Writing, deleting, and moving files are not part of it. + +**Parameters:** + +| Name | Type | Required | Default | Description | +|------|------|----------|---------|-------------| +| `operation` | string | yes | — | What to do with the path: `read`, `list`, or `stat`. | +| `path` | string | yes | — | The path to read, list, or describe. A relative path is resolved against the project root. | +| `recursive` | boolean | no | `false` | Whether a listing descends into subdirectories. | +| `format` | string | no | `json` | How the content of a read file is represented: `json` or `text`. | + +**Results:** + +| Name | Type | Required | Description | +|------|------|----------|-------------| +| `path` | string | yes | The absolute path the operation applied to. | +| `content` | any or string | no | The content of the file, for the read operation: parsed JSON in the json format, a plain string in the text format. | +| `size` | number | no | The size of the file in bytes, for the read and stat operations. | +| `entries` | array of objects | no | The entries of the directory, for the list operation. Each entry has `name`, `path`, `type` (`file`, `directory`, `link`, `other`), and `size`. | +| `exists` | boolean | no | Whether the path exists, for the stat operation. | +| `type` | string | no | What the path is: `file`, `directory`, `link`, or `other`, for the stat operation. | +| `entries_truncated` | boolean | no | Present when the listing was shortened to the entry limit. | + +**Constraints:** + +- Refuses a path that resolves outside the permitted roots, and a symbolic link cannot be used to leave them. +- Reads only: no operation writes, deletes, or moves anything. +- Bounds the content a single read returns and the entries a single listing returns. + +--- + +## Using built-ins in TypeScript + +```ts +import { set, http, command, filesystem } from "@neuronruntime/sdk/core"; + +const fetchUser = http("users.get"); +const listFiles = filesystem("files.list"); +const runTool = command("tools.build"); +const values = set("config.values"); +``` + +Each export is a factory that takes the capability's name and returns a definition with the built-in's contract already applied, so the params and results are typed. + +--- + +## Removed built-ins + +| Built-in | Removed because | Replacement | +| --- | --- | --- | +| `neuron:core:ai` | Mock/placeholder; not a real capability | — | +| `neuron:core:log` | Logging is an observability concern, not a capability | — | +| `neuron:core:delay` | Waiting is a sequencing concern, not a capability | Explicit dependency between the capabilities that must run in order | + +A reference to a removed built-in is reported as an actionable error: + +``` +capability runtime "neuron:core:delay" does not exist: it was removed because it only waited, which is a sequencing concern rather than a capability; use an explicit dependency between the capabilities that must run in order instead. available built-in capability runtimes are: neuron:core:command, neuron:core:filesystem, neuron:core:http, neuron:core:set +``` \ No newline at end of file diff --git a/docs/MIGRATION.md b/docs/MIGRATION.md index 31ad348..340ff05 100644 --- a/docs/MIGRATION.md +++ b/docs/MIGRATION.md @@ -57,6 +57,11 @@ build record. - Internal storage keys (including `capabilityRuntimes.storeDir`) and `providers.storeDir` are rejected from project configuration; use the documented environment overrides where a non-default location is required. +- The internal storage provider now names a registered storage driver and + defaults to `sqlite`. The retired `file` value is not a driver and is + rejected. Select a driver with `--storage-driver` or + `NEURON_STORAGE_PROVIDER`; storage settings remain engine-owned and are not + accepted in project configuration files. - The old configuration file names `neuron.yaml` and `neuron.yml` are no longer read. Rename the file to `neuron.config.yaml` (or `neuron.config.json`). - The global configuration file may not declare Assembly identity or entries @@ -118,6 +123,29 @@ params or result fields. Add `paramsSchema(...)` / `resultSchema(...)` plus structured reference (`capability.result.x`, `assembly.params.x`, or a literal); arbitrary expression strings are not evaluated for result fields. +### Built-in capability runtimes + +The SDK now exports every built-in capability runtime from `@neuronruntime/sdk/core`: + +```ts +import { set, http, command, filesystem } from "@neuronruntime/sdk/core"; + +const fetchUser = http("users.get"); +const listFiles = filesystem("files.list"); +``` + +- `set` is the default runtime for a capability that declares none; most + assemblies never name it explicitly. +- `http` now accepts `query`, `contentType`, `auth`, and `retry` params, and + produces `content_type` and `duration_ms` results — matching the runtime's + published contract. +- `delay` is removed: it only waited, which is a sequencing concern rather than + a capability. Use an explicit dependency between the capabilities that must + run in order instead. +- A built-in capability runtime runs in-process and is never resolved or + installed, so its manifest declaration carries no version and no registry. + The SDK no longer writes `version: "latest", registry: "local"` for a built-in. + ## Expressions and structured references The canonical manifest stores references structurally: diff --git a/docs/MIGRATIONS.md b/docs/MIGRATIONS.md new file mode 100644 index 0000000..8b5147b --- /dev/null +++ b/docs/MIGRATIONS.md @@ -0,0 +1,21 @@ +# Storage Migrations, Backup, and Restore + +## Migrations +- Storage backends use a versioned migration framework in `nore/internal/storage/migrate/`. +- The framework tracks applied versions in a `schema_migrations` table and applies pending migrations transactionally where possible. +- If the database schema version is newer than the binary supports, startup is refused with an error (do not downgrade). +- Migrations run during backend initialization before the store is returned/pinged. + +## SQLite +- Default location: `~/.neuron/nore/nore.db` (or configured data dir/path). +- Backup: copy the database file while the store is closed, or use SQLite's backup mechanism. For a live database, ensure no active writes or copy the WAL-consistent snapshot. +- Restore: replace the database file with a backup copy and ensure correct permissions. + +## PostgreSQL +- Configure via `NEURON_DATABASE_URL` or `--storage-dsn`. DSN must not be logged. +- Backup: use `pg_dump` (e.g. `pg_dump -Fc $NEURON_DATABASE_URL > backup.dump`). +- Restore: use `pg_restore` to a prepared database. +- The `schema_migrations` table tracks versions; migrations are applied on startup. + +## Upgrade path from existing SQLite databases +Existing SQLite databases created before the versioned migration framework are compatible: migration v1 uses `CREATE TABLE IF NOT EXISTS` and the framework will record it in `schema_migrations` on first run. diff --git a/docs/MODULES.md b/docs/MODULES.md index 15dd909..998b639 100644 --- a/docs/MODULES.md +++ b/docs/MODULES.md @@ -411,11 +411,26 @@ neuron capability runtime inspect example:echo@1.0.0 ## Built-in modules -The Neuron runtime ships a small set of built-in modules for common operations. They run **in-process** inside the runtime engine and require no resolution or installation — referencing one in a Capability is a plain module reference, and the runtime dispatches it directly to the in-process implementation. +The Neuron runtime ships four built-in modules for common operations. They run **in-process** inside the runtime engine and require no resolution or installation — referencing one in a Capability is a plain module reference, and the runtime dispatches it directly to the in-process implementation. + +| Built-in | Purpose | +| --- | --- | +| `neuron:core:set` | Materializes a capability's value set: the capability's configuration with its params overlaid | +| `neuron:core:http` | Performs an HTTP request with SSRF protection, host allowlist, and optional retry policy | +| `neuron:core:command` | Runs a local program with working-directory confinement, environment allowlist, and process-group termination | +| `neuron:core:filesystem` | Reads files and lists directories, confined to the project root and any extra roots the project declares | > [!IMPORTANT] > Built-ins are the deliberate exception to out-of-process hosting: they are part of the Neuron runtime itself (and thereby of the trusted, tested runtime). Everything else follows the external-module path above, with verification and process/WASM isolation. +The `neuron builtins` command reports the contract of every built-in, or the contract of one built-in when named: + +```bash +neuron builtins # every built-in +neuron builtins http # one built-in +neuron builtins http -v # with the full descriptions +``` + --- ## Security model diff --git a/nore/cmd/nore/main.go b/nore/cmd/nore/main.go index 3d5d33a..e547be1 100644 --- a/nore/cmd/nore/main.go +++ b/nore/cmd/nore/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "flag" "fmt" "log" @@ -10,6 +11,7 @@ import ( "os" "os/signal" "path/filepath" + "strings" "syscall" "time" @@ -22,8 +24,10 @@ import ( "github.com/neuron-runtime/neuron/nore/internal/plugin" "github.com/neuron-runtime/neuron/nore/internal/resolver" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" + _ "github.com/neuron-runtime/neuron/nore/internal/storage/postgres" + _ "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" "github.com/neuron-runtime/neuron/shared/types/apitoken" + "github.com/neuron-runtime/neuron/shared/types/builtins" "github.com/neuron-runtime/neuron/shared/types/projectruntime" "github.com/neuron-runtime/neuron/shared/version" ) @@ -39,6 +43,9 @@ func main() { socket string workers int dataDir string + storageDriver string + storagePath string + storageDSN string tokenArg string detachedDrainTimeout time.Duration capabilityStore string @@ -47,12 +54,16 @@ func main() { projectMode bool projectRoot string host string + builtinPolicyArg string ) flag.StringVar(&port, "port", "", "TCP address for the N.O.R.E. API; empty disables TCP (default: Unix socket only)") flag.StringVar(&socket, "socket", defaultSocket(), "Unix socket for local CLI clients; empty disables Unix socket") flag.IntVar(&workers, "workers", 8, "capability runtime worker count") flag.StringVar(&dataDir, "data-dir", defaultDataDir(), "persistent data directory") + flag.StringVar(&storageDriver, "storage-driver", defaultStorageDriver(), "storage backend driver (default: sqlite)") + flag.StringVar(&storagePath, "storage-path", "", "storage path override (sqlite)") + flag.StringVar(&storageDSN, "storage-dsn", os.Getenv("NEURON_DATABASE_URL"), "storage DSN (postgres)") flag.StringVar(&tokenArg, "token", "", "API token for authenticating requests; empty loads the token from the socket's token file") flag.DurationVar(&detachedDrainTimeout, "detached-drain-timeout", engine.DefaultDetachedDrainTimeout, "how long detached capability work may run after the daemon begins shutting down") flag.StringVar(&capabilityStore, "capability-runtime-store", defaultCapabilityRuntimeStore(), "capability runtime store directory for resolving installed artifacts") @@ -61,19 +72,36 @@ func main() { flag.BoolVar(&projectMode, "project", false, "run in project runtime mode (foreground, loopback TCP, auto port)") flag.StringVar(&projectRoot, "project-root", "", "project root directory (required with --project)") flag.StringVar(&host, "host", "127.0.0.1", "host interface to bind to in project mode (default: 127.0.0.1)") + flag.StringVar(&builtinPolicyArg, "builtin-policy", "", "JSON policy bounding the built-in capability runtimes (neuron:core:*)") flag.Parse() + if envDriver := os.Getenv("NEURON_STORAGE_DRIVER"); envDriver != "" { + storageDriver = envDriver + } + if envDSN := os.Getenv("NEURON_DATABASE_URL"); envDSN != "" { + storageDSN = envDSN + } + if showVer { fmt.Println(version.String()) return } + // The built-in capability policy is resolved once, at the process boundary: + // the policy may arrive from a project configuration over the command line, + // and it is resolved against the ceilings compiled into this binary so a + // policy that arrived from outside cannot widen what a capability reaches. + builtinPolicy, err := resolveBuiltinPolicy(builtinPolicyArg) + if err != nil { + log.Fatalf("resolve built-in capability policy: %v", err) + } + // Direct invocation mode: host a single installed Capability Runtime for one // execution and exit. No storage, assemblies, or API are involved. if invokeMode { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := invoke.Run(ctx, capabilityStore, os.Stdin, os.Stdout); err != nil { + if err := invoke.Run(ctx, capabilityStore, builtinPolicy, os.Stdin, os.Stdout); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -85,7 +113,7 @@ func main() { if projectMode { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - runProjectMode(ctx, host, port, projectRoot, workers, dataDir, capabilityStore, detachedDrainTimeout, tokenArg) + runProjectMode(ctx, host, port, projectRoot, workers, dataDir, storageDriver, storagePath, storageDSN, capabilityStore, detachedDrainTimeout, tokenArg, builtinPolicy) return } @@ -110,11 +138,17 @@ func main() { plugin.SetStoreRoot(capabilityStore) - if err := os.MkdirAll(dataDir, 0o755); err != nil { - log.Fatalf("create data directory %s: %v", dataDir, err) + storePath := dataDir + if storagePath != "" { + storePath = storagePath + } + if storageDriver == "" || storageDriver == "sqlite" { + if err := os.MkdirAll(storePath, 0o755); err != nil { + log.Fatalf("create data directory %s: %v", storePath, err) + } } - store, err := sqlite.New(storage.Config{DataDir: dataDir}) + store, err := storage.Open(ctx, storage.Config{Driver: storageDriver, DataDir: storePath, Path: storePath, DSN: storageDSN}) if err != nil { log.Fatalf("init storage: %v", err) } @@ -131,7 +165,11 @@ func main() { log.Fatalf("init planner: %v", err) } - inst := instance.NewManager(ctx, workers, detachedDrainTimeout, store, assemblies) + inst, err := instance.NewManager(ctx, workers, detachedDrainTimeout, store, assemblies, + instance.WithBuiltinPolicy(builtinPolicy)) + if err != nil { + log.Fatalf("init instance manager: %v", err) + } srv := api.NewServer(inst, assemblies, compiler, token) type listenerEntry struct { @@ -232,6 +270,32 @@ func shutdownRuntimes(srv *api.Server) { } } +// resolveBuiltinPolicy decodes the built-in capability policy the runtime was +// started with. +// +// The policy travels as JSON on the command line because the runtime is a +// separate process from the CLI. It is resolved against the ceilings compiled +// into this binary, so a policy that arrives over a transport it did not come +// from cannot widen what a capability may reach, and it is validated so a +// policy that cannot be applied stops the runtime instead of being ignored. +func resolveBuiltinPolicy(argument string) (builtins.Policy, error) { + var policy builtins.Policy + if strings.TrimSpace(argument) != "" { + if err := json.Unmarshal([]byte(argument), &policy); err != nil { + return builtins.Policy{}, fmt.Errorf("decode the built-in capability policy: %w", err) + } + } + + effective, err := builtins.Effective(policy) + if err != nil { + return builtins.Policy{}, err + } + if err := effective.Validate(); err != nil { + return builtins.Policy{}, err + } + return effective, nil +} + // resolveAPIToken determines the token the API will require. // // The token is taken from the flag first, then from the token file beside the @@ -285,6 +349,13 @@ func defaultSocket() string { return filepath.Join(home, ".neuron", "nore.sock") } +func defaultStorageDriver() string { + if value := os.Getenv("NEURON_STORAGE_PROVIDER"); value != "" { + return value + } + return storage.DefaultDriver +} + func defaultDataDir() string { if value := os.Getenv("NEURON_DATA_DIR"); value != "" { return value @@ -309,7 +380,7 @@ func defaultCapabilityRuntimeStore() string { // runProjectMode runs N.O.R.E. as a foreground project runtime with auto port // allocation and discovery metadata. -func runProjectMode(ctx context.Context, host, port, projectRoot string, workers int, dataDir, capabilityStore string, detachedDrainTimeout time.Duration, tokenArg string) { +func runProjectMode(ctx context.Context, host, port, projectRoot string, workers int, dataDir, storageDriver, storagePath, storageDSN, capabilityStore string, detachedDrainTimeout time.Duration, tokenArg string, builtinPolicy builtins.Policy) { if projectRoot == "" { log.Fatal("--project-root is required when --project is set") } @@ -320,6 +391,12 @@ func runProjectMode(ctx context.Context, host, port, projectRoot string, workers log.Fatalf("resolve project root: %v", err) } + // The project root is the root the built-in capability runtimes are + // confined to. The runtime knows the project it was started for, so it + // states the root itself rather than trusting the one that arrived with + // the policy. + builtinPolicy.ProjectRoot = absRoot + // Generate an API token for this project runtime token, err := generateProjectToken() if err != nil { @@ -340,10 +417,24 @@ func runProjectMode(ctx context.Context, host, port, projectRoot string, workers } // Create data directory - if err := os.MkdirAll(dataDir, 0o755); err != nil { - log.Fatalf("create data directory %s: %v", dataDir, err) + storePath := dataDir + if storagePath != "" { + storePath = storagePath + } + if storageDriver == "" || storageDriver == "sqlite" { + if err := os.MkdirAll(storePath, 0o755); err != nil { + log.Fatalf("create data directory %s: %v", storePath, err) + } } + // Open storage before publishing the discovery record. A runtime that + // cannot initialize its durable state must not advertise itself as ready. + store, err := storage.Open(ctx, storage.Config{Driver: storageDriver, DataDir: storePath, Path: storePath, DSN: storageDSN}) + if err != nil { + log.Fatalf("init storage: %v", err) + } + defer store.Close() + // Publish the discovery record so `neuron run` can find this runtime. The // record is owner-only: it carries the API token. Each session writes its // own record file, so concurrent sessions for the same project do not @@ -360,12 +451,6 @@ func runProjectMode(ctx context.Context, host, port, projectRoot string, workers plugin.SetStoreRoot(capabilityStore) - store, err := sqlite.New(storage.Config{DataDir: dataDir}) - if err != nil { - log.Fatalf("init storage: %v", err) - } - defer store.Close() - assemblies := assembly.NewRepository(store) celCompiler, err := resolver.NewCELCompiler(resolver.DefaultCELConfig()) @@ -377,7 +462,11 @@ func runProjectMode(ctx context.Context, host, port, projectRoot string, workers log.Fatalf("init planner: %v", err) } - inst := instance.NewManager(ctx, workers, detachedDrainTimeout, store, assemblies) + inst, err := instance.NewManager(ctx, workers, detachedDrainTimeout, store, assemblies, + instance.WithBuiltinPolicy(builtinPolicy)) + if err != nil { + log.Fatalf("init instance manager: %v", err) + } srv := api.NewServer(inst, assemblies, compiler, token) // Serve on the listener diff --git a/nore/cmd/nore/main_test.go b/nore/cmd/nore/main_test.go index a18f5d3..a924d20 100644 --- a/nore/cmd/nore/main_test.go +++ b/nore/cmd/nore/main_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/neuron-runtime/neuron/shared/types/apitoken" + "github.com/neuron-runtime/neuron/shared/types/builtins" ) func TestResolveAPITokenPrefersExplicitFlag(t *testing.T) { @@ -156,3 +157,41 @@ func TestListenProjectOccupiedPortFails(t *testing.T) { t.Fatalf("binding occupied port %s succeeded, want an error", port) } } + +func TestResolveBuiltinPolicyWithoutAnArgumentUsesTheCeilings(t *testing.T) { + policy, err := resolveBuiltinPolicy("") + if err != nil { + t.Fatalf("resolveBuiltinPolicy: %v", err) + } + + ceilings := builtins.DefaultPolicy() + if policy.MaxCommandOutputBytes != ceilings.MaxCommandOutputBytes { + t.Errorf("maxCommandOutputBytes = %d, want %d", policy.MaxCommandOutputBytes, ceilings.MaxCommandOutputBytes) + } + if policy.DefaultTimeout != ceilings.DefaultTimeout { + t.Errorf("defaultTimeout = %q, want %q", policy.DefaultTimeout, ceilings.DefaultTimeout) + } +} + +func TestResolveBuiltinPolicyAppliesAConfiguredPolicy(t *testing.T) { + policy, err := resolveBuiltinPolicy(`{"allowedHTTPHosts":["api.example.com"],"maxHTTPResponseBytes":1024}`) + if err != nil { + t.Fatalf("resolveBuiltinPolicy: %v", err) + } + + if len(policy.AllowedHTTPHosts) != 1 || policy.AllowedHTTPHosts[0] != "api.example.com" { + t.Errorf("allowedHTTPHosts = %v, want [api.example.com]", policy.AllowedHTTPHosts) + } + if policy.MaxHTTPResponseBytes != 1024 { + t.Errorf("maxHTTPResponseBytes = %d, want 1024", policy.MaxHTTPResponseBytes) + } +} + +func TestResolveBuiltinPolicyRefusesAnUnusablePolicy(t *testing.T) { + if _, err := resolveBuiltinPolicy("{not json"); err == nil { + t.Fatal("resolveBuiltinPolicy accepted a malformed policy") + } + if _, err := resolveBuiltinPolicy(`{"maxHTTPResponseBytes":1073741824}`); err == nil { + t.Fatal("resolveBuiltinPolicy accepted a limit above the compiled-in ceiling") + } +} diff --git a/nore/go.mod b/nore/go.mod index 86d9302..4cd91e0 100644 --- a/nore/go.mod +++ b/nore/go.mod @@ -5,6 +5,7 @@ go 1.26.5 require ( github.com/coder/websocket v1.8.15 github.com/google/cel-go v0.30.0 + github.com/lib/pq v1.10.9 github.com/neuron-runtime/neuron/shared v0.0.0-20260905200234-63564fa7a003 github.com/tetratelabs/wazero v1.12.0 google.golang.org/grpc v1.83.2 diff --git a/nore/go.sum b/nore/go.sum index aacc538..0175702 100644 --- a/nore/go.sum +++ b/nore/go.sum @@ -24,6 +24,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= diff --git a/nore/internal/api/bridge/bridge.go b/nore/internal/api/bridge/bridge.go index 00b70a3..326d39b 100644 --- a/nore/internal/api/bridge/bridge.go +++ b/nore/internal/api/bridge/bridge.go @@ -56,8 +56,8 @@ func (b *EventBridge) Stream(ctx context.Context, room string) (<-chan protocol. if !ok { return nil, fmt.Errorf("instance %s not found", instanceID) } - if _, ok := inst.GetExecution(executionID); !ok { - return nil, fmt.Errorf("execution %s not found", executionID) + if _, err := inst.GetExecution(ctx, executionID); err != nil { + return nil, err } messages, err := inst.Events(ctx, executionID, "") diff --git a/nore/internal/api/handlers/instances/handler.go b/nore/internal/api/handlers/instances/handler.go index e7decbc..54b1f19 100644 --- a/nore/internal/api/handlers/instances/handler.go +++ b/nore/internal/api/handlers/instances/handler.go @@ -26,13 +26,14 @@ func New(m *instance.Manager, assemblies *assembly.Repository, compiler *planner // instance ID (inst_*) or a colon-encoded assembly key // (assemblyID:version:hash[:env]); keys resolve to an existing runtime without // creating one. -func (h *Handler) resolveInstance(r *http.Request, id string) (*instance.Instance, bool) { +func (h *Handler) resolveInstance(r *http.Request, id string) (*instance.Instance, bool, error) { if strings.HasPrefix(id, "inst_") { - return h.instances.GetByID(id) + i, ok := h.instances.GetByID(id) + return i, ok, nil } key, err := protocol.ParseKey(id) if err != nil { - return nil, false + return nil, false, nil } return h.instances.Resolve(r.Context(), key) } @@ -62,9 +63,13 @@ func (h *Handler) runningInstance(r *http.Request, id string) (*instance.Instanc // registered key. This is what makes `neuron run` re-run an existing assembly // instead of failing with "instance is not running". func (h *Handler) runningInstanceByKey(ctx context.Context, key protocol.InstanceKey) (*instance.Instance, error) { - if i, ok := h.instances.Resolve(ctx, key); ok && i.Status() == instance.StatusRunning { + i, ok, err := h.instances.Resolve(ctx, key) + if err != nil { + return nil, err + } + if ok && i.Status() == instance.StatusRunning { return i, nil } - i, _, err := h.instances.GetOrCreate(ctx, key) + i, _, err = h.instances.GetOrCreate(ctx, key) return i, err } diff --git a/nore/internal/api/handlers/instances/instance_by_id.go b/nore/internal/api/handlers/instances/instance_by_id.go index c9caaf6..debb9c8 100644 --- a/nore/internal/api/handlers/instances/instance_by_id.go +++ b/nore/internal/api/handlers/instances/instance_by_id.go @@ -10,7 +10,11 @@ import ( func (h *Handler) GetInstanceByID(w http.ResponseWriter, r *http.Request) { id := utils.PathID(r.PathValue("id")) - i, ok := h.resolveInstance(r, id) + i, ok, err := h.resolveInstance(r, id) + if err != nil { + utils.ErrorJSON(w, http.StatusInternalServerError, err) + return + } if !ok { utils.ErrorJSON(w, http.StatusNotFound, fmt.Errorf("instance %s not found", id)) return diff --git a/nore/internal/api/handlers/instances/instance_cancel_test.go b/nore/internal/api/handlers/instances/instance_cancel_test.go index 1f67783..d73b18b 100644 --- a/nore/internal/api/handlers/instances/instance_cancel_test.go +++ b/nore/internal/api/handlers/instances/instance_cancel_test.go @@ -24,7 +24,7 @@ import ( func newCancellationHandler(t *testing.T) (*Handler, *instance.Instance) { t.Helper() - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } @@ -47,7 +47,10 @@ func newCancellationHandler(t *testing.T) (*Handler, *instance.Instance) { t.Fatalf("register assembly: %v", err) } - m := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + m, err := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + if err != nil { + t.Fatalf("new manager: %v", err) + } cel, err := resolver.NewCELCompiler(resolver.DefaultCELConfig()) if err != nil { t.Fatalf("new CEL compiler: %v", err) diff --git a/nore/internal/api/handlers/instances/instance_execution_events.go b/nore/internal/api/handlers/instances/instance_execution_events.go index 29e47b0..f3dc44a 100644 --- a/nore/internal/api/handlers/instances/instance_execution_events.go +++ b/nore/internal/api/handlers/instances/instance_execution_events.go @@ -2,11 +2,13 @@ package instances import ( "encoding/json" + "errors" "fmt" "net/http" "strings" "github.com/neuron-runtime/neuron/nore/internal/api/utils" + "github.com/neuron-runtime/neuron/nore/internal/instance" "github.com/neuron-runtime/neuron/shared/types/core" "github.com/neuron-runtime/neuron/shared/types/protocol" ) @@ -28,8 +30,12 @@ func (h *Handler) StreamExecutionEvents(w http.ResponseWriter, r *http.Request) utils.ErrorJSON(w, http.StatusNotFound, fmt.Errorf("instance %s not found", instID)) return } - if _, ok := i.GetExecution(execID); !ok { - utils.ErrorJSON(w, http.StatusNotFound, fmt.Errorf("execution %s not found", execID)) + if _, err := i.GetExecution(r.Context(), execID); err != nil { + status := http.StatusInternalServerError + if errors.Is(err, instance.ErrExecutionNotFound) { + status = http.StatusNotFound + } + utils.ErrorJSON(w, status, err) return } diff --git a/nore/internal/api/handlers/instances/instance_executions.go b/nore/internal/api/handlers/instances/instance_executions.go index 76f147a..ad09b45 100644 --- a/nore/internal/api/handlers/instances/instance_executions.go +++ b/nore/internal/api/handlers/instances/instance_executions.go @@ -18,7 +18,11 @@ import ( func (h *Handler) ListExecutions(w http.ResponseWriter, r *http.Request) { id := utils.PathID(r.PathValue("id")) - i, ok := h.resolveInstance(r, id) + i, ok, resolveErr := h.resolveInstance(r, id) + if resolveErr != nil { + utils.ErrorJSON(w, http.StatusInternalServerError, resolveErr) + return + } if !ok { // The target may still be addressable: a assembly key that names a // registered but never-instantiated assembly has no runtime, yet its @@ -27,7 +31,12 @@ func (h *Handler) ListExecutions(w http.ResponseWriter, r *http.Request) { // before the first execution. Unknown instances and assemblies 404. if !strings.HasPrefix(id, "inst_") { if key, err := protocol.ParseKey(id); err == nil { - if exists, existsErr := h.assemblies.Exists(r.Context(), key); existsErr == nil && exists { + exists, existsErr := h.assemblies.Exists(r.Context(), key) + if existsErr != nil { + utils.ErrorJSON(w, http.StatusInternalServerError, existsErr) + return + } + if exists { utils.WriteJSON(w, http.StatusOK, protocol.Response{ Message: "executions", Status: http.StatusOK, @@ -41,8 +50,14 @@ func (h *Handler) ListExecutions(w http.ResponseWriter, r *http.Request) { return } + executions, err := i.ListExecutions(r.Context()) + if err != nil { + utils.ErrorJSON(w, http.StatusInternalServerError, err) + return + } + items := make([]protocol.ExecutionItem, 0) - for _, exec := range i.ListExecutions() { + for _, exec := range executions { item := protocol.ExecutionItem{ ID: exec.ID, CorrelationID: exec.CorrelationID, @@ -146,9 +161,13 @@ func (h *Handler) GetExecutionState(w http.ResponseWriter, r *http.Request) { return } - exec, ok := i.GetExecution(core.ID(execID)) - if !ok { - utils.ErrorJSON(w, http.StatusNotFound, fmt.Errorf("execution %s not found", execID)) + exec, err := i.GetExecution(r.Context(), core.ID(execID)) + if err != nil { + status := http.StatusInternalServerError + if errors.Is(err, instance.ErrExecutionNotFound) { + status = http.StatusNotFound + } + utils.ErrorJSON(w, status, err) return } @@ -217,7 +236,7 @@ func (h *Handler) CancelExecution(w http.ResponseWriter, r *http.Request) { return } - exec, _ := i.GetExecution(core.ID(execID)) + exec, _ := i.GetExecution(r.Context(), core.ID(execID)) item := protocol.ExecutionItem{ID: core.ID(execID)} if exec != nil { item.Status = string(exec.Status()) diff --git a/nore/internal/api/handlers/instances/instance_executions_test.go b/nore/internal/api/handlers/instances/instance_executions_test.go index 036ae55..895ac82 100644 --- a/nore/internal/api/handlers/instances/instance_executions_test.go +++ b/nore/internal/api/handlers/instances/instance_executions_test.go @@ -24,14 +24,17 @@ import ( func newTestHandler(t *testing.T) *Handler { t.Helper() - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } t.Cleanup(func() { _ = store.Close() }) assembliesRepo := assembly.NewRepository(store) - m := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + m, err := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + if err != nil { + t.Fatalf("new manager: %v", err) + } cel, err := resolver.NewCELCompiler(resolver.DefaultCELConfig()) if err != nil { @@ -137,7 +140,7 @@ func TestListExecutionsUnknownInstanceIDReturns404(t *testing.T) { // Execute rather than rejected with "instance is not running". This is what // makes `neuron run` re-run an already-instantiated assembly. func TestExecuteRecreatesStaleRestoredInstance(t *testing.T) { - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } @@ -182,7 +185,10 @@ func TestExecuteRecreatesStaleRestoredInstance(t *testing.T) { // A fresh manager reconciles the record as metadata-only; the runtime is // intentionally absent. - m := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + m, err := instance.NewManager(context.Background(), 2, 0, store, assembliesRepo) + if err != nil { + t.Fatalf("new manager: %v", err) + } stale, ok := m.GetByID("inst_stale") if !ok { t.Fatal("stale instance was not restored") diff --git a/nore/internal/assembly/index.go b/nore/internal/assembly/index.go index 9cb6f2a..e64a946 100644 --- a/nore/internal/assembly/index.go +++ b/nore/internal/assembly/index.go @@ -9,7 +9,6 @@ import ( "time" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" "github.com/neuron-runtime/neuron/shared/types/protocol" ) @@ -68,14 +67,19 @@ func (idx *VersionIndex) Select(version string) (string, VersionEntry, error) { // indexKey is the storage key of a assembly's version index. // It lives at assemblies/{assemblyID}/versions.json, outside the artifact tree. -func (r *Repository) indexKey(assemblyID string) string { - return sqlite.SanitizeKey("assemblies", assemblyID, indexFileName) +func (r *Repository) indexKey(assemblyID string) (string, error) { + return storage.Key("assemblies", assemblyID, indexFileName) } -// loadIndex reads the assembly's version index. When absent it is rebuilt from -// the registered artifacts and persisted, so pre-index data stays addressable. -func (r *Repository) loadIndex(ctx context.Context, assemblyID string) (*VersionIndex, error) { - data, err := r.store.Get(ctx, r.indexKey(assemblyID)) +// loadIndex reads the assembly's version index through kv. When absent it is +// rebuilt from the registered artifacts and persisted, so pre-index data stays +// addressable. +func (r *Repository) loadIndex(ctx context.Context, kv storage.KV, assemblyID string) (*VersionIndex, error) { + indexKey, err := r.indexKey(assemblyID) + if err != nil { + return nil, err + } + data, err := kv.Get(ctx, indexKey) if err == nil { var idx VersionIndex if err := json.Unmarshal(data, &idx); err != nil { @@ -86,14 +90,17 @@ func (r *Repository) loadIndex(ctx context.Context, assemblyID string) (*Version if !errors.Is(err, storage.ErrNotFound) { return nil, err } - return r.buildIndex(ctx, assemblyID) + return r.buildIndex(ctx, kv, assemblyID) } // buildIndex scans every artifact of a assembly and reconstructs its version // index. Rebuilding is self-healing: it only runs when no index exists yet. -func (r *Repository) buildIndex(ctx context.Context, assemblyID string) (*VersionIndex, error) { - prefix := sqlite.SanitizeKey("assemblies", assemblyID) - keys, err := r.store.List(ctx, prefix+"/") +func (r *Repository) buildIndex(ctx context.Context, kv storage.KV, assemblyID string) (*VersionIndex, error) { + prefix, err := storage.Prefix("assemblies", assemblyID) + if err != nil { + return nil, err + } + keys, err := kv.List(ctx, prefix) if err != nil { return nil, err } @@ -103,9 +110,9 @@ func (r *Repository) buildIndex(ctx context.Context, assemblyID string) (*Versio if !strings.HasSuffix(storageKey, "/manifest") { continue } - reg, err := r.getByStorageKey(ctx, storageKey) + reg, err := r.getByStorageKey(ctx, kv, storageKey) if err != nil { - continue + return nil, fmt.Errorf("rebuild index for %s: load artifact %q: %w", assemblyID, storageKey, err) } entry := VersionEntry{ Hash: reg.Key.Hash, @@ -120,13 +127,13 @@ func (r *Repository) buildIndex(ctx context.Context, assemblyID string) (*Versio } } - if err := r.saveIndex(ctx, assemblyID, idx); err != nil { + if err := r.saveIndex(ctx, kv, assemblyID, idx); err != nil { return nil, err } return idx, nil } -func (r *Repository) saveIndex(ctx context.Context, assemblyID string, idx *VersionIndex) error { +func (r *Repository) saveIndex(ctx context.Context, kv storage.KV, assemblyID string, idx *VersionIndex) error { if idx == nil { idx = newVersionIndex() } @@ -134,7 +141,11 @@ func (r *Repository) saveIndex(ctx context.Context, assemblyID string, idx *Vers if err != nil { return fmt.Errorf("marshal version index for %s: %w", assemblyID, err) } - return r.store.Put(ctx, r.indexKey(assemblyID), data) + indexKey, err := r.indexKey(assemblyID) + if err != nil { + return err + } + return kv.Put(ctx, indexKey, data) } // newerEntry reports whether (verA, entryA) is newer than (verB, entryB). diff --git a/nore/internal/assembly/repository.go b/nore/internal/assembly/repository.go index c0a2954..d0ceb4f 100644 --- a/nore/internal/assembly/repository.go +++ b/nore/internal/assembly/repository.go @@ -9,18 +9,29 @@ import ( "time" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" "github.com/neuron-runtime/neuron/shared/types/protocol" ) +var ( + // ErrArtifactMismatch reports that the artifact stored under a requested key + // does not describe the identity that key names. Resolution never silently + // falls back to a different artifact. + ErrArtifactMismatch = errors.New("assembly: artifact identity mismatch") + + // ErrAmbiguousArtifact reports that a content hash without an environment + // matches more than one stored artifact. + ErrAmbiguousArtifact = errors.New("assembly: ambiguous artifact") +) + // Repository persists RegisteredAssembly artifacts under // assemblies/{assemblyID}/{version}/{hash}/{env}/manifest keys in the shared // storage.Store, plus a per-assembly version index (versions.json) that makes // (assemblyID, version) the source of truth. // -// Each content hash is an immutable artifact; registering the same version -// again REPLACES the previous artifact for that version, so a given -// name:version always addresses exactly one artifact — the newest one. +// Registering the same version again REPLACES the previous artifact(s) for that +// version: earlier hashes are pruned, so a given name:version always addresses +// exactly one artifact, the newest one. Artifacts are therefore not +// content-addressed immutable; that retention limitation remains until Phase 4. type Repository struct { store storage.Store } @@ -32,8 +43,10 @@ func NewRepository(store storage.Store) *Repository { // Register persists reg durably and makes (assemblyID, version) point at it. // Registering the same version clears the previously recorded artifact(s) for // that version, regardless of hash or environment, then writes the new one and -// sets it as the latest version. The request is idempotent when the exact same -// artifact is already current: created=false, replaced=false. +// sets it as the latest version. The artifact write, pruning, and index update +// happen in one transaction, so a failed registration leaves no partial state. +// The request is idempotent when the exact same artifact is already current: +// created=false, replaced=false. func (r *Repository) Register(ctx context.Context, reg RegisteredAssembly) (created bool, replaced bool, err error) { if reg.Key.AssemblyID == "" { return false, false, fmt.Errorf("register requires a assembly id") @@ -48,41 +61,53 @@ func (r *Repository) Register(ctx context.Context, reg RegisteredAssembly) (crea reg.UpdatedAt = reg.RegisteredAt } - idx, err := r.loadIndex(ctx, reg.Key.AssemblyID) + storageKey, err := r.key(reg.Key) if err != nil { return false, false, err } - old, existed := idx.Versions[reg.Key.Version] + data := mustJSON(reg) - storageKey := r.key(reg.Key) - if err := r.store.Put(ctx, storageKey, mustJSON(reg)); err != nil { - return false, false, err - } + err = r.store.WithTx(ctx, func(tx storage.Tx) error { + idx, err := r.loadIndex(ctx, tx, reg.Key.AssemblyID) + if err != nil { + return err + } + old, existed := idx.Versions[reg.Key.Version] - // Enforce a single artifact per version: clear every other artifact - // previously recorded under assemblies/{id}/{version}/*. - if err := r.pruneVersionArtifacts(ctx, reg.Key); err != nil { - return false, false, err - } + if err := tx.Put(ctx, storageKey, data); err != nil { + return err + } - entry := VersionEntry{ - Hash: reg.Key.Hash, - Env: reg.Key.Env, - RegisteredAt: reg.RegisteredAt, - } - idx.Versions[reg.Key.Version] = entry - idx.Latest = &VersionRef{Version: reg.Key.Version, VersionEntry: entry} - if err := r.saveIndex(ctx, reg.Key.AssemblyID, idx); err != nil { - return false, false, err - } + // Enforce a single artifact per version: clear every other artifact + // previously recorded under assemblies/{id}/{version}/*. + if err := r.pruneVersionArtifacts(ctx, tx, reg.Key); err != nil { + return err + } - if existed { - if old.Hash == entry.Hash && old.Env == entry.Env { - return false, false, nil + entry := VersionEntry{ + Hash: reg.Key.Hash, + Env: reg.Key.Env, + RegisteredAt: reg.RegisteredAt, } - return false, true, nil + idx.Versions[reg.Key.Version] = entry + idx.Latest = &VersionRef{Version: reg.Key.Version, VersionEntry: entry} + if err := r.saveIndex(ctx, tx, reg.Key.AssemblyID, idx); err != nil { + return err + } + + if existed { + created = false + replaced = old.Hash != entry.Hash || old.Env != entry.Env + } else { + created = true + replaced = false + } + return nil + }) + if err != nil { + return false, false, err } - return true, false, nil + return created, replaced, nil } // Get resolves the durable artifact for key. See Resolve for how partial keys @@ -95,7 +120,13 @@ func (r *Repository) Get(ctx context.Context, key protocol.InstanceKey) (Registe // forms are assemblyID, assemblyID:latest, assemblyID:version, and // assemblyID:version:hash, each optionally suffixed with :env. // -// - A fully-specified hash maps directly to its immutable artifact. +// - When a hash is supplied it must be paired with a concrete version. The +// hash addresses exactly that artifact: a key with an environment loads the +// literal manifest key, while a key without one requires exactly one +// artifact under the version/hash prefix. Missing artifacts return +// storage.ErrNotFound, ambiguous ones return ErrAmbiguousArtifact, and +// stored artifacts whose identity disagrees with the key return +// ErrArtifactMismatch. Hash resolution never falls back to the index. // - Otherwise the version is resolved through the assembly's version index // (latest resolves to the most recently registered version) and the hash // and environment come from that index entry. @@ -104,14 +135,10 @@ func (r *Repository) Get(ctx context.Context, key protocol.InstanceKey) (Registe // addressable. func (r *Repository) Resolve(ctx context.Context, key protocol.InstanceKey) (RegisteredAssembly, error) { if key.Hash != "" { - reg, err := r.resolveInVersion(ctx, key.AssemblyID, key.Version, key.Env) - if err != nil { - return RegisteredAssembly{}, err - } - return reg, nil + return r.resolveByHash(ctx, key) } - idx, err := r.loadIndex(ctx, key.AssemblyID) + idx, err := r.loadIndex(ctx, r.store, key.AssemblyID) if err != nil { return RegisteredAssembly{}, err } @@ -121,9 +148,12 @@ func (r *Repository) Resolve(ctx context.Context, key protocol.InstanceKey) (Reg } if key.Env != "" && key.Env != entry.Env { - reg, scanErr := r.resolveInVersion(ctx, key.AssemblyID, version, key.Env) + reg, scanErr := r.resolveInVersion(ctx, r.store, key.AssemblyID, version, key.Env) if scanErr != nil { - return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: env %q not registered", key.String(), key.Env) + if errors.Is(scanErr, storage.ErrNotFound) { + return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: env %q not registered", key.String(), key.Env) + } + return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: %w", key.String(), scanErr) } return reg, nil } @@ -133,19 +163,102 @@ func (r *Repository) Resolve(ctx context.Context, key protocol.InstanceKey) (Reg full.Hash = entry.Hash full.Env = entry.Env - reg, err := r.getByStorageKey(ctx, r.key(full)) + storageKey, err := r.key(full) + if err != nil { + return RegisteredAssembly{}, err + } + reg, err := r.getByStorageKey(ctx, r.store, storageKey) if err != nil { return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: %w", key.String(), err) } return reg, nil } +// resolveByHash loads the exact artifact named by key.Hash. A content hash is +// only meaningful together with a concrete version. When an environment is +// supplied it addresses one literal key; otherwise the version/hash prefix must +// contain exactly one manifest. +func (r *Repository) resolveByHash(ctx context.Context, key protocol.InstanceKey) (RegisteredAssembly, error) { + if key.Version == "" || key.Version == protocol.VersionLatest { + return RegisteredAssembly{}, fmt.Errorf( + "registered assembly %s: a content hash requires a concrete version, got %q", + key.String(), key.Version, + ) + } + + if key.Env != "" { + storageKey, err := r.key(key) + if err != nil { + return RegisteredAssembly{}, err + } + reg, err := r.getByStorageKey(ctx, r.store, storageKey) + if err != nil { + return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: %w", key.String(), err) + } + if err := verifyArtifact(key, reg.Key, true); err != nil { + return RegisteredAssembly{}, err + } + return reg, nil + } + + prefix, err := storage.Prefix("assemblies", key.AssemblyID, key.Version, key.Hash) + if err != nil { + return RegisteredAssembly{}, err + } + keys, err := r.store.List(ctx, prefix) + if err != nil { + return RegisteredAssembly{}, err + } + + matches := make([]string, 0, 1) + for _, storageKey := range keys { + if strings.HasSuffix(storageKey, "/manifest") { + matches = append(matches, storageKey) + } + } + + switch len(matches) { + case 0: + return RegisteredAssembly{}, fmt.Errorf("%w: registered assembly %s", storage.ErrNotFound, key.String()) + case 1: + reg, err := r.getByStorageKey(ctx, r.store, matches[0]) + if err != nil { + return RegisteredAssembly{}, fmt.Errorf("registered assembly %s: %w", key.String(), err) + } + if err := verifyArtifact(key, reg.Key, false); err != nil { + return RegisteredAssembly{}, err + } + return reg, nil + default: + return RegisteredAssembly{}, fmt.Errorf( + "%w: registered assembly %s matches %d artifacts; specify an environment", + ErrAmbiguousArtifact, key.String(), len(matches), + ) + } +} + +// verifyArtifact checks that a stored artifact describes the requested key. +// The environment is only compared when the caller requested one. +func verifyArtifact(want, got protocol.InstanceKey, checkEnv bool) error { + if got.AssemblyID != want.AssemblyID || + got.Version != want.Version || + got.Hash != want.Hash || + (checkEnv && got.Env != want.Env) { + return fmt.Errorf("%w: requested %s, stored artifact is %s", ErrArtifactMismatch, want.String(), got.String()) + } + return nil +} + // resolveInVersion scans the artifacts registered under // assemblies/{id}/{version}/* and returns the newest one. When env is non-empty -// only artifacts of that environment are considered. -func (r *Repository) resolveInVersion(ctx context.Context, assemblyID, version, env string) (RegisteredAssembly, error) { - prefix := sqlite.SanitizeKey("assemblies", assemblyID, version) - keys, err := r.store.List(ctx, prefix+"/") +// only artifacts of that environment are considered. Read or decode failures +// are returned rather than treated as absence. +func (r *Repository) resolveInVersion(ctx context.Context, kv storage.KV, assemblyID, version, env string) (RegisteredAssembly, error) { + prefix, err := storage.Prefix("assemblies", assemblyID, version) + if err != nil { + return RegisteredAssembly{}, err + } + keys, err := kv.List(ctx, prefix) if err != nil { return RegisteredAssembly{}, err } @@ -155,9 +268,9 @@ func (r *Repository) resolveInVersion(ctx context.Context, assemblyID, version, if !strings.HasSuffix(storageKey, "/manifest") { continue } - candidate, err := r.getByStorageKey(ctx, storageKey) + candidate, err := r.getByStorageKey(ctx, kv, storageKey) if err != nil { - continue + return RegisteredAssembly{}, fmt.Errorf("load artifact %q: %w", storageKey, err) } if env != "" && candidate.Key.Env != env { continue @@ -196,32 +309,43 @@ func (r *Repository) Exists(ctx context.Context, key protocol.InstanceKey) (bool // Delete removes version's artifact(s) and its version index entry, then // recomputes the latest pointer. A key may be partial (assemblyID:version); when -// the version is empty every version of the assembly is removed. +// the version is empty every version of the assembly is removed. The artifact +// deletes and index update happen in one transaction. func (r *Repository) Delete(ctx context.Context, key protocol.InstanceKey) error { - idx, err := r.loadIndex(ctx, key.AssemblyID) - if err != nil { - return err - } - - if key.Version == "" || key.Version == protocol.VersionLatest { - if err := r.deleteVersionArtifacts(ctx, key.AssemblyID, ""); err != nil { + return r.store.WithTx(ctx, func(tx storage.Tx) error { + idx, err := r.loadIndex(ctx, tx, key.AssemblyID) + if err != nil { return err } - return r.store.Delete(ctx, r.indexKey(key.AssemblyID)) - } - if err := r.deleteVersionArtifacts(ctx, key.AssemblyID, key.Version); err != nil { - return err - } + if key.Version == "" || key.Version == protocol.VersionLatest { + if err := r.deleteVersionArtifacts(ctx, tx, key.AssemblyID, ""); err != nil { + return err + } + indexKey, err := r.indexKey(key.AssemblyID) + if err != nil { + return err + } + return tx.Delete(ctx, indexKey) + } + + if err := r.deleteVersionArtifacts(ctx, tx, key.AssemblyID, key.Version); err != nil { + return err + } - delete(idx.Versions, key.Version) - idx.Latest = recomputeLatest(idx) - return r.saveIndex(ctx, key.AssemblyID, idx) + delete(idx.Versions, key.Version) + idx.Latest = recomputeLatest(idx) + return r.saveIndex(ctx, tx, key.AssemblyID, idx) + }) } // List returns every registered artifact, one per indexing assembly/version/hash. func (r *Repository) List(ctx context.Context) ([]RegisteredAssembly, error) { - keys, err := r.store.List(ctx, "assemblies/") + prefix, err := storage.Prefix("assemblies") + if err != nil { + return nil, err + } + keys, err := r.store.List(ctx, prefix) if err != nil { return nil, err } @@ -231,9 +355,9 @@ func (r *Repository) List(ctx context.Context) ([]RegisteredAssembly, error) { if !strings.HasSuffix(key, "/manifest") { continue } - rec, err := r.getByStorageKey(ctx, key) + rec, err := r.getByStorageKey(ctx, r.store, key) if err != nil { - continue + return nil, fmt.Errorf("load artifact %q: %w", key, err) } identity := rec.Key.AssemblyID + "\x00" + rec.Key.Version + "\x00" + rec.Key.Hash + "\x00" + rec.Key.Env if seen[identity] { @@ -247,17 +371,24 @@ func (r *Repository) List(ctx context.Context) ([]RegisteredAssembly, error) { // pruneVersionArtifacts deletes every artifact under assemblies/{id}/{version}/* // except the just-written one for new. -func (r *Repository) pruneVersionArtifacts(ctx context.Context, new protocol.InstanceKey) error { - prefix := sqlite.SanitizeKey("assemblies", new.AssemblyID, new.Version) - keys, err := r.store.List(ctx, prefix+"/") +func (r *Repository) pruneVersionArtifacts(ctx context.Context, kv storage.KV, new protocol.InstanceKey) error { + prefix, err := storage.Prefix("assemblies", new.AssemblyID, new.Version) + if err != nil { + return err + } + newKey, err := r.key(new) + if err != nil { + return err + } + keys, err := kv.List(ctx, prefix) if err != nil { return err } for _, storageKey := range keys { - if !strings.HasSuffix(storageKey, "/manifest") || storageKey == r.key(new) { + if !strings.HasSuffix(storageKey, "/manifest") || storageKey == newKey { continue } - if err := r.store.Delete(ctx, storageKey); err != nil { + if err := kv.Delete(ctx, storageKey); err != nil { return err } } @@ -266,12 +397,18 @@ func (r *Repository) pruneVersionArtifacts(ctx context.Context, new protocol.Ins // deleteVersionArtifacts removes every artifact under assemblies/{id}/{version}/*. // An empty version deletes every artifact of the assembly (but never the index). -func (r *Repository) deleteVersionArtifacts(ctx context.Context, assemblyID, version string) error { - prefix := sqlite.SanitizeKey("assemblies", assemblyID) +func (r *Repository) deleteVersionArtifacts(ctx context.Context, kv storage.KV, assemblyID, version string) error { + var prefix string + var err error if version != "" { - prefix += "/" + version + prefix, err = storage.Prefix("assemblies", assemblyID, version) + } else { + prefix, err = storage.Prefix("assemblies", assemblyID) + } + if err != nil { + return err } - keys, err := r.store.List(ctx, prefix+"/") + keys, err := kv.List(ctx, prefix) if err != nil { return err } @@ -279,7 +416,7 @@ func (r *Repository) deleteVersionArtifacts(ctx context.Context, assemblyID, ver if !strings.HasSuffix(storageKey, "/manifest") { continue } - if err := r.store.Delete(ctx, storageKey); err != nil { + if err := kv.Delete(ctx, storageKey); err != nil { return err } } @@ -300,18 +437,18 @@ func recomputeLatest(idx *VersionIndex) *VersionRef { return latest } -func (r *Repository) key(key protocol.InstanceKey) string { - return sqlite.SanitizeKey("assemblies", key.AssemblyID, key.Version, key.Hash, key.Env, "manifest") +func (r *Repository) key(key protocol.InstanceKey) (string, error) { + return storage.Key("assemblies", key.AssemblyID, key.Version, key.Hash, key.Env, "manifest") } -func (r *Repository) getByStorageKey(ctx context.Context, storageKey string) (RegisteredAssembly, error) { +func (r *Repository) getByStorageKey(ctx context.Context, kv storage.KV, storageKey string) (RegisteredAssembly, error) { var reg RegisteredAssembly - data, err := r.store.Get(ctx, storageKey) + data, err := kv.Get(ctx, storageKey) if err != nil { return reg, err } if err := json.Unmarshal(data, ®); err != nil { - return reg, err + return reg, fmt.Errorf("unmarshal registered assembly %q: %w", storageKey, err) } return reg, nil } diff --git a/nore/internal/assembly/repository_test.go b/nore/internal/assembly/repository_test.go new file mode 100644 index 0000000..f125755 --- /dev/null +++ b/nore/internal/assembly/repository_test.go @@ -0,0 +1,342 @@ +package assembly + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" + "github.com/neuron-runtime/neuron/shared/types/protocol" +) + +func newTestRepository(t *testing.T) (*Repository, storage.Store) { + t.Helper() + + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("open sqlite store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + return NewRepository(store), store +} + +func testRegistration(assemblyID, version, hash, env string) RegisteredAssembly { + return RegisteredAssembly{ + Key: protocol.InstanceKey{ + AssemblyID: assemblyID, + Version: version, + Hash: hash, + Env: env, + }, + } +} + +func register(t *testing.T, repo *Repository, assemblyID, version, hash, env string) { + t.Helper() + + if _, _, err := repo.Register(context.Background(), testRegistration(assemblyID, version, hash, env)); err != nil { + t.Fatalf("register %s:%s#%s:%s: %v", assemblyID, version, hash, env, err) + } +} + +func TestResolveByExactHash(t *testing.T) { + ctx := context.Background() + repo, _ := newTestRepository(t) + register(t, repo, "app", "1.0.0", "hash-one", "dev") + + got, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-one", + Env: "dev", + }) + if err != nil { + t.Fatalf("resolve hash with env: %v", err) + } + if got.Key.Hash != "hash-one" || got.Key.Env != "dev" { + t.Fatalf("resolved unexpected artifact: %+v", got.Key) + } + + got, err = repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-one", + }) + if err != nil { + t.Fatalf("resolve hash without env: %v", err) + } + if got.Key.Hash != "hash-one" { + t.Fatalf("resolved unexpected hash: %+v", got.Key) + } +} + +func TestSameVersionRebuildReplacesOldHash(t *testing.T) { + ctx := context.Background() + repo, _ := newTestRepository(t) + register(t, repo, "app", "1.0.0", "old-hash", "dev") + register(t, repo, "app", "1.0.0", "new-hash", "dev") + + _, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "old-hash", + Env: "dev", + }) + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("old hash should be gone, got %v", err) + } + + got, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "new-hash", + Env: "dev", + }) + if err != nil { + t.Fatalf("resolve new hash: %v", err) + } + if got.Key.Hash != "new-hash" { + t.Fatalf("resolved unexpected hash: %+v", got.Key) + } + + latest, err := repo.Resolve(ctx, protocol.InstanceKey{AssemblyID: "app"}) + if err != nil { + t.Fatalf("resolve latest: %v", err) + } + if latest.Key.Hash != "new-hash" { + t.Fatalf("latest should point at new hash, got %+v", latest.Key) + } +} + +func TestDeleteRemovesVersionAndKeepsOthers(t *testing.T) { + ctx := context.Background() + repo, _ := newTestRepository(t) + register(t, repo, "app", "1.0.0", "hash-v1", "dev") + register(t, repo, "app", "2.0.0", "hash-v2", "dev") + + if err := repo.Delete(ctx, protocol.InstanceKey{AssemblyID: "app", Version: "1.0.0"}); err != nil { + t.Fatalf("delete version: %v", err) + } + + _, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-v1", + Env: "dev", + }) + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("deleted version hash should be gone, got %v", err) + } + + got, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "2.0.0", + Hash: "hash-v2", + Env: "dev", + }) + if err != nil { + t.Fatalf("other version should remain: %v", err) + } + if got.Key.Version != "2.0.0" { + t.Fatalf("resolved unexpected version: %+v", got.Key) + } + + // A key with no version removes every version of the assembly. + if err := repo.Delete(ctx, protocol.InstanceKey{AssemblyID: "app"}); err != nil { + t.Fatalf("delete all versions: %v", err) + } + _, err = repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "2.0.0", + Hash: "hash-v2", + Env: "dev", + }) + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("force-deleted hash should be gone, got %v", err) + } +} + +func TestDistinctVersionsResolveIndependently(t *testing.T) { + ctx := context.Background() + repo, _ := newTestRepository(t) + register(t, repo, "app", "1.0.0", "hash-v1", "dev") + register(t, repo, "app", "2.0.0", "hash-v2", "prod") + + for _, key := range []protocol.InstanceKey{ + {AssemblyID: "app", Version: "1.0.0", Hash: "hash-v1", Env: "dev"}, + {AssemblyID: "app", Version: "2.0.0", Hash: "hash-v2", Env: "prod"}, + } { + got, err := repo.Resolve(ctx, key) + if err != nil { + t.Fatalf("resolve %s: %v", key.String(), err) + } + if got.Key != key { + t.Fatalf("resolve %s returned %s", key.String(), got.Key.String()) + } + } + + for _, key := range []protocol.InstanceKey{ + {AssemblyID: "app", Version: "1.0.0", Hash: "hash-v1"}, + {AssemblyID: "app", Version: "2.0.0", Hash: "hash-v2"}, + } { + got, err := repo.Resolve(ctx, key) + if err != nil { + t.Fatalf("resolve %s: %v", key.String(), err) + } + if got.Key.AssemblyID != key.AssemblyID || got.Key.Version != key.Version || got.Key.Hash != key.Hash { + t.Fatalf("resolve %s returned %s", key.String(), got.Key.String()) + } + } +} + +func TestHashRequiresConcreteVersion(t *testing.T) { + ctx := context.Background() + repo, _ := newTestRepository(t) + register(t, repo, "app", "1.0.0", "hash-v1", "dev") + + for _, version := range []string{"", protocol.VersionLatest} { + _, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: version, + Hash: "hash-v1", + Env: "dev", + }) + if err == nil { + t.Fatalf("hash with version %q should fail", version) + } + if !strings.Contains(err.Error(), "requires a concrete version") { + t.Fatalf("unexpected error for version %q: %v", version, err) + } + } +} + +func TestHashAbsentAndMismatchedArtifacts(t *testing.T) { + ctx := context.Background() + repo, store := newTestRepository(t) + register(t, repo, "app", "1.0.0", "hash-v1", "dev") + + for _, key := range []protocol.InstanceKey{ + {AssemblyID: "app", Version: "1.0.0", Hash: "missing", Env: "dev"}, + {AssemblyID: "app", Version: "1.0.0", Hash: "missing"}, + } { + _, err := repo.Resolve(ctx, key) + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("absent hash %s should be ErrNotFound, got %v", key.String(), err) + } + } + + // Store a manifest under a key whose embedded identity disagrees with it. + mismatched := testRegistration("app", "1.0.0", "embedded-hash", "dev") + key, err := storage.Key("assemblies", "app", "1.0.0", "hash-v2", "dev", "manifest") + if err != nil { + t.Fatalf("build storage key: %v", err) + } + if err := store.Put(ctx, key, mustJSON(mismatched)); err != nil { + t.Fatalf("write mismatched artifact: %v", err) + } + + _, err = repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-v2", + Env: "dev", + }) + if !errors.Is(err, ErrArtifactMismatch) { + t.Fatalf("mismatched artifact should be ErrArtifactMismatch, got %v", err) + } +} + +func TestAmbiguousHashWithoutEnv(t *testing.T) { + ctx := context.Background() + repo, store := newTestRepository(t) + + for _, env := range []string{"dev", "prod"} { + reg := testRegistration("app", "1.0.0", "hash-v1", env) + key, err := storage.Key("assemblies", "app", "1.0.0", "hash-v1", env, "manifest") + if err != nil { + t.Fatalf("build storage key: %v", err) + } + if err := store.Put(ctx, key, mustJSON(reg)); err != nil { + t.Fatalf("write %s artifact: %v", env, err) + } + } + + _, err := repo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-v1", + }) + if !errors.Is(err, ErrAmbiguousArtifact) { + t.Fatalf("two env artifacts should be ambiguous, got %v", err) + } +} + +// failingIndexStore wraps a real store and injects a failure when the +// transaction writes the version index. The artifact write before it must roll +// back. +type failingIndexStore struct { + storage.Store + failSuffix string +} + +func (s *failingIndexStore) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + return s.Store.WithTx(ctx, func(tx storage.Tx) error { + return fn(failingIndexTx{Tx: tx, failSuffix: s.failSuffix}) + }) +} + +type failingIndexTx struct { + storage.Tx + failSuffix string +} + +func (tx failingIndexTx) Put(ctx context.Context, key string, data []byte) error { + if strings.HasSuffix(key, tx.failSuffix) { + return errors.New("injected index write failure") + } + return tx.Tx.Put(ctx, key, data) +} + +func TestRegisterIsAtomicWhenIndexWriteFails(t *testing.T) { + ctx := context.Background() + store, err := sqlite.New(ctx, storage.Config{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("open sqlite store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + // Seed an empty index so Register reaches the artifact write before the + // injected index failure. + baseRepo := NewRepository(store) + indexKey, err := baseRepo.indexKey("app") + if err != nil { + t.Fatalf("build index key: %v", err) + } + data, err := json.Marshal(newVersionIndex()) + if err != nil { + t.Fatalf("marshal index: %v", err) + } + if err := store.Put(ctx, indexKey, data); err != nil { + t.Fatalf("seed index: %v", err) + } + + failingRepo := NewRepository(&failingIndexStore{Store: store, failSuffix: indexFileName}) + _, _, err = failingRepo.Register(ctx, testRegistration("app", "1.0.0", "hash-v1", "dev")) + if err == nil { + t.Fatal("registration should fail when the index write fails") + } + + _, err = baseRepo.Resolve(ctx, protocol.InstanceKey{ + AssemblyID: "app", + Version: "1.0.0", + Hash: "hash-v1", + Env: "dev", + }) + if !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("artifact should have rolled back, got %v", err) + } +} diff --git a/nore/internal/builtins/ai.go b/nore/internal/builtins/ai.go deleted file mode 100644 index 0427b0f..0000000 --- a/nore/internal/builtins/ai.go +++ /dev/null @@ -1,39 +0,0 @@ -package builtins - -import ( - "context" - "fmt" - "time" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// AIMock simulates an AI capability: it resolves a "prompt" from the -// capability configuration and returns a canned response. In a real -// deployment this would be replaced by a runtime that calls an LLM provider. -type AIMock struct{} - -// Execute resolves the configured prompt and emits it as a CapabilityLog -// before returning a mock response. -func (AIMock) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - prompt, ok := execution.CapabilityConfigurations["prompt"].(string) - if !ok || prompt == "" { - return nil, fmt.Errorf("AI capability requires a resolved prompt string") - } - execution.Logger.Info(ctx, "resolved AI prompt", contracts.F("prompt", prompt)) - - // Simulate AI processing delay - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(500 * time.Millisecond): - } - - return map[string]any{"content": "[mock AI response] " + prompt}, nil -} diff --git a/nore/internal/builtins/command.go b/nore/internal/builtins/command.go deleted file mode 100644 index d2c3308..0000000 --- a/nore/internal/builtins/command.go +++ /dev/null @@ -1,61 +0,0 @@ -package builtins - -import ( - "bytes" - "context" - "errors" - "fmt" - "maps" - "os/exec" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// Command runs a local command and returns stdout/stderr/exit_code. It is -// cancelled automatically when the execution context ends. -type Command struct{} - -func (Command) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - params := make(map[string]any) - maps.Copy(params, execution.CapabilityConfigurations) - maps.Copy(params, execution.Params) - - cmdString, _ := params["command"].(string) - if cmdString == "" { - return nil, fmt.Errorf("command runtime requires a 'command' string") - } - - // Parse arguments safely - var cmdArgs []string - if argsRaw, ok := params["args"].([]any); ok { - for _, arg := range argsRaw { - cmdArgs = append(cmdArgs, fmt.Sprintf("%v", arg)) - } - } - - // Create command with context (this auto-kills the process if N.O.R.E times out) - cmd := exec.CommandContext(ctx, cmdString, cmdArgs...) - - // Capture outputs - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - - err := cmd.Run() - - exitCode := 0 - if err != nil { - var exitError *exec.ExitError - if errors.As(err, &exitError) { - exitCode = exitError.ExitCode() - } - } - - return map[string]any{ - "stdout": stdout.String(), - "stderr": stderr.String(), - "exit_code": float64(exitCode), // JSON numbers are float64 by default in Go - "success": exitCode == 0, - }, nil -} diff --git a/nore/internal/builtins/command/command.go b/nore/internal/builtins/command/command.go new file mode 100644 index 0000000..4679126 --- /dev/null +++ b/nore/internal/builtins/command/command.go @@ -0,0 +1,467 @@ +// Package command implements the neuron:core:command capability runtime. +package command + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/builtins/input" + "github.com/neuron-runtime/neuron/nore/internal/builtins/paths" + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// Command runs a local program and returns its exit code, standard output and +// standard error. +// +// It is the boundary between an assembly and the programs already installed on +// the machine. The program and its arguments are passed to the operating +// system separately, so an argument can never become a command of its own. +// +// The policy bounds the run: which programs it may run, where it may run +// them, what environment it inherits, how long it may take, and how much +// output is retained. +type Command struct { + // Policy bounds what a run may do. A zero policy means the compiled-in + // ceilings. + Policy builtins.Policy +} + +// inheritedVariables are the environment variables a program inherits from the +// runtime. +// +// The environment of the runtime process is not the environment of the +// program: a program that inherits everything inherits the credentials of +// whoever started the runtime. Only the variables a program needs to behave +// like a program are inherited, and everything else is dropped. +var inheritedVariables = []string{"PATH", "HOME", "LANG", "LC_ALL", "TZ", "TMPDIR"} + +// terminateGrace is how long a terminated program is given to exit before the +// runtime stops waiting for it. +const terminateGrace = 5 * time.Second + +// Execute runs the program named by the capability's input. +func (c Command) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { + // The policy is resolved here rather than trusted, because a runtime may + // also be constructed outside the registry. The resolution only narrows, + // so a policy that has already been resolved is unchanged. + policy, err := builtins.Effective(c.Policy) + if err != nil { + return nil, fmt.Errorf("neuron:core:command: %w", err) + } + + values := input.New("neuron:core:command", execution.CapabilityConfigurations, execution.Params) + logger := contracts.LoggerOrNop(execution.Logger) + + program, err := values.RequiredString("command") + if err != nil { + return nil, err + } + + args, _, err := values.StringSlice("args") + if err != nil { + return nil, err + } + + timeout, err := runTimeout(values, policy) + if err != nil { + return nil, err + } + + failOnFailure, err := values.Bool("failOnNonZeroExit", true) + if err != nil { + return nil, err + } + + format, err := values.Format("format", input.FormatText) + if err != nil { + return nil, err + } + + limit, err := outputLimit(values, policy) + if err != nil { + return nil, err + } + + environment, err := buildEnvironment(values) + if err != nil { + return nil, err + } + + // The working directory is confined to the permitted roots, so a program + // cannot be pointed at a directory the policy does not allow. + roots := paths.NewRoots(policy.ProjectRoot, policy.FilesystemRoots) + base := roots.Base() + + workingDirectory, err := resolveWorkingDirectory(values, roots, base) + if err != nil { + return nil, err + } + + // The program is resolved against the program path the program itself + // runs with, so the program that is checked is the program that runs. + resolved, err := resolveProgram(program, environment["PATH"], workingDirectory, policy.AllowedExecutables) + if err != nil { + return nil, err + } + + runCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + cmd := exec.CommandContext(runCtx, resolved, args...) + cmd.Dir = workingDirectory + cmd.Env = environmentSlice(environment) + + // The program runs in its own process group, and the whole group is + // stopped when the run ends. A program that starts a child does not get + // to keep it once the run it belongs to is over. + runInItsOwnGroup(cmd) + cmd.Cancel = func() error { return terminate(cmd) } + cmd.WaitDelay = terminateGrace + + if stdin, present, err := values.String("stdin"); err != nil { + return nil, err + } else if present { + cmd.Stdin = strings.NewReader(stdin) + } + + // The output is bounded while it is produced, not after: a program that + // writes without limit must not be able to exhaust the runtime's memory. + stdout := &cappedBuffer{limit: limit} + stderr := &cappedBuffer{limit: limit} + cmd.Stdout = stdout + cmd.Stderr = stderr + + logger.Debug(ctx, "neuron:core:command: running the program", + contracts.F("program", resolved), + contracts.F("working_directory", workingDirectory), + ) + + started := time.Now() + runErr := cmd.Run() + elapsed := time.Since(started) + + // A cancelled execution is not a program that finished. The program was + // stopped because the work it belonged to is gone, so the cancellation is + // reported rather than an exit code the program never produced. + if ctxErr := ctx.Err(); ctxErr != nil { + return nil, ctxErr + } + + // The deadline is the runtime's own bound on the run, which is different + // from a cancellation: the program was still wanted, it simply took too + // long. + timedOut := errors.Is(runCtx.Err(), context.DeadlineExceeded) + + exitCode := 0 + if runErr != nil { + var exitError *exec.ExitError + if !errors.As(runErr, &exitError) { + // The program never ran: it could not be found, was not + // executable, or could not be started. Reporting exit code zero + // here would claim a success that never happened. + return nil, fmt.Errorf("neuron:core:command: run %q: %w", program, runErr) + } + exitCode = exitError.ExitCode() + } + + output, err := representOutput(stdout.String(), format) + if err != nil { + return nil, err + } + + result := map[string]any{ + "stdout": output, + "stderr": stderr.String(), + "exit_code": float64(exitCode), // JSON numbers are float64 by default in Go + "success": exitCode == 0 && !timedOut, + "timed_out": timedOut, + "duration_ms": float64(elapsed.Milliseconds()), + } + // The truncation is reported, so a caller can tell a shortened stream from + // a complete one. + if stdout.Truncated() { + result["stdout_truncated"] = true + } + if stderr.Truncated() { + result["stderr_truncated"] = true + } + + if result["success"] == false && failOnFailure { + if timedOut { + return nil, fmt.Errorf("neuron:core:command: %q was terminated by its deadline of %s", program, timeout) + } + return nil, fmt.Errorf("neuron:core:command: %q exited with code %d%s", program, exitCode, stderrSummary(stderr.String())) + } + + return result, nil +} + +// runTimeout resolves how long the program may run. +// +// A capability may ask for a shorter timeout than the policy allows, but not +// for a longer one: the policy is the ceiling, and a capability is part of the +// assembly's input. +func runTimeout(values input.Values, policy builtins.Policy) (time.Duration, error) { + limit, err := policy.Timeout() + if err != nil { + return 0, fmt.Errorf("neuron:core:command: %w", err) + } + + requested, err := values.Duration("timeout", limit) + if err != nil { + return 0, err + } + if requested <= 0 { + return 0, fmt.Errorf("neuron:core:command: parameter \"timeout\" must be positive") + } + if requested > limit { + return 0, fmt.Errorf("neuron:core:command: the requested timeout %s exceeds the limit of %s", requested, limit) + } + return requested, nil +} + +// outputLimit resolves how much output is retained per stream. +// +// The capability may ask for less than the policy allows, never for more. +func outputLimit(values input.Values, policy builtins.Policy) (int64, error) { + requested, err := values.Int("maxOutputBytes", 0) + if err != nil { + return 0, err + } + if requested < 0 { + return 0, fmt.Errorf("neuron:core:command: parameter \"maxOutputBytes\" must not be negative, got %d", requested) + } + if requested == 0 { + return policy.MaxCommandOutputBytes, nil + } + if int64(requested) > policy.MaxCommandOutputBytes { + return 0, fmt.Errorf("neuron:core:command: the requested output limit of %d bytes exceeds the limit of %d bytes", + requested, policy.MaxCommandOutputBytes) + } + return int64(requested), nil +} + +// resolveWorkingDirectory resolves the directory the program runs in. +// +// A directory outside the permitted roots is refused: the program is bounded +// by the same reach as everything else the built-ins do. +func resolveWorkingDirectory(values input.Values, roots paths.Roots, base string) (string, error) { + cwd, present, err := values.String("cwd") + if err != nil { + return "", err + } + if !present || strings.TrimSpace(cwd) == "" { + return base, nil + } + + resolved, err := roots.Resolve(base, cwd) + if err != nil { + return "", fmt.Errorf("neuron:core:command: %w", err) + } + return resolved, nil +} + +// buildEnvironment builds the environment the program runs with. +// +// The inherited variables come first, and the variables the capability asked +// for override them. +func buildEnvironment(values input.Values) (map[string]string, error) { + environment := make(map[string]string, len(inheritedVariables)) + + for _, name := range inheritedVariables { + if value, ok := os.LookupEnv(name); ok { + environment[name] = value + } + } + + requested, _, err := values.StringMap("env") + if err != nil { + return nil, err + } + for name, value := range requested { + if strings.TrimSpace(name) == "" { + return nil, fmt.Errorf("neuron:core:command: an environment variable name is empty") + } + environment[name] = value + } + + return environment, nil +} + +// environmentSlice renders the environment as the list the operating system +// expects, sorted so that a run is reproducible. +func environmentSlice(environment map[string]string) []string { + names := make([]string, 0, len(environment)) + for name := range environment { + names = append(names, name) + } + sort.Strings(names) + + rendered := make([]string, 0, len(names)) + for _, name := range names { + rendered = append(rendered, name+"="+environment[name]) + } + return rendered +} + +// resolveProgram resolves the program to run and checks it against the +// allowlist. +// +// The check is made on the resolved path rather than on the name, so a name +// cannot be used to reach a program that is not permitted. +func resolveProgram(program, programPath, workingDirectory string, allowed []string) (string, error) { + resolved, err := lookupProgram(program, programPath, workingDirectory) + if err != nil { + return "", fmt.Errorf("neuron:core:command: %w", err) + } + + if len(allowed) == 0 { + return resolved, nil + } + + for _, entry := range allowed { + permitted, err := lookupProgram(entry, programPath, workingDirectory) + if err != nil { + continue + } + if permitted == resolved { + return resolved, nil + } + } + + return "", fmt.Errorf("neuron:core:command: the program %q is not permitted; the permitted programs are %s", + resolved, strings.Join(allowed, ", ")) +} + +// lookupProgram resolves a program name against a program path. +// +// A name that contains a separator is a path, and is resolved against the +// working directory the program will run in. A name that does not is looked up +// in each directory of the program path, in order. +func lookupProgram(program, programPath, workingDirectory string) (string, error) { + if strings.ContainsRune(program, filepath.Separator) { + candidate := program + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(workingDirectory, candidate) + } + if !isExecutable(candidate) { + return "", fmt.Errorf("the program %q is not an executable file", candidate) + } + return filepath.Clean(candidate), nil + } + + for _, directory := range filepath.SplitList(programPath) { + if directory == "" { + directory = "." + } + candidate := filepath.Join(directory, program) + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(workingDirectory, candidate) + } + if isExecutable(candidate) { + return filepath.Clean(candidate), nil + } + } + + return "", fmt.Errorf("the program %q was not found on the program path", program) +} + +// isExecutable reports whether the path is a file that can be executed. +func isExecutable(path string) bool { + info, err := os.Stat(path) + if err != nil || info.IsDir() { + return false + } + return info.Mode().Perm()&0o111 != 0 +} + +// representOutput converts the program's standard output into the +// representation the caller asked for. +// +// The rule is the same as for the http built-in: a JSON output is returned as +// the value it represents, and an output that is not JSON is refused in the +// JSON format rather than handed over as a string the caller cannot bind. +func representOutput(text string, format input.Format) (any, error) { + if strings.TrimSpace(text) == "" { + if format == input.FormatText { + return "", nil + } + return nil, nil + } + + if format == input.FormatText { + return text, nil + } + + var value any + if err := json.Unmarshal([]byte(text), &value); err != nil { + return nil, fmt.Errorf("neuron:core:command: the program's standard output is not JSON; use format \"text\" to receive it as text") + } + return value, nil +} + +// stderrSummary returns the last meaningful line of the error output, for an +// error message that says why the program failed. +func stderrSummary(text string) string { + lines := strings.Split(strings.TrimSpace(text), "\n") + for index := len(lines) - 1; index >= 0; index-- { + line := strings.TrimSpace(lines[index]) + if line == "" { + continue + } + const limit = 200 + if len(line) > limit { + line = line[:limit] + "..." + } + return ": " + line + } + return "" +} + +// cappedBuffer is a writer that retains at most limit bytes and remembers how +// much was discarded. +// +// The bytes it retains are a prefix of the stream, so the output still starts +// where the program wrote it. Writing past the limit succeeds — the program is +// not the one being punished for exceeding a limit — but the excess is +// dropped. +type cappedBuffer struct { + buf bytes.Buffer + limit int64 + discarded int64 +} + +// Write retains the bytes that fit and discards the rest. +func (c *cappedBuffer) Write(p []byte) (int, error) { + if remaining := c.limit - int64(c.buf.Len()); remaining > 0 { + if int64(len(p)) <= remaining { + c.buf.Write(p) + return len(p), nil + } + c.buf.Write(p[:remaining]) + c.discarded += int64(len(p)) - remaining + return len(p), nil + } + c.discarded += int64(len(p)) + return len(p), nil +} + +// String returns the retained bytes. +func (c *cappedBuffer) String() string { + return c.buf.String() +} + +// Truncated reports whether any bytes were discarded. +func (c *cappedBuffer) Truncated() bool { + return c.discarded > 0 +} diff --git a/nore/internal/builtins/command/command_test.go b/nore/internal/builtins/command/command_test.go new file mode 100644 index 0000000..d399f1c --- /dev/null +++ b/nore/internal/builtins/command/command_test.go @@ -0,0 +1,512 @@ +package command + +import ( + "context" + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// commandHelperMarker is the argument that tells a re-executed test binary to +// act as the command built-in's helper program. +// +// The built-in passes the program and its arguments to the operating system +// separately, so the helper is selected by an argument rather than by an +// environment variable or by a program that happens to be installed. +const commandHelperMarker = "neuron-command-helper" + +// TestCommandHelperProcess is the program the command built-in runs. +// +// It is not a test in its own right: it only does anything when the test +// binary has been re-executed with commandHelperMarker among its arguments. +func TestCommandHelperProcess(t *testing.T) { + if !isCommandHelperProcess() { + t.Skip("helper process: only meaningful when re-executed by the command built-in test") + } + + for _, arg := range os.Args { + switch arg { + case "exit0": + os.Exit(0) + case "exit3": + os.Exit(3) + case "stderr": + os.Stderr.WriteString("something went wrong") + os.Exit(0) + case "sleep": + time.Sleep(time.Minute) + os.Exit(0) + case "flood": + // Write far more than any output limit allows, so the test can + // observe that the output is bounded while it is produced. + for i := 0; i < 4096; i++ { + os.Stdout.WriteString("0123456789") + } + os.Exit(0) + case "json": + os.Stdout.WriteString(`{"answer":42}`) + os.Exit(0) + case "text": + os.Stdout.WriteString("not json") + os.Exit(0) + case "environment": + os.Stdout.WriteString(strings.Join(os.Environ(), "\n")) + os.Exit(0) + case "directory": + directory, err := os.Getwd() + if err != nil { + os.Stderr.WriteString(err.Error()) + os.Exit(1) + } + os.Stdout.WriteString(directory) + os.Exit(0) + case "spawn": + // Start a child that would outlive this program, so the test can + // observe that the whole process group is stopped when the run + // ends. The program then waits, so the run ends on its deadline + // rather than on the program's own exit. + child := exec.Command(os.Args[0], "-test.run=^TestCommandHelperProcess$", commandHelperMarker, "sleep") + child.Start() + os.Stdout.WriteString(strconv.Itoa(child.Process.Pid)) + time.Sleep(time.Minute) + os.Exit(0) + } + } + os.Exit(0) +} + +// isCommandHelperProcess reports whether this test binary was re-executed to +// act as the helper program. +func isCommandHelperProcess() bool { + for _, arg := range os.Args { + if arg == commandHelperMarker { + return true + } + } + return false +} + +// helperParams runs the test binary again with the requested behaviour. +func helperParams(t *testing.T, behaviour string) map[string]any { + t.Helper() + return map[string]any{ + "command": os.Args[0], + "args": []any{"-test.run=^TestCommandHelperProcess$", commandHelperMarker, behaviour}, + } +} + +// helperParamsWith runs the test binary again with the requested behaviour and +// additional params. +func helperParamsWith(t *testing.T, behaviour string, extra map[string]any) map[string]any { + t.Helper() + params := helperParams(t, behaviour) + for name, value := range extra { + params[name] = value + } + return params +} + +func TestCommandRunsTheProgramAndReportsItsResult(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "exit0"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["success"] != true { + t.Errorf("success = %v, want true", result["success"]) + } + if result["exit_code"] != float64(0) { + t.Errorf("exit_code = %v, want 0", result["exit_code"]) + } + if result["timed_out"] != false { + t.Errorf("timed_out = %v, want false", result["timed_out"]) + } +} + +// A program that failed is a failed capability: the assembly is told that the +// program did not succeed, rather than receiving a result that says so. +func TestCommandFailsOnANonZeroExit(t *testing.T) { + _, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "exit3"), + }) + if err == nil { + t.Fatal("Execute succeeded for a program that exited with a non-zero code") + } + if !strings.Contains(err.Error(), "exited with code 3") { + t.Errorf("error = %v, want it to state the exit code", err) + } +} + +// A capability that asks to inspect a failure gets the result instead. +func TestCommandReportsANonZeroExitWhenTheFailureIsAllowed(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "exit3", map[string]any{"failOnNonZeroExit": false}), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["success"] != false { + t.Errorf("success = %v, want false", result["success"]) + } + if result["exit_code"] != float64(3) { + t.Errorf("exit_code = %v, want 3", result["exit_code"]) + } +} + +func TestCommandCapturesStandardError(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "stderr"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if got, _ := result["stderr"].(string); got != "something went wrong" { + t.Errorf("stderr = %q, want %q", got, "something went wrong") + } +} + +// A program that never ran must not be reported as a program that succeeded. +func TestCommandFailsWhenTheProgramCannotBeStarted(t *testing.T) { + _, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"command": "neuron-command-that-does-not-exist"}, + }) + if err == nil { + t.Fatal("Execute succeeded for a program that does not exist") + } + if !strings.Contains(err.Error(), "neuron-command-that-does-not-exist") { + t.Errorf("error = %v, want it to name the missing program", err) + } +} + +// A cancelled execution is not a program that finished. Reporting an exit +// code here would claim a result the program never produced. +func TestCommandReportsCancellationRatherThanSuccess(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() + + _, err := Command{}.Execute(ctx, contracts.ExecutionContext{ + Params: helperParams(t, "sleep"), + }) + if err == nil { + t.Fatal("Execute succeeded for a cancelled command") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("error %v does not report the cancellation", err) + } +} + +func TestCommandRequiresAProgram(t *testing.T) { + _, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{}) + if err == nil { + t.Fatal("Execute succeeded without a command") + } +} + +func TestCommandHelperProcessIsSkippedInTheOrdinaryRun(t *testing.T) { + if isCommandHelperProcess() { + t.Skip("this is the helper process") + } + if strings.Contains(strings.Join(os.Args, " "), commandHelperMarker) { + t.Errorf("the ordinary run carries the helper marker %q", commandHelperMarker) + } +} + +// Output beyond the limit is dropped while it is produced, and the truncation +// is reported so a caller can tell a shortened stream from a complete one. +func TestCommandBoundsTheOutputAndReportsTheTruncation(t *testing.T) { + result, err := Command{Policy: builtins.Policy{MaxCommandOutputBytes: 16}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "flood"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + stdout, _ := result["stdout"].(string) + if len(stdout) != 16 { + t.Errorf("stdout is %d bytes, want 16", len(stdout)) + } + if result["stdout_truncated"] != true { + t.Errorf("stdout_truncated = %v, want true", result["stdout_truncated"]) + } + if result["stderr_truncated"] == true { + t.Errorf("stderr_truncated = %v, want no truncation on an empty stream", result["stderr_truncated"]) + } +} + +// A capability may ask for less output than the policy allows, never for more. +func TestCommandRefusesAnOutputLimitAboveThePolicy(t *testing.T) { + _, err := Command{Policy: builtins.Policy{MaxCommandOutputBytes: 16}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "exit0", map[string]any{"maxOutputBytes": 1024}), + }) + if err == nil { + t.Fatal("Execute accepted an output limit above the policy") + } + if !strings.Contains(err.Error(), "exceeds the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +// The program runs with an allowlisted environment, not with the environment +// of the runtime: a program that inherited everything would inherit the +// credentials of whoever started the runtime. +func TestCommandRunsWithAnAllowlistedEnvironment(t *testing.T) { + t.Setenv("NEURON_COMMAND_SECRET", "secret") + + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "environment"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + environment, _ := result["stdout"].(string) + if strings.Contains(environment, "NEURON_COMMAND_SECRET") { + t.Error("the program inherited a variable that is not on the allowlist") + } + if !strings.Contains(environment, "PATH=") { + t.Error("the program did not inherit the program path") + } +} + +// The environment the capability asks for is added to the program's +// environment. +func TestCommandAddsTheRequestedEnvironment(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "environment", map[string]any{ + "env": map[string]any{"NEURON_COMMAND_EXTRA": "value"}, + }), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + environment, _ := result["stdout"].(string) + if !strings.Contains(environment, "NEURON_COMMAND_EXTRA=value") { + t.Errorf("the program did not receive the requested variable: %q", environment) + } +} + +// The program runs in the project root, which is where the assembly lives. +func TestCommandRunsInTheProjectRoot(t *testing.T) { + root := t.TempDir() + + result, err := Command{Policy: builtins.Policy{ProjectRoot: root}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "directory"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + directory, _ := result["stdout"].(string) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil { + t.Fatalf("resolve the working directory: %v", err) + } + expected, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatalf("resolve the project root: %v", err) + } + if resolved != expected { + t.Errorf("the program ran in %q, want %q", resolved, expected) + } +} + +// A working directory outside the permitted roots is refused: the program is +// bounded by the same reach as everything else the built-ins do. +func TestCommandRefusesAWorkingDirectoryOutsideThePermittedRoots(t *testing.T) { + root := t.TempDir() + + _, err := Command{Policy: builtins.Policy{ProjectRoot: root}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "directory", map[string]any{"cwd": t.TempDir()}), + }) + if err == nil { + t.Fatal("Execute ran a program outside the permitted roots") + } + if !strings.Contains(err.Error(), "not reachable") { + t.Errorf("error = %v, want it to state that the path is not reachable", err) + } +} + +// A working directory inside the project root is permitted. +func TestCommandRunsInAWorkingDirectoryInsideTheProjectRoot(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "nested") + if err := os.Mkdir(nested, 0o755); err != nil { + t.Fatalf("create the nested directory: %v", err) + } + + result, err := Command{Policy: builtins.Policy{ProjectRoot: root}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "directory", map[string]any{"cwd": nested}), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + directory, _ := result["stdout"].(string) + resolved, err := filepath.EvalSymlinks(directory) + if err != nil { + t.Fatalf("resolve the working directory: %v", err) + } + expected, err := filepath.EvalSymlinks(nested) + if err != nil { + t.Fatalf("resolve the nested directory: %v", err) + } + if resolved != expected { + t.Errorf("the program ran in %q, want %q", resolved, expected) + } +} + +// A program that is not on the allowlist is refused, even when it exists. +func TestCommandRefusesAProgramThatIsNotAllowed(t *testing.T) { + _, err := Command{Policy: builtins.Policy{AllowedExecutables: []string{"/bin/true"}}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "exit0"), + }) + if err == nil { + t.Fatal("Execute ran a program that is not on the allowlist") + } + if !strings.Contains(err.Error(), "not permitted") { + t.Errorf("error = %v, want it to state that the program is not permitted", err) + } +} + +// A program that is on the allowlist runs. +func TestCommandRunsAProgramThatIsAllowed(t *testing.T) { + result, err := Command{Policy: builtins.Policy{AllowedExecutables: []string{os.Args[0]}}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "exit0"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["success"] != true { + t.Errorf("success = %v, want true", result["success"]) + } +} + +// A program that outlasts its deadline is stopped, and the deadline is +// reported rather than an exit code the program never produced. +func TestCommandStopsTheProgramAtItsDeadline(t *testing.T) { + _, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "sleep", map[string]any{"timeout": "200ms"}), + }) + if err == nil { + t.Fatal("Execute succeeded for a program that outlasted its deadline") + } + if !strings.Contains(err.Error(), "deadline") { + t.Errorf("error = %v, want it to state the deadline", err) + } +} + +// A capability may ask for a shorter timeout than the policy allows, never +// for a longer one. +func TestCommandRefusesATimeoutAboveThePolicyLimit(t *testing.T) { + _, err := Command{Policy: builtins.Policy{DefaultTimeout: "1s"}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "exit0", map[string]any{"timeout": "10s"}), + }) + if err == nil { + t.Fatal("Execute accepted a timeout above the policy limit") + } + if !strings.Contains(err.Error(), "exceeds the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +// The JSON format returns the value the program produced, not a stringified +// copy of it. +func TestCommandReportsJSONOutput(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "json", map[string]any{"format": "json"}), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + output, ok := result["stdout"].(map[string]any) + if !ok { + t.Fatalf("stdout = %#v, want a parsed object", result["stdout"]) + } + if output["answer"] != float64(42) { + t.Errorf("answer = %v, want 42", output["answer"]) + } +} + +// Output that is not JSON is refused in the JSON format rather than handed +// over as a string the caller cannot bind. +func TestCommandRefusesOutputThatIsNotJSON(t *testing.T) { + _, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "text", map[string]any{"format": "json"}), + }) + if err == nil { + t.Fatal("Execute accepted output that is not JSON") + } + if !strings.Contains(err.Error(), "not JSON") { + t.Errorf("error = %v, want it to state that the output is not JSON", err) + } +} + +// The program receives the text it was given on its standard input. +func TestCommandWritesToTheProgramsStandardInput(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "exit0", map[string]any{"stdin": "input"}), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["success"] != true { + t.Errorf("success = %v, want true", result["success"]) + } +} + +// The result of a run says how long the program took. +func TestCommandReportsTheDuration(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "exit0"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if _, ok := result["duration_ms"].(float64); !ok { + t.Errorf("duration_ms = %#v, want a number", result["duration_ms"]) + } +} + +// The helper program is written as JSON so the output assertions above do not +// depend on the order of the environment variables. +func TestCommandHelperOutputIsStable(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParams(t, "environment"), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + var lines []string + for _, line := range strings.Split(result["stdout"].(string), "\n") { + if strings.Contains(line, "=") { + lines = append(lines, line) + } + } + if len(lines) == 0 { + t.Fatal("the program received no environment") + } + + encoded, err := json.Marshal(lines) + if err != nil { + t.Fatalf("encode the environment: %v", err) + } + if !strings.Contains(string(encoded), "PATH=") { + t.Errorf("the environment does not carry the program path: %s", encoded) + } +} diff --git a/nore/internal/builtins/command/process_unix.go b/nore/internal/builtins/command/process_unix.go new file mode 100644 index 0000000..9a31218 --- /dev/null +++ b/nore/internal/builtins/command/process_unix.go @@ -0,0 +1,37 @@ +//go:build !windows + +package command + +import ( + "errors" + "os/exec" + "syscall" +) + +// runInItsOwnGroup starts the program in its own process group. +// +// The group is what makes a deadline or a cancellation reach everything the +// program started: a program that leaves a child behind does not get to keep +// it once the run it belongs to is over. +func runInItsOwnGroup(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} +} + +// terminate stops the program and everything it started. +// +// The group is signalled rather than the program alone, so a child that +// outlived its parent is stopped as well. +func terminate(cmd *exec.Cmd) error { + if cmd.Process == nil { + return nil + } + + // A negative pid addresses the process group the program leads. + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL); err != nil { + if errors.Is(err, syscall.ESRCH) { + return nil + } + return cmd.Process.Kill() + } + return nil +} diff --git a/nore/internal/builtins/command/process_unix_test.go b/nore/internal/builtins/command/process_unix_test.go new file mode 100644 index 0000000..eada005 --- /dev/null +++ b/nore/internal/builtins/command/process_unix_test.go @@ -0,0 +1,50 @@ +//go:build !windows + +package command + +import ( + "context" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" +) + +// The program runs in its own process group, and the whole group is stopped +// when the run ends. A child that outlived its parent would otherwise keep +// running inside the runtime. +func TestCommandStopsTheWholeProcessGroup(t *testing.T) { + result, err := Command{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: helperParamsWith(t, "spawn", map[string]any{ + "timeout": "300ms", + "failOnNonZeroExit": false, + }), + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["timed_out"] != true { + t.Errorf("timed_out = %v, want true", result["timed_out"]) + } + + // The helper reports the pid of the child it started before it went on + // to wait, so the test can check that the child is gone. + pid, err := strconv.Atoi(strings.TrimSpace(result["stdout"].(string))) + if err != nil { + t.Fatalf("read the child's pid: %v", err) + } + + // The group is stopped with the program, so the child is gone with it. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if err := syscall.Kill(pid, 0); err != nil { + return + } + time.Sleep(20 * time.Millisecond) + } + + t.Errorf("the child process %d survived the run", pid) +} diff --git a/nore/internal/builtins/command/process_windows.go b/nore/internal/builtins/command/process_windows.go new file mode 100644 index 0000000..4020e3f --- /dev/null +++ b/nore/internal/builtins/command/process_windows.go @@ -0,0 +1,21 @@ +//go:build windows + +package command + +import ( + "os/exec" +) + +// runInItsOwnGroup starts the program in its own process group. +// +// Windows has no process group in the POSIX sense, so the program is started +// as it is and only the program itself can be stopped. +func runInItsOwnGroup(cmd *exec.Cmd) {} + +// terminate stops the program. +func terminate(cmd *exec.Cmd) error { + if cmd.Process == nil { + return nil + } + return cmd.Process.Kill() +} diff --git a/nore/internal/builtins/delay.go b/nore/internal/builtins/delay.go deleted file mode 100644 index 6df09fc..0000000 --- a/nore/internal/builtins/delay.go +++ /dev/null @@ -1,44 +0,0 @@ -package builtins - -import ( - "context" - "fmt" - "time" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// Delay blocks for a configured "duration" (e.g. "5s"), honouring context -// cancellation. -type Delay struct{} - -func (Delay) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - - // Look for duration in params or config - var durationStr string - if d, ok := execution.Params["duration"].(string); ok { - durationStr = d - } else if d, ok := execution.CapabilityConfigurations["duration"].(string); ok { - durationStr = d - } else { - return nil, fmt.Errorf("delay runtime requires a 'duration' string (e.g., '5s', '2m')") - } - - duration, err := time.ParseDuration(durationStr) - if err != nil { - return nil, fmt.Errorf("invalid duration format: %w", err) - } - - // Wait for the duration, or until the context is cancelled - select { - case <-time.After(duration): - // Delay completed successfully - return map[string]any{ - "delayed_for": duration.String(), - }, nil - - case <-ctx.Done(): - // Execution was cancelled (e.g., HTTP timeout) before delay finished - return nil, ctx.Err() - } -} diff --git a/nore/internal/builtins/filesystem/filesystem.go b/nore/internal/builtins/filesystem/filesystem.go new file mode 100644 index 0000000..09e41ac --- /dev/null +++ b/nore/internal/builtins/filesystem/filesystem.go @@ -0,0 +1,275 @@ +// Package filesystem implements the neuron:core:filesystem capability +// runtime. +package filesystem + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/builtins/input" + "github.com/neuron-runtime/neuron/nore/internal/builtins/paths" + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// Filesystem reads files and directories. +// +// It is the read-only view of the machine the assembly runs on. The reach is +// bounded by the policy: the project root, and the directories the project +// added to it. A path that resolves outside them is refused, and a symbolic +// link inside them does not grant reach beyond them. +type Filesystem struct { + // Policy bounds what the built-in may reach and how much it may return. + // A zero policy means the compiled-in ceilings. + Policy builtins.Policy +} + +// errListingComplete ends a listing that reached the entry limit. It is not a +// failure: the listing is complete as far as the policy allows, and the +// result says so. +var errListingComplete = errors.New("the listing reached its limit") + +// Execute performs the operation named by the capability's input. +func (f Filesystem) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { + // The policy is resolved here rather than trusted, because a runtime may + // also be constructed outside the registry. The resolution only narrows, + // so a policy that has already been resolved is unchanged. + policy, err := builtins.Effective(f.Policy) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: %w", err) + } + + values := input.New("neuron:core:filesystem", execution.CapabilityConfigurations, execution.Params) + logger := contracts.LoggerOrNop(execution.Logger) + + operation, err := values.RequiredString("operation") + if err != nil { + return nil, err + } + + target, err := values.RequiredString("path") + if err != nil { + return nil, err + } + + roots := paths.NewRoots(policy.ProjectRoot, policy.FilesystemRoots) + + resolved, err := roots.Resolve(roots.Base(), target) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: %w", err) + } + + logger.Debug(ctx, "neuron:core:filesystem: performing the operation", + contracts.F("operation", operation), + contracts.F("path", resolved), + ) + + switch strings.ToLower(strings.TrimSpace(operation)) { + case "read": + return f.read(values, resolved, policy) + case "list": + return f.list(values, resolved, policy) + case "stat": + return f.stat(resolved) + default: + return nil, fmt.Errorf("neuron:core:filesystem: unknown operation %q; supported operations are read, list, stat", operation) + } +} + +// read returns the content of a file. +func (f Filesystem) read(values input.Values, resolved string, policy builtins.Policy) (map[string]any, error) { + format, err := values.Format("format", input.FormatJSON) + if err != nil { + return nil, err + } + + info, err := os.Stat(resolved) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: read %q: %w", resolved, err) + } + if info.IsDir() { + return nil, fmt.Errorf("neuron:core:filesystem: %q is a directory; use the list operation", resolved) + } + + file, err := os.Open(resolved) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: read %q: %w", resolved, err) + } + defer file.Close() + + // The content is read one byte past the limit so a file that is exactly at + // the limit is read and a file that exceeds it is refused rather than + // silently shortened: a caller that receives a shortened file cannot tell + // it apart from a complete one. + content, err := io.ReadAll(io.LimitReader(file, policy.MaxFileReadBytes+1)) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: read %q: %w", resolved, err) + } + if int64(len(content)) > policy.MaxFileReadBytes { + return nil, fmt.Errorf("neuron:core:filesystem: the file %q is larger than the limit of %d bytes", + resolved, policy.MaxFileReadBytes) + } + + value, err := representContent(content, format) + if err != nil { + return nil, err + } + + return map[string]any{ + "path": resolved, + "content": value, + "size": float64(info.Size()), + }, nil +} + +// list returns the entries of a directory. +func (f Filesystem) list(values input.Values, resolved string, policy builtins.Policy) (map[string]any, error) { + recursive, err := values.Bool("recursive", false) + if err != nil { + return nil, err + } + + info, err := os.Stat(resolved) + if err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: list %q: %w", resolved, err) + } + if !info.IsDir() { + return nil, fmt.Errorf("neuron:core:filesystem: %q is not a directory; use the read operation", resolved) + } + + entries := make([]any, 0, 16) + truncated := false + + appendEntry := func(path string, entry fs.DirEntry) error { + if len(entries) >= policy.MaxDirectoryEntries { + truncated = true + return errListingComplete + } + info, err := entry.Info() + if err != nil { + return err + } + entries = append(entries, map[string]any{ + "name": entry.Name(), + "path": path, + "type": entryType(entry), + "size": float64(info.Size()), + }) + return nil + } + + err = filepath.WalkDir(resolved, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if path == resolved { + return nil + } + if !recursive && entry.IsDir() { + // The directory is listed, but its contents are not. + if err := appendEntry(path, entry); err != nil { + return err + } + return filepath.SkipDir + } + return appendEntry(path, entry) + }) + + if err != nil && !errors.Is(err, errListingComplete) { + return nil, fmt.Errorf("neuron:core:filesystem: list %q: %w", resolved, err) + } + + result := map[string]any{ + "path": resolved, + "entries": entries, + } + // The truncation is reported, so a caller can tell a shortened listing + // from a complete one. + if truncated { + result["entries_truncated"] = true + } + return result, nil +} + +// stat reports what a path is. +func (f Filesystem) stat(resolved string) (map[string]any, error) { + info, err := os.Stat(resolved) + if err != nil { + if os.IsNotExist(err) { + // A path that is not there is an answer, not a failure: the + // caller asked what is there. + return map[string]any{ + "path": resolved, + "exists": false, + }, nil + } + return nil, fmt.Errorf("neuron:core:filesystem: stat %q: %w", resolved, err) + } + + return map[string]any{ + "path": resolved, + "exists": true, + "type": fileType(info), + "size": float64(info.Size()), + "modified": info.ModTime().UTC().Format(time.RFC3339), + }, nil +} + +// representContent converts the content of a file into the representation the +// caller asked for. +// +// The rule is the same as for the other built-ins: a JSON file is returned as +// the value it represents, and a file that is not JSON is refused in the JSON +// format rather than handed over as a string the caller cannot bind. +func representContent(content []byte, format input.Format) (any, error) { + if len(strings.TrimSpace(string(content))) == 0 { + if format == input.FormatText { + return "", nil + } + return nil, nil + } + + if format == input.FormatText { + return string(content), nil + } + + var value any + if err := json.Unmarshal(content, &value); err != nil { + return nil, fmt.Errorf("neuron:core:filesystem: the file is not JSON; use format \"text\" to receive it as text") + } + return value, nil +} + +// entryType names the kind of a directory entry. +func entryType(entry fs.DirEntry) string { + switch { + case entry.IsDir(): + return "directory" + case entry.Type().IsRegular(): + return "file" + case entry.Type()&fs.ModeSymlink != 0: + return "link" + default: + return "other" + } +} + +// fileType names the kind of a file. +func fileType(info fs.FileInfo) string { + switch { + case info.IsDir(): + return "directory" + case info.Mode().IsRegular(): + return "file" + default: + return "other" + } +} diff --git a/nore/internal/builtins/filesystem/filesystem_test.go b/nore/internal/builtins/filesystem/filesystem_test.go new file mode 100644 index 0000000..29bf487 --- /dev/null +++ b/nore/internal/builtins/filesystem/filesystem_test.go @@ -0,0 +1,326 @@ +package filesystem + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// project builds a project root with a few files in it and returns the +// runtime confined to it. +func project(t *testing.T) (string, Filesystem) { + t.Helper() + + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "data.json"), []byte(`{"answer":42}`), 0o600); err != nil { + t.Fatalf("write the data file: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "notes.txt"), []byte("plain notes"), 0o600); err != nil { + t.Fatalf("write the notes file: %v", err) + } + if err := os.Mkdir(filepath.Join(root, "nested"), 0o755); err != nil { + t.Fatalf("create the nested directory: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "nested", "inner.json"), []byte(`{"inner":true}`), 0o600); err != nil { + t.Fatalf("write the inner file: %v", err) + } + + return root, Filesystem{Policy: builtins.Policy{ProjectRoot: root}} +} + +// execute performs an operation against the runtime. +func execute(t *testing.T, runtime Filesystem, params map[string]any) (map[string]any, error) { + t.Helper() + return runtime.Execute(context.Background(), contracts.ExecutionContext{Params: params}) +} + +func TestFilesystemReadsAFile(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{"operation": "read", "path": "data.json"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + content, ok := result["content"].(map[string]any) + if !ok { + t.Fatalf("content = %#v, want a parsed object", result["content"]) + } + if content["answer"] != float64(42) { + t.Errorf("answer = %v, want 42", content["answer"]) + } + if result["size"] != float64(13) { + t.Errorf("size = %v, want 13", result["size"]) + } +} + +// A file that is not JSON is refused in the JSON format rather than handed +// over as a string the caller cannot bind. +func TestFilesystemRefusesAFileThatIsNotJSON(t *testing.T) { + _, runtime := project(t) + + _, err := execute(t, runtime, map[string]any{"operation": "read", "path": "notes.txt"}) + if err == nil { + t.Fatal("Execute accepted a file that is not JSON") + } + if !strings.Contains(err.Error(), "not JSON") { + t.Errorf("error = %v, want it to state that the file is not JSON", err) + } +} + +// The text format is how a caller asks for a file that is not JSON. +func TestFilesystemReadsAFileAsText(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{ + "operation": "read", + "path": "notes.txt", + "format": "text", + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["content"] != "plain notes" { + t.Errorf("content = %#v, want the raw text", result["content"]) + } +} + +// A path outside the permitted roots is refused. +func TestFilesystemRefusesAPathOutsideThePermittedRoots(t *testing.T) { + _, runtime := project(t) + + _, err := execute(t, runtime, map[string]any{"operation": "read", "path": t.TempDir()}) + if err == nil { + t.Fatal("Execute read a path outside the permitted roots") + } + if !strings.Contains(err.Error(), "not reachable") { + t.Errorf("error = %v, want it to state that the path is not reachable", err) + } +} + +// A symbolic link inside the permitted roots does not grant reach beyond +// them. +func TestFilesystemRefusesALinkThatLeavesThePermittedRoots(t *testing.T) { + root, runtime := project(t) + + outside := t.TempDir() + secret := filepath.Join(outside, "secret.txt") + if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil { + t.Fatalf("write the secret: %v", err) + } + + link := filepath.Join(root, "link.txt") + if err := os.Symlink(secret, link); err != nil { + t.Skipf("the platform does not support symbolic links: %v", err) + } + + _, err := execute(t, runtime, map[string]any{"operation": "read", "path": "link.txt"}) + if err == nil { + t.Fatal("Execute read through a link that leaves the permitted roots") + } +} + +func TestFilesystemListsADirectory(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{"operation": "list", "path": "."}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + entries, ok := result["entries"].([]any) + if !ok { + t.Fatalf("entries = %#v, want a list", result["entries"]) + } + + names := map[string]string{} + for _, entry := range entries { + item, _ := entry.(map[string]any) + name, _ := item["name"].(string) + kind, _ := item["type"].(string) + names[name] = kind + } + + if names["data.json"] != "file" { + t.Errorf("data.json = %q, want a file", names["data.json"]) + } + if names["nested"] != "directory" { + t.Errorf("nested = %q, want a directory", names["nested"]) + } +} + +// A listing that does not descend stops at the directory it was given. +func TestFilesystemListsADirectoryWithoutDescending(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{"operation": "list", "path": "."}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + for _, entry := range result["entries"].([]any) { + item, _ := entry.(map[string]any) + if name, _ := item["name"].(string); name == "inner.json" { + t.Error("the listing descended into a subdirectory") + } + } +} + +// A recursive listing descends. +func TestFilesystemListsADirectoryRecursively(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{ + "operation": "list", + "path": ".", + "recursive": true, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + found := false + for _, entry := range result["entries"].([]any) { + item, _ := entry.(map[string]any) + if name, _ := item["name"].(string); name == "inner.json" { + found = true + } + } + if !found { + t.Error("the recursive listing did not reach the nested file") + } +} + +// A listing that reaches the limit reports the truncation rather than hiding +// it. +func TestFilesystemReportsATruncatedListing(t *testing.T) { + root := t.TempDir() + for _, name := range []string{"a.txt", "b.txt", "c.txt"} { + if err := os.WriteFile(filepath.Join(root, name), []byte("x"), 0o600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + runtime := Filesystem{Policy: builtins.Policy{ProjectRoot: root, MaxDirectoryEntries: 2}} + + result, err := execute(t, runtime, map[string]any{"operation": "list", "path": "."}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if len(result["entries"].([]any)) != 2 { + t.Errorf("entries = %d, want 2", len(result["entries"].([]any))) + } + if result["entries_truncated"] != true { + t.Errorf("entries_truncated = %v, want true", result["entries_truncated"]) + } +} + +// A file that is larger than the limit is refused rather than shortened. +func TestFilesystemRefusesAFileAboveTheLimit(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "big.json"), []byte(`{"a":"0123456789"}`), 0o600); err != nil { + t.Fatalf("write the file: %v", err) + } + + runtime := Filesystem{Policy: builtins.Policy{ProjectRoot: root, MaxFileReadBytes: 4}} + + _, err := execute(t, runtime, map[string]any{"operation": "read", "path": "big.json"}) + if err == nil { + t.Fatal("Execute accepted a file above the limit") + } + if !strings.Contains(err.Error(), "larger than the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +func TestFilesystemReportsWhatAPathIs(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{"operation": "stat", "path": "data.json"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["exists"] != true { + t.Errorf("exists = %v, want true", result["exists"]) + } + if result["type"] != "file" { + t.Errorf("type = %v, want file", result["type"]) + } +} + +// A path that is not there is an answer, not a failure. +func TestFilesystemReportsAPathThatIsNotThere(t *testing.T) { + _, runtime := project(t) + + result, err := execute(t, runtime, map[string]any{"operation": "stat", "path": "missing.json"}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["exists"] != false { + t.Errorf("exists = %v, want false", result["exists"]) + } +} + +func TestFilesystemRequiresAnOperationAndAPath(t *testing.T) { + _, runtime := project(t) + + if _, err := execute(t, runtime, map[string]any{"path": "data.json"}); err == nil { + t.Error("Execute succeeded without an operation") + } + if _, err := execute(t, runtime, map[string]any{"operation": "read"}); err == nil { + t.Error("Execute succeeded without a path") + } +} + +// An operation that does not exist is refused, and the error names the ones +// that do. +func TestFilesystemRefusesAnUnknownOperation(t *testing.T) { + _, runtime := project(t) + + _, err := execute(t, runtime, map[string]any{"operation": "delete", "path": "data.json"}) + if err == nil { + t.Fatal("Execute accepted an unknown operation") + } + if !strings.Contains(err.Error(), "supported operations are read, list, stat") { + t.Errorf("error = %v, want it to name the supported operations", err) + } +} + +// Reading a directory with the read operation is refused: the caller asked +// for the wrong thing. +func TestFilesystemRefusesToReadADirectory(t *testing.T) { + _, runtime := project(t) + + _, err := execute(t, runtime, map[string]any{"operation": "read", "path": "nested"}) + if err == nil { + t.Fatal("Execute read a directory") + } + if !strings.Contains(err.Error(), "list operation") { + t.Errorf("error = %v, want it to point at the list operation", err) + } +} + +// An extra root extends the reach. +func TestFilesystemReadsFromAnExtraRoot(t *testing.T) { + root := t.TempDir() + extra := t.TempDir() + if err := os.WriteFile(filepath.Join(extra, "data.json"), []byte(`{"ok":true}`), 0o600); err != nil { + t.Fatalf("write the file: %v", err) + } + + runtime := Filesystem{Policy: builtins.Policy{ProjectRoot: root, FilesystemRoots: []string{extra}}} + + result, err := execute(t, runtime, map[string]any{"operation": "read", "path": filepath.Join(extra, "data.json")}) + if err != nil { + t.Fatalf("Execute: %v", err) + } + content, _ := result["content"].(map[string]any) + if content["ok"] != true { + t.Errorf("content = %#v, want the file from the extra root", result["content"]) + } +} diff --git a/nore/internal/builtins/http.go b/nore/internal/builtins/http.go deleted file mode 100644 index 49589a2..0000000 --- a/nore/internal/builtins/http.go +++ /dev/null @@ -1,97 +0,0 @@ -package builtins - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "maps" - "net/http" - "strings" - "time" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// HTTP performs a single HTTP request from the merged params/config and -// returns status/body. -type HTTP struct{} - -func (HTTP) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - - // 1. Merge Config and Params (Params override Config) - params := make(map[string]any) - maps.Copy(params, execution.CapabilityConfigurations) - maps.Copy(params, execution.Params) - - // 2. Extract parameters safely - url, ok := params["url"].(string) - if !ok || url == "" { - return nil, fmt.Errorf("http runtime requires a valid 'url' string") - } - - method, _ := params["method"].(string) - if method == "" { - method = http.MethodGet - } - - // 3. Handle Body (if provided) - var bodyReader io.Reader - if body, exists := params["body"]; exists { - bodyBytes, err := json.Marshal(body) - if err != nil { - return nil, fmt.Errorf("failed to encode body: %w", err) - } - bodyReader = bytes.NewReader(bodyBytes) - } - - // 4. Create the Request with Context - req, err := http.NewRequestWithContext(ctx, strings.ToUpper(method), url, bodyReader) - if err != nil { - return nil, err - } - - // 5. Handle Headers - if headers, ok := params["headers"].(map[string]any); ok { - for k, v := range headers { - if strVal, isStr := v.(string); isStr { - req.Header.Set(k, strVal) - } - } - } - if bodyReader != nil && req.Header.Get("Content-Type") == "" { - req.Header.Set("Content-Type", "application/json") - } - - // 6. Execute Request - client := &http.Client{Timeout: 30 * time.Second} - - // Simulate realistic network delay for streaming demo - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(500 * time.Millisecond): - } - - resp, err := client.Do(req) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - // 7. Parse Response - respBody, _ := io.ReadAll(resp.Body) - var jsonResponse any - - // Try to parse as JSON, fallback to raw string if it's not JSON - if err := json.Unmarshal(respBody, &jsonResponse); err != nil { - jsonResponse = string(respBody) - } - - // 8. Return standardized result - return map[string]any{ - "status_code": resp.StatusCode, - "body": jsonResponse, - }, nil -} diff --git a/nore/internal/builtins/http/http.go b/nore/internal/builtins/http/http.go new file mode 100644 index 0000000..6e578c6 --- /dev/null +++ b/nore/internal/builtins/http/http.go @@ -0,0 +1,559 @@ +// Package http implements the neuron:core:http capability runtime. +package http + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + nethttp "net/http" + neturl "net/url" + "strings" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/builtins/input" + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// HTTP performs a single HTTP request and returns the status, the response +// headers and the body. +// +// It is the general-purpose integration boundary: an author who needs to talk +// to an HTTP API should not have to write a capability runtime for it. +// +// The policy bounds the request: which hosts it may reach, how long it may +// take, and how large a body it may send and read. +type HTTP struct { + // Policy bounds what a request may do. A zero policy means the + // compiled-in ceilings. + Policy builtins.Policy +} + +// maxRedirects bounds how many redirects a single request may follow when +// redirects are enabled. +const maxRedirects = 5 + +// maxRetryDelay bounds how long the runtime waits before a retry. +const maxRetryDelay = 2 * time.Second + +// defaultRetryBackoff is how long the runtime waits before the first retry. +const defaultRetryBackoff = 100 * time.Millisecond + +// Execute performs the request named by the capability's input. +func (h HTTP) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { + // The policy is resolved here rather than trusted, because a runtime may + // also be constructed outside the registry. The resolution only narrows, + // so a policy that has already been resolved is unchanged. + policy, err := builtins.Effective(h.Policy) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: %w", err) + } + + values := input.New("neuron:core:http", execution.CapabilityConfigurations, execution.Params) + logger := contracts.LoggerOrNop(execution.Logger) + + target, err := values.RequiredString("url") + if err != nil { + return nil, err + } + + query, _, err := values.StringMap("query") + if err != nil { + return nil, err + } + if len(query) > 0 { + target, err = appendQuery(target, query) + if err != nil { + return nil, err + } + } + + if err := validateURL(target); err != nil { + return nil, err + } + + format, err := values.Format("format", input.FormatJSON) + if err != nil { + return nil, err + } + + method, _, err := values.String("method") + if err != nil { + return nil, err + } + if strings.TrimSpace(method) == "" { + method = nethttp.MethodGet + } + method = strings.ToUpper(strings.TrimSpace(method)) + + timeout, err := requestTimeout(values, policy) + if err != nil { + return nil, err + } + + followRedirects, err := values.Bool("followRedirects", false) + if err != nil { + return nil, err + } + + retry, err := readRetry(values) + if err != nil { + return nil, err + } + + contentType, _, err := values.String("contentType") + if err != nil { + return nil, err + } + if strings.TrimSpace(contentType) == "" { + contentType = "application/json" + } + + var bodyReader io.Reader + if body, exists := values.Raw("body"); exists && body != nil { + bodyBytes, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: encode the request body: %w", err) + } + if int64(len(bodyBytes)) > policy.MaxHTTPRequestBodyBytes { + return nil, fmt.Errorf("neuron:core:http: the request body is %d bytes, which exceeds the limit of %d bytes", + len(bodyBytes), policy.MaxHTTPRequestBodyBytes) + } + bodyReader = bytes.NewReader(bodyBytes) + } + + req, err := nethttp.NewRequestWithContext(ctx, method, target, bodyReader) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: %w", err) + } + + headers, _, err := values.StringMap("headers") + if err != nil { + return nil, err + } + for name, value := range headers { + req.Header.Set(name, value) + } + if bodyReader != nil && req.Header.Get("Content-Type") == "" { + req.Header.Set("Content-Type", contentType) + } + + // The credentials are applied to the request rather than to the URL, so + // they never end up in a log line or in an error that names the URL. + auth, hasAuth, err := values.Nested("auth") + if err != nil { + return nil, err + } + if hasAuth { + if err := applyAuthentication(req, auth); err != nil { + return nil, err + } + } + + // The deadline bounds the whole operation, retries included, rather than + // each attempt: a request that is retried must not be able to take + // longer than the timeout it asked for. + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + client := newClient(policy, timeout, followRedirects) + + logger.Debug(ctx, "neuron:core:http: performing the request", + contracts.F("method", method), + contracts.F("url", target), + ) + + started := time.Now() + resp, err := perform(ctx, client, req, retry) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: %w", err) + } + defer resp.Body.Close() + elapsed := time.Since(started) + + // The response is read one byte past the limit so a body that is exactly + // at the limit is accepted and a body that exceeds it is refused rather + // than silently truncated: a caller that receives a truncated body cannot + // tell it apart from a complete one. + respBody, err := io.ReadAll(io.LimitReader(resp.Body, policy.MaxHTTPResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: read the response body: %w", err) + } + if int64(len(respBody)) > policy.MaxHTTPResponseBytes { + return nil, fmt.Errorf("neuron:core:http: the response body exceeds the limit of %d bytes", + policy.MaxHTTPResponseBytes) + } + + body, err := representBody(respBody, format) + if err != nil { + return nil, err + } + + logger.Debug(ctx, "neuron:core:http: the request completed", + contracts.F("status_code", resp.StatusCode), + ) + + return map[string]any{ + "status_code": resp.StatusCode, + "headers": responseHeaders(resp.Header), + "body": body, + "content_type": resp.Header.Get("Content-Type"), + "duration_ms": float64(elapsed.Milliseconds()), + }, nil +} + +// newClient builds the HTTP client the request is performed with. +// +// The client refuses to reach a host the policy does not permit, and it +// refuses to follow a redirect unless the capability asked for it: a redirect +// is a second request to a host the caller never named, and it is checked +// against the same rules as the first one. +func newClient(policy builtins.Policy, timeout time.Duration, followRedirects bool) *nethttp.Client { + guard := newHostGuard(policy.AllowedHTTPHosts, policy.AllowPrivateNetworks) + + transport := &nethttp.Transport{ + DialContext: guard.dialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 16, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + } + + return &nethttp.Client{ + Transport: transport, + Timeout: timeout, + CheckRedirect: func(req *nethttp.Request, via []*nethttp.Request) error { + if !followRedirects { + // The redirect is not followed: the 3xx response is the + // answer the caller receives, with the target in its + // Location header. + return nethttp.ErrUseLastResponse + } + if len(via) >= maxRedirects { + return fmt.Errorf("neuron:core:http: the request was redirected more than %d times", maxRedirects) + } + return guard.refuses(req.URL.Hostname()) + }, + } +} + +// perform issues the request, retrying it when the capability asked for +// retries. +// +// A retry is only made for a failure that a second attempt can plausibly +// repair — a transport error, or a status that says the server is busy — and +// only for the methods the capability named. +func perform(ctx context.Context, client *nethttp.Client, req *nethttp.Request, retry retryPolicy) (*nethttp.Response, error) { + var lastErr error + + for attempt := 0; attempt < retry.attempts; attempt++ { + if attempt > 0 { + if err := waitForRetry(ctx, retry.backoff, attempt); err != nil { + return nil, err + } + // The body is read again for the retry: a consumed body cannot + // be sent a second time. + if req.GetBody != nil { + body, err := req.GetBody() + if err != nil { + return nil, fmt.Errorf("neuron:core:http: prepare the request body for a retry: %w", err) + } + req.Body = body + } + } + + resp, err := client.Do(req) + if err != nil { + lastErr = err + if attempt < retry.attempts-1 && retry.allows(req.Method) { + continue + } + break + } + + if attempt < retry.attempts-1 && retry.allows(req.Method) && retryableStatus(resp.StatusCode) { + resp.Body.Close() + lastErr = fmt.Errorf("the server answered %d", resp.StatusCode) + continue + } + return resp, nil + } + + if lastErr == nil { + lastErr = fmt.Errorf("the request was not performed") + } + return nil, lastErr +} + +// retryPolicy describes how a failed request is retried. +type retryPolicy struct { + // attempts is how many attempts are made in total, including the first. + attempts int + + // backoff is how long the runtime waits before the second attempt. The + // wait grows with the attempt. + backoff time.Duration + + // methods are the request methods that may be retried. + methods map[string]bool +} + +// allows reports whether the method may be retried. +func (r retryPolicy) allows(method string) bool { + return r.methods[strings.ToUpper(strings.TrimSpace(method))] +} + +// readRetry reads the retry policy from the capability's input. +// +// Retrying is off unless the capability asks for it, and only the methods it +// names may be retried: a retried POST may repeat an effect the server +// already performed. +func readRetry(values input.Values) (retryPolicy, error) { + policy := retryPolicy{attempts: 1, backoff: defaultRetryBackoff} + + nested, present, err := values.Nested("retry") + if err != nil || !present { + return policy, err + } + + attempts, err := nested.Int("attempts", 1) + if err != nil { + return policy, err + } + if attempts < 1 { + return policy, fmt.Errorf("neuron:core:http: the retry attempts must be at least 1, got %d", attempts) + } + policy.attempts = attempts + + backoff, err := nested.Duration("backoff", policy.backoff) + if err != nil { + return policy, err + } + if backoff < 0 { + return policy, fmt.Errorf("neuron:core:http: the retry backoff must not be negative") + } + policy.backoff = backoff + + methods, _, err := nested.StringSlice("methods") + if err != nil { + return policy, err + } + if len(methods) == 0 { + policy.methods = defaultRetryMethods() + return policy, nil + } + + policy.methods = make(map[string]bool, len(methods)) + for _, method := range methods { + trimmed := strings.ToUpper(strings.TrimSpace(method)) + if trimmed == "" { + continue + } + policy.methods[trimmed] = true + } + if len(policy.methods) == 0 { + policy.methods = defaultRetryMethods() + } + return policy, nil +} + +// defaultRetryMethods are the methods that are retried when the capability +// did not name any. +// +// They are the methods that are idempotent by definition: a retried request +// may repeat the effect of the first one, and only for these methods that +// effect is the same. +func defaultRetryMethods() map[string]bool { + return map[string]bool{ + nethttp.MethodGet: true, + nethttp.MethodHead: true, + nethttp.MethodPut: true, + nethttp.MethodDelete: true, + nethttp.MethodOptions: true, + } +} + +// retryableStatus reports whether a status says the server is busy rather +// than that it answered. +func retryableStatus(status int) bool { + switch status { + case nethttp.StatusTooManyRequests, + nethttp.StatusBadGateway, + nethttp.StatusServiceUnavailable, + nethttp.StatusGatewayTimeout: + return true + default: + return false + } +} + +// waitForRetry waits before the next attempt, or returns the reason the wait +// was cut short. +func waitForRetry(ctx context.Context, backoff time.Duration, attempt int) error { + delay := backoff * time.Duration(attempt) + if delay > maxRetryDelay { + delay = maxRetryDelay + } + if delay <= 0 { + return nil + } + + timer := time.NewTimer(delay) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// requestTimeout resolves how long the request may take. +// +// A capability may ask for a shorter timeout than the policy allows, but not +// for a longer one: the policy is the ceiling, and a capability is part of the +// assembly's input. +func requestTimeout(values input.Values, policy builtins.Policy) (time.Duration, error) { + limit, err := policy.Timeout() + if err != nil { + return 0, fmt.Errorf("neuron:core:http: %w", err) + } + + requested, err := values.Duration("timeout", limit) + if err != nil { + return 0, err + } + if requested <= 0 { + return 0, fmt.Errorf("neuron:core:http: parameter \"timeout\" must be positive") + } + if requested > limit { + return 0, fmt.Errorf("neuron:core:http: the requested timeout %s exceeds the limit of %s", requested, limit) + } + return requested, nil +} + +// validateURL rejects a target the request could not be made against. +func validateURL(target string) error { + parsed, err := neturl.Parse(target) + if err != nil { + return fmt.Errorf("neuron:core:http: the url %q is not a valid URL: %w", target, err) + } + switch parsed.Scheme { + case "http", "https": + default: + return fmt.Errorf("neuron:core:http: the url scheme %q is not supported; use http or https", parsed.Scheme) + } + if parsed.Host == "" { + return fmt.Errorf("neuron:core:http: the url %q has no host", target) + } + return nil +} + +// appendQuery adds the query parameters to the URL. +func appendQuery(target string, query map[string]string) (string, error) { + parsed, err := neturl.Parse(target) + if err != nil { + return "", fmt.Errorf("neuron:core:http: the url %q is not a valid URL: %w", target, err) + } + + values := parsed.Query() + for name, value := range query { + values.Set(name, value) + } + parsed.RawQuery = values.Encode() + + return parsed.String(), nil +} + +// applyAuthentication adds the credentials to the request. +func applyAuthentication(req *nethttp.Request, auth input.Values) error { + kind, err := auth.RequiredString("type") + if err != nil { + return err + } + + switch strings.ToLower(strings.TrimSpace(kind)) { + case "bearer": + token, err := auth.RequiredString("token") + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+token) + case "basic": + username, err := auth.RequiredString("username") + if err != nil { + return err + } + password, _, err := auth.String("password") + if err != nil { + return err + } + credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + req.Header.Set("Authorization", "Basic "+credentials) + case "apikey": + name, err := auth.RequiredString("name") + if err != nil { + return err + } + value, err := auth.RequiredString("value") + if err != nil { + return err + } + req.Header.Set(name, value) + default: + return fmt.Errorf("neuron:core:http: unknown authentication type %q; supported types are bearer, basic, apiKey", kind) + } + + return nil +} + +// representBody converts the response body into the representation the caller +// asked for. +// +// A JSON body is returned as the value it represents, never as a stringified +// copy of it, because a stringified object cannot be bound into a downstream +// capability. A body that is not JSON is refused in the JSON format, and +// returned as text in the text format. +func representBody(body []byte, format input.Format) (any, error) { + if len(bytes.TrimSpace(body)) == 0 { + if format == input.FormatText { + return "", nil + } + return nil, nil + } + + if format == input.FormatText { + return string(body), nil + } + + var value any + if err := json.Unmarshal(body, &value); err != nil { + return nil, fmt.Errorf("neuron:core:http: the response body is not JSON (it starts with %q); use format \"text\" to receive it as text", + preview(body)) + } + return value, nil +} + +// responseHeaders returns the response headers as a plain map. +func responseHeaders(header nethttp.Header) map[string]string { + headers := make(map[string]string, len(header)) + for name, values := range header { + headers[name] = strings.Join(values, ", ") + } + return headers +} + +// preview returns the beginning of a body, for an error message. +func preview(body []byte) string { + const limit = 40 + text := strings.TrimSpace(string(body)) + if len(text) > limit { + return text[:limit] + "..." + } + return text +} diff --git a/nore/internal/builtins/http/http_test.go b/nore/internal/builtins/http/http_test.go new file mode 100644 index 0000000..9903950 --- /dev/null +++ b/nore/internal/builtins/http/http_test.go @@ -0,0 +1,498 @@ +package http + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// localHTTP returns a runtime that may reach the loopback interface, which +// is where the test servers live. The runtime refuses such an address by +// default, so a test that wants to talk to a local server has to say so. +func localHTTP() HTTP { + return HTTP{Policy: builtins.Policy{AllowedHTTPHosts: []string{"127.0.0.1"}}} +} + +func TestHTTPReturnsTheStatusAndTheParsedBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + w.Write([]byte(`{"id":"abc"}`)) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if result["status_code"] != 201 { + t.Errorf("status_code = %v, want 201", result["status_code"]) + } + body, ok := result["body"].(map[string]any) + if !ok { + t.Fatalf("body = %#v, want a parsed object", result["body"]) + } + if body["id"] != "abc" { + t.Errorf("body id = %v, want %q", body["id"], "abc") + } +} + +// The response headers are part of the answer: a caller that cannot see them +// cannot read a rate-limit header or a content type. +func TestHTTPReturnsTheResponseHeaders(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Trace", "abc") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + headers, ok := result["headers"].(map[string]string) + if !ok { + t.Fatalf("headers = %#v, want a map", result["headers"]) + } + if headers["X-Trace"] != "abc" { + t.Errorf("X-Trace = %q, want %q", headers["X-Trace"], "abc") + } +} + +// A body that is not JSON is refused in the JSON format: a caller that asked +// for a structured answer cannot be handed a string and told it is a value. +func TestHTTPRefusesANonJSONBodyInTheJSONFormat(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("plain text")) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err == nil { + t.Fatal("Execute accepted a body that is not JSON") + } + if !strings.Contains(err.Error(), "not JSON") { + t.Errorf("error = %v, want it to state that the body is not JSON", err) + } +} + +// The text format is how a caller asks for a body that is not JSON. +func TestHTTPKeepsANonJSONBodyAsText(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("plain text")) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL, "format": "text"}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["body"] != "plain text" { + t.Errorf("body = %#v, want the raw text", result["body"]) + } +} + +// An empty body is an answer, not a malformed one. +func TestHTTPAcceptsAnEmptyBody(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["body"] != nil { + t.Errorf("body = %#v, want no body", result["body"]) + } +} + +// The status is part of the result, not a failure: a 404 is an answer the +// caller asked for. +func TestHTTPReturnsANonSuccessStatusWithoutFailing(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["status_code"] != 404 { + t.Errorf("status_code = %v, want 404", result["status_code"]) + } +} + +func TestHTTPSendsTheMethodHeadersAndBody(t *testing.T) { + var received struct { + method string + header string + payload string + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := make([]byte, r.ContentLength) + r.Body.Read(body) + received.method = r.Method + received.header = r.Header.Get("X-Trace") + received.payload = string(body) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "method": "post", + "headers": map[string]any{"X-Trace": "abc"}, + "body": map[string]any{"name": "neuron"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if received.method != "POST" { + t.Errorf("method = %q, want POST", received.method) + } + if received.header != "abc" { + t.Errorf("X-Trace = %q, want %q", received.header, "abc") + } + if received.payload != `{"name":"neuron"}` { + t.Errorf("body = %q, want the encoded JSON", received.payload) + } +} + +// A url names a GET. A caller that wants another method asks for it. +func TestHTTPDefaultsToGET(t *testing.T) { + var method string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method = r.Method + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if method != http.MethodGet { + t.Errorf("method = %q, want GET", method) + } +} + +func TestHTTPRequiresAURL(t *testing.T) { + _, err := HTTP{}.Execute(context.Background(), contracts.ExecutionContext{}) + if err == nil { + t.Fatal("Execute succeeded without a url") + } +} + +func TestHTTPRejectsAnUnsupportedScheme(t *testing.T) { + _, err := HTTP{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": "ftp://example.com/file"}, + }) + if err == nil { + t.Fatal("Execute accepted an unsupported url scheme") + } + if !strings.Contains(err.Error(), "not supported") { + t.Errorf("error = %v, want it to state that the scheme is not supported", err) + } +} + +func TestHTTPReportsAnUnreachableServer(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + url := server.URL + server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": url}, + }) + if err == nil { + t.Fatal("Execute succeeded for a server that is not listening") + } +} + +// A response body that exceeds the limit is refused rather than truncated: a +// caller that receives a shortened body cannot tell it apart from a complete +// one. +func TestHTTPRefusesAResponseBodyAboveTheLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("0123456789")) + })) + defer server.Close() + + _, err := HTTP{Policy: builtins.Policy{ + AllowedHTTPHosts: []string{"127.0.0.1"}, + MaxHTTPResponseBytes: 4, + }}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err == nil { + t.Fatal("Execute accepted a response body above the limit") + } + if !strings.Contains(err.Error(), "exceeds the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +func TestHTTPAcceptsAResponseBodyAtTheLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("0123456789")) + })) + defer server.Close() + + result, err := HTTP{Policy: builtins.Policy{ + AllowedHTTPHosts: []string{"127.0.0.1"}, + MaxHTTPResponseBytes: 10, + }}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL, "format": "text"}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["body"] != "0123456789" { + t.Errorf("body = %#v, want the whole body", result["body"]) + } +} + +func TestHTTPRefusesARequestBodyAboveTheLimit(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer server.Close() + + _, err := HTTP{Policy: builtins.Policy{ + AllowedHTTPHosts: []string{"127.0.0.1"}, + MaxHTTPRequestBodyBytes: 4, + }}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "body": map[string]any{"name": "neuron"}, + }, + }) + if err == nil { + t.Fatal("Execute accepted a request body above the limit") + } + if !strings.Contains(err.Error(), "exceeds the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +// The query parameters are appended to the URL, so a caller can shape the +// request without building the URL itself. +func TestHTTPAppendsTheQueryParameters(t *testing.T) { + var query string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + query = r.URL.RawQuery + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "query": map[string]any{"page": "2"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if query != "page=2" { + t.Errorf("query = %q, want %q", query, "page=2") + } +} + +// The bearer credentials are sent as an authorization header, and never as +// part of the URL. +func TestHTTPSendsBearerCredentials(t *testing.T) { + var authorization string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authorization = r.Header.Get("Authorization") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "auth": map[string]any{"type": "bearer", "token": "abc"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if authorization != "Bearer abc" { + t.Errorf("Authorization = %q, want %q", authorization, "Bearer abc") + } +} + +// The basic credentials are sent as an authorization header. +func TestHTTPSendsBasicCredentials(t *testing.T) { + var authorization string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + authorization = r.Header.Get("Authorization") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "auth": map[string]any{"type": "basic", "username": "neuron", "password": "secret"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if authorization != "Basic bmV1cm9uOnNlY3JldA==" { + t.Errorf("Authorization = %q, want the basic credentials", authorization) + } +} + +// An API key is sent in the header it names. +func TestHTTPSendsAnAPIKey(t *testing.T) { + var key string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key = r.Header.Get("X-Api-Key") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "auth": map[string]any{"type": "apiKey", "name": "X-Api-Key", "value": "abc"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if key != "abc" { + t.Errorf("X-Api-Key = %q, want %q", key, "abc") + } +} + +// An authentication type that does not exist is refused rather than ignored. +func TestHTTPRefusesAnUnknownAuthenticationType(t *testing.T) { + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": "https://example.com", + "auth": map[string]any{"type": "magic"}, + }, + }) + if err == nil { + t.Fatal("Execute accepted an unknown authentication type") + } + if !strings.Contains(err.Error(), "unknown authentication type") { + t.Errorf("error = %v, want it to state that the type is unknown", err) + } +} + +// The request carries the content type the caller asked for. +func TestHTTPSendsTheContentType(t *testing.T) { + var contentType string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType = r.Header.Get("Content-Type") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "method": "POST", + "contentType": "application/xml", + "body": map[string]any{"name": "neuron"}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if contentType != "application/xml" { + t.Errorf("Content-Type = %q, want %q", contentType, "application/xml") + } +} + +// The answer carries the content type of the response, so a caller can tell +// what it received. +func TestHTTPReportsTheResponseContentType(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(`{}`)) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["content_type"] != "application/json" { + t.Errorf("content_type = %v, want %q", result["content_type"], "application/json") + } +} + +// The answer says how long the request took. +func TestHTTPReportsTheDuration(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if _, ok := result["duration_ms"].(float64); !ok { + t.Errorf("duration_ms = %#v, want a number", result["duration_ms"]) + } +} + +// The retry policy names the methods that may be retried. +func TestHTTPRetriesOnlyTheNamedMethods(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "url": server.URL, + "method": "POST", + "retry": map[string]any{"attempts": 3, "backoff": "10ms", "methods": []any{"POST"}}, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if got := attempts.Load(); got != 3 { + t.Errorf("the server was asked %d times, want 3", got) + } +} diff --git a/nore/internal/builtins/http/network.go b/nore/internal/builtins/http/network.go new file mode 100644 index 0000000..ea0424a --- /dev/null +++ b/nore/internal/builtins/http/network.go @@ -0,0 +1,188 @@ +package http + +import ( + "context" + "fmt" + "net" + "strings" + "time" +) + +// hostGuard decides which hosts a request may reach. +// +// The guard exists because a capability's URL may be derived from untrusted +// input. A capability that can reach any address on the machine, or on the +// network the runtime runs in, is a pivot: it can read the cloud metadata +// service, reach an internal API that trusts the network, or reach the +// runtime's own control surface. +// +// There are exactly two rules. +// +// When the policy lists no hosts, any host may be reached, but only through +// an address that is publicly routable. A loopback, private, link-local, +// multicast, or otherwise unroutable address is refused. +// +// When the policy lists hosts, only those hosts may be reached. A listed host +// is permitted even when it resolves to a private address, because listing it +// is the explicit and auditable decision to let a capability reach an +// internal service. +type hostGuard struct { + // allowedHosts are the hosts the policy permits. + allowedHosts []string + + // allowPrivate lifts the address restriction. It is set for the paths + // where the request is authored by the operator rather than derived from + // an assembly's input. + allowPrivate bool +} + +// newHostGuard builds the guard for a policy. +func newHostGuard(allowedHosts []string, allowPrivate bool) hostGuard { + hosts := make([]string, 0, len(allowedHosts)) + for _, host := range allowedHosts { + trimmed := strings.TrimSpace(host) + if trimmed == "" { + continue + } + hosts = append(hosts, trimmed) + } + return hostGuard{allowedHosts: hosts, allowPrivate: allowPrivate} +} + +// restricts reports whether the guard names a fixed set of hosts. +func (g hostGuard) restricts() bool { + return len(g.allowedHosts) > 0 +} + +// lists reports whether the host is one of the permitted hosts. +// +// A host matches an entry exactly, or through a leading "*." wildcard, which +// covers the named domain and its subdomains. +func (g hostGuard) lists(host string) bool { + host = normalizeHost(host) + for _, allowed := range g.allowedHosts { + allowed = normalizeHost(allowed) + if allowed == "" { + continue + } + if strings.HasPrefix(allowed, "*.") { + if host == allowed[2:] || strings.HasSuffix(host, allowed[1:]) { + return true + } + continue + } + if host == allowed { + return true + } + } + return false +} + +// refuses reports whether the host may not be reached. +// +// It is the name-level check: the address-level check happens when the +// connection is made, so a host that resolves to a private address is +// refused there. +func (g hostGuard) refuses(host string) error { + if !g.restricts() { + return nil + } + if g.lists(host) { + return nil + } + return fmt.Errorf("neuron:core:http: the host %q is not permitted; the permitted hosts are %s", + host, strings.Join(g.allowedHosts, ", ")) +} + +// dialContext connects to addr, refusing an address the policy does not +// permit. +// +// The address is resolved here rather than trusted from the URL: a host name +// is a claim about an address, and the claim is checked against the addresses +// it actually resolves to. +func (g hostGuard) dialContext(ctx context.Context, network, addr string) (net.Conn, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: %w", err) + } + + if g.restricts() { + if err := g.refuses(host); err != nil { + return nil, err + } + return dialer.DialContext(ctx, network, addr) + } + + if g.allowPrivate { + return dialer.DialContext(ctx, network, addr) + } + + addresses, err := net.DefaultResolver.LookupIPAddr(ctx, host) + if err != nil { + return nil, fmt.Errorf("neuron:core:http: resolve %q: %w", host, err) + } + + var refusal error + for _, address := range addresses { + if !isPubliclyRoutable(address.IP) { + refusal = fmt.Errorf( + "neuron:core:http: the host %q resolves to %s, which is not a publicly routable address; list the host in builtins.http.allowedHosts to permit it", + host, address.IP) + continue + } + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(address.IP.String(), port)) + if err == nil { + return conn, nil + } + refusal = err + } + + if refusal == nil { + refusal = fmt.Errorf("neuron:core:http: the host %q has no address to connect to", host) + } + return nil, refusal +} + +// dialer is the base dialer. It carries no timeout of its own: the request's +// deadline is carried by the context, so the timeout applies to the whole +// operation rather than to each attempt. +var dialer = &net.Dialer{KeepAlive: 30 * time.Second} + +// isPubliclyRoutable reports whether an address may be reached when the +// policy names no hosts. +// +// The refusal covers the loopback interface, the private ranges, the +// link-local ranges (which carry the cloud metadata address), the shared +// address space used by carrier-grade NAT, the multicast ranges, and the +// reserved range. +func isPubliclyRoutable(ip net.IP) bool { + if ip == nil { + return false + } + if ip.IsLoopback() || + ip.IsPrivate() || + ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || + ip.IsInterfaceLocalMulticast() || + ip.IsMulticast() || + ip.IsUnspecified() { + return false + } + if v4 := ip.To4(); v4 != nil { + // The shared address space (100.64.0.0/10) is not reachable from the + // public internet. + if v4[0] == 100 && v4[1] >= 64 && v4[1] <= 127 { + return false + } + // The reserved range (240.0.0.0/4) is not routable either. + if v4[0] >= 240 { + return false + } + } + return true +} + +// normalizeHost prepares a host name for comparison. +func normalizeHost(host string) string { + return strings.ToLower(strings.TrimSuffix(strings.TrimSpace(host), ".")) +} diff --git a/nore/internal/builtins/http/network_test.go b/nore/internal/builtins/http/network_test.go new file mode 100644 index 0000000..7f1fc56 --- /dev/null +++ b/nore/internal/builtins/http/network_test.go @@ -0,0 +1,279 @@ +package http + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" +) + +// A capability whose url comes from an assembly's input must not be able to +// reach the machine it runs on. +func TestHTTPRefusesALoopbackHostByDefault(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := HTTP{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err == nil { + t.Fatal("Execute reached a loopback address without an allowlist") + } + if !strings.Contains(err.Error(), "not a publicly routable address") { + t.Errorf("error = %v, want it to state that the address is not routable", err) + } +} + +// The operator may author a request that reaches an internal service. That +// request is not derived from an assembly's input, so the restriction is +// lifted for it. +func TestHTTPPermitsAPrivateAddressWhenThePolicySaysSo(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := HTTP{Policy: builtins.Policy{AllowPrivateNetworks: true}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } +} + +// An allowlist names the hosts a capability may reach. A host that is not +// named is refused even when it is a public one. +func TestHTTPRefusesAHostThatIsNotAllowed(t *testing.T) { + _, err := (HTTP{Policy: builtins.Policy{AllowedHTTPHosts: []string{"api.example.com"}}}).Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": "https://other.example.com/data"}, + }) + if err == nil { + t.Fatal("Execute reached a host that is not on the allowlist") + } + if !strings.Contains(err.Error(), "not permitted") { + t.Errorf("error = %v, want it to state that the host is not permitted", err) + } +} + +// A wildcard covers the named domain and its subdomains, and nothing else. +func TestHTTPAllowsAWildcardHost(t *testing.T) { + guard := newHostGuard([]string{"*.example.com"}, false) + + for _, host := range []string{"example.com", "api.example.com", "deep.api.example.com"} { + if !guard.lists(host) { + t.Errorf("lists(%q) = false, want true", host) + } + } + for _, host := range []string{"notexample.com", "example.org", "example.com.evil.test"} { + if guard.lists(host) { + t.Errorf("lists(%q) = true, want false", host) + } + } +} + +// A host name is compared without regard to case, and a trailing root dot +// does not make it a different host. +func TestHTTPComparesHostsWithoutRegardToCase(t *testing.T) { + guard := newHostGuard([]string{"API.Example.COM"}, false) + + if !guard.lists("api.example.com") { + t.Error("lists refused a host that differs only in case") + } + if !guard.lists("api.example.com.") { + t.Error("lists refused a host with a trailing root dot") + } +} + +func TestHTTPIsPubliclyRoutable(t *testing.T) { + cases := []struct { + address string + public bool + }{ + {"93.184.216.34", true}, + {"8.8.8.8", true}, + {"127.0.0.1", false}, + {"127.1.2.3", false}, + {"::1", false}, + {"10.0.0.1", false}, + {"172.16.0.1", false}, + {"192.168.1.1", false}, + {"169.254.169.254", false}, + {"100.64.0.1", false}, + {"0.0.0.0", false}, + {"224.0.0.1", false}, + {"240.0.0.1", false}, + {"fd00::1", false}, + {"fe80::1", false}, + } + + for _, test := range cases { + ip := net.ParseIP(test.address) + if ip == nil { + t.Fatalf("%q is not an address", test.address) + } + if got := isPubliclyRoutable(ip); got != test.public { + t.Errorf("isPubliclyRoutable(%q) = %v, want %v", test.address, got, test.public) + } + } +} + +// A redirect names a host the caller never asked for, so it is not followed +// unless the capability asked for it. +func TestHTTPDoesNotFollowARedirectByDefault(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"followed":true}`)) + })) + defer target.Close() + + redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer redirector.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": redirector.URL, "format": "text"}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["status_code"] != 302 { + t.Errorf("status_code = %v, want 302", result["status_code"]) + } +} + +// A redirect that is followed is checked against the allowlist as well: the +// second request is a request to a host the policy has to permit. +func TestHTTPDoesNotFollowARedirectToAHostThatIsNotAllowed(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(`{"followed":true}`)) + })) + defer target.Close() + + redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusFound) + })) + defer redirector.Close() + + _, err := HTTP{Policy: builtins.Policy{AllowedHTTPHosts: []string{"127.0.0.1"}}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": redirector.URL, "followRedirects": true}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } +} + +// A retry is not made unless the capability asked for one: a request is +// performed once by default, so a capability that is not idempotent cannot +// cause a side effect twice by accident. +func TestHTTPDoesNotRetryByDefault(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if got := attempts.Load(); got != 1 { + t.Errorf("the server was asked %d times, want 1", got) + } +} + +// A retry is made when the capability asked for one and the method is +// idempotent. +func TestHTTPRetriesAnIdempotentRequest(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) < 3 { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + w.Write([]byte(`{"ok":true}`)) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL, "retry": map[string]any{"attempts": 3, "backoff": "10ms"}}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["status_code"] != 200 { + t.Errorf("status_code = %v, want 200", result["status_code"]) + } + if got := attempts.Load(); got != 3 { + t.Errorf("the server was asked %d times, want 3", got) + } +} + +// A method that is not idempotent is not retried: the server may already +// have performed the effect the caller asked for. +func TestHTTPDoesNotRetryANonIdempotentRequest(t *testing.T) { + var attempts atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + result, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL, "method": "POST", "retry": map[string]any{"attempts": 3, "backoff": "10ms"}}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + if result["status_code"] != 503 { + t.Errorf("status_code = %v, want 503", result["status_code"]) + } + if got := attempts.Load(); got != 1 { + t.Errorf("the server was asked %d times, want 1", got) + } +} + +// The timeout the capability asks for is bounded by the policy: a capability +// is part of the assembly's input and cannot widen what the policy allows. +func TestHTTPRefusesATimeoutAboveThePolicyLimit(t *testing.T) { + _, err := HTTP{Policy: builtins.Policy{DefaultTimeout: "1s"}}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": "https://example.com", "timeout": "10s"}, + }) + if err == nil { + t.Fatal("Execute accepted a timeout above the policy limit") + } + if !strings.Contains(err.Error(), "exceeds the limit") { + t.Errorf("error = %v, want it to state the limit", err) + } +} + +// A request that outlasts its timeout fails with the deadline, not with a +// transport error. +func TestHTTPReturnsTheDeadlineWhenTheServerIsTooSlow(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(500 * time.Millisecond) + w.Write([]byte(`{}`)) + })) + defer server.Close() + + _, err := localHTTP().Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"url": server.URL, "timeout": "50ms"}, + }) + if err == nil { + t.Fatal("Execute succeeded after the deadline") + } + if !strings.Contains(err.Error(), "deadline") { + t.Errorf("error = %v, want it to state the deadline", err) + } +} diff --git a/nore/internal/builtins/input/input.go b/nore/internal/builtins/input/input.go new file mode 100644 index 0000000..71caf11 --- /dev/null +++ b/nore/internal/builtins/input/input.go @@ -0,0 +1,381 @@ +// Package input defines how a built-in capability runtime reads its input. +// +// Every built-in capability runtime receives the same two inputs: the params +// bound to the capability invocation, and the capability's own resolved +// configuration. This package owns the single merge rule that combines them +// and the typed accessors that read a value out of the result, so that every +// built-in interprets its input the same way. +// +// The merge rule is: params override configuration. A param is an explicit +// argument supplied at the call site; a configuration value is the ambient +// default declared alongside the capability. The explicit value wins. +// +// The engine-owned RuntimeConfig is deliberately not part of this input. It +// instructs N.O.R.E. how to drive the capability and is never sent to the +// capability runtime, so a built-in must never read its options from it. +package input + +import ( + "fmt" + "sort" + "strings" + "time" +) + +// Format selects how a built-in capability runtime represents the payload it +// produced. +// +// The format is chosen by the caller through the capability's own input, and +// every built-in that produces a payload honors it. JSON is the default +// because a structured value is what an assembly can bind into a downstream +// capability; a stringified object is never a substitute for it. +type Format string + +const ( + // FormatJSON represents the payload as structured JSON. The payload is + // parsed as JSON and returned as the value it represents. + FormatJSON Format = "json" + + // FormatText represents the payload as a plain string. + FormatText Format = "text" +) + +// Formats lists every supported format, in the order they are reported to a +// user who asks what a built-in accepts. +func Formats() []Format { + return []Format{FormatJSON, FormatText} +} + +// ParseFormat converts a caller-supplied format name into a Format. An empty +// name resolves to fallback, which is how a built-in expresses its default. +// +// An unknown name is an error rather than a silent fallback: a capability that +// asked for a format that does not exist must not quietly receive another one. +func ParseFormat(name string, fallback Format) (Format, error) { + switch strings.TrimSpace(strings.ToLower(name)) { + case "": + return fallback, nil + case string(FormatJSON): + return FormatJSON, nil + case string(FormatText): + return FormatText, nil + default: + return fallback, fmt.Errorf("unknown output format %q; supported formats are %s", name, formatList()) + } +} + +// Values is a merged, read-only view of a built-in capability runtime's input. +// +// The zero value is not usable; build one with New. +type Values struct { + // runtime names the built-in that is reading these values. It is used to + // make every error self-describing, because a capability failure is + // reported against the capability, not against the runtime engine. + runtime string + + // values is the merged input: configuration first, params second. + values map[string]any +} + +// New merges a capability's configuration and its params into one value set. +// +// Configuration supplies the ambient defaults declared alongside the +// capability. Params supply the explicit arguments bound to the invocation +// and override them, so the same capability behaves the same way whether it +// is invoked with explicit arguments or left to its declared defaults. +// +// Neither map is retained or mutated. +func New(runtime string, configurations map[string]any, params map[string]any) Values { + merged := make(map[string]any, len(configurations)+len(params)) + for key, value := range configurations { + merged[key] = value + } + for key, value := range params { + merged[key] = value + } + return Values{runtime: runtime, values: merged} +} + +// Runtime returns the name of the built-in reading these values. +func (v Values) Runtime() string { + return v.runtime +} + +// Has reports whether the named key carries a value. +func (v Values) Has(name string) bool { + _, ok := v.values[name] + return ok +} + +// Keys returns the names present in the merged input, sorted, so that an +// error listing the available names is stable. +func (v Values) Keys() []string { + keys := make([]string, 0, len(v.values)) + for key := range v.values { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} + +// All returns the merged input as a plain map. +// +// The returned map is a copy, so a caller that returns it as a capability +// result cannot be affected by a later change to the input. +func (v Values) All() map[string]any { + merged := make(map[string]any, len(v.values)) + for key, value := range v.values { + merged[key] = value + } + return merged +} + +// Raw returns the value stored under name and whether it was present. It is +// the escape hatch for values this package does not model, such as an +// arbitrary JSON body. +func (v Values) Raw(name string) (any, bool) { + value, ok := v.values[name] + return value, ok +} + +// String returns the value stored under name as a string. +// +// A present value that is not a string is an error rather than a coercion: a +// number silently turned into text is how a duration ends up parsed as +// "30000000000" and a URL ends up sent with a missing scheme. +func (v Values) String(name string) (string, bool, error) { + value, ok := v.values[name] + if !ok || value == nil { + return "", false, nil + } + text, ok := value.(string) + if !ok { + return "", true, v.typeError(name, "a string", value) + } + return text, true, nil +} + +// RequiredString returns the value stored under name as a string and fails +// when it is absent or empty. +func (v Values) RequiredString(name string) (string, error) { + text, present, err := v.String(name) + if err != nil { + return "", err + } + if !present { + return "", v.missingError(name) + } + if strings.TrimSpace(text) == "" { + return "", fmt.Errorf("%s: required parameter %q is empty", v.runtime, name) + } + return text, nil +} + +// Bool returns the value stored under name as a boolean. +// +// Only a real boolean is accepted. The strings "true" and "false" are not, +// because a capability that receives "false" as text has a defect in the +// capability's data, and coercing it hides that defect. +func (v Values) Bool(name string, fallback bool) (bool, error) { + value, ok := v.values[name] + if !ok || value == nil { + return fallback, nil + } + flag, ok := value.(bool) + if !ok { + return false, v.typeError(name, "a boolean", value) + } + return flag, nil +} + +// Int returns the value stored under name as an integer. +// +// JSON has a single number type, so an integer arrives as a float64. A value +// with a fractional part is rejected, because truncating it would silently +// change what the author asked for. +func (v Values) Int(name string, fallback int) (int, error) { + value, ok := v.values[name] + if !ok || value == nil { + return fallback, nil + } + switch typed := value.(type) { + case int: + return typed, nil + case int32: + return int(typed), nil + case int64: + return int(typed), nil + case float64: + if typed != float64(int(typed)) { + return 0, v.typeError(name, "a whole number", value) + } + return int(typed), nil + case float32: + if typed != float32(int(typed)) { + return 0, v.typeError(name, "a whole number", value) + } + return int(typed), nil + default: + return 0, v.typeError(name, "a whole number", value) + } +} + +// Duration returns the value stored under name as a duration and falls back +// when it is absent. +// +// A duration is written as a string such as "30s" or "2m30s" rather than as a +// number of nanoseconds, so the value survives the JSON round trip between +// the compiler, the manifest, and the runtime without losing its meaning. +func (v Values) Duration(name string, fallback time.Duration) (time.Duration, error) { + value, ok := v.values[name] + if !ok || value == nil { + return fallback, nil + } + text, ok := value.(string) + if !ok { + return 0, v.typeError(name, `a duration string such as "30s"`, value) + } + text = strings.TrimSpace(text) + if text == "" { + return fallback, nil + } + duration, err := time.ParseDuration(text) + if err != nil { + return 0, fmt.Errorf("%s: parameter %q must be a duration string such as \"30s\", got %q", v.runtime, name, text) + } + return duration, nil +} + +// StringMap returns the value stored under name as a map of strings. +// +// Every value must be a string. A header map or an environment map with a +// number in it is a defect in the capability's data, and dropping the +// offending entry silently would produce a request that is missing exactly +// the part the author cared about. +func (v Values) StringMap(name string) (map[string]string, bool, error) { + value, ok := v.values[name] + if !ok || value == nil { + return nil, false, nil + } + source, ok := value.(map[string]any) + if !ok { + return nil, true, v.typeError(name, "an object of string values", value) + } + result := make(map[string]string, len(source)) + for key, item := range source { + text, ok := item.(string) + if !ok { + return nil, true, fmt.Errorf("%s: parameter %q entry %q must be a string, got %s", v.runtime, name, key, describe(item)) + } + result[key] = text + } + return result, true, nil +} + +// StringSlice returns the value stored under name as a list of strings. +// +// A JSON array arrives as []any. Every element must be a string; a number in +// an argument list is a defect in the capability's data. +func (v Values) StringSlice(name string) ([]string, bool, error) { + value, ok := v.values[name] + if !ok || value == nil { + return nil, false, nil + } + source, ok := value.([]any) + if !ok { + return nil, true, v.typeError(name, "an array of strings", value) + } + result := make([]string, 0, len(source)) + for index, item := range source { + text, ok := item.(string) + if !ok { + return nil, true, fmt.Errorf("%s: parameter %q entry %d must be a string, got %s", v.runtime, name, index, describe(item)) + } + result = append(result, text) + } + return result, true, nil +} + +// Object returns the value stored under name as an object. +// +// The returned map is the nested value itself, not a copy of it. +func (v Values) Object(name string) (map[string]any, bool, error) { + value, ok := v.values[name] + if !ok || value == nil { + return nil, false, nil + } + object, ok := value.(map[string]any) + if !ok { + return nil, true, v.typeError(name, "an object", value) + } + return object, true, nil +} + +// Nested returns the object stored under name as a set of values. +// +// The nested values are read with the same rules as the outer ones, so a +// built-in does not have to know how the fields of a nested object are typed. +func (v Values) Nested(name string) (Values, bool, error) { + object, present, err := v.Object(name) + if err != nil || !present { + return Values{runtime: v.runtime}, present, err + } + return New(v.runtime, nil, object), true, nil +} + +// Format returns the output format selected by the parameter named name, +// falling back when it is absent. +func (v Values) Format(name string, fallback Format) (Format, error) { + text, present, err := v.String(name) + if err != nil { + return fallback, err + } + if !present { + return fallback, nil + } + return ParseFormat(text, fallback) +} + +// missingError reports an absent required parameter and names the values that +// are present, so the author can see what the capability did receive. +func (v Values) missingError(name string) error { + keys := v.Keys() + if len(keys) == 0 { + return fmt.Errorf("%s: required parameter %q is missing and the capability received no input", v.runtime, name) + } + return fmt.Errorf("%s: required parameter %q is missing; received %s", v.runtime, name, strings.Join(keys, ", ")) +} + +// typeError reports a value of the wrong type and states what was expected. +func (v Values) typeError(name, expected string, value any) error { + return fmt.Errorf("%s: parameter %q must be %s, got %s", v.runtime, name, expected, describe(value)) +} + +// describe names the type of a value so an error can say what it received. +func describe(value any) string { + switch value.(type) { + case nil: + return "null" + case string: + return "a string" + case bool: + return "a boolean" + case float64, float32, int, int32, int64: + return "a number" + case []any: + return "an array" + case map[string]any: + return "an object" + default: + return fmt.Sprintf("%T", value) + } +} + +// formatList names every supported format for an error message. +func formatList() string { + names := make([]string, 0, len(Formats())) + for _, format := range Formats() { + names = append(names, string(format)) + } + return strings.Join(names, ", ") +} diff --git a/nore/internal/builtins/input/input_test.go b/nore/internal/builtins/input/input_test.go new file mode 100644 index 0000000..138dbd6 --- /dev/null +++ b/nore/internal/builtins/input/input_test.go @@ -0,0 +1,250 @@ +package input + +import ( + "strings" + "testing" + "time" +) + +func TestNewParamsOverrideConfiguration(t *testing.T) { + values := New("neuron:core:test", + map[string]any{"mode": "config", "only_config": "kept"}, + map[string]any{"mode": "param"}, + ) + + mode, present, err := values.String("mode") + if err != nil { + t.Fatalf("String(mode): %v", err) + } + if !present { + t.Fatal("String(mode) reported no value") + } + if mode != "param" { + t.Errorf("mode = %q, want %q", mode, "param") + } + + kept, present, err := values.String("only_config") + if err != nil { + t.Fatalf("String(only_config): %v", err) + } + if !present || kept != "kept" { + t.Errorf("only_config = %q (present %v), want %q", kept, present, "kept") + } +} + +func TestNewDoesNotMutateItsInputs(t *testing.T) { + configuration := map[string]any{"mode": "config"} + params := map[string]any{"mode": "param"} + + values := New("neuron:core:test", configuration, params) + if _, _, err := values.String("mode"); err != nil { + t.Fatalf("String(mode): %v", err) + } + + if configuration["mode"] != "config" { + t.Errorf("configuration was mutated: %v", configuration) + } + if params["mode"] != "param" { + t.Errorf("params were mutated: %v", params) + } +} + +func TestRequiredStringReportsWhatWasReceived(t *testing.T) { + values := New("neuron:core:http", map[string]any{"method": "GET"}, nil) + + _, err := values.RequiredString("url") + if err == nil { + t.Fatal("RequiredString(url) succeeded, want an error") + } + if !strings.Contains(err.Error(), "neuron:core:http") { + t.Errorf("error %q does not name the runtime", err) + } + if !strings.Contains(err.Error(), `"url"`) { + t.Errorf("error %q does not name the parameter", err) + } + if !strings.Contains(err.Error(), "method") { + t.Errorf("error %q does not report the received input", err) + } +} + +func TestRequiredStringRejectsEmptyValue(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"url": " "}) + + if _, err := values.RequiredString("url"); err == nil { + t.Fatal("RequiredString(url) succeeded for a blank value") + } else if !strings.Contains(err.Error(), "empty") { + t.Errorf("error %q does not report the value as empty", err) + } +} + +func TestStringRejectsNonStringValue(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"url": 42}) + + _, _, err := values.String("url") + if err == nil { + t.Fatal("String(url) succeeded for a number") + } + if !strings.Contains(err.Error(), "must be a string") { + t.Errorf("error %q does not state the expected type", err) + } + if !strings.Contains(err.Error(), "a number") { + t.Errorf("error %q does not report the received type", err) + } +} + +func TestBoolRejectsStringValue(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"followRedirects": "true"}) + + if _, err := values.Bool("followRedirects", false); err == nil { + t.Fatal("Bool(followRedirects) succeeded for the string \"true\"") + } +} + +func TestIntAcceptsJSONNumbersAndRejectsFractions(t *testing.T) { + values := New("neuron:core:test", nil, map[string]any{"count": 7.0, "ratio": 1.5}) + + count, err := values.Int("count", 0) + if err != nil { + t.Fatalf("Int(count): %v", err) + } + if count != 7 { + t.Errorf("count = %d, want 7", count) + } + + if _, err := values.Int("ratio", 0); err == nil { + t.Fatal("Int(ratio) succeeded for a fractional number") + } +} + +func TestIntFallsBackWhenAbsent(t *testing.T) { + values := New("neuron:core:test", nil, nil) + + count, err := values.Int("count", 3) + if err != nil { + t.Fatalf("Int(count): %v", err) + } + if count != 3 { + t.Errorf("count = %d, want the fallback 3", count) + } +} + +func TestDurationParsesDurationStrings(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"timeout": "90s"}) + + timeout, err := values.Duration("timeout", time.Second) + if err != nil { + t.Fatalf("Duration(timeout): %v", err) + } + if timeout != 90*time.Second { + t.Errorf("timeout = %s, want 90s", timeout) + } +} + +func TestDurationRejectsInvalidValue(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"timeout": "soon"}) + + if _, err := values.Duration("timeout", time.Second); err == nil { + t.Fatal("Duration(timeout) succeeded for an unparseable duration") + } else if !strings.Contains(err.Error(), "30s") { + t.Errorf("error %q does not show an example of a valid duration", err) + } +} + +func TestStringMapRejectsNonStringEntries(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"headers": map[string]any{"X-Count": 3}}) + + if _, _, err := values.StringMap("headers"); err == nil { + t.Fatal("StringMap(headers) succeeded with a numeric entry") + } +} + +func TestStringSliceRejectsNonStringEntries(t *testing.T) { + values := New("neuron:core:command", nil, map[string]any{"args": []any{"--flag", 2}}) + + if _, _, err := values.StringSlice("args"); err == nil { + t.Fatal("StringSlice(args) succeeded with a numeric entry") + } +} + +func TestParseFormatDefaultsAndRejectsUnknown(t *testing.T) { + format, err := ParseFormat("", FormatJSON) + if err != nil { + t.Fatalf("ParseFormat(\"\"): %v", err) + } + if format != FormatJSON { + t.Errorf("format = %q, want %q", format, FormatJSON) + } + + if _, err := ParseFormat("yaml", FormatJSON); err == nil { + t.Fatal("ParseFormat(yaml) succeeded, want an error") + } else if !strings.Contains(err.Error(), "json") || !strings.Contains(err.Error(), "text") { + t.Errorf("error %q does not list the supported formats", err) + } +} + +func TestFormatReadsTheSelectedFormat(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"format": "text"}) + + format, err := values.Format("format", FormatJSON) + if err != nil { + t.Fatalf("Format(format): %v", err) + } + if format != FormatText { + t.Errorf("format = %q, want %q", format, FormatText) + } +} + +// A nested object is read with the same rules as the outer values, so a +// built-in does not have to know how the fields of a nested object are typed. +func TestNestedReadsAnObjectWithTheSameRules(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{ + "retry": map[string]any{"attempts": float64(3), "backoff": "200ms"}, + }) + + nested, present, err := values.Nested("retry") + if err != nil { + t.Fatalf("Nested: %v", err) + } + if !present { + t.Fatal("Nested did not find the object") + } + + attempts, err := nested.Int("attempts", 1) + if err != nil { + t.Fatalf("Int: %v", err) + } + if attempts != 3 { + t.Errorf("attempts = %d, want 3", attempts) + } + + backoff, err := nested.Duration("backoff", 0) + if err != nil { + t.Fatalf("Duration: %v", err) + } + if backoff.String() != "200ms" { + t.Errorf("backoff = %s, want 200ms", backoff) + } +} + +// A value that is not an object is refused rather than read as one. +func TestNestedRejectsANonObjectValue(t *testing.T) { + values := New("neuron:core:http", nil, map[string]any{"retry": "often"}) + + _, _, err := values.Nested("retry") + if err == nil { + t.Fatal("Nested accepted a value that is not an object") + } +} + +// A nested object that is absent is reported as absent. +func TestNestedReportsAnAbsentObject(t *testing.T) { + values := New("neuron:core:http", nil, nil) + + _, present, err := values.Nested("retry") + if err != nil { + t.Fatalf("Nested: %v", err) + } + if present { + t.Error("Nested found an object that is not there") + } +} diff --git a/nore/internal/builtins/log.go b/nore/internal/builtins/log.go deleted file mode 100644 index 90e0923..0000000 --- a/nore/internal/builtins/log.go +++ /dev/null @@ -1,28 +0,0 @@ -package builtins - -import ( - "context" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// Log surfaces the capability's params as a CapabilityLog event and passes -// the params through unchanged, acting as a pass-through observability node. -type Log struct{} - -// Execute emits a structured log of the capability params/config and returns -// the params as its result so downstream capabilities keep receiving the same -// data. -func (Log) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - execution.Logger.Info(ctx, - "[log] params received", - contracts.F("params", execution.Params), - contracts.F("config", execution.CapabilityConfigurations), - ) - return execution.Params, nil -} diff --git a/nore/internal/builtins/paths/paths.go b/nore/internal/builtins/paths/paths.go new file mode 100644 index 0000000..0da9f79 --- /dev/null +++ b/nore/internal/builtins/paths/paths.go @@ -0,0 +1,175 @@ +// Package paths resolves the paths the built-in capability runtimes may reach. +// +// The built-ins that touch the filesystem — the filesystem built-in and the +// command built-in — must agree on what is reachable, so the rule lives here +// rather than in each of them. +// +// The rule is containment: a path is reachable when it resolves inside one of +// the permitted roots. The check is made on the resolved path, after symbolic +// links are followed, because a link inside a root is a way out of it. +package paths + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// Roots is the set of directories a built-in capability runtime may reach. +type Roots struct { + // roots are the permitted directories as they were configured, made + // absolute. + roots []string + + // resolved are the permitted directories with their symbolic links + // followed, where that was possible. A target is checked against both + // lists, because a path that reaches a root through a link is still + // inside it. + resolved []string +} + +// NewRoots builds the permitted roots. +// +// The project root is the directory the assembly lives in, and it is always +// permitted. When there is no project — a capability runtime invoked +// directly — the working directory of the process is the root, because that +// is the directory the operator invoked it from. +// +// The extra roots extend the reach to the directories the assembly is +// expected to work with. +func NewRoots(projectRoot string, extra []string) Roots { + roots := make([]string, 0, len(extra)+1) + + if strings.TrimSpace(projectRoot) != "" { + roots = append(roots, absolute(projectRoot)) + } else if working, err := os.Getwd(); err == nil { + roots = append(roots, absolute(working)) + } + + for _, root := range extra { + if strings.TrimSpace(root) == "" { + continue + } + roots = append(roots, absolute(root)) + } + + resolved := make([]string, 0, len(roots)) + for _, root := range roots { + if followed, err := filepath.EvalSymlinks(root); err == nil { + resolved = append(resolved, filepath.Clean(followed)) + } + } + + return Roots{roots: roots, resolved: resolved} +} + +// Base returns the directory a relative path is resolved against. +// +// It is the project root when there is one, and the working directory of the +// process otherwise. +func (r Roots) Base() string { + if len(r.roots) == 0 { + return "" + } + return r.roots[0] +} + +// Roots returns the permitted roots, for an error message that says what is +// reachable. +func (r Roots) Roots() []string { + roots := make([]string, len(r.roots)) + copy(roots, r.roots) + return roots +} + +// Resolve resolves target against base and returns it when it is reachable. +// +// A relative target is resolved against base. The resolved path is checked +// twice: once as it was written, and once through the symbolic links it +// contains. A target that leaves the permitted roots through a link is +// refused even though the link itself is inside them. +// +// A target that does not exist is returned as an absolute path: the caller +// reports the missing path, which is a more useful error than a refusal. +func (r Roots) Resolve(base, target string) (string, error) { + if strings.TrimSpace(target) == "" { + return "", fmt.Errorf("the path is empty") + } + + absolute := target + if !filepath.IsAbs(absolute) { + if base == "" { + base = "." + } + absolute = filepath.Join(base, absolute) + } + absolute = filepath.Clean(absolute) + + if !r.contains(absolute) { + return "", r.refuse(absolute) + } + + followed, err := filepath.EvalSymlinks(absolute) + if err != nil { + return absolute, nil + } + followed = filepath.Clean(followed) + + if !r.contains(followed) { + return "", r.refuse(followed) + } + + return followed, nil +} + +// Contains reports whether the path is reachable. +func (r Roots) Contains(path string) bool { + if !filepath.IsAbs(path) { + path = filepath.Clean(filepath.Join(r.Base(), path)) + } + return r.contains(filepath.Clean(path)) +} + +// contains reports whether the path is inside one of the permitted roots. +func (r Roots) contains(path string) bool { + for _, root := range r.roots { + if within(root, path) { + return true + } + } + for _, root := range r.resolved { + if within(root, path) { + return true + } + } + return false +} + +// refuse reports a path that is not reachable, and names the roots that are. +func (r Roots) refuse(path string) error { + if len(r.roots) == 0 { + return fmt.Errorf("the path %q is not reachable: no directory is permitted", path) + } + return fmt.Errorf("the path %q is not reachable: it is outside %s", path, strings.Join(r.roots, ", ")) +} + +// within reports whether path is root or is inside it. +func within(root, path string) bool { + relative, err := filepath.Rel(root, path) + if err != nil { + return false + } + if relative == "." { + return true + } + return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) +} + +// absolute makes a path absolute and clean. +func absolute(path string) string { + if resolved, err := filepath.Abs(path); err == nil { + return filepath.Clean(resolved) + } + return filepath.Clean(path) +} diff --git a/nore/internal/builtins/paths/paths_test.go b/nore/internal/builtins/paths/paths_test.go new file mode 100644 index 0000000..7b15903 --- /dev/null +++ b/nore/internal/builtins/paths/paths_test.go @@ -0,0 +1,154 @@ +package paths + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestRootsResolveAPathInsideTheProjectRoot(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "nested") + if err := os.Mkdir(nested, 0o755); err != nil { + t.Fatalf("create the nested directory: %v", err) + } + + roots := NewRoots(root, nil) + + resolved, err := roots.Resolve(root, nested) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if resolved != nested { + t.Errorf("resolved = %q, want %q", resolved, nested) + } +} + +// A relative path is resolved against the base directory. +func TestRootsResolveARelativePathAgainstTheBase(t *testing.T) { + root := t.TempDir() + + roots := NewRoots(root, nil) + + resolved, err := roots.Resolve(root, "file.txt") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if resolved != filepath.Join(root, "file.txt") { + t.Errorf("resolved = %q, want it inside the project root", resolved) + } +} + +// A path outside the permitted roots is refused. +func TestRootsRefuseAPathOutsideThePermittedRoots(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + + roots := NewRoots(root, nil) + + _, err := roots.Resolve(root, outside) + if err == nil { + t.Fatal("Resolve accepted a path outside the permitted roots") + } + if !strings.Contains(err.Error(), "not reachable") { + t.Errorf("error = %v, want it to state that the path is not reachable", err) + } +} + +// A path that climbs out of the permitted roots with ".." is refused: the +// path is cleaned before it is checked. +func TestRootsRefuseAPathThatClimbsOut(t *testing.T) { + root := t.TempDir() + + roots := NewRoots(root, nil) + + _, err := roots.Resolve(root, filepath.Join("..", "elsewhere")) + if err == nil { + t.Fatal("Resolve accepted a path that climbs out of the permitted roots") + } +} + +// An extra root extends the reach. +func TestRootsAcceptAnExtraRoot(t *testing.T) { + root := t.TempDir() + extra := t.TempDir() + + roots := NewRoots(root, []string{extra}) + + if _, err := roots.Resolve(root, extra); err != nil { + t.Errorf("Resolve refused a path inside an extra root: %v", err) + } +} + +// A symbolic link inside a permitted root does not grant reach outside it. +func TestRootsRefuseALinkThatLeavesThePermittedRoots(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + + secret := filepath.Join(outside, "secret.txt") + if err := os.WriteFile(secret, []byte("secret"), 0o600); err != nil { + t.Fatalf("write the secret: %v", err) + } + + link := filepath.Join(root, "link.txt") + if err := os.Symlink(secret, link); err != nil { + t.Skipf("the platform does not support symbolic links: %v", err) + } + + roots := NewRoots(root, nil) + + if _, err := roots.Resolve(root, link); err == nil { + t.Fatal("Resolve accepted a link that leaves the permitted roots") + } +} + +// The base is the project root when there is one. +func TestRootsUseTheProjectRootAsTheBase(t *testing.T) { + root := t.TempDir() + + roots := NewRoots(root, nil) + + if roots.Base() != root { + t.Errorf("Base = %q, want %q", roots.Base(), root) + } +} + +// Without a project, the working directory of the process is the root: that +// is the directory a directly invoked capability runtime was invoked from. +func TestRootsFallBackToTheWorkingDirectory(t *testing.T) { + working, err := os.Getwd() + if err != nil { + t.Fatalf("read the working directory: %v", err) + } + + roots := NewRoots("", nil) + + if roots.Base() != working { + t.Errorf("Base = %q, want %q", roots.Base(), working) + } +} + +// A path that does not exist is returned as an absolute path: the caller +// reports the missing path, which is more useful than a refusal. +func TestRootsReturnAMissingPathAsAnAbsolutePath(t *testing.T) { + root := t.TempDir() + + roots := NewRoots(root, nil) + + resolved, err := roots.Resolve(root, "missing.txt") + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if resolved != filepath.Join(root, "missing.txt") { + t.Errorf("resolved = %q, want %q", resolved, filepath.Join(root, "missing.txt")) + } +} + +func TestRootsRefuseAnEmptyPath(t *testing.T) { + roots := NewRoots(t.TempDir(), nil) + + if _, err := roots.Resolve(t.TempDir(), " "); err == nil { + t.Fatal("Resolve accepted an empty path") + } +} diff --git a/nore/internal/builtins/set.go b/nore/internal/builtins/set.go deleted file mode 100644 index c3a2d62..0000000 --- a/nore/internal/builtins/set.go +++ /dev/null @@ -1,25 +0,0 @@ -package builtins - -import ( - "context" - "maps" - - "github.com/neuron-runtime/neuron/nore/internal/contracts" -) - -// Set merges the capability's params with its configuration and returns the -// combination as its result. It is the canonical way to assign or override -// values for downstream capabilities. -type Set struct{} - -func (Set) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - result := make(map[string]any, len(execution.Params)+len(execution.CapabilityConfigurations)) - maps.Copy(result, execution.Params) - maps.Copy(result, execution.CapabilityConfigurations) - return result, nil -} diff --git a/nore/internal/builtins/set/set.go b/nore/internal/builtins/set/set.go new file mode 100644 index 0000000..7d48ffe --- /dev/null +++ b/nore/internal/builtins/set/set.go @@ -0,0 +1,94 @@ +// Package set implements the neuron:core:set capability runtime. +package set + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/neuron-runtime/neuron/nore/internal/builtins/input" + "github.com/neuron-runtime/neuron/nore/internal/contracts" +) + +// Set materializes a capability's value set. +// +// It combines the capability's own configuration with the params bound to the +// invocation and returns the combination. It is the default runtime for a +// capability that declares none, which makes it the way an assembly supplies a +// value set to a downstream capability without writing a capability runtime. +// +// The merge follows the shared rule: params override configuration. A param is +// an explicit argument at the call site, and a configuration value is the +// ambient default declared alongside the capability, so the explicit value +// wins. +type Set struct{} + +// Execute returns the merged value set. +// +// The result carries the names of the merged set, so a capability that +// declares a result schema declares exactly those names. +func (Set) Execute(ctx context.Context, execution contracts.ExecutionContext) (map[string]any, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + + values := input.New("neuron:core:set", execution.CapabilityConfigurations, execution.Params) + + format, err := values.Format("format", input.FormatJSON) + if err != nil { + return nil, err + } + + merged := values.All() + // The format is an instruction to the built-in rather than a value of the + // set, so it is not part of what the set returns. + delete(merged, "format") + + if format == input.FormatText { + return renderText(merged), nil + } + return merged, nil +} + +// renderText represents every value of the set as text. +// +// The names of the set are kept, so the result of the capability has the same +// shape in both formats: only the representation of the values changes. +func renderText(values map[string]any) map[string]any { + rendered := make(map[string]any, len(values)) + for name, value := range values { + rendered[name] = renderValue(value) + } + return rendered +} + +// renderValue represents one value as text. +func renderValue(value any) string { + switch typed := value.(type) { + case nil: + return "" + case string: + return typed + case bool: + return strconv.FormatBool(typed) + case float64: + return strconv.FormatFloat(typed, 'f', -1, 64) + case float32: + return strconv.FormatFloat(float64(typed), 'f', -1, 32) + case int: + return strconv.Itoa(typed) + case int32: + return strconv.FormatInt(int64(typed), 10) + case int64: + return strconv.FormatInt(typed, 10) + default: + // A nested value has no text form of its own, so it is written as + // JSON rather than being dropped. + encoded, err := json.Marshal(typed) + if err != nil { + return fmt.Sprintf("%v", typed) + } + return string(encoded) + } +} diff --git a/nore/internal/builtins/set/set_test.go b/nore/internal/builtins/set/set_test.go new file mode 100644 index 0000000..3567306 --- /dev/null +++ b/nore/internal/builtins/set/set_test.go @@ -0,0 +1,108 @@ +package set + +import ( + "context" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/contracts" +) + +func TestSetMergesConfigurationAndParams(t *testing.T) { + result, err := Set{}.Execute(context.Background(), contracts.ExecutionContext{ + CapabilityConfigurations: map[string]any{"region": "eu", "retries": float64(3)}, + Params: map[string]any{"region": "us"}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if result["region"] != "us" { + t.Errorf("region = %v, want the param to override the configuration", result["region"]) + } + if result["retries"] != float64(3) { + t.Errorf("retries = %v, want the configuration value", result["retries"]) + } +} + +func TestSetReturnsACopy(t *testing.T) { + configuration := map[string]any{"region": "eu"} + result, err := Set{}.Execute(context.Background(), contracts.ExecutionContext{ + CapabilityConfigurations: configuration, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + result["region"] = "changed" + result["added"] = true + + if configuration["region"] != "eu" { + t.Errorf("the configuration was mutated: %v", configuration) + } + if _, ok := configuration["added"]; ok { + t.Errorf("the configuration gained a key: %v", configuration) + } +} + +func TestSetReportsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := (Set{}).Execute(ctx, contracts.ExecutionContext{}); err == nil { + t.Fatal("Execute succeeded for a cancelled execution") + } +} + +// The format is an instruction to the built-in rather than a value of the +// set, so it is not returned as one. +func TestSetDoesNotReturnTheFormat(t *testing.T) { + result, err := Set{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"region": "us", "format": "json"}, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if _, ok := result["format"]; ok { + t.Errorf("the result carries the format: %v", result) + } + if result["region"] != "us" { + t.Errorf("region = %v, want us", result["region"]) + } +} + +// The text format keeps the names of the set and represents the values as +// text, so the shape of the result does not change between the formats. +func TestSetRendersTheValuesAsText(t *testing.T) { + result, err := Set{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{ + "format": "text", + "name": "World", + "count": float64(42), + "ready": true, + }, + }) + if err != nil { + t.Fatalf("Execute: %v", err) + } + + if result["name"] != "World" { + t.Errorf("name = %#v, want the text of the value", result["name"]) + } + if result["count"] != "42" { + t.Errorf("count = %#v, want the text of the value", result["count"]) + } + if result["ready"] != "true" { + t.Errorf("ready = %#v, want the text of the value", result["ready"]) + } +} + +// A format that does not exist is refused rather than silently ignored. +func TestSetRefusesAnUnknownFormat(t *testing.T) { + _, err := Set{}.Execute(context.Background(), contracts.ExecutionContext{ + Params: map[string]any{"format": "yaml"}, + }) + if err == nil { + t.Fatal("Execute accepted a format that does not exist") + } +} diff --git a/nore/internal/contracts/capabilityruntime.go b/nore/internal/contracts/capabilityruntime.go index 4be52e4..4469488 100644 --- a/nore/internal/contracts/capabilityruntime.go +++ b/nore/internal/contracts/capabilityruntime.go @@ -27,6 +27,32 @@ type Logger interface { Error(ctx context.Context, message string, fields ...Field) } +// NopLogger discards every log message. +// +// It is the logger for the execution paths that have no event bus to publish +// onto — a direct invocation, for instance, which runs a capability runtime +// outside the engine. Handing those paths a nil logger instead would turn +// every diagnostic a capability runtime writes into a panic, so the seam is +// always satisfied. +type NopLogger struct{} + +func (NopLogger) Debug(context.Context, string, ...Field) {} +func (NopLogger) Info(context.Context, string, ...Field) {} +func (NopLogger) Warn(context.Context, string, ...Field) {} +func (NopLogger) Error(context.Context, string, ...Field) {} + +// LoggerOrNop returns l, or a logger that discards everything when l is nil. +// +// The logger is a seam that some execution paths do not provide — a direct +// invocation has no event bus to publish onto — so a capability runtime that +// logs must not have to know which path it runs under. +func LoggerOrNop(l Logger) Logger { + if l == nil { + return NopLogger{} + } + return l +} + type ExecutionContext struct { ExecutionID core2.ID CorrelationID core2.ID diff --git a/nore/internal/contracts/execution-repository.go b/nore/internal/contracts/execution-repository.go index 5c255e5..769343f 100644 --- a/nore/internal/contracts/execution-repository.go +++ b/nore/internal/contracts/execution-repository.go @@ -7,11 +7,16 @@ import ( "github.com/neuron-runtime/neuron/shared/types/core" ) +// ExecutionRepository is the durable execution boundary shared by the +// scheduler, the instance persister, and the API. +// +// Reads report storage.ErrNotFound for absence and return backend or decode +// failures as errors. A miss must never be synthesized from a failure. type ExecutionRepository interface { - Add(execution *executionmodel.Execution) error - Get(executionID core.ID) (*executionmodel.Execution, bool) + Add(ctx context.Context, execution *executionmodel.Execution) error + Get(ctx context.Context, executionID core.ID) (*executionmodel.Execution, error) Save(ctx context.Context, execution *executionmodel.Execution) error - Delete(executionID core.ID) - List() []*executionmodel.Execution - ListByInstance(instanceID core.ID) []*executionmodel.Execution + Delete(ctx context.Context, executionID core.ID) error + List(ctx context.Context) ([]*executionmodel.Execution, error) + ListByInstance(ctx context.Context, instanceID core.ID) ([]*executionmodel.Execution, error) } diff --git a/nore/internal/event/delete_test.go b/nore/internal/event/delete_test.go index ccbe9d7..0c86b6c 100644 --- a/nore/internal/event/delete_test.go +++ b/nore/internal/event/delete_test.go @@ -2,6 +2,8 @@ package event import ( "context" + "errors" + "strings" "testing" "github.com/neuron-runtime/neuron/nore/internal/storage" @@ -10,7 +12,7 @@ import ( ) func TestDeleteExecutionRemovesOnlyThatExecution(t *testing.T) { - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } @@ -60,3 +62,67 @@ func newTestEvent(executionID core.ID) Event { ExecutionStartedPayload{Params: map[string]any{"execution": string(executionID)}}, ) } + +// failingDeleteStore wraps a real store and fails one event delete inside the +// transaction, proving DeleteExecution cannot leave a partially deleted history. +type failingDeleteStore struct { + storage.Store + failSuffix string +} + +func (s *failingDeleteStore) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + return s.Store.WithTx(ctx, func(tx storage.Tx) error { + return fn(failingDeleteTx{Tx: tx, failSuffix: s.failSuffix}) + }) +} + +type failingDeleteTx struct { + storage.Tx + failSuffix string +} + +func (tx failingDeleteTx) Delete(ctx context.Context, key string) error { + if strings.HasSuffix(key, tx.failSuffix) { + return errors.New("injected event delete failure") + } + return tx.Tx.Delete(ctx, key) +} + +func TestDeleteExecutionRollsBackOnFailure(t *testing.T) { + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("open sqlite store: %v", err) + } + defer store.Close() + + ctx := context.Background() + execID := core.ID("exec_atomic") + base := NewStore(store) + for i := 0; i < 3; i++ { + if err := base.Save(ctx, newTestEvent(execID)); err != nil { + t.Fatalf("save event: %v", err) + } + } + + keys, err := store.List(ctx, "events/"+string(execID)+"/") + if err != nil { + t.Fatalf("list event keys: %v", err) + } + if len(keys) != 3 { + t.Fatalf("event keys = %d, want 3", len(keys)) + } + suffix := keys[len(keys)-1] + failing := NewStore(&failingDeleteStore{Store: store, failSuffix: suffix}) + + if err := failing.DeleteExecution(ctx, execID); err == nil { + t.Fatal("DeleteExecution should fail when one delete fails") + } + + remaining, err := base.List(ctx, execID) + if err != nil { + t.Fatalf("list after failed delete: %v", err) + } + if len(remaining) != 3 { + t.Fatalf("failed DeleteExecution removed %d events, want 0", 3-len(remaining)) + } +} diff --git a/nore/internal/event/store.go b/nore/internal/event/store.go index c59c363..9b22249 100644 --- a/nore/internal/event/store.go +++ b/nore/internal/event/store.go @@ -8,7 +8,6 @@ import ( "time" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" "github.com/neuron-runtime/neuron/shared/types/core" ) @@ -51,11 +50,19 @@ func (s *Store) Save(ctx context.Context, evt Event) error { return fmt.Errorf("marshal persisted event: %w", err) } - return s.store.Put(ctx, s.key(evt.Metadata.ExecutionID, evt.Metadata.EventID), data) + key, err := s.key(evt.Metadata.ExecutionID, evt.Metadata.EventID) + if err != nil { + return err + } + + return s.store.Put(ctx, key, data) } func (s *Store) List(ctx context.Context, executionID core.ID) ([]Event, error) { - prefix := sqlite.SanitizeKey("events", string(executionID)) + prefix, err := storage.Prefix("events", string(executionID)) + if err != nil { + return nil, err + } keys, err := s.store.List(ctx, prefix) if err != nil { return nil, err @@ -65,7 +72,7 @@ func (s *Store) List(ctx context.Context, executionID core.ID) ([]Event, error) for _, key := range keys { evt, err := s.load(ctx, key) if err != nil { - continue + return nil, fmt.Errorf("load event %q: %w", key, err) } events = append(events, evt) } @@ -126,23 +133,30 @@ func (s *Store) load(ctx context.Context, key string) (Event, error) { }, nil } -func (s *Store) key(executionID, eventID core.ID) string { - return sqlite.SanitizeKey("events", string(executionID), string(eventID)) +func (s *Store) key(executionID, eventID core.ID) (string, error) { + return storage.Key("events", string(executionID), string(eventID)) } // DeleteExecution removes every persisted event recorded for the execution. +// The listing and deletes run in one transaction so a failure cannot leave a +// partially deleted history. func (s *Store) DeleteExecution(ctx context.Context, executionID core.ID) error { - prefix := sqlite.SanitizeKey("events", string(executionID)) - keys, err := s.store.List(ctx, prefix) + prefix, err := storage.Prefix("events", string(executionID)) if err != nil { return err } - for _, key := range keys { - if err := s.store.Delete(ctx, key); err != nil { + return s.store.WithTx(ctx, func(tx storage.Tx) error { + keys, err := tx.List(ctx, prefix) + if err != nil { return err } - } - return nil + for _, key := range keys { + if err := tx.Delete(ctx, key); err != nil { + return fmt.Errorf("delete event %q: %w", key, err) + } + } + return nil + }) } // unixNanoToTime intentionally deleted; time.Unix is used directly in load. diff --git a/nore/internal/event/store_test.go b/nore/internal/event/store_test.go index 7194346..cb03887 100644 --- a/nore/internal/event/store_test.go +++ b/nore/internal/event/store_test.go @@ -2,6 +2,7 @@ package event import ( "context" + "strings" "testing" "github.com/neuron-runtime/neuron/nore/internal/storage" @@ -11,7 +12,7 @@ import ( func newTestStore(t *testing.T) *Store { t.Helper() - s, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + s, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatal(err) } @@ -68,3 +69,65 @@ func TestStoreRoundTripAndListAfter(t *testing.T) { t.Fatalf("expected second first after cursor, got %s", after[0].Metadata.EventID) } } + +func TestListReportsMalformedPersistedEvent(t *testing.T) { + ctx := context.Background() + raw, err := sqlite.New(ctx, storage.Config{DataDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = raw.Close() }) + + store := NewStore(raw) + execID := core.ID("exec_malformed") + eventID := core.ID("evt_corrupt") + + key, err := storage.Key("events", string(execID), string(eventID)) + if err != nil { + t.Fatal(err) + } + if err := raw.Put(ctx, key, []byte("{not valid json")); err != nil { + t.Fatal(err) + } + + _, err = store.List(ctx, execID) + if err == nil { + t.Fatal("expected List to fail on malformed event") + } + if !strings.Contains(err.Error(), key) { + t.Fatalf("expected error to name key %q, got %v", key, err) + } +} + +func TestListSeparatesPrefixCollidingExecutions(t *testing.T) { + ctx := context.Background() + store := newTestStore(t) + + execA := core.ID("exec_a") + execAB := core.ID("exec_ab") + + a := newTestEvent(execA) + if err := store.Save(ctx, a); err != nil { + t.Fatal(err) + } + ab := newTestEvent(execAB) + if err := store.Save(ctx, ab); err != nil { + t.Fatal(err) + } + + listA, err := store.List(ctx, execA) + if err != nil { + t.Fatal(err) + } + if len(listA) != 1 || listA[0].Metadata.EventID != a.Metadata.EventID { + t.Fatalf("exec_a returned the wrong events: %+v", listA) + } + + listAB, err := store.List(ctx, execAB) + if err != nil { + t.Fatal(err) + } + if len(listAB) != 1 || listAB[0].Metadata.EventID != ab.Metadata.EventID { + t.Fatalf("exec_ab returned the wrong events: %+v", listAB) + } +} diff --git a/nore/internal/execution/engine/config_params_test.go b/nore/internal/execution/engine/config_params_test.go index 5c49123..99ad2b9 100644 --- a/nore/internal/execution/engine/config_params_test.go +++ b/nore/internal/execution/engine/config_params_test.go @@ -107,7 +107,7 @@ func TestCapabilityConfigParamsVariableExposesSnakeCaseKeys(t *testing.T) { if err != nil { t.Fatalf("new execution: %v", err) } - if err := store.Add(execution); err != nil { + if err := store.Add(context.Background(), execution); err != nil { t.Fatalf("store.Add: %v", err) } diff --git a/nore/internal/execution/engine/executor_engine.go b/nore/internal/execution/engine/executor_engine.go index 08332c1..fa132e3 100644 --- a/nore/internal/execution/engine/executor_engine.go +++ b/nore/internal/execution/engine/executor_engine.go @@ -2,7 +2,9 @@ package engine import ( "context" + "errors" "fmt" + "log/slog" "time" "sync" @@ -14,6 +16,7 @@ import ( "github.com/neuron-runtime/neuron/nore/internal/data" "github.com/neuron-runtime/neuron/nore/internal/event" "github.com/neuron-runtime/neuron/nore/internal/resolver" + "github.com/neuron-runtime/neuron/nore/internal/storage" ) // DefaultDetachedDrainTimeout bounds how long N.O.R.E. keeps running detached @@ -82,8 +85,19 @@ func (e *CapabilityRuntimeEngine) Run(ctx context.Context) error { } func (e *CapabilityRuntimeEngine) executeCapability(ctx context.Context, received event.Event) { - execution, exists := e.executions.Get(received.Metadata.ExecutionID) - if !exists || execution.IsTerminal() { + execution, err := e.executions.Get(ctx, received.Metadata.ExecutionID) + if errors.Is(err, storage.ErrNotFound) { + return + } + if err != nil { + slog.Error("load execution for capability", + slog.String("execution_id", string(received.Metadata.ExecutionID)), + slog.String("capability_id", string(received.Metadata.CapabilityID)), + slog.String("error", err.Error()), + ) + return + } + if execution.IsTerminal() { return } capabilityID := received.Metadata.CapabilityID diff --git a/nore/internal/execution/engine/timeout_test.go b/nore/internal/execution/engine/timeout_test.go index 174c2fd..ae65f32 100644 --- a/nore/internal/execution/engine/timeout_test.go +++ b/nore/internal/execution/engine/timeout_test.go @@ -158,7 +158,7 @@ func TestCapabilityTimeoutIsAFailureNotACancellation(t *testing.T) { if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(execution); err != nil { + if err := store.Add(context.Background(), execution); err != nil { t.Fatalf("store.Add: %v", err) } diff --git a/nore/internal/execution/execution.go b/nore/internal/execution/execution.go index f9557b8..c638435 100644 --- a/nore/internal/execution/execution.go +++ b/nore/internal/execution/execution.go @@ -79,6 +79,20 @@ type Execution struct { // state; Wait and Done poll it. It is a runtime-only device and is not // serialized in snapshots. done chan struct{} + + // revision is the durable compare-and-swap version of this execution. It is + // serialized in snapshots and incremented only after a successful save. + revision uint64 + + // saveMu serializes saves of this Execution value. Two writers sharing the + // same value cannot produce a false conflict; only a distinct copy with an + // older revision triggers storage.ErrConflict. + saveMu sync.Mutex + + // joinState tracks completion of predecessor bindings for join targets. + // Each key is a target capability ID; the value records which predecessor + // bindings have completed and what inputs they contributed. + joinState map[shared.ID]*joinStateInfo } func NewExecution(blueprint *types.ExecutionBlueprint, correlationID shared.ID, instanceID shared.ID) (*Execution, error) { @@ -96,7 +110,8 @@ func NewExecution(blueprint *types.ExecutionBlueprint, correlationID shared.ID, ID: shared.NewID("exec_"), CorrelationID: correlationID, InstanceID: instanceID, Blueprint: blueprint, status: StatusPending, initialParams: make(map[string]any), params: make(map[shared.ID]map[string]any), results: make(map[shared.ID]map[string]any), states: states, - done: make(chan struct{}), + done: make(chan struct{}), + joinState: make(map[shared.ID]*joinStateInfo), }, nil } @@ -265,7 +280,9 @@ func (e *Execution) CompleteCurrentAndSchedule(nextCapabilityCount int) (int, er if e.inFlight <= 0 { return e.inFlight, errors.New("execution in-flight counter is invalid") } - e.inFlight += nextCapabilityCount - 1 + if nextCapabilityCount > 0 { + e.inFlight += nextCapabilityCount - 1 + } return e.inFlight, nil } @@ -353,6 +370,20 @@ func (e *Execution) IsTerminal() bool { return e.isTerminalLocked() } +// Revision reports the durable revision this execution was last saved at. A +// newly created execution has revision 0 until its first successful Add. +func (e *Execution) Revision() uint64 { + e.mu.RLock() + defer e.mu.RUnlock() + return e.revision +} + +func (e *Execution) setRevision(revision uint64) { + e.mu.Lock() + defer e.mu.Unlock() + e.revision = revision +} + func (e *Execution) Params(capabilityID shared.ID) map[string]any { e.mu.RLock() defer e.mu.RUnlock() @@ -449,3 +480,157 @@ func pathValue(payload any, path []string, source *shared.ValueRef) (any, error) } return nil, fmt.Errorf("%s reference path %q: unexpected end", source.Kind, strings.Join(path, ".")) } + +// joinStateInfo holds the state for a join target capability. +type joinStateInfo struct { + CompletedBindings map[shared.ID]bool + PredecessorInputs map[shared.ID]map[string]any + Dispatched bool +} + +// RecordCompletedBinding marks a binding as completed for a join target. +// It is safe to call even if the binding was already completed; it is idempotent. +// Returns true if the join target is now ready (all predecessors completed). +func (e *Execution) RecordCompletedBinding(capabilityID shared.ID, bindingID shared.ID) bool { + e.mu.Lock() + defer e.mu.Unlock() + + ji, exists := e.joinState[capabilityID] + if !exists { + return false + } + + if ji.CompletedBindings == nil { + ji.CompletedBindings = make(map[shared.ID]bool) + } + ji.CompletedBindings[bindingID] = true + + node := e.Blueprint.Nodes[capabilityID] + required := len(node.Predecessors) + + if required == 0 { + ji.Dispatched = true + return true + } + + allCompleted := true + for i := 0; i < required; i++ { + found := false + for completedID := range ji.CompletedBindings { + if completedID == node.Predecessors[i].BindingID { + found = true + break + } + } + if !found { + allCompleted = false + break + } + } + + if allCompleted && !ji.Dispatched { + ji.Dispatched = true + } + + return allCompleted +} + +// RecordPredecessorInput stores the output of a completed predecessor capability +// as an input for the join target. +func (e *Execution) RecordPredecessorInput(capabilityID shared.ID, bindingID shared.ID, input map[string]any) { + e.mu.Lock() + defer e.mu.Unlock() + + ji, exists := e.joinState[capabilityID] + if !exists { + return + } + + if ji.PredecessorInputs == nil { + ji.PredecessorInputs = make(map[shared.ID]map[string]any) + } + ji.PredecessorInputs[bindingID] = input +} + +// ReadyForJoin checks if a join target capability is ready to execute. +func (e *Execution) ReadyForJoin(capabilityID shared.ID) bool { + e.mu.RLock() + defer e.mu.RUnlock() + + ji, exists := e.joinState[capabilityID] + if !exists { + return false + } + + node := e.Blueprint.Nodes[capabilityID] + if len(node.Predecessors) == 0 { + return true + } + + required := len(node.Predecessors) + completed := 0 + for _, pred := range node.Predecessors { + if ji.CompletedBindings != nil { + for completedID := range ji.CompletedBindings { + if completedID == pred.BindingID { + completed++ + break + } + } + } + } + + return completed >= required +} + +// MergeCompletedInputs gathers all inputs from completed predecessors. +func (e *Execution) MergeCompletedInputs(capabilityID shared.ID) map[string]any { + e.mu.RLock() + ji, exists := e.joinState[capabilityID] + e.mu.RUnlock() + if !exists { + return nil + } + + if ji.PredecessorInputs == nil { + return map[string]any{} + } + + merged := make(map[string]any, len(ji.PredecessorInputs)) + for _, input := range ji.PredecessorInputs { + for k, v := range input { + merged[k] = v + } + } + return merged +} + +// DispatchedOnce checks if work has already been dispatched for a join target. +func (e *Execution) DispatchedOnce(capabilityID shared.ID) bool { + e.mu.Lock() + defer e.mu.Unlock() + + ji, exists := e.joinState[capabilityID] + if !exists { + return false + } + + if ji.Dispatched { + return true + } + + ji.Dispatched = true + return true +} + +// mergeWithTransitionInput combines merged predecessor inputs with transition mappings. +func mergeWithTransitionInput(merged map[string]any, transitionInput map[string]any, transition types.ExecutionTransition) map[string]any { + result := map[string]any{} + for k, v := range merged { + result[k] = v + } + for k, v := range transitionInput { + result[k] = v + } + return result +} diff --git a/nore/internal/execution/memstore.go b/nore/internal/execution/memstore.go index 2474625..d24b92b 100644 --- a/nore/internal/execution/memstore.go +++ b/nore/internal/execution/memstore.go @@ -5,9 +5,13 @@ import ( "fmt" "sync" + "github.com/neuron-runtime/neuron/nore/internal/storage" "github.com/neuron-runtime/neuron/shared/types/core" ) +// MemoryStore is the in-process execution repository. It implements the same +// boundary as ExecutionStore so scheduler and engine tests exercise the real +// interface instead of a looser test-only shape. type MemoryStore struct { mu sync.RWMutex executions map[core.ID]*Execution @@ -17,24 +21,27 @@ func NewMemoryStore() *MemoryStore { return &MemoryStore{executions: make(map[core.ID]*Execution)} } -func (s *MemoryStore) Add(execution *Execution) error { +func (s *MemoryStore) Add(_ context.Context, execution *Execution) error { if execution == nil { return fmt.Errorf("execution is required") } s.mu.Lock() defer s.mu.Unlock() if _, exists := s.executions[execution.ID]; exists { - return fmt.Errorf("execution %s already exists", execution.ID) + return fmt.Errorf("%w: execution %s", storage.ErrAlreadyExists, execution.ID) } s.executions[execution.ID] = execution return nil } -func (s *MemoryStore) Get(executionID core.ID) (*Execution, bool) { +func (s *MemoryStore) Get(_ context.Context, executionID core.ID) (*Execution, error) { s.mu.RLock() defer s.mu.RUnlock() execution, exists := s.executions[executionID] - return execution, exists + if !exists { + return nil, fmt.Errorf("%w: execution %s", storage.ErrNotFound, executionID) + } + return execution, nil } // Save records the current state of an execution, inserting it when it is not @@ -55,13 +62,14 @@ func (s *MemoryStore) Save(_ context.Context, execution *Execution) error { return nil } -func (s *MemoryStore) Delete(executionID core.ID) { +func (s *MemoryStore) Delete(_ context.Context, executionID core.ID) error { s.mu.Lock() defer s.mu.Unlock() delete(s.executions, executionID) + return nil } -func (s *MemoryStore) List() []*Execution { +func (s *MemoryStore) List(_ context.Context) ([]*Execution, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -69,13 +77,12 @@ func (s *MemoryStore) List() []*Execution { for _, exec := range s.executions { list = append(list, exec) } - return list + return list, nil } -// ListByInstance returns every execution that belongs to instanceID. It is the -// single implementation of the ownership filter, shared with the persistent -// store so the two can never disagree about which executions an instance owns. -func (s *MemoryStore) ListByInstance(instanceID core.ID) []*Execution { +// ListByInstance returns every execution that belongs to instanceID. The +// persistent store applies the same ownership predicate to its merged view. +func (s *MemoryStore) ListByInstance(_ context.Context, instanceID core.ID) ([]*Execution, error) { s.mu.RLock() defer s.mu.RUnlock() @@ -85,5 +92,5 @@ func (s *MemoryStore) ListByInstance(instanceID core.ID) []*Execution { list = append(list, exec) } } - return list + return list, nil } diff --git a/nore/internal/execution/scheduler/cancel_test.go b/nore/internal/execution/scheduler/cancel_test.go index 54bde88..d5bfd68 100644 --- a/nore/internal/execution/scheduler/cancel_test.go +++ b/nore/internal/execution/scheduler/cancel_test.go @@ -82,7 +82,7 @@ func startExecution(t *testing.T, ctx context.Context, bus *event.Bus, store *ex if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(exec); err != nil { + if err := store.Add(ctx, exec); err != nil { t.Fatalf("store.Add: %v", err) } if err := bus.Publish(ctx, event.New(event.ExecutionStarted, exec.ID, exec.CorrelationID, "", event.ExecutionStartedPayload{Params: map[string]any{}})); err != nil { @@ -133,7 +133,11 @@ func TestCancelExecutionStopsTheRunningCapability(t *testing.T) { scheduler, exec, runtime := newCancelHarness(t, ctx, singleCapabilityBlueprint("slow")) runtime.awaitStart(t) - if !scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")) { + cancelled, err := scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")) + if err != nil { + t.Fatalf("CancelExecution: %v", err) + } + if !cancelled { t.Fatal("CancelExecution() = false, want true for a running execution") } runtime.awaitCancellation(t) @@ -175,7 +179,11 @@ func TestCancelExecutionAnnouncesTheCancellation(t *testing.T) { events := subscribeToExecution(t, bus, exec.ID) runtime.awaitStart(t) - if !scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")) { + cancelled, err := scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")) + if err != nil { + t.Fatalf("CancelExecution: %v", err) + } + if !cancelled { t.Fatal("CancelExecution() = false, want true") } @@ -214,7 +222,9 @@ func TestTheAbortedInvocationDoesNotReportTheCapabilityAsFailed(t *testing.T) { scheduler, exec, runtime := newCancelHarness(t, ctx, singleCapabilityBlueprint("slow")) runtime.awaitStart(t) - scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")) + if _, err := scheduler.CancelExecution(ctx, exec.ID, errors.New("stopped by operator")); err != nil { + t.Fatalf("CancelExecution: %v", err) + } runtime.awaitCancellation(t) // The engine needs a moment to unwind and report the aborted invocation. @@ -247,7 +257,11 @@ func TestCancelExecutionStopsThePlanFromAdvancing(t *testing.T) { scheduler, exec, runtime := newCancelHarness(t, ctx, plan) runtime.awaitStart(t) - if !scheduler.CancelExecution(ctx, exec.ID, errors.New("stop")) { + cancelled, err := scheduler.CancelExecution(ctx, exec.ID, errors.New("stop")) + if err != nil { + t.Fatalf("CancelExecution: %v", err) + } + if !cancelled { t.Fatal("CancelExecution() = false, want true") } runtime.awaitCancellation(t) @@ -276,15 +290,15 @@ func TestCancelExecutionRefusesUnknownAndTerminalExecutions(t *testing.T) { t.Fatalf("New scheduler: %v", err) } - if scheduler.CancelExecution(ctx, "exec_missing", errors.New("stop")) { - t.Error("CancelExecution() = true for an unknown execution, want false") + if cancelled, err := scheduler.CancelExecution(ctx, "exec_missing", errors.New("stop")); err != nil || cancelled { + t.Errorf("CancelExecution() = (%v, %v), want (false, nil)", cancelled, err) } exec, err := executionmodel.NewExecution(singleCapabilityBlueprint("slow"), "corr_1", "inst_1") if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(exec); err != nil { + if err := store.Add(ctx, exec); err != nil { t.Fatalf("store.Add: %v", err) } if err := exec.Start(map[string]any{}, 1); err != nil { @@ -294,8 +308,8 @@ func TestCancelExecutionRefusesUnknownAndTerminalExecutions(t *testing.T) { t.Fatal("MarkCompleted() = false, want true") } - if scheduler.CancelExecution(ctx, exec.ID, errors.New("stop")) { - t.Error("CancelExecution() = true for a completed execution, want false") + if cancelled, err := scheduler.CancelExecution(ctx, exec.ID, errors.New("stop")); err != nil || cancelled { + t.Errorf("CancelExecution() = (%v, %v), want (false, nil)", cancelled, err) } if got := exec.Status(); got != executionmodel.StatusCompleted { t.Errorf("Status() = %s, want %s (a refused cancellation must not rewrite the outcome)", got, executionmodel.StatusCompleted) diff --git a/nore/internal/execution/scheduler/detach_test.go b/nore/internal/execution/scheduler/detach_test.go index 50ccda0..75d3cee 100644 --- a/nore/internal/execution/scheduler/detach_test.go +++ b/nore/internal/execution/scheduler/detach_test.go @@ -137,16 +137,16 @@ func TestSchedulerHandsDetachedWorkToATrackedChildExecution(t *testing.T) { if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(parent); err != nil { - t.Fatalf("store.Add(parent): %v", err) + if err := store.Add(ctx, parent); err != nil { + t.Fatalf("store.Add(ctx, parent): %v", err) } if err := bus.Publish(ctx, event.New(event.ExecutionStarted, parent.ID, parent.CorrelationID, "", event.ExecutionStartedPayload{Params: map[string]any{}})); err != nil { t.Fatalf("publish ExecutionStarted: %v", err) } waitFor(t, "parent execution to finish", func() bool { - current, ok := store.Get(parent.ID) - return ok && current.IsTerminal() + current, err := store.Get(context.Background(), parent.ID) + return err == nil && current.IsTerminal() }) // The parent must never run the detached capability or its descendants. @@ -159,8 +159,8 @@ func TestSchedulerHandsDetachedWorkToATrackedChildExecution(t *testing.T) { task := waitForChild(t, store, parent.ID) waitFor(t, "detached task to finish", func() bool { - current, ok := store.Get(task.ID) - return ok && current.IsTerminal() + current, err := store.Get(context.Background(), task.ID) + return err == nil && current.IsTerminal() }) // The work actually happened exactly once, inside the task. @@ -171,9 +171,9 @@ func TestSchedulerHandsDetachedWorkToATrackedChildExecution(t *testing.T) { t.Fatalf("the detached descendant ran %d times, want 1", got) } - storedTask, ok := store.Get(task.ID) - if !ok { - t.Fatal("detached task disappeared") + storedTask, err := store.Get(context.Background(), task.ID) + if err != nil { + t.Fatalf("detached task disappeared: %v", err) } if storedTask.ParentExecutionID != parent.ID { t.Fatalf("task parent = %q, want %q", storedTask.ParentExecutionID, parent.ID) @@ -188,7 +188,11 @@ func waitForChild(t *testing.T, store *executionmodel.MemoryStore, parent core.I t.Helper() var found *executionmodel.Execution waitFor(t, "detached child execution to be created", func() bool { - for _, candidate := range store.List() { + candidates, err := store.List(context.Background()) + if err != nil { + return false + } + for _, candidate := range candidates { if candidate.ParentExecutionID == parent { found = candidate return true @@ -310,8 +314,8 @@ func TestDetachedWorkSurvivesInstanceShutdown(t *testing.T) { if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(parent); err != nil { - t.Fatalf("store.Add(parent): %v", err) + if err := store.Add(ctx, parent); err != nil { + t.Fatalf("store.Add(ctx, parent): %v", err) } if err := bus.Publish(ctx, event.New(event.ExecutionStarted, parent.ID, parent.CorrelationID, "", event.ExecutionStartedPayload{Params: map[string]any{}})); err != nil { t.Fatalf("publish ExecutionStarted: %v", err) @@ -395,8 +399,8 @@ func TestOrdinaryWorkIsStillCancelledAtShutdown(t *testing.T) { if err != nil { t.Fatalf("NewExecution: %v", err) } - if err := store.Add(root); err != nil { - t.Fatalf("store.Add(root): %v", err) + if err := store.Add(ctx, root); err != nil { + t.Fatalf("store.Add: %v", err) } if err := bus.Publish(ctx, event.New(event.ExecutionStarted, root.ID, root.CorrelationID, "", event.ExecutionStartedPayload{Params: map[string]any{}})); err != nil { t.Fatalf("publish ExecutionStarted: %v", err) diff --git a/nore/internal/execution/scheduler/scheduler.go b/nore/internal/execution/scheduler/scheduler.go index f48a1e6..2afc5da 100644 --- a/nore/internal/execution/scheduler/scheduler.go +++ b/nore/internal/execution/scheduler/scheduler.go @@ -4,11 +4,13 @@ import ( "context" "errors" "fmt" + "log/slog" "github.com/neuron-runtime/neuron/nore/internal/contracts" "github.com/neuron-runtime/neuron/nore/internal/data" "github.com/neuron-runtime/neuron/nore/internal/event" executionmodel "github.com/neuron-runtime/neuron/nore/internal/execution" + "github.com/neuron-runtime/neuron/nore/internal/storage" "github.com/neuron-runtime/neuron/shared/types/core" ) @@ -113,10 +115,13 @@ func (s *Scheduler) Run(ctx context.Context) error { // The task's input is the detached capability's params, not the root execution's, // because the task's entry is that one capability rather than the assembly. func (s *Scheduler) onCapabilityDetached(ctx context.Context, received event.Event) error { - execution, exists := s.executions.Get(received.Metadata.ExecutionID) - if !exists { + execution, err := s.executions.Get(ctx, received.Metadata.ExecutionID) + if errors.Is(err, storage.ErrNotFound) { return fmt.Errorf("execution %s was not found", received.Metadata.ExecutionID) } + if err != nil { + return fmt.Errorf("load execution %s: %w", received.Metadata.ExecutionID, err) + } if execution.IsTerminal() { return nil } @@ -131,7 +136,7 @@ func (s *Scheduler) onCapabilityDetached(ctx context.Context, received event.Eve if err != nil { return err } - if err := s.executions.Add(task); err != nil { + if err := s.executions.Add(ctx, task); err != nil { return fmt.Errorf("persist detached task for capability %s: %w", capabilityID, err) } @@ -139,14 +144,18 @@ func (s *Scheduler) onCapabilityDetached(ctx context.Context, received event.Eve if err := execution.MarkCapabilityDetached(capabilityID); err != nil { // The task was never started, so leaving it behind would strand work no // one will run. - s.executions.Delete(task.ID) + if deleteErr := s.executions.Delete(ctx, task.ID); deleteErr != nil { + return errors.Join(err, fmt.Errorf("remove unstarted detached task %s: %w", task.ID, deleteErr)) + } return err } remaining, err := execution.CompleteCurrentAndSchedule(0) if err != nil { // The parent is corrupt, but the task has not started. Remove it rather // than leave an execution no one will ever run. - s.executions.Delete(task.ID) + if deleteErr := s.executions.Delete(ctx, task.ID); deleteErr != nil { + return errors.Join(err, fmt.Errorf("remove unstarted detached task %s: %w", task.ID, deleteErr)) + } return err } @@ -168,23 +177,26 @@ func (s *Scheduler) onCapabilityDetached(ctx context.Context, received event.Eve // It reports whether this call cancelled the execution. A false return means the // execution was already terminal or unknown, and the caller must not claim it // cancelled something that had already finished. -func (s *Scheduler) CancelExecution(ctx context.Context, executionID core.ID, reason error) bool { - execution, exists := s.executions.Get(executionID) - if !exists { - return false +func (s *Scheduler) CancelExecution(ctx context.Context, executionID core.ID, reason error) (bool, error) { + execution, err := s.executions.Get(ctx, executionID) + if errors.Is(err, storage.ErrNotFound) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("load execution %s: %w", executionID, err) } // Stop the work even when the state transition loses the race below: a // capability must not keep running just because the execution completed in // the same instant the cancellation arrived. s.scopes.Cancel(executionID) if !execution.MarkCancelled(reason) { - return false + return false, nil } s.scopes.Release(executionID) _ = s.bus.Publish(ctx, event.New(event.ExecutionCancelled, execution.ID, execution.CorrelationID, "", event.ExecutionCancelledPayload{ Message: reasonMessage(reason), })) - return true + return true, nil } // reasonMessage renders a cancellation reason for the event payload, which is @@ -197,10 +209,13 @@ func reasonMessage(reason error) string { } func (s *Scheduler) onExecutionStarted(ctx context.Context, received event.Event) error { - execution, exists := s.executions.Get(received.Metadata.ExecutionID) - if !exists { + execution, err := s.executions.Get(ctx, received.Metadata.ExecutionID) + if errors.Is(err, storage.ErrNotFound) { return fmt.Errorf("execution %s was not found", received.Metadata.ExecutionID) } + if err != nil { + return fmt.Errorf("load execution %s: %w", received.Metadata.ExecutionID, err) + } payload, ok := received.Payload.(event.ExecutionStartedPayload) if !ok { return fmt.Errorf("invalid ExecutionStarted payload") @@ -238,10 +253,13 @@ func (s *Scheduler) onExecutionStarted(ctx context.Context, received event.Event } func (s *Scheduler) onCapabilityCompleted(ctx context.Context, received event.Event) error { - execution, exists := s.executions.Get(received.Metadata.ExecutionID) - if !exists { + execution, err := s.executions.Get(ctx, received.Metadata.ExecutionID) + if errors.Is(err, storage.ErrNotFound) { return fmt.Errorf("execution %s was not found", received.Metadata.ExecutionID) } + if err != nil { + return fmt.Errorf("load execution %s: %w", received.Metadata.ExecutionID, err) + } if execution.IsTerminal() { return nil } @@ -261,6 +279,20 @@ func (s *Scheduler) onCapabilityCompleted(ctx context.Context, received event.Ev if !exists { return fmt.Errorf("target capability %s is missing", transition.TargetCapabilityID) } + targetNode := execution.Blueprint.Nodes[transition.TargetCapabilityID] + // For fan-in (JoinPolicyAll with multiple predecessors), check that all + // predecessors have completed before scheduling. RecordCompletedBinding + // is idempotent and tracks completion of each predecessor binding. + if len(targetNode.Predecessors) > 1 { + allCompleted := execution.RecordCompletedBinding(transition.TargetCapabilityID, transition.BindingID) + if !allCompleted { + continue + } + } else { + // For single-predecessor targets, record the binding completion so + // the join state is updated, but schedule immediately (original behavior). + execution.RecordCompletedBinding(transition.TargetCapabilityID, transition.BindingID) + } environment := buildTransitionEnvironment(execution, node, output) if err := validateTransition(ctx, environment, transition); err != nil { return err @@ -314,8 +346,18 @@ func (s *Scheduler) completeIfDrained(ctx context.Context, execution *executionm } func (s *Scheduler) failExecution(ctx context.Context, executionID core.ID, err error) { - execution, exists := s.executions.Get(executionID) - if !exists || !execution.MarkFailed(err) { + execution, loadErr := s.executions.Get(ctx, executionID) + if errors.Is(loadErr, storage.ErrNotFound) { + return + } + if loadErr != nil { + slog.Error("load execution for failure transition", + slog.String("execution_id", string(executionID)), + slog.String("error", loadErr.Error()), + ) + return + } + if !execution.MarkFailed(err) { return } s.scopes.Release(execution.ID) diff --git a/nore/internal/execution/scoping_test.go b/nore/internal/execution/scoping_test.go index 821a068..41b2356 100644 --- a/nore/internal/execution/scoping_test.go +++ b/nore/internal/execution/scoping_test.go @@ -12,7 +12,8 @@ import ( func openTestStore(t *testing.T) storage.Store { t.Helper() - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + ctx := context.Background() + store, err := sqlite.New(ctx, storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } @@ -47,16 +48,22 @@ func TestListByInstanceScopesExecutions(t *testing.T) { b1 := newTestExecution(t, bp, "inst_b") for _, exec := range []*Execution{a1, a2, b1} { - if err := s.Add(exec); err != nil { + if err := s.Add(context.Background(), exec); err != nil { t.Fatalf("add execution: %v", err) } } - gotA := s.ListByInstance("inst_a") + gotA, err := s.ListByInstance(context.Background(), "inst_a") + if err != nil { + t.Fatalf("list inst_a: %v", err) + } if len(gotA) != 2 { t.Fatalf("ListByInstance(inst_a) = %d executions, want 2", len(gotA)) } - gotB := s.ListByInstance("inst_b") + gotB, err := s.ListByInstance(context.Background(), "inst_b") + if err != nil { + t.Fatalf("list inst_b: %v", err) + } if len(gotB) != 1 { t.Fatalf("ListByInstance(inst_b) = %d executions, want 1", len(gotB)) } @@ -68,7 +75,7 @@ func TestExecutionInstanceIDRoundTripsSnapshot(t *testing.T) { bp := &types.ExecutionBlueprint{Nodes: map[shared.ID]types.ExecutionNode{}} exec := newTestExecution(t, bp, "inst_x") - if err := s.Add(exec); err != nil { + if err := s.Add(context.Background(), exec); err != nil { t.Fatalf("add execution: %v", err) } if err := exec.Start(nil, 1); err != nil { @@ -81,9 +88,9 @@ func TestExecutionInstanceIDRoundTripsSnapshot(t *testing.T) { t.Fatalf("save execution: %v", err) } - loaded, ok := s.Get(exec.ID) - if !ok { - t.Fatalf("get execution: not found") + loaded, err := s.Get(context.Background(), exec.ID) + if err != nil { + t.Fatalf("get execution: %v", err) } if loaded.InstanceID != "inst_x" { t.Errorf("loaded InstanceID = %q, want inst_x", loaded.InstanceID) diff --git a/nore/internal/execution/snapshot.go b/nore/internal/execution/snapshot.go index 0967f47..ac01890 100644 --- a/nore/internal/execution/snapshot.go +++ b/nore/internal/execution/snapshot.go @@ -13,6 +13,7 @@ type ExecutionSnapshot struct { CorrelationID core.ID `json:"correlation_id"` InstanceID core.ID `json:"instance_id,omitempty"` ParentExecutionID core.ID `json:"parent_execution_id,omitempty"` + Revision uint64 `json:"revision"` Status Status `json:"status"` InitialParams map[string]any `json:"initial_params,omitempty"` Params map[core.ID]map[string]any `json:"params,omitempty"` @@ -27,7 +28,17 @@ type ExecutionSnapshot struct { func (e *Execution) Snapshot() *ExecutionSnapshot { e.mu.RLock() defer e.mu.RUnlock() + return e.snapshotLocked(e.revision) +} + +func (e *Execution) snapshotAt(revision uint64) *ExecutionSnapshot { + e.mu.RLock() + defer e.mu.RUnlock() + return e.snapshotLocked(revision) +} +// snapshotLocked builds a snapshot while the caller holds e.mu. +func (e *Execution) snapshotLocked(revision uint64) *ExecutionSnapshot { states := make(map[core.ID]CapabilityExecutionState, len(e.states)) for id, state := range e.states { states[id] = state @@ -38,6 +49,7 @@ func (e *Execution) Snapshot() *ExecutionSnapshot { CorrelationID: e.CorrelationID, InstanceID: e.InstanceID, ParentExecutionID: e.ParentExecutionID, + Revision: revision, Status: e.status, InitialParams: data.CanonicalMap(e.initialParams), Params: cloneParamsMap(e.params), @@ -55,6 +67,7 @@ func (e *Execution) Restore(snapshot *ExecutionSnapshot) { defer e.mu.Unlock() e.status = snapshot.Status + e.revision = snapshot.Revision e.initialParams = data.CanonicalMap(snapshot.InitialParams) e.params = cloneParamsMap(snapshot.Params) e.results = cloneParamsMap(snapshot.Results) @@ -82,6 +95,7 @@ func UnmarshalExecution(data []byte) (*Execution, error) { CorrelationID: snap.CorrelationID, InstanceID: snap.InstanceID, ParentExecutionID: snap.ParentExecutionID, + revision: snap.Revision, status: snap.Status, initialParams: snap.InitialParams, params: snap.Params, diff --git a/nore/internal/execution/store.go b/nore/internal/execution/store.go index 284a786..04aac8d 100644 --- a/nore/internal/execution/store.go +++ b/nore/internal/execution/store.go @@ -2,161 +2,270 @@ package execution import ( "context" + "encoding/json" + "errors" "fmt" - "sync" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" shared "github.com/neuron-runtime/neuron/shared/types/core" ) +// ExecutionStore is the durable execution repository. Memory is a read-through +// cache for live executions; durable state is authoritative for everything not +// currently live. +// +// Concurrency contract: Save serializes writes for one Execution value with a +// per-execution mutex, then compares the snapshot revision with the stored +// revision inside one transaction. Two writers sharing the same Execution value +// therefore cannot conflict with one another; a conflict means another store or +// process holds a stale copy. That distinction is why the revision guard is +// meaningful instead of merely rejecting ordinary same-process races. type ExecutionStore struct { - mu sync.RWMutex mem *MemoryStore store storage.Store } func NewExecutionStore(store storage.Store) *ExecutionStore { - return &ExecutionStore{ - mem: NewMemoryStore(), - store: store, - } + return &ExecutionStore{mem: NewMemoryStore(), store: store} } -func (s *ExecutionStore) Add(execution *Execution) error { - if err := s.mem.Add(execution); err != nil { - return err +// Add persists a new execution and only then inserts it into memory. It is +// insert-only: an existing durable record makes Add fail with +// storage.ErrAlreadyExists rather than overwriting a previous execution. +func (s *ExecutionStore) Add(ctx context.Context, execution *Execution) error { + if execution == nil { + return fmt.Errorf("execution is required") } + execution.saveMu.Lock() + defer execution.saveMu.Unlock() - data, err := MarshalExecution(execution) + key, err := s.key(execution.ID) + if err != nil { + return err + } + data, err := marshalExecutionAt(execution, 1) if err != nil { return fmt.Errorf("marshal execution: %w", err) } - if err := s.store.Put(context.Background(), s.key(execution.ID), data); err != nil { - return fmt.Errorf("persist execution: %w", err) + err = s.store.WithTx(ctx, func(tx storage.Tx) error { + if _, err := tx.Get(ctx, key); err == nil { + return fmt.Errorf("%w: execution %s", storage.ErrAlreadyExists, execution.ID) + } else if !errors.Is(err, storage.ErrNotFound) { + return fmt.Errorf("read execution %s: %w", execution.ID, err) + } + return tx.Put(ctx, key, data) + }) + if err != nil { + return fmt.Errorf("persist execution %s: %w", execution.ID, err) } + if err := s.mem.Add(ctx, execution); err != nil { + // Memory refused the execution after the durable write succeeded. Remove + // the durable record so a later restart does not surface an execution the + // live store never accepted. + if cleanupErr := s.store.Delete(ctx, key); cleanupErr != nil { + return errors.Join(err, fmt.Errorf("remove unaccepted execution %s: %w", execution.ID, cleanupErr)) + } + return err + } + + execution.setRevision(1) return nil } -func (s *ExecutionStore) Get(executionID shared.ID) (*Execution, bool) { - exec, ok := s.mem.Get(executionID) - if ok { - return exec, true +// Get returns the live execution when present, otherwise the durable record. A +// missing execution reports storage.ErrNotFound; a backend or decode failure is +// returned as an error and never converted into absence. +func (s *ExecutionStore) Get(ctx context.Context, executionID shared.ID) (*Execution, error) { + exec, err := s.mem.Get(ctx, executionID) + if err == nil { + return exec, nil } - - return s.loadFromStore(executionID) + if !errors.Is(err, storage.ErrNotFound) { + return nil, err + } + return s.loadFromStore(ctx, executionID) } +// Save persists execution's current snapshot with revision compare-and-swap and +// only then records it in memory. The snapshot revision is incremented on +// success. A stale revision reports storage.ErrConflict and leaves both durable +// and in-memory state untouched. +// +// Persisting before the memory update is deliberate: a caller must never observe +// an execution as live that durable storage rejected. func (s *ExecutionStore) Save(ctx context.Context, execution *Execution) error { - if err := s.mem.Save(ctx, execution); err != nil { - return err + if execution == nil { + return fmt.Errorf("execution is required") } + execution.saveMu.Lock() + defer execution.saveMu.Unlock() - data, err := MarshalExecution(execution) + key, err := s.key(execution.ID) if err != nil { - return fmt.Errorf("marshal execution: %w", err) + return err } - if err := s.store.Put(ctx, s.key(execution.ID), data); err != nil { - return fmt.Errorf("persist execution: %w", err) + var nextRevision uint64 + err = s.store.WithTx(ctx, func(tx storage.Tx) error { + currentRevision, err := storedRevision(ctx, tx, key) + if err != nil { + return err + } + if execution.Revision() != currentRevision { + return fmt.Errorf("%w: execution %s is at revision %d, stored revision is %d", + storage.ErrConflict, execution.ID, execution.Revision(), currentRevision) + } + + nextRevision = currentRevision + 1 + data, err := marshalExecutionAt(execution, nextRevision) + if err != nil { + return err + } + return tx.Put(ctx, key, data) + }) + if err != nil { + return fmt.Errorf("persist execution %s: %w", execution.ID, err) } + execution.setRevision(nextRevision) + if err := s.mem.Save(ctx, execution); err != nil { + return fmt.Errorf("record execution %s in memory after durable save: %w", execution.ID, err) + } return nil } -func (s *ExecutionStore) Delete(executionID shared.ID) { - s.mu.Lock() - defer s.mu.Unlock() - - s.mem.Delete(executionID) - _ = s.store.Delete(context.Background(), s.key(executionID)) - +// Delete removes the durable record before the in-memory record. A failed +// durable delete leaves memory intact so the execution remains readable. +func (s *ExecutionStore) Delete(ctx context.Context, executionID shared.ID) error { + key, err := s.key(executionID) + if err != nil { + return err + } + if err := s.store.Delete(ctx, key); err != nil { + return fmt.Errorf("delete execution %s: %w", executionID, err) + } + if err := s.mem.Delete(ctx, executionID); err != nil { + return fmt.Errorf("remove execution %s from memory: %w", executionID, err) + } + return nil } -func (s *ExecutionStore) List() []*Execution { - s.mu.RLock() - keys, err := s.store.List(context.Background(), "executions/") - s.mu.RUnlock() - +// List merges live executions with durable records. A storage or decode failure +// is returned; it is never replaced with a partial in-memory list. +func (s *ExecutionStore) List(ctx context.Context) ([]*Execution, error) { + prefix, err := storage.Prefix("executions") if err != nil { - return s.mem.List() + return nil, err + } + keys, err := s.store.List(ctx, prefix) + if err != nil { + return nil, fmt.Errorf("list executions: %w", err) } result := make([]*Execution, 0, len(keys)) seen := make(map[shared.ID]bool) - live := s.mem.List() + live, err := s.mem.List(ctx) + if err != nil { + return nil, err + } for _, exec := range live { result = append(result, exec) seen[exec.ID] = true } for _, key := range keys { - id := shared.ID(extractID(key)) + id, err := executionIDFromKey(key) + if err != nil { + return nil, err + } if seen[id] { continue } - exec, ok := s.loadFromStore(id) - if ok { - result = append(result, exec) + exec, err := s.loadFromStore(ctx, id) + if errors.Is(err, storage.ErrNotFound) { + // The record was deleted between List and Get. + continue + } + if err != nil { + return nil, err } + result = append(result, exec) + seen[id] = true } - return result + return result, nil } -func (s *ExecutionStore) ListByInstance(instanceID shared.ID) []*Execution { - s.mu.RLock() - keys, err := s.store.List(context.Background(), "executions/") - s.mu.RUnlock() - - result := s.mem.ListByInstance(instanceID) - seen := make(map[shared.ID]bool, len(result)) - for _, exec := range result { - seen[exec.ID] = true - } - +// ListByInstance returns every execution belonging to instanceID from both +// memory and durable storage. Storage and decode failures are returned. +func (s *ExecutionStore) ListByInstance(ctx context.Context, instanceID shared.ID) ([]*Execution, error) { + all, err := s.List(ctx) if err != nil { - return result + return nil, err } - for _, key := range keys { - id := shared.ID(extractID(key)) - if seen[id] { - continue - } - exec, ok := s.loadFromStore(id) - if !ok || exec.InstanceID != instanceID { - continue + result := make([]*Execution, 0) + for _, exec := range all { + if exec.InstanceID == instanceID { + result = append(result, exec) } - result = append(result, exec) } - return result + return result, nil } -func (s *ExecutionStore) loadFromStore(executionID shared.ID) (*Execution, bool) { - data, err := s.store.Get(context.Background(), s.key(executionID)) +func (s *ExecutionStore) loadFromStore(ctx context.Context, executionID shared.ID) (*Execution, error) { + key, err := s.key(executionID) if err != nil { - return nil, false + return nil, err + } + data, err := s.store.Get(ctx, key) + if err != nil { + return nil, err } - exec, err := UnmarshalExecution(data) if err != nil { - return nil, false + return nil, fmt.Errorf("decode execution %s: %w", executionID, err) + } + if exec.ID != executionID { + return nil, fmt.Errorf("decode execution %s: stored id is %s", executionID, exec.ID) } + return exec, nil +} - return exec, true +func (s *ExecutionStore) key(id shared.ID) (string, error) { + return storage.Key("executions", string(id)) } -func (s *ExecutionStore) key(id shared.ID) string { - return sqlite.SanitizeKey("executions", string(id)) +func executionIDFromKey(key string) (shared.ID, error) { + prefix, err := storage.Prefix("executions") + if err != nil { + return "", err + } + if len(key) <= len(prefix) || key[:len(prefix)] != prefix { + return "", fmt.Errorf("decode execution key %q: unexpected prefix", key) + } + return shared.ID(key[len(prefix):]), nil } -func extractID(key string) string { - prefix := "executions/" - if len(key) > len(prefix) { - return key[len(prefix):] +func storedRevision(ctx context.Context, tx storage.Tx, key string) (uint64, error) { + data, err := tx.Get(ctx, key) + if errors.Is(err, storage.ErrNotFound) { + return 0, nil + } + if err != nil { + return 0, fmt.Errorf("read persisted execution revision: %w", err) } - return key + var record struct { + Revision uint64 `json:"revision"` + } + if err := json.Unmarshal(data, &record); err != nil { + return 0, fmt.Errorf("decode persisted execution revision: %w", err) + } + return record.Revision, nil +} + +func marshalExecutionAt(execution *Execution, revision uint64) ([]byte, error) { + return json.Marshal(execution.snapshotAt(revision)) } diff --git a/nore/internal/execution/store_failure_test.go b/nore/internal/execution/store_failure_test.go new file mode 100644 index 0000000..dc42df3 --- /dev/null +++ b/nore/internal/execution/store_failure_test.go @@ -0,0 +1,145 @@ +package execution + +import ( + "context" + "errors" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/types" + shared "github.com/neuron-runtime/neuron/shared/types/core" +) + +var errInjected = errors.New("injected storage failure") + +// faultStore lets a test fail one storage operation while leaving the rest +// behaving like an empty backend. It is used to prove that runtime code reports +// failures instead of converting them into absence, partial data, or success. +type faultStore struct { + get func(string) ([]byte, error) + put func(string, []byte) error + del func(string) error + list func(string) ([]string, error) + withTx func(context.Context, func(storage.Tx) error) error +} + +func (f *faultStore) Get(_ context.Context, key string) ([]byte, error) { + if f.get != nil { + return f.get(key) + } + return nil, storage.ErrNotFound +} + +func (f *faultStore) Put(_ context.Context, key string, data []byte) error { + if f.put != nil { + return f.put(key, data) + } + return nil +} + +func (f *faultStore) Delete(_ context.Context, key string) error { + if f.del != nil { + return f.del(key) + } + return nil +} + +func (f *faultStore) List(_ context.Context, prefix string) ([]string, error) { + if f.list != nil { + return f.list(prefix) + } + return nil, nil +} + +func (f *faultStore) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + if f.withTx != nil { + return f.withTx(ctx, fn) + } + return fn(f) +} + +func (f *faultStore) Ping(context.Context) error { return nil } +func (f *faultStore) Close() error { return nil } + +func testExecution(t *testing.T) *Execution { + t.Helper() + exec, err := NewExecution(&types.ExecutionBlueprint{Nodes: map[shared.ID]types.ExecutionNode{}}, shared.NewID("corr_"), "inst_test") + if err != nil { + t.Fatalf("new execution: %v", err) + } + return exec +} + +func TestExecutionStoreSurfacesReadFailure(t *testing.T) { + s := NewExecutionStore(&faultStore{get: func(string) ([]byte, error) { return nil, errInjected }}) + _, err := s.Get(context.Background(), "exec_missing") + if !errors.Is(err, errInjected) { + t.Fatalf("Get error = %v, want injected failure", err) + } +} + +func TestExecutionStoreReportsDecodeFailureAsError(t *testing.T) { + s := NewExecutionStore(&faultStore{get: func(string) ([]byte, error) { return []byte("{not json"), nil }}) + _, err := s.Get(context.Background(), "exec_bad") + if err == nil || errors.Is(err, storage.ErrNotFound) { + t.Fatalf("Get malformed execution = %v, want a decode error", err) + } +} + +func TestExecutionStoreAddDoesNotCacheWhenTransactionFails(t *testing.T) { + store := &faultStore{withTx: func(context.Context, func(storage.Tx) error) error { return errInjected }} + s := NewExecutionStore(store) + exec := testExecution(t) + + if err := s.Add(context.Background(), exec); !errors.Is(err, errInjected) { + t.Fatalf("Add error = %v, want injected failure", err) + } + if _, err := s.mem.Get(context.Background(), exec.ID); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("failed Add left execution in memory: %v", err) + } +} + +func TestExecutionStoreAddDoesNotCacheWhenPutFails(t *testing.T) { + store := &faultStore{put: func(string, []byte) error { return errInjected }} + s := NewExecutionStore(store) + exec := testExecution(t) + + if err := s.Add(context.Background(), exec); !errors.Is(err, errInjected) { + t.Fatalf("Add error = %v, want injected failure", err) + } + if _, err := s.mem.Get(context.Background(), exec.ID); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("failed Add left execution in memory: %v", err) + } +} + +func TestExecutionStoreListSurfacesStorageFailure(t *testing.T) { + store := &faultStore{list: func(string) ([]string, error) { return nil, errInjected }} + s := NewExecutionStore(store) + exec := testExecution(t) + if err := s.mem.Save(context.Background(), exec); err != nil { + t.Fatalf("seed memory: %v", err) + } + + if _, err := s.List(context.Background()); !errors.Is(err, errInjected) { + t.Fatalf("List error = %v, want injected failure", err) + } + if _, err := s.ListByInstance(context.Background(), exec.InstanceID); !errors.Is(err, errInjected) { + t.Fatalf("ListByInstance error = %v, want injected failure", err) + } +} + +func TestExecutionStoreDeleteKeepsMemoryWhenBackendFails(t *testing.T) { + store := &faultStore{del: func(string) error { return errInjected }} + s := NewExecutionStore(store) + exec := testExecution(t) + if err := s.mem.Save(context.Background(), exec); err != nil { + t.Fatalf("seed memory: %v", err) + } + + if err := s.Delete(context.Background(), exec.ID); !errors.Is(err, errInjected) { + t.Fatalf("Delete error = %v, want injected failure", err) + } + if _, err := s.mem.Get(context.Background(), exec.ID); err != nil { + t.Fatalf("failed Delete removed execution from memory: %v", err) + } +} diff --git a/nore/internal/execution/store_revision_test.go b/nore/internal/execution/store_revision_test.go new file mode 100644 index 0000000..7bfee36 --- /dev/null +++ b/nore/internal/execution/store_revision_test.go @@ -0,0 +1,114 @@ +package execution + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/types" + shared "github.com/neuron-runtime/neuron/shared/types/core" +) + +func TestSaveRejectsStaleWriter(t *testing.T) { + store := openTestStore(t) + s := NewExecutionStore(store) + ctx := context.Background() + exec, err := NewExecution(&types.ExecutionBlueprint{Nodes: map[shared.ID]types.ExecutionNode{}}, shared.NewID("corr_"), "inst_test") + if err != nil { + t.Fatalf("new execution: %v", err) + } + if err := s.Add(ctx, exec); err != nil { + t.Fatalf("add execution: %v", err) + } + + first, err := UnmarshalExecution(mustMarshal(t, exec)) + if err != nil { + t.Fatalf("copy execution: %v", err) + } + second, err := UnmarshalExecution(mustMarshal(t, exec)) + if err != nil { + t.Fatalf("copy execution: %v", err) + } + + start := make(chan struct{}) + errs := make(chan error, 2) + var wg sync.WaitGroup + for _, candidate := range []*Execution{first, second} { + candidate := candidate + wg.Add(1) + go func() { + defer wg.Done() + <-start + errs <- s.Save(ctx, candidate) + }() + } + close(start) + wg.Wait() + close(errs) + + var successes, conflicts int + for err := range errs { + switch { + case err == nil: + successes++ + case errors.Is(err, storage.ErrConflict): + conflicts++ + default: + t.Fatalf("save error = %v, want success or ErrConflict", err) + } + } + if successes != 1 || conflicts != 1 { + t.Fatalf("concurrent saves: %d successes, %d conflicts; want 1 and 1", successes, conflicts) + } +} + +func TestSaveUsesCallerCancellation(t *testing.T) { + store := openTestStore(t) + s := NewExecutionStore(store) + ctx := context.Background() + exec, err := NewExecution(&types.ExecutionBlueprint{Nodes: map[shared.ID]types.ExecutionNode{}}, shared.NewID("corr_"), "inst_test") + if err != nil { + t.Fatalf("new execution: %v", err) + } + if err := s.Add(ctx, exec); err != nil { + t.Fatalf("add execution: %v", err) + } + + cancelled, cancel := context.WithCancel(ctx) + cancel() + if err := s.Save(cancelled, exec); err == nil { + t.Fatal("Save with cancelled context succeeded") + } + + durable, err := store.Get(ctx, mustExecutionKey(t, exec.ID)) + if err != nil { + t.Fatalf("read durable execution: %v", err) + } + loaded, err := UnmarshalExecution(durable) + if err != nil { + t.Fatalf("decode durable execution: %v", err) + } + if loaded.Revision() != 1 { + t.Fatalf("durable revision after cancelled Save = %d, want 1", loaded.Revision()) + } +} + +func mustMarshal(t *testing.T, exec *Execution) []byte { + t.Helper() + data, err := MarshalExecution(exec) + if err != nil { + t.Fatalf("marshal execution: %v", err) + } + return data +} + +func mustExecutionKey(t *testing.T, id shared.ID) string { + t.Helper() + key, err := storage.Key("executions", string(id)) + if err != nil { + t.Fatalf("execution key: %v", err) + } + return key +} diff --git a/nore/internal/instance/cancel_test.go b/nore/internal/instance/cancel_test.go index c9cfc44..ec6158a 100644 --- a/nore/internal/instance/cancel_test.go +++ b/nore/internal/instance/cancel_test.go @@ -13,13 +13,13 @@ import ( "github.com/neuron-runtime/neuron/shared/types/protocol" ) -// Cancel is only honest if the work actually stops. The delay runtime blocks until +// Cancel is only honest if the work actually stops. The command runtime blocks until // its context dies, so an execution observed in flight is genuinely running code -- // which makes this the boundary at which "cancelled" can be proven end to end // rather than inferred from a status string. func TestCancelExecutionStopsTheRunningWork(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.CancelExecution(context.Background(), exec.ID, errors.New("stopped by operator")); err != nil { @@ -49,7 +49,7 @@ func TestCancelExecutionStopsTheRunningWork(t *testing.T) { // deliberately, and blaming it would misdirect whoever reads the execution. func TestCancelExecutionDoesNotReportTheAbortedCapabilityAsFailed(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.CancelExecution(context.Background(), exec.ID, errors.New("stopped by operator")); err != nil { @@ -59,7 +59,7 @@ func TestCancelExecutionDoesNotReportTheAbortedCapabilityAsFailed(t *testing.T) // The engine unwinds asynchronously after the cancellation is recorded. time.Sleep(250 * time.Millisecond) - if got := exec.CapabilityState("cap_wait").Status; got != execution.CapabilityCancelled { + if got := exec.CapabilityState("cap_blocking").Status; got != execution.CapabilityCancelled { t.Errorf("capability status = %s, want %s", got, execution.CapabilityCancelled) } } @@ -69,8 +69,8 @@ func TestCancelExecutionDoesNotReportTheAbortedCapabilityAsFailed(t *testing.T) // took effect when the work had already concluded on its own. func TestCancelExecutionRefusesACompletedExecution(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) - exec, err := i.Execute(context.Background(), map[string]any{"duration": "1ms"}) + i := startBlockingInstance(t, store) + exec, err := i.Execute(context.Background(), shortParams()) if err != nil { t.Fatalf("execute: %v", err) } @@ -101,7 +101,7 @@ func TestCancelExecutionRefusesACompletedExecution(t *testing.T) { // able to tell that apart from a refusal. func TestCancelExecutionReportsAnUnknownExecution(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) err := i.CancelExecution(context.Background(), "exec_does_not_exist", errors.New("stop")) if !errors.Is(err, ErrExecutionNotFound) { @@ -121,7 +121,7 @@ func TestCancelExecutionReportsAnUnknownExecution(t *testing.T) { // already happened -- stopping the instance first would race it and prove nothing. func TestCancellationIsPersisted(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.CancelExecution(context.Background(), exec.ID, errors.New("stopped by operator")); err != nil { @@ -172,16 +172,16 @@ func waitForPersistedStatus(t *testing.T, store *sqlite.Store, executionID share // and the honest answer there is that there is nothing left to stop. func TestCancelOnARestoredInstanceReportsTheExecutionAsFinished(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.Stop(); err != nil { t.Fatalf("stop instance: %v", err) } - assembly := delayAssembly() + assembly := blockingAssembly() restored, err := New(context.Background(), i.ID, protocol.InstanceKey{ - AssemblyID: "sys_delay", Version: "1.0.0", + AssemblyID: "sys_blocking", Version: "1.0.0", }, &assembly, 2, store) if err != nil { t.Fatalf("restore instance: %v", err) @@ -199,7 +199,7 @@ func TestCancelOnARestoredInstanceReportsTheExecutionAsFinished(t *testing.T) { // until its own timeout. func TestCancelExecutionPublishesTheCancellation(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) subscription, err := i.Bus().SubscribeExecution(exec.ID, 16) diff --git a/nore/internal/instance/drain.go b/nore/internal/instance/drain.go index 1eedc4b..73f070a 100644 --- a/nore/internal/instance/drain.go +++ b/nore/internal/instance/drain.go @@ -2,8 +2,9 @@ package instance import ( "context" + "errors" "fmt" - "log/slog" + "time" "github.com/neuron-runtime/neuron/nore/internal/contracts" executionmodel "github.com/neuron-runtime/neuron/nore/internal/execution" @@ -32,6 +33,12 @@ import ( // shutdown race. Exactly one transition can take effect, because MarkCompleted // and MarkFailed both refuse an execution that is already terminal. +// sweepPersistenceTimeout bounds persistence work that intentionally outlives +// the request or instance context it was triggered by (shutdown sweeps and +// restored metadata). It is a policy for that write, not a claim that the work +// can resume. +const sweepPersistenceTimeout = 10 * time.Second + // sweepUnresumableExecutions drives every execution of an instance to failed, // persisting each one. // @@ -40,8 +47,8 @@ import ( // them as running would be a permanent lie rather than a temporary inaccuracy. // Detached executions are included here for exactly that reason: without the // runtime that owned them they cannot resume at all. -func sweepUnresumableExecutions(store contracts.ExecutionRepository, instanceID shared.ID, reason error) { - sweepExecutions(store, instanceID, reason, nil) +func sweepUnresumableExecutions(ctx context.Context, store contracts.ExecutionRepository, instanceID shared.ID, reason error) error { + return sweepExecutions(ctx, store, instanceID, reason, nil) } // sweepAbandonedExecutions drives the executions an instance owned to failed @@ -54,8 +61,8 @@ func sweepUnresumableExecutions(store contracts.ExecutionRepository, instanceID // while the work it describes may well have completed moments later. It is left // instead for the restore path, which records the accurate reason -- that the // process ended with no runtime to resume it -- rather than a misleading one. -func sweepAbandonedExecutions(store contracts.ExecutionRepository, instanceID shared.ID, reason error) { - sweepExecutions(store, instanceID, reason, func(e *executionmodel.Execution) bool { +func sweepAbandonedExecutions(ctx context.Context, store contracts.ExecutionRepository, instanceID shared.ID, reason error) error { + return sweepExecutions(ctx, store, instanceID, reason, func(e *executionmodel.Execution) bool { return e.ParentExecutionID != "" }) } @@ -68,11 +75,17 @@ func sweepAbandonedExecutions(store contracts.ExecutionRepository, instanceID sh // the event persister has already returned, and it would save through the // cancelled instance context in any case. There is also nothing left streaming to // inform, so a publish would be discarded rather than observed. -func sweepExecutions(store contracts.ExecutionRepository, instanceID shared.ID, reason error, skip func(*executionmodel.Execution) bool) { +func sweepExecutions(ctx context.Context, store contracts.ExecutionRepository, instanceID shared.ID, reason error, skip func(*executionmodel.Execution) bool) error { if store == nil { - return + return nil + } + executions, err := store.ListByInstance(ctx, instanceID) + if err != nil { + return fmt.Errorf("list executions for instance %s: %w", instanceID, err) } - for _, exec := range store.ListByInstance(instanceID) { + + var failures []error + for _, exec := range executions { if exec.IsTerminal() { continue } @@ -85,16 +98,13 @@ func sweepExecutions(store contracts.ExecutionRepository, instanceID shared.ID, continue } // The instance context is already cancelled, so persistence cannot be - // bounded by it; a bounded context would risk abandoning the write and - // leaving the stranded snapshot on disk. - if err := store.Save(context.Background(), exec); err != nil { - slog.Error("persist terminal execution state", - slog.String("execution_id", string(exec.ID)), - slog.String("instance_id", string(instanceID)), - slog.String("error", err.Error()), - ) + // bounded by it. Callers pass a bounded background context for shutdown + // sweeps; restore passes the live startup context. + if err := store.Save(ctx, exec); err != nil { + failures = append(failures, fmt.Errorf("persist failed execution %s: %w", exec.ID, err)) } } + return errors.Join(failures...) } // stoppedBeforeFinished explains that an execution lost its instance before it diff --git a/nore/internal/instance/drain_test.go b/nore/internal/instance/drain_test.go index a135614..3ece46b 100644 --- a/nore/internal/instance/drain_test.go +++ b/nore/internal/instance/drain_test.go @@ -2,6 +2,7 @@ package instance import ( "context" + "os" "testing" "time" @@ -16,7 +17,7 @@ import ( func newTestStore(t *testing.T) *sqlite.Store { t.Helper() - store, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatalf("open sqlite store: %v", err) } @@ -24,31 +25,96 @@ func newTestStore(t *testing.T) *sqlite.Store { return store } -// delayAssembly is the assembly behind startDelayInstance: one capability whose -// implementation blocks until its context dies. -func delayAssembly() shared.Assembly { +// blockingHelperMarker is the argument that tells a re-executed test binary to +// act as the drain test's helper program. +// +// The built-in passes the program and its arguments to the operating system +// separately, so the helper is selected by an argument rather than by an +// environment variable or by a program that happens to be installed. +const blockingHelperMarker = "neuron-drain-helper" + +// TestBlockingHelperProcess is the program the drain test's capability runs. +// +// It is not a test in its own right. It only blocks when the test binary has +// been re-executed with blockingHelperMarker among its arguments, which is +// how the drain test produces a capability that stays in flight until its +// context dies without depending on a program being installed on the machine. +func TestBlockingHelperProcess(t *testing.T) { + if !isBlockingHelperProcess() { + t.Skip("helper process: only meaningful when re-executed by the drain test") + } + for _, arg := range os.Args { + if arg == "complete" { + os.Exit(0) + } + } + time.Sleep(time.Minute) + os.Exit(0) +} + +// isBlockingHelperProcess reports whether this test binary was re-executed to +// act as the helper program. +func isBlockingHelperProcess() bool { + for _, arg := range os.Args { + if arg == blockingHelperMarker { + return true + } + } + return false +} + +// blockingAssembly is the assembly behind startBlockingInstance: one +// capability whose implementation blocks until its context dies. +func blockingAssembly() shared.Assembly { return shared.Assembly{ - Metadata: shared.Metadata{ID: "sys_delay", Name: "sys_delay", Version: "1.0.0"}, + Metadata: shared.Metadata{ID: "sys_blocking", Name: "sys_blocking", Version: "1.0.0"}, Specification: shared.AssemblySpec{ Capabilities: []shared.Capability{{ - Metadata: shared.Metadata{ID: "cap_wait", Name: "cap_wait", Version: "1.0.0"}, - Type: shared.CoreName("delay"), - Params: []shared.Port{{Name: "duration", Type: shared.ValueString, Required: true}}, - Results: []shared.Port{{Name: "delayed_for", Type: shared.ValueString}}, + Metadata: shared.Metadata{ID: "cap_blocking", Name: "cap_blocking", Version: "1.0.0"}, + Type: shared.CoreName("command"), + Params: []shared.Port{ + {Name: "command", Type: shared.ValueString, Required: true}, + {Name: "args", Type: shared.ValueArray}, + {Name: "env", Type: shared.ValueObject}, + }, + Results: []shared.Port{ + {Name: "exit_code", Type: shared.ValueNumber}, + {Name: "stdout", Type: shared.ValueString}, + {Name: "stderr", Type: shared.ValueString}, + {Name: "success", Type: shared.ValueBool}, + }, }}, }, } } -// startDelayInstance brings up an instance whose single capability blocks on the -// in-process delay runtime. A capability that blocks until its context dies is -// what makes an execution genuinely in flight, so stopping the instance -// exercises the window in which the scheduler can no longer report its outcome. -func startDelayInstance(t *testing.T, store *sqlite.Store) *Instance { +// blockingParams runs the test binary again as the blocking helper. +func blockingParams() map[string]any { + return map[string]any{ + "command": os.Args[0], + "args": []any{"-test.run=^TestBlockingHelperProcess$", blockingHelperMarker}, + } +} + +// shortParams runs the test binary again as a helper that returns +// immediately, which produces an execution that completes on its own. +func shortParams() map[string]any { + return map[string]any{ + "command": os.Args[0], + "args": []any{"-test.run=^TestBlockingHelperProcess$", blockingHelperMarker, "complete"}, + } +} + +// startBlockingInstance brings up an instance whose single capability blocks on +// the in-process command runtime. A capability that blocks until its context +// dies is what makes an execution genuinely in flight, so stopping the +// instance exercises the window in which the scheduler can no longer report +// its outcome. +func startBlockingInstance(t *testing.T, store *sqlite.Store) *Instance { t.Helper() - assembly := delayAssembly() - i, err := New(context.Background(), "inst_delay", protocol.InstanceKey{ - AssemblyID: "sys_delay", Version: "1.0.0", + assembly := blockingAssembly() + i, err := New(context.Background(), "inst_blocking", protocol.InstanceKey{ + AssemblyID: "sys_blocking", Version: "1.0.0", }, &assembly, 2, store) if err != nil { t.Fatalf("create instance: %v", err) @@ -62,7 +128,7 @@ func startDelayInstance(t *testing.T, store *sqlite.Store) *Instance { func startBlockingExecution(t *testing.T, i *Instance) *execution.Execution { t.Helper() - exec, err := i.Execute(context.Background(), map[string]any{"duration": "1m"}) + exec, err := i.Execute(context.Background(), blockingParams()) if err != nil { t.Fatalf("execute: %v", err) } @@ -85,7 +151,7 @@ func startBlockingExecution(t *testing.T, i *Instance) *execution.Execution { // the only thing that can give a still-running execution a terminal state. func TestStopDrivesInFlightExecutionsToTerminal(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.Stop(); err != nil { @@ -115,7 +181,7 @@ func TestStopDrivesInFlightExecutionsToTerminal(t *testing.T) { // terminal. func TestStopIsIdempotentForExecutions(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) for attempt := 0; attempt < 3; attempt++ { @@ -133,7 +199,7 @@ func TestStopIsIdempotentForExecutions(t *testing.T) { // the user-visible half of the same bug. func TestStopPersistsTerminalStateToStorage(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) if err := i.Stop(); err != nil { @@ -141,9 +207,9 @@ func TestStopPersistsTerminalStateToStorage(t *testing.T) { } // Read through a fresh repository so the in-memory copy cannot answer. - reloaded, ok := execution.NewExecutionStore(store).Get(exec.ID) - if !ok { - t.Fatalf("execution %s not found in storage after the instance stopped", exec.ID) + reloaded, err := execution.NewExecutionStore(store).Get(context.Background(), exec.ID) + if err != nil { + t.Fatalf("execution %s not found in storage after the instance stopped: %v", exec.ID, err) } if got := reloaded.Status(); got != execution.StatusFailed { t.Fatalf("persisted status = %s, want %s", got, execution.StatusFailed) @@ -174,23 +240,26 @@ func TestRestoreInstanceDrainsExecutionsLeftRunning(t *testing.T) { if err := exec.Start(map[string]any{}, 1); err != nil { t.Fatalf("start execution: %v", err) } - if err := execution.NewExecutionStore(store).Add(exec); err != nil { + if err := execution.NewExecutionStore(store).Add(context.Background(), exec); err != nil { t.Fatalf("add execution: %v", err) } - restored := restoreInstance( + restored, err := restoreInstance( context.Background(), metadata{ID: string(instanceID), Status: StatusRunning}, execution.NewExecutionStore(store), event.NewStore(store), ) + if err != nil { + t.Fatalf("restore instance: %v", err) + } if restored.Status() != StatusFailed { t.Fatalf("restored instance status = %s, want %s", restored.Status(), StatusFailed) } - reloaded, ok := execution.NewExecutionStore(store).Get(exec.ID) - if !ok { - t.Fatalf("execution %s not found after restore", exec.ID) + reloaded, err := execution.NewExecutionStore(store).Get(context.Background(), exec.ID) + if err != nil { + t.Fatalf("execution %s not found after restore: %v", exec.ID, err) } if got := reloaded.Status(); got != execution.StatusFailed { t.Fatalf("restored execution status = %s, want %s", got, execution.StatusFailed) @@ -216,7 +285,7 @@ func TestRestoreInstanceDrainsExecutionsLeftRunning(t *testing.T) { // correct Stop has not swept yet, so the execution is still unfinished. func TestStopSweepsOnlyAfterRuntimeGoroutinesFinish(t *testing.T) { store := newTestStore(t) - i := startDelayInstance(t, store) + i := startBlockingInstance(t, store) exec := startBlockingExecution(t, i) sweptDuringDrain := make(chan bool, 1) @@ -278,7 +347,7 @@ func TestSweepLeavesDetachedExecutionsForRestore(t *testing.T) { if err := detached.Start(map[string]any{}, 1); err != nil { t.Fatalf("start detached task: %v", err) } - if err := executions.Add(detached); err != nil { + if err := executions.Add(context.Background(), detached); err != nil { t.Fatalf("add detached task: %v", err) } @@ -289,11 +358,11 @@ func TestSweepLeavesDetachedExecutionsForRestore(t *testing.T) { if err := ordinary.Start(map[string]any{}, 1); err != nil { t.Fatalf("start execution: %v", err) } - if err := executions.Add(ordinary); err != nil { + if err := executions.Add(context.Background(), ordinary); err != nil { t.Fatalf("add execution: %v", err) } - sweepAbandonedExecutions(executions, instanceID, stoppedBeforeFinished(string(instanceID))) + sweepAbandonedExecutions(context.Background(), executions, instanceID, stoppedBeforeFinished(string(instanceID))) if !ordinary.IsTerminal() || ordinary.Status() != execution.StatusFailed { t.Errorf("ordinary execution status = %s, want %s: work the instance owns must still be given a terminal state", ordinary.Status(), execution.StatusFailed) @@ -302,9 +371,9 @@ func TestSweepLeavesDetachedExecutionsForRestore(t *testing.T) { t.Errorf("the detached task was driven to %s; the sweep must leave it for the restore path, which can report accurately", detached.Status()) } - reloaded, ok := executions.Get(detached.ID) - if !ok { - t.Fatalf("detached task %s not found", detached.ID) + reloaded, err := executions.Get(context.Background(), detached.ID) + if err != nil { + t.Fatalf("detached task %s not found: %v", detached.ID, err) } if reloaded.IsTerminal() { t.Error("the detached task was persisted as terminal, so its truthful unfinished state would be lost on restart") @@ -312,17 +381,20 @@ func TestSweepLeavesDetachedExecutionsForRestore(t *testing.T) { // A restart is what finally gives it a terminal state, and the reason it // records is the accurate one rather than "the instance stopped". - restored := restoreInstance( + restored, err := restoreInstance( context.Background(), metadata{ID: string(instanceID), Status: StatusRunning}, execution.NewExecutionStore(store), event.NewStore(store), ) + if err != nil { + t.Fatalf("restore instance: %v", err) + } t.Cleanup(func() { _ = restored.Stop() }) - recovered, ok := execution.NewExecutionStore(store).Get(detached.ID) - if !ok { - t.Fatalf("detached task %s not found after restore", detached.ID) + recovered, err := execution.NewExecutionStore(store).Get(context.Background(), detached.ID) + if err != nil { + t.Fatalf("detached task %s not found after restore: %v", detached.ID, err) } if !recovered.IsTerminal() { t.Fatalf("restored detached task status = %s, want a terminal state: nothing can resume it", recovered.Status()) diff --git a/nore/internal/instance/execution_persister.go b/nore/internal/instance/execution_persister.go index a0f15fe..307bb31 100644 --- a/nore/internal/instance/execution_persister.go +++ b/nore/internal/instance/execution_persister.go @@ -2,11 +2,13 @@ package instance import ( "context" + "errors" "fmt" "log/slog" "github.com/neuron-runtime/neuron/nore/internal/contracts" "github.com/neuron-runtime/neuron/nore/internal/event" + "github.com/neuron-runtime/neuron/nore/internal/storage" ) // stateChangingEventTypes are the event kinds after which an execution @@ -64,8 +66,15 @@ func (p *executionPersister) Run(ctx context.Context) error { if !isStateChanging(received.Type) { continue } - execution, exists := p.store.Get(received.Metadata.ExecutionID) - if !exists { + execution, err := p.store.Get(ctx, received.Metadata.ExecutionID) + if errors.Is(err, storage.ErrNotFound) { + continue + } + if err != nil { + slog.Error("load execution for persistence", + slog.String("execution_id", string(received.Metadata.ExecutionID)), + slog.String("error", err.Error()), + ) continue } if err := p.store.Save(ctx, execution); err != nil { diff --git a/nore/internal/instance/instance.go b/nore/internal/instance/instance.go index d50963b..63011d7 100644 --- a/nore/internal/instance/instance.go +++ b/nore/internal/instance/instance.go @@ -22,6 +22,7 @@ import ( "github.com/neuron-runtime/neuron/nore/internal/storage" "github.com/neuron-runtime/neuron/nore/internal/stream" "github.com/neuron-runtime/neuron/nore/internal/types" + "github.com/neuron-runtime/neuron/shared/types/builtins" capabilityrt "github.com/neuron-runtime/neuron/shared/types/capabilityruntime" shared "github.com/neuron-runtime/neuron/shared/types/core" "github.com/neuron-runtime/neuron/shared/types/protocol" @@ -48,6 +49,10 @@ type options struct { // detachedDrainTimeout is how long detached work may keep running after the // instance stops. A non-positive value selects the engine default. detachedDrainTimeout time.Duration + + // builtinPolicy bounds the built-in capability runtimes this instance + // registers. + builtinPolicy builtins.Policy } // WithResolvedCapabilityRuntimes supplies the frozen capability runtime set persisted with the @@ -68,6 +73,20 @@ func WithDetachedDrainTimeout(timeout time.Duration) Option { } } +// WithBuiltinPolicy sets the policy the built-in capability runtimes run +// under. +// +// The policy bounds what the built-ins may reach: the directories they may +// read, the programs they may run, the hosts they may contact. It is +// resolved against the compiled-in ceilings when the runtimes are registered, +// so a policy that reaches the runtime from outside cannot widen what a +// capability may do. +func WithBuiltinPolicy(policy builtins.Policy) Option { + return func(o *options) { + o.builtinPolicy = policy + } +} + type Instance struct { ID string Key protocol.InstanceKey @@ -131,7 +150,11 @@ func New( scopes := execution.NewScopeRegistry(ctx) reg := registry.New() - reg.RegisterCoreRuntimes() + if err := reg.RegisterCoreRuntimes(optsApplied.builtinPolicy); err != nil { + cancel() + bus.Close() + return nil, fmt.Errorf("register core capability runtimes: %w", err) + } if err := plugin.RegisterResolvedCapabilityRuntimes(reg, optsApplied.resolvedCapabilityRuntimes); err != nil { cancel() @@ -293,7 +316,9 @@ func (i *Instance) Stop() error { i.wg.Wait() } - sweepAbandonedExecutions(i.store, shared.ID(i.ID), stoppedBeforeFinished(i.ID)) + sweepCtx, cancelSweep := context.WithTimeout(context.Background(), sweepPersistenceTimeout) + defer cancelSweep() + sweepErr := sweepAbandonedExecutions(sweepCtx, i.store, shared.ID(i.ID), stoppedBeforeFinished(i.ID)) // Release capability runtime-backed resources (wasm runtimes) now that no execution // can be in flight. @@ -310,7 +335,7 @@ func (i *Instance) Stop() error { i.mu.Lock() i.status = StatusStopped i.mu.Unlock() - return nil + return sweepErr } func (i *Instance) Status() Status { @@ -343,7 +368,7 @@ func (i *Instance) Execute(ctx context.Context, input map[string]any) (*executio if err != nil { return nil, err } - if err := i.store.Add(exec); err != nil { + if err := i.store.Add(ctx, exec); err != nil { return nil, err } @@ -381,10 +406,13 @@ var ErrExecutionNotCancellable = errors.New("execution already reached a termina // decision. A restored instance has no scheduler -- and therefore no work to // stop -- so there the execution's record is simply corrected and persisted. func (i *Instance) CancelExecution(ctx context.Context, executionID shared.ID, reason error) error { - exec, exists := i.store.Get(executionID) - if !exists { + exec, err := i.store.Get(ctx, executionID) + if errors.Is(err, storage.ErrNotFound) { return fmt.Errorf("%w: %s", ErrExecutionNotFound, executionID) } + if err != nil { + return fmt.Errorf("load execution %s: %w", executionID, err) + } if exec.IsTerminal() { return fmt.Errorf("%w: %s is %s", ErrExecutionNotCancellable, executionID, exec.Status()) } @@ -393,24 +421,35 @@ func (i *Instance) CancelExecution(ctx context.Context, executionID shared.ID, r if !exec.MarkCancelled(reason) { return fmt.Errorf("%w: %s is %s", ErrExecutionNotCancellable, executionID, exec.Status()) } - if err := i.store.Save(context.Background(), exec); err != nil { + if err := i.store.Save(ctx, exec); err != nil { return fmt.Errorf("persist cancelled execution %s: %w", executionID, err) } return nil } - if !i.scheduler.CancelExecution(ctx, executionID, reason) { + cancelled, err := i.scheduler.CancelExecution(ctx, executionID, reason) + if err != nil { + return err + } + if !cancelled { return fmt.Errorf("%w: %s", ErrExecutionNotCancellable, executionID) } return nil } -func (i *Instance) ListExecutions() []*execution.Execution { - return i.store.ListByInstance(shared.ID(i.ID)) +func (i *Instance) ListExecutions(ctx context.Context) ([]*execution.Execution, error) { + return i.store.ListByInstance(ctx, shared.ID(i.ID)) } -func (i *Instance) GetExecution(id shared.ID) (*execution.Execution, bool) { - return i.store.Get(id) +func (i *Instance) GetExecution(ctx context.Context, id shared.ID) (*execution.Execution, error) { + exec, err := i.store.Get(ctx, id) + if errors.Is(err, storage.ErrNotFound) { + return nil, fmt.Errorf("%w: %s", ErrExecutionNotFound, id) + } + if err != nil { + return nil, err + } + return exec, nil } func (i *Instance) ListExecutionEvents(ctx context.Context, executionID shared.ID) ([]event.Event, error) { diff --git a/nore/internal/instance/manager.go b/nore/internal/instance/manager.go index 3bf6c3d..c1c9ed7 100644 --- a/nore/internal/instance/manager.go +++ b/nore/internal/instance/manager.go @@ -2,8 +2,8 @@ package instance import ( "context" + "errors" "fmt" - "log" "strings" "sync" "time" @@ -13,6 +13,7 @@ import ( "github.com/neuron-runtime/neuron/nore/internal/execution" "github.com/neuron-runtime/neuron/nore/internal/plugin" "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/shared/types/builtins" core2 "github.com/neuron-runtime/neuron/shared/types/core" "github.com/neuron-runtime/neuron/shared/types/protocol" ) @@ -33,15 +34,25 @@ type Manager struct { // finish after the instance stops. A non-positive value selects the engine // default. detachedDrainTimeout time.Duration + + // builtinPolicy bounds the built-in capability runtimes every instance + // this manager creates registers. + builtinPolicy builtins.Policy } -func NewManager(parent context.Context, workers int, detachedDrainTimeout time.Duration, store storage.Store, assemblies *assembly.Repository) *Manager { +func NewManager(parent context.Context, workers int, detachedDrainTimeout time.Duration, store storage.Store, assemblies *assembly.Repository, opts ...Option) (*Manager, error) { if parent == nil { parent = context.Background() } if workers <= 0 { workers = 8 } + var optsApplied options + for _, opt := range opts { + if opt != nil { + opt(&optsApplied) + } + } m := &Manager{ instancesByKey: make(map[protocol.InstanceKey]*Instance), instancesByID: make(map[string]*Instance), @@ -51,30 +62,36 @@ func NewManager(parent context.Context, workers int, detachedDrainTimeout time.D metadata: newMetadataStore(store), assemblies: assemblies, detachedDrainTimeout: detachedDrainTimeout, + builtinPolicy: optsApplied.builtinPolicy, } - m.reconcile() - return m + if err := m.reconcile(parent); err != nil { + return nil, err + } + return m, nil } // reconcile registers instances whose metadata was persisted by a previous // process so their executions and events remain queryable after a restart. // Instances are restored as metadata-only records and are not started. -func (m *Manager) reconcile() { - records, err := m.metadata.List(context.Background()) +func (m *Manager) reconcile(ctx context.Context) error { + records, err := m.metadata.List(ctx) if err != nil { - log.Printf("load instance metadata: %v", err) - return + return fmt.Errorf("load instance metadata: %w", err) } for _, rec := range records { - i := restoreInstance( + i, err := restoreInstance( m.parent, rec, execution.NewExecutionStore(m.store), event.NewStore(m.store), ) + if err != nil { + return fmt.Errorf("restore instance %s: %w", rec.ID, err) + } m.instancesByID[rec.ID] = i m.instancesByKey[i.Key] = i } + return nil } func (m *Manager) Get(key protocol.InstanceKey) (*Instance, bool) { @@ -96,23 +113,27 @@ func (m *Manager) GetByID(id string) (*Instance, bool) { // or env) is resolved through the registered assemblies so version-without-hash // addresses the most recently registered artifact. Returns false when no // instance exists for the key. -func (m *Manager) Resolve(ctx context.Context, key protocol.InstanceKey) (*Instance, bool) { +func (m *Manager) Resolve(ctx context.Context, key protocol.InstanceKey) (*Instance, bool, error) { m.mu.RLock() if i, ok := m.instancesByKey[key]; ok { m.mu.RUnlock() - return i, true + return i, true, nil } m.mu.RUnlock() - if reg, err := m.assemblies.Resolve(ctx, key); err == nil { - m.mu.RLock() - if i, ok := m.instancesByKey[reg.Key]; ok { - m.mu.RUnlock() - return i, true - } - m.mu.RUnlock() + reg, err := m.assemblies.Resolve(ctx, key) + if errors.Is(err, storage.ErrNotFound) { + return nil, false, nil } - return nil, false + if err != nil { + return nil, false, err + } + m.mu.RLock() + defer m.mu.RUnlock() + if i, ok := m.instancesByKey[reg.Key]; ok { + return i, true, nil + } + return nil, false, nil } // GetOrCreate returns the live runtime for key, lazily constructing it from @@ -150,7 +171,7 @@ func (m *Manager) GetOrCreate(ctx context.Context, key protocol.InstanceKey) (*I return nil, false, err } i, err := New(m.parent, string(id), canonical, ®.Assembly, m.workers, m.store, - execOpt, WithDetachedDrainTimeout(m.detachedDrainTimeout)) + execOpt, WithDetachedDrainTimeout(m.detachedDrainTimeout), WithBuiltinPolicy(m.builtinPolicy)) if err != nil { return nil, false, err } @@ -160,8 +181,14 @@ func (m *Manager) GetOrCreate(ctx context.Context, key protocol.InstanceKey) (*I m.instancesByKey[canonical] = i m.instancesByID[string(id)] = i - if err := m.metadata.Save(context.Background(), recordFor(i, i.Status())); err != nil { - log.Printf("persist instance metadata for %s: %v", i.ID, err) + if err := m.metadata.Save(ctx, recordFor(i, i.Status())); err != nil { + delete(m.instancesByKey, canonical) + delete(m.instancesByID, string(id)) + stopErr := i.Stop() + return nil, false, errors.Join( + fmt.Errorf("persist instance metadata for %s: %w", i.ID, err), + stopErr, + ) } return i, true, nil } @@ -199,7 +226,7 @@ func withCapabilityRuntimes(reg assembly.RegisteredAssembly) (Option, error) { return WithResolvedCapabilityRuntimes(resolved), nil } -func (m *Manager) Stop(id string) error { +func (m *Manager) Stop(ctx context.Context, id string) error { m.mu.RLock() i, ok := m.instancesByID[id] m.mu.RUnlock() @@ -213,8 +240,8 @@ func (m *Manager) Stop(id string) error { m.mu.Lock() defer m.mu.Unlock() - if err := m.metadata.Save(context.Background(), recordFor(i, StatusStopped)); err != nil { - log.Printf("persist instance metadata for %s: %v", i.ID, err) + if err := m.metadata.Save(ctx, recordFor(i, StatusStopped)); err != nil { + return fmt.Errorf("persist instance metadata for %s: %w", i.ID, err) } return nil } @@ -236,7 +263,10 @@ func (m *Manager) Remove(ctx context.Context, target string) (removed bool, err if err != nil { return false, fmt.Errorf("invalid instance target %q: %w", target, err) } - inst, ok := m.Resolve(ctx, key) + inst, ok, err := m.Resolve(ctx, key) + if err != nil { + return false, err + } if !ok { return false, nil } @@ -245,26 +275,66 @@ func (m *Manager) Remove(ctx context.Context, target string) (removed bool, err // removeInstance deletes an instance's runtime, executions, events and // metadata. It is safe for both live and metadata-only (restored) instances. +// All deletions are performed atomically in a single storage transaction when +// possible; in-memory state is only removed after successful commit. func (m *Manager) removeInstance(ctx context.Context, inst *Instance) error { if err := inst.Stop(); err != nil { return err } - for _, exec := range inst.ListExecutions() { - inst.Store().Delete(exec.ID) - if err := inst.EventStore().DeleteExecution(ctx, exec.ID); err != nil { - log.Printf("delete events for execution %s on %s: %v", exec.ID, inst.ID, err) + executions, err := inst.ListExecutions(ctx) + if err != nil { + return fmt.Errorf("list executions for instance %s: %w", inst.ID, err) + } + + var keys []string + for _, exec := range executions { + k, err := storage.Key("executions", string(exec.ID)) + if err != nil { + return err + } + keys = append(keys, k) + prefix, err := storage.Prefix("events", string(exec.ID)) + if err != nil { + return err + } + ekeys, err := m.store.List(ctx, prefix) + if err != nil { + return fmt.Errorf("list events for execution %s on %s: %w", exec.ID, inst.ID, err) + } + keys = append(keys, ekeys...) + } + mkey, err := storage.Key("instances", inst.ID) + if err != nil { + return err + } + keys = append(keys, mkey) + + if err := m.store.WithTx(ctx, func(tx storage.Tx) error { + for _, k := range keys { + if err := tx.Delete(ctx, k); err != nil { + return err + } } + return nil + }); err != nil { + return err } + // Clear memory caches if present (best effort) + for _, exec := range executions { + if es := inst.Store(); es != nil { + _ = es.Delete(ctx, exec.ID) + } + _ = inst.EventStore().DeleteExecution(ctx, exec.ID) + } + // metadata already deleted in tx; ensure mem cleared + _ = m.metadata.Delete(ctx, inst.ID) + m.mu.Lock() delete(m.instancesByID, inst.ID) delete(m.instancesByKey, inst.Key) m.mu.Unlock() - - if err := m.metadata.Delete(ctx, inst.ID); err != nil { - log.Printf("delete instance metadata for %s: %v", inst.ID, err) - } return nil } @@ -302,11 +372,11 @@ func (m *Manager) Clear(ctx context.Context) error { } m.mu.RUnlock() - var firstErr error + var failures []error for _, inst := range instances { - if err := m.removeInstance(ctx, inst); err != nil && firstErr == nil { - firstErr = err + if err := m.removeInstance(ctx, inst); err != nil { + failures = append(failures, err) } } - return firstErr + return errors.Join(failures...) } diff --git a/nore/internal/instance/metadata_store_test.go b/nore/internal/instance/metadata_store_test.go new file mode 100644 index 0000000..d8886f6 --- /dev/null +++ b/nore/internal/instance/metadata_store_test.go @@ -0,0 +1,70 @@ +package instance + +import ( + "context" + "errors" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" +) + +var errMetadata = errors.New("injected metadata failure") + +type metadataFaultStore struct { + get func(string) ([]byte, error) + list func(string) ([]string, error) +} + +func (f *metadataFaultStore) Get(_ context.Context, key string) ([]byte, error) { + if f.get != nil { + return f.get(key) + } + return nil, storage.ErrNotFound +} + +func (f *metadataFaultStore) Put(context.Context, string, []byte) error { return nil } +func (f *metadataFaultStore) Delete(context.Context, string) error { return nil } + +func (f *metadataFaultStore) List(_ context.Context, prefix string) ([]string, error) { + if f.list != nil { + return f.list(prefix) + } + return nil, nil +} + +func (f *metadataFaultStore) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + return fn(f) +} + +func (f *metadataFaultStore) Ping(context.Context) error { return nil } +func (f *metadataFaultStore) Close() error { return nil } + +func TestMetadataGetSurfacesBackendFailure(t *testing.T) { + ms := newMetadataStore(&metadataFaultStore{get: func(string) ([]byte, error) { return nil, errMetadata }}) + if _, err := ms.Get(context.Background(), "inst_a"); !errors.Is(err, errMetadata) { + t.Fatalf("Get error = %v, want injected failure", err) + } +} + +func TestMetadataGetReportsDecodeFailureAsError(t *testing.T) { + ms := newMetadataStore(&metadataFaultStore{get: func(string) ([]byte, error) { return []byte("{not json"), nil }}) + if _, err := ms.Get(context.Background(), "inst_a"); err == nil || errors.Is(err, storage.ErrNotFound) { + t.Fatalf("Get malformed metadata = %v, want a decode error", err) + } +} + +func TestMetadataListSurfacesBackendFailure(t *testing.T) { + ms := newMetadataStore(&metadataFaultStore{list: func(string) ([]string, error) { return nil, errMetadata }}) + if _, err := ms.List(context.Background()); !errors.Is(err, errMetadata) { + t.Fatalf("List error = %v, want injected failure", err) + } +} + +func TestMetadataListReportsMalformedKey(t *testing.T) { + ms := newMetadataStore(&metadataFaultStore{list: func(string) ([]string, error) { + return []string{"other/inst_a"}, nil + }}) + if _, err := ms.List(context.Background()); err == nil { + t.Fatal("List with malformed key succeeded") + } +} diff --git a/nore/internal/instance/restore.go b/nore/internal/instance/restore.go index 5c68f23..d81ff97 100644 --- a/nore/internal/instance/restore.go +++ b/nore/internal/instance/restore.go @@ -40,13 +40,15 @@ func restoreInstance( rec metadata, store contracts.ExecutionRepository, eventStore *event.Store, -) *Instance { +) (*Instance, error) { status := rec.Status if status == StatusStarting || status == StatusRunning { status = StatusFailed } - sweepUnresumableExecutions(store, shared.ID(rec.ID), restoredWithoutRuntime(rec.ID)) + if err := sweepUnresumableExecutions(parent, store, shared.ID(rec.ID), restoredWithoutRuntime(rec.ID)); err != nil { + return nil, err + } ctx, cancel := context.WithCancel(parent) return &Instance{ @@ -64,5 +66,5 @@ func restoreInstance( createdAt: rec.CreatedAt, store: store, eventStore: eventStore, - } + }, nil } diff --git a/nore/internal/instance/store.go b/nore/internal/instance/store.go index b4e3690..f6171ee 100644 --- a/nore/internal/instance/store.go +++ b/nore/internal/instance/store.go @@ -3,11 +3,11 @@ package instance import ( "context" "encoding/json" + "errors" "fmt" "time" "github.com/neuron-runtime/neuron/nore/internal/storage" - "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" shared "github.com/neuron-runtime/neuron/shared/types/core" ) @@ -41,45 +41,78 @@ func (ms *metadataStore) Save(ctx context.Context, rec metadata) error { if err != nil { return fmt.Errorf("marshal instance metadata: %w", err) } - return ms.store.Put(ctx, ms.key(rec.ID), data) + key, err := ms.key(rec.ID) + if err != nil { + return err + } + return ms.store.Put(ctx, key, data) } -func (ms *metadataStore) Get(ctx context.Context, id string) (metadata, bool) { - data, err := ms.store.Get(ctx, ms.key(id)) +// Get reports storage.ErrNotFound for absence. A backend or decode failure is +// returned and must not be interpreted as a missing instance. +func (ms *metadataStore) Get(ctx context.Context, id string) (metadata, error) { + key, err := ms.key(id) + if err != nil { + return metadata{}, err + } + data, err := ms.store.Get(ctx, key) if err != nil { - return metadata{}, false + return metadata{}, err } var rec metadata if err := json.Unmarshal(data, &rec); err != nil { - return metadata{}, false + return metadata{}, fmt.Errorf("decode instance metadata %q: %w", id, err) } - return rec, true + return rec, nil } func (ms *metadataStore) List(ctx context.Context) ([]metadata, error) { - keys, err := ms.store.List(ctx, "instances/") + prefix, err := storage.Prefix("instances") if err != nil { return nil, err } + keys, err := ms.store.List(ctx, prefix) + if err != nil { + return nil, fmt.Errorf("list instance metadata: %w", err) + } records := make([]metadata, 0, len(keys)) for _, key := range keys { - rec, ok := ms.Get(ctx, instanceIDFromKey(key)) - if !ok { + id, err := instanceIDFromKey(key) + if err != nil { + return nil, err + } + rec, err := ms.Get(ctx, id) + if errors.Is(err, storage.ErrNotFound) { + // The record was deleted between List and Get. continue } + if err != nil { + return nil, err + } records = append(records, rec) } return records, nil } func (ms *metadataStore) Delete(ctx context.Context, id string) error { - return ms.store.Delete(ctx, ms.key(id)) + key, err := ms.key(id) + if err != nil { + return err + } + return ms.store.Delete(ctx, key) } -func (ms *metadataStore) key(id string) string { - return sqlite.SanitizeKey("instances", id) +func (ms *metadataStore) key(id string) (string, error) { + return storage.Key("instances", id) } -func instanceIDFromKey(key string) string { - return key[len("instances/"):] +func instanceIDFromKey(key string) (string, error) { + prefix, err := storage.Prefix("instances") + if err != nil { + return "", err + } + if len(key) <= len(prefix) || key[:len(prefix)] != prefix { + return "", fmt.Errorf("decode instance key %q: unexpected prefix", key) + } + return key[len(prefix):], nil } diff --git a/nore/internal/invoke/invoke.go b/nore/internal/invoke/invoke.go index 51e6aa8..db11818 100644 --- a/nore/internal/invoke/invoke.go +++ b/nore/internal/invoke/invoke.go @@ -23,6 +23,7 @@ import ( "github.com/neuron-runtime/neuron/nore/internal/contracts" "github.com/neuron-runtime/neuron/nore/internal/plugin" "github.com/neuron-runtime/neuron/nore/internal/registry" + "github.com/neuron-runtime/neuron/shared/types/builtins" capabilityrt "github.com/neuron-runtime/neuron/shared/types/capabilityruntime" core "github.com/neuron-runtime/neuron/shared/types/core" ) @@ -39,7 +40,11 @@ const shutdownTimeout = 10 * time.Second // // storeRoot is the capability runtime store root the artifact was installed // into; it is a property of the host, not the request. -func Run(ctx context.Context, storeRoot string, in io.Reader, out io.Writer) error { +// +// policy bounds the built-in capability runtimes; it is a property of the host +// as well, because it describes what the machine running the invocation is +// willing to let a capability reach. +func Run(ctx context.Context, storeRoot string, policy builtins.Policy, in io.Reader, out io.Writer) error { req, err := decode(in) if err != nil { _ = write(out, &capabilityrt.InvokeResponse{Error: err.Error()}) @@ -57,7 +62,7 @@ func Run(ctx context.Context, storeRoot string, in io.Reader, out io.Writer) err } }() - resp := Invoke(ctx, storeRoot, req) + resp := Invoke(ctx, storeRoot, policy, req) if err := write(out, resp); err != nil { return err } @@ -74,9 +79,11 @@ func Run(ctx context.Context, storeRoot string, in io.Reader, out io.Writer) err // missing provider, or a runtime error — are returned as a structured error // response, never as a panic, so the parent always receives a parseable // document. -func Invoke(ctx context.Context, storeRoot string, req capabilityrt.InvokeRequest) *capabilityrt.InvokeResponse { +// +// The policy bounds the built-in capability runtimes the request may address. +func Invoke(ctx context.Context, storeRoot string, policy builtins.Policy, req capabilityrt.InvokeRequest) *capabilityrt.InvokeResponse { if req.CoreType != "" { - return invokeCore(ctx, req) + return invokeCore(ctx, policy, req) } plugin.SetStoreRoot(storeRoot) @@ -91,7 +98,10 @@ func Invoke(ctx context.Context, storeRoot string, req capabilityrt.InvokeReques } }() - result, err := adapter.Execute(ctx, contracts.ExecutionContext{Params: req.Params}) + result, err := adapter.Execute(ctx, contracts.ExecutionContext{ + Params: req.Params, + Logger: contracts.NopLogger{}, + }) if err != nil { return &capabilityrt.InvokeResponse{Error: err.Error()} } @@ -102,9 +112,11 @@ func Invoke(ctx context.Context, storeRoot string, req capabilityrt.InvokeReques // registry Assembly execution uses and executes it once in-process. No store, // provider, or installed artifact is involved; an unknown core name is // reported as a structured error. -func invokeCore(ctx context.Context, req capabilityrt.InvokeRequest) *capabilityrt.InvokeResponse { +func invokeCore(ctx context.Context, policy builtins.Policy, req capabilityrt.InvokeRequest) *capabilityrt.InvokeResponse { reg := registry.New() - reg.RegisterCoreRuntimes() + if err := reg.RegisterCoreRuntimes(policy); err != nil { + return &capabilityrt.InvokeResponse{Error: err.Error()} + } defer func() { if err := reg.Close(); err != nil { slog.Warn("close core capability runtimes", slog.String("error", err.Error())) @@ -116,7 +128,10 @@ func invokeCore(ctx context.Context, req capabilityrt.InvokeRequest) *capability return &capabilityrt.InvokeResponse{Error: err.Error()} } - result, err := runtime.Execute(ctx, contracts.ExecutionContext{Params: req.Params}) + result, err := runtime.Execute(ctx, contracts.ExecutionContext{ + Params: req.Params, + Logger: contracts.NopLogger{}, + }) if err != nil { return &capabilityrt.InvokeResponse{Error: err.Error()} } diff --git a/nore/internal/planner/compiler.go b/nore/internal/planner/compiler.go index 10d0f3f..22b973c 100644 --- a/nore/internal/planner/compiler.go +++ b/nore/internal/planner/compiler.go @@ -2,6 +2,7 @@ package planner import ( "fmt" + "sort" "strings" "github.com/neuron-runtime/neuron/nore/internal/resolver" @@ -56,12 +57,21 @@ func (c *Compiler) Compile(assembly shared.Assembly) (*types.ExecutionBlueprint, nodes := make(map[shared.ID]types.ExecutionNode, len(capabilities)) incoming := make(map[shared.ID]int, len(capabilities)) + predecessors := make(map[shared.ID][]types.ExecutionPredecessor, len(capabilities)) + joinPolicies := make(map[shared.ID]shared.JoinPolicy, len(capabilities)) for id, capability := range capabilities { compiledConfig, err := c.expressions.CompileCapabilityConfigurations(capability.CapabilityConfigurations) if err != nil { return nil, fmt.Errorf("compile configurations for capability %s: %w", id, err) } - nodes[id] = types.ExecutionNode{Capability: capability, Configurations: compiledConfig} + // A capability with no incoming bindings is an entry point; every + // capability otherwise joins its predecessors under the same default + // policy until an explicit declaration overrides it below. + nodes[id] = types.ExecutionNode{ + Capability: capability, + Configurations: compiledConfig, + JoinPolicy: shared.JoinPolicyAll, + } incoming[id] = 0 } @@ -82,10 +92,22 @@ func (c *Compiler) Compile(assembly shared.Assembly) (*types.ExecutionBlueprint, return nil, fmt.Errorf("target input port %q does not exist on capability %s", binding.To.Port, toCapability.Metadata.ID) } - incoming[binding.To.CapabilityID]++ - if incoming[binding.To.CapabilityID] > 1 { - return nil, fmt.Errorf("capability %s has multiple incoming bindings; use an aggregation capability", binding.To.CapabilityID) + joinPolicy := binding.JoinPolicy.Effective() + if err := joinPolicy.Validate(); err != nil { + return nil, fmt.Errorf("binding %s: %w", binding.Metadata.ID, err) + } + if existing, exists := joinPolicies[binding.To.CapabilityID]; exists && existing != joinPolicy { + return nil, fmt.Errorf( + "capability %s has conflicting join policies %q and %q", + binding.To.CapabilityID, existing, joinPolicy) } + joinPolicies[binding.To.CapabilityID] = joinPolicy + + incoming[binding.To.CapabilityID]++ + predecessors[binding.To.CapabilityID] = append(predecessors[binding.To.CapabilityID], types.ExecutionPredecessor{ + BindingID: binding.Metadata.ID, + CapabilityID: binding.From.CapabilityID, + }) transition, err := c.compileTransition(binding) if err != nil { @@ -96,6 +118,39 @@ func (c *Compiler) Compile(assembly shared.Assembly) (*types.ExecutionBlueprint, nodes[binding.From.CapabilityID] = node } + for capabilityID, capabilityPredecessors := range predecessors { + // Binding IDs are the stable edge identity. Sorting here means readiness + // and merged inputs are deterministic regardless of map iteration or the + // order the bindings were declared in. + sort.Slice(capabilityPredecessors, func(i, j int) bool { + if capabilityPredecessors[i].BindingID != capabilityPredecessors[j].BindingID { + return capabilityPredecessors[i].BindingID < capabilityPredecessors[j].BindingID + } + return capabilityPredecessors[i].CapabilityID < capabilityPredecessors[j].CapabilityID + }) + node := nodes[capabilityID] + node.Predecessors = capabilityPredecessors + node.JoinPolicy = joinPolicies[capabilityID] + nodes[capabilityID] = node + } + + // R3-16: A detached capability must not participate in a fan-in (multiple + // incoming bindings). Detached branches run in a separate execution that the + // parent does not wait for; a join spanning detached and non-detached work + // would create ambiguous ownership. Reject such configurations at build time. + for capabilityID, count := range incoming { + if count > 1 { + n, ex := nodes[capabilityID] + if ex && n.Capability.RuntimeConfig != nil && n.Capability.RuntimeConfig.Execution != nil && n.Capability.RuntimeConfig.Execution.Mode == shared.RuntimeExecutionModeDetach { + return nil, fmt.Errorf( + "capability %s is detached and has %d incoming bindings (fan-in); "+ + "a detached branch may not participate in a join; "+ + "use an aggregation capability or restructure the graph", + capabilityID, count) + } + } + } + entryIDs := triggerIDs if len(entryIDs) == 0 { for capabilityID, count := range incoming { diff --git a/nore/internal/planner/predecessor_test.go b/nore/internal/planner/predecessor_test.go new file mode 100644 index 0000000..794420a --- /dev/null +++ b/nore/internal/planner/predecessor_test.go @@ -0,0 +1,56 @@ +package planner + +import ( + "testing" + + shared "github.com/neuron-runtime/neuron/shared/types/core" +) + +func TestCompileBuildsPredecessorsAndJoinPolicy(t *testing.T) { + assembly := planningAssembly(nil) + assembly.Specification.Bindings[0].Metadata.ID = "a-b" + assembly.Specification.Bindings[0].JoinPolicy = shared.JoinPolicyAll + + blueprint := compileAssembly(t, assembly) + + a := blueprint.Nodes["a"] + if a.JoinPolicy != shared.JoinPolicyAll { + t.Fatalf("entry join policy = %q, want %q", a.JoinPolicy, shared.JoinPolicyAll) + } + if len(a.Predecessors) != 0 { + t.Fatalf("entry predecessors = %+v, want none", a.Predecessors) + } + + b := blueprint.Nodes["b"] + if b.JoinPolicy != shared.JoinPolicyAll { + t.Fatalf("target join policy = %q, want %q", b.JoinPolicy, shared.JoinPolicyAll) + } + if len(b.Predecessors) != 1 { + t.Fatalf("target predecessors = %+v, want one", b.Predecessors) + } + predecessor := b.Predecessors[0] + if predecessor.BindingID != "a-b" || predecessor.CapabilityID != "a" { + t.Fatalf("predecessor = %+v, want binding a-b from a", predecessor) + } +} + +func TestCompileDefaultsJoinPolicyToAll(t *testing.T) { + blueprint := compileAssembly(t, planningAssembly(nil)) + + if got := blueprint.Nodes["b"].JoinPolicy; got != shared.JoinPolicyAll { + t.Fatalf("default join policy = %q, want %q", got, shared.JoinPolicyAll) + } +} + +func TestCompileRejectsUnsupportedJoinPolicy(t *testing.T) { + assembly := planningAssembly(nil) + assembly.Specification.Bindings[0].JoinPolicy = shared.JoinPolicy("any") + + compiler, err := NewCompiler(stubExpressionCompiler{}) + if err != nil { + t.Fatalf("new compiler: %v", err) + } + if _, err := compiler.Compile(assembly); err == nil { + t.Fatal("an unsupported join policy must be rejected at plan time") + } +} diff --git a/nore/internal/registry/registry.go b/nore/internal/registry/registry.go index 2f06e41..af40bb8 100644 --- a/nore/internal/registry/registry.go +++ b/nore/internal/registry/registry.go @@ -2,18 +2,27 @@ package registry import ( "fmt" - "sort" "strings" "sync" - "github.com/neuron-runtime/neuron/nore/internal/builtins" + "github.com/neuron-runtime/neuron/nore/internal/builtins/command" + "github.com/neuron-runtime/neuron/nore/internal/builtins/filesystem" + "github.com/neuron-runtime/neuron/nore/internal/builtins/http" + "github.com/neuron-runtime/neuron/nore/internal/builtins/set" "github.com/neuron-runtime/neuron/nore/internal/contracts" + "github.com/neuron-runtime/neuron/shared/types/builtins" "github.com/neuron-runtime/neuron/shared/types/core" ) type Registry struct { mu sync.RWMutex runtimes map[core.CapabilityRuntimeType]contracts.CapabilityRuntime + + // instances holds every registered runtime exactly once, in registration + // order. A runtime registered under more than one name — a built-in is + // reachable under its canonical name and under its legacy bare name — + // appears here once, so it is closed once. + instances []contracts.CapabilityRuntime } func New() *Registry { @@ -30,6 +39,7 @@ func (r *Registry) Register(runtimeType core.CapabilityRuntimeType, runtime cont return fmt.Errorf("runtime for runtime type %q is already registered", runtimeType) } r.runtimes[runtimeType] = runtime + r.instances = append(r.instances, runtime) return nil } @@ -38,52 +48,90 @@ func (r *Registry) Resolve(runtimeType core.CapabilityRuntimeType) (contracts.Ca defer r.mu.RUnlock() runtime, exists := r.runtimes[runtimeType] if !exists { + // A reference to a built-in that does not exist is a mistake in the + // assembly, and saying so is more useful than reporting a missing + // runtime. + if err := core.UnknownBuiltinError(runtimeType); err != nil { + return nil, err + } return nil, fmt.Errorf("runtime for runtime type %q was not found", runtimeType) } return runtime, nil } -func (r *Registry) RegisterCoreRuntimes() { - builtins := map[string]contracts.CapabilityRuntime{ - "set": builtins.Set{}, - "ai": builtins.AIMock{}, - "log": builtins.Log{}, - "http": builtins.HTTP{}, - "delay": builtins.Delay{}, - "command": builtins.Command{}, +// RegisterCoreRuntimes registers the built-in capability runtimes that the +// runtime provides in-process. +// +// The set of built-ins is declared in the shared core package, so the runtime, +// the compiler, and the CLI agree on which built-ins exist and what they +// accept. Every declared built-in must have an implementation here: a +// declared built-in with no implementation is a defect in the runtime, and it +// is reported rather than leaving the built-in silently unavailable. +// +// The policy bounds what the built-ins may reach. It is resolved here, against +// the ceilings compiled into the runtime, so a policy that arrives over a +// transport it did not come from — a command line, for instance — still +// cannot widen what a capability may reach. +// +// Each built-in is registered under its canonical namespaced name +// (neuron:core:http) and under its legacy bare name (http), so an assembly +// authored before the namespace existed still resolves. +func (r *Registry) RegisterCoreRuntimes(policy builtins.Policy) error { + effective, err := builtins.Effective(policy) + if err != nil { + return fmt.Errorf("resolve the built-in capability policy: %w", err) + } + if err := effective.Validate(); err != nil { + return fmt.Errorf("validate the built-in capability policy: %w", err) } - // Register each in-process runtime under its canonical namespaced name - // (neuron:core:set, ...) plus the legacy bare name so assemblies authored - // before the namespace existed still resolve. - for name, runtime := range builtins { - must(r.Register(core.CoreName(name), runtime)) - must(r.Register(core.CapabilityRuntimeType(name), runtime)) + implementations := map[string]contracts.CapabilityRuntime{ + "set": set.Set{}, + "http": http.HTTP{Policy: effective}, + "command": command.Command{Policy: effective}, + "filesystem": filesystem.Filesystem{Policy: effective}, } + + for _, name := range core.BuiltinTypeNames() { + implementation, ok := implementations[name] + if !ok { + return fmt.Errorf("built-in capability runtime %q is declared but has no implementation", name) + } + if err := r.Register(core.CoreName(name), implementation); err != nil { + return fmt.Errorf("register built-in capability runtime %q: %w", name, err) + } + if err := r.Register(core.CapabilityRuntimeType(name), implementation); err != nil { + return fmt.Errorf("register built-in capability runtime alias %q: %w", name, err) + } + } + + for name := range implementations { + if !core.IsBuiltinRuntimeType(core.CapabilityRuntimeType(name)) { + return fmt.Errorf("built-in capability runtime %q is implemented but not declared", name) + } + } + + return nil } // Close releases resources held by registered runtimes that implement // contracts.CapabilityRuntimeCloser (e.g. wasm backends). It is idempotent // and safe to call from an instance shutdown path once no execution is in // flight. +// +// A runtime registered under several names is closed once. func (r *Registry) Close() error { r.mu.RLock() defer r.mu.RUnlock() var failures []string - types := make([]string, 0, len(r.runtimes)) - for runtimeType := range r.runtimes { - types = append(types, string(runtimeType)) - } - sort.Strings(types) - - for _, capabilityType := range types { - closer, ok := r.runtimes[core.CapabilityRuntimeType(capabilityType)].(contracts.CapabilityRuntimeCloser) + for _, runtime := range r.instances { + closer, ok := runtime.(contracts.CapabilityRuntimeCloser) if !ok { continue } if err := closer.Close(); err != nil { - failures = append(failures, fmt.Sprintf("%s: %v", capabilityType, err)) + failures = append(failures, fmt.Sprintf("%T: %v", runtime, err)) } } @@ -92,9 +140,3 @@ func (r *Registry) Close() error { } return nil } - -func must(err error) { - if err != nil { - panic(err) - } -} diff --git a/nore/internal/registry/registry_test.go b/nore/internal/registry/registry_test.go new file mode 100644 index 0000000..d95eb7e --- /dev/null +++ b/nore/internal/registry/registry_test.go @@ -0,0 +1,47 @@ +package registry + +import ( + "testing" + + "github.com/neuron-runtime/neuron/shared/types/builtins" + "github.com/neuron-runtime/neuron/shared/types/core" +) + +func TestRegisterCoreRuntimesRegistersTheCanonicalAndLegacyNames(t *testing.T) { + reg := New() + if err := reg.RegisterCoreRuntimes(builtins.Policy{}); err != nil { + t.Fatalf("RegisterCoreRuntimes: %v", err) + } + + for _, name := range core.BuiltinTypeNames() { + canonical := core.CoreName(name) + if _, err := reg.Resolve(canonical); err != nil { + t.Errorf("resolve %q: %v", canonical, err) + } + if _, err := reg.Resolve(core.CapabilityRuntimeType(name)); err != nil { + t.Errorf("resolve %q: %v", name, err) + } + } +} + +// A policy that reaches the runtime from outside must not be able to widen +// what a capability may reach: the registry resolves it against the ceilings +// compiled into the runtime. +func TestRegisterCoreRuntimesRefusesAPolicyAboveTheCeiling(t *testing.T) { + reg := New() + err := reg.RegisterCoreRuntimes(builtins.Policy{MaxCommandOutputBytes: 1 << 40}) + if err == nil { + t.Fatal("RegisterCoreRuntimes accepted a limit above the compiled-in ceiling") + } +} + +// A runtime registered under two names is one runtime, and it is closed once. +func TestCloseClosesARuntimeRegisteredUnderTwoNamesOnce(t *testing.T) { + reg := New() + if err := reg.RegisterCoreRuntimes(builtins.Policy{}); err != nil { + t.Fatalf("RegisterCoreRuntimes: %v", err) + } + if err := reg.Close(); err != nil { + t.Fatalf("Close: %v", err) + } +} diff --git a/nore/internal/storage/driver.go b/nore/internal/storage/driver.go new file mode 100644 index 0000000..b12a930 --- /dev/null +++ b/nore/internal/storage/driver.go @@ -0,0 +1,95 @@ +package storage + +import ( + "context" + "fmt" + "sort" + "strings" + "sync" +) + +// DefaultDriver is selected when Config.Driver is empty. It keeps the simple +// deployment path working without making runtime code choose a backend. +const DefaultDriver = "sqlite" + +// DriverFunc constructs a ready-to-use backend. It must open the underlying +// resource and initialize its schema; Open performs the health check and closes +// the backend if construction or the health check fails. +type DriverFunc func(ctx context.Context, cfg Config) (Store, error) + +var ( + driversMu sync.RWMutex + drivers = make(map[string]DriverFunc) +) + +// RegisterDriver makes a backend selectable by name. Backends register +// themselves from init so only the composition root needs to import them. +// Registering an empty or duplicate name is a programmer error. +func RegisterDriver(name string, fn DriverFunc) { + name = normalizeDriver(name) + if name == "" { + panic("storage: driver name is required") + } + if fn == nil { + panic("storage: driver constructor is required") + } + + driversMu.Lock() + defer driversMu.Unlock() + if _, exists := drivers[name]; exists { + panic("storage: duplicate driver " + name) + } + drivers[name] = fn +} + +// Open constructs the configured backend, verifies it responds to Ping, and +// returns it. An unknown driver is an actionable error rather than a silent +// fallback. Open is the only construction path used by the runtime. +func Open(ctx context.Context, cfg Config) (Store, error) { + name := normalizeDriver(cfg.Driver) + if name == "" { + name = DefaultDriver + } + cfg.Driver = name + + driversMu.RLock() + driver, ok := drivers[name] + driversMu.RUnlock() + if !ok { + return nil, fmt.Errorf("%w: %q (available: %s)", ErrUnknownDriver, name, availableDrivers()) + } + + store, err := driver(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("open storage driver %q: %w", name, err) + } + if store == nil { + return nil, fmt.Errorf("open storage driver %q: constructor returned nil", name) + } + + if err := store.Ping(ctx); err != nil { + _ = store.Close() + return nil, fmt.Errorf("ping storage driver %q: %w", name, err) + } + + return store, nil +} + +func normalizeDriver(name string) string { + return strings.ToLower(strings.TrimSpace(name)) +} + +func availableDrivers() string { + driversMu.RLock() + defer driversMu.RUnlock() + + names := make([]string, 0, len(drivers)) + for name := range drivers { + names = append(names, name) + } + sort.Strings(names) + if len(names) == 0 { + return "none" + } + return strings.Join(names, ", ") +} diff --git a/nore/internal/storage/guard_test.go b/nore/internal/storage/guard_test.go new file mode 100644 index 0000000..b5380cf --- /dev/null +++ b/nore/internal/storage/guard_test.go @@ -0,0 +1,37 @@ +package storage_test + +import ( + "os/exec" + "strings" + "testing" +) + +// TestRuntimePackagesDoNotImportConcreteBackends keeps the storage boundary +// honest. Generic runtime packages may depend on storage.Store; only the +// composition root may depend on a concrete backend. +func TestRuntimePackagesDoNotImportConcreteBackends(t *testing.T) { + const module = "github.com/neuron-runtime/neuron" + + packages := []string{ + module + "/nore/internal/execution", + module + "/nore/internal/event", + module + "/nore/internal/instance", + module + "/nore/internal/assembly", + module + "/nore/internal/planner", + } + + for _, pkg := range packages { + pkg := pkg + t.Run(strings.TrimPrefix(pkg, module+"/"), func(t *testing.T) { + out, err := exec.Command("go", "list", "-deps", "-f", "{{.ImportPath}}", pkg).Output() + if err != nil { + t.Fatalf("go list -deps %s: %v", pkg, err) + } + for _, dep := range strings.Fields(string(out)) { + if strings.HasPrefix(dep, module+"/nore/internal/storage/") { + t.Errorf("%s depends on concrete storage backend %s", pkg, dep) + } + } + }) + } +} diff --git a/nore/internal/storage/key.go b/nore/internal/storage/key.go new file mode 100644 index 0000000..a9a6190 --- /dev/null +++ b/nore/internal/storage/key.go @@ -0,0 +1,33 @@ +package storage + +import ( + "fmt" + "strings" +) + +// Key joins key segments with '/'. It deliberately accepts empty segments: +// assembly artifacts carry an optional environment segment, and a partial key +// is represented by the same layout as its resolved form. Segments containing +// '/' or NUL are rejected because they would make the prefix boundaries +// ambiguous. +func Key(parts ...string) (string, error) { + if len(parts) == 0 { + return "", fmt.Errorf("%w: at least one segment is required", ErrInvalidKey) + } + for _, part := range parts { + if strings.ContainsAny(part, "/\x00") { + return "", fmt.Errorf("%w: segment %q contains '/' or NUL", ErrInvalidKey, part) + } + } + return strings.Join(parts, "/"), nil +} + +// Prefix returns Key(parts...) with a trailing separator. Use it for List +// prefixes so "exec_a" does not also match keys belonging to "exec_ab". +func Prefix(parts ...string) (string, error) { + key, err := Key(parts...) + if err != nil { + return "", err + } + return key + "/", nil +} diff --git a/nore/internal/storage/key_test.go b/nore/internal/storage/key_test.go new file mode 100644 index 0000000..6db4961 --- /dev/null +++ b/nore/internal/storage/key_test.go @@ -0,0 +1,45 @@ +package storage + +import ( + "errors" + "testing" +) + +func TestKeyJoinsSegments(t *testing.T) { + key, err := Key("executions", "exec_1") + if err != nil { + t.Fatalf("Key: %v", err) + } + if key != "executions/exec_1" { + t.Fatalf("Key = %q, want executions/exec_1", key) + } + + prefix, err := Prefix("executions") + if err != nil { + t.Fatalf("Prefix: %v", err) + } + if prefix != "executions/" { + t.Fatalf("Prefix = %q, want executions/", prefix) + } +} + +func TestKeyPreservesEmptySegments(t *testing.T) { + key, err := Key("assemblies", "sys", "1.0.0", "hash", "") + if err != nil { + t.Fatalf("Key with empty environment: %v", err) + } + if key != "assemblies/sys/1.0.0/hash/" { + t.Fatalf("Key = %q, want assemblies/sys/1.0.0/hash/", key) + } +} + +func TestKeyRejectsAmbiguousSegments(t *testing.T) { + for _, part := range []string{"a/b", "a\x00b"} { + if _, err := Key("prefix", part); !errors.Is(err, ErrInvalidKey) { + t.Errorf("Key(%q) = %v, want ErrInvalidKey", part, err) + } + } + if _, err := Key(); !errors.Is(err, ErrInvalidKey) { + t.Errorf("Key() = %v, want ErrInvalidKey", err) + } +} diff --git a/nore/internal/storage/migrate/migrate.go b/nore/internal/storage/migrate/migrate.go new file mode 100644 index 0000000..de7a7b3 --- /dev/null +++ b/nore/internal/storage/migrate/migrate.go @@ -0,0 +1,185 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "sort" + "strings" +) + +var ( + // ErrNewerSchema indicates the database schema version is newer than + // this binary supports. The version policy requires refusing to start + // in this case (do not downgrade). + ErrNewerSchema = errors.New("storage/migrate: schema version newer than supported") + + // ErrDirty indicates a migration was left in a dirty/incomplete state. + ErrDirty = errors.New("storage/migrate: migration is dirty") + + // ErrNoMigrations indicates no migrations were provided to the runner. + ErrNoMigrations = errors.New("storage/migrate: no migrations registered") +) + +// Migration represents one schema change. +type Migration struct { + Version int64 + Description string + Up func(context.Context, *sql.Tx) error +} + +// Driver identifies the SQL dialect for driver-specific behavior. +type Driver string + +const ( + DriverSQLite Driver = "sqlite" + DriverPostgres Driver = "postgres" +) + +// NormalizeDriver normalizes driver names to the migrate package's forms. +func NormalizeDriver(d string) Driver { + switch strings.ToLower(strings.TrimSpace(d)) { + case "sqlite": + return DriverSQLite + case "postgres", "postgresql": + return DriverPostgres + default: + return Driver(strings.ToLower(strings.TrimSpace(d))) + } +} + +// Config configures migration execution. +type Config struct { + DB *sql.DB + Driver Driver + Migrations []Migration + MaxVersion int64 + TableName string +} + +// Migrate ensures the schema is up to date. +func Migrate(ctx context.Context, cfg Config) error { + if cfg.DB == nil { + return fmt.Errorf("migrate: db is required") + } + if len(cfg.Migrations) == 0 { + return ErrNoMigrations + } + table := cfg.TableName + if table == "" { + table = "schema_migrations" + } + + if err := ensureMigrationsTable(ctx, cfg.DB, cfg.Driver, table); err != nil { + return err + } + + current, dirty, err := getCurrentVersion(ctx, cfg.DB, cfg.Driver, table) + if err != nil { + return err + } + if dirty { + return ErrDirty + } + if cfg.MaxVersion >= 0 && current > cfg.MaxVersion { + return fmt.Errorf("%w: db version %d > supported max %d", ErrNewerSchema, current, cfg.MaxVersion) + } + + migs := make([]Migration, len(cfg.Migrations)) + copy(migs, cfg.Migrations) + sort.Slice(migs, func(i, j int) bool { return migs[i].Version < migs[j].Version }) + + for _, m := range migs { + if m.Version <= current { + continue + } + if err := applyMigration(ctx, cfg.DB, cfg.Driver, table, m); err != nil { + return err + } + } + return nil +} + +func ensureMigrationsTable(ctx context.Context, db *sql.DB, drv Driver, table string) error { + var q string + if drv == DriverPostgres { + q = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + version BIGINT PRIMARY KEY, + description TEXT, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + dirty BOOLEAN NOT NULL DEFAULT FALSE + )`, table) + } else { + q = fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s ( + version INTEGER PRIMARY KEY, + description TEXT, + applied_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + dirty INTEGER NOT NULL DEFAULT 0 + )`, table) + } + if _, err := db.ExecContext(ctx, q); err != nil { + return fmt.Errorf("migrate: create migrations table: %w", err) + } + return nil +} + +func getCurrentVersion(ctx context.Context, db *sql.DB, drv Driver, table string) (int64, bool, error) { + q := fmt.Sprintf(`SELECT version, dirty FROM %s ORDER BY version DESC LIMIT 1`, table) + var v int64 + var dirtyVal interface{} + err := db.QueryRowContext(ctx, q).Scan(&v, &dirtyVal) + if err == sql.ErrNoRows { + return 0, false, nil + } + if err != nil { + return 0, false, fmt.Errorf("migrate: read current version: %w", err) + } + switch d := dirtyVal.(type) { + case bool: + return v, d, nil + case int64: + return v, d != 0, nil + case int: + return v, d != 0, nil + default: + return v, false, nil + } +} + +func applyMigration(ctx context.Context, db *sql.DB, drv Driver, table string, m Migration) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("migrate: begin tx for v%d: %w", m.Version, err) + } + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + if err := m.Up(ctx, tx); err != nil { + return fmt.Errorf("migrate: up v%d (%s): %w", m.Version, m.Description, err) + } + if err := recordMigration(ctx, tx, drv, table, m); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("migrate: commit v%d: %w", m.Version, err) + } + committed = true + return nil +} + +func recordMigration(ctx context.Context, tx *sql.Tx, drv Driver, table string, m Migration) error { + var q string + if drv == DriverPostgres { + q = fmt.Sprintf(`INSERT INTO %s (version, description, dirty) VALUES ($1, $2, FALSE)`, table) + } else { + q = fmt.Sprintf(`INSERT INTO %s (version, description, dirty) VALUES (?, ?, 0)`, table) + } + if _, err := tx.ExecContext(ctx, q, m.Version, m.Description); err != nil { + return fmt.Errorf("migrate: record v%d: %w", m.Version, err) + } + return nil +} diff --git a/nore/internal/storage/migrate/migrate_test.go b/nore/internal/storage/migrate/migrate_test.go new file mode 100644 index 0000000..cbbe428 --- /dev/null +++ b/nore/internal/storage/migrate/migrate_test.go @@ -0,0 +1,79 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +func TestMigrateEmptySQLite(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dbPath := filepath.Join(dir, "nore.db") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + migs := KVStoreMigrations(DriverSQLite) + maxv := int64(0) + for _, m := range migs { + if m.Version > maxv { + maxv = m.Version + } + } + if err := Migrate(ctx, Config{DB: db, Driver: DriverSQLite, Migrations: migs, MaxVersion: maxv}); err != nil { + t.Fatalf("migrate: %v", err) + } + if _, err := db.ExecContext(ctx, `INSERT INTO kv_store (key, value) VALUES ('k','v')`); err != nil { + t.Fatalf("insert: %v", err) + } +} + +func TestMigrateIdempotent(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dbPath := filepath.Join(dir, "nore.db") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + migs := KVStoreMigrations(DriverSQLite) + maxv := int64(0) + for _, m := range migs { + if m.Version > maxv { + maxv = m.Version + } + } + if err := Migrate(ctx, Config{DB: db, Driver: DriverSQLite, Migrations: migs, MaxVersion: maxv}); err != nil { + t.Fatalf("migrate: %v", err) + } + if err := Migrate(ctx, Config{DB: db, Driver: DriverSQLite, Migrations: migs, MaxVersion: maxv}); err != nil { + t.Fatalf("migrate again: %v", err) + } +} + +func TestMigrateRefuseNewerSchema(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + dbPath := filepath.Join(dir, "nore.db") + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open: %v", err) + } + defer db.Close() + migs := KVStoreMigrations(DriverSQLite) + if err := Migrate(ctx, Config{DB: db, Driver: DriverSQLite, Migrations: migs, MaxVersion: 5}); err != nil { + t.Fatalf("migrate: %v", err) + } + if err := Migrate(ctx, Config{DB: db, Driver: DriverSQLite, Migrations: migs, MaxVersion: 0}); err == nil { + t.Fatal("expected error for newer schema") + } else if !errors.Is(err, ErrNewerSchema) { + t.Fatalf("expected ErrNewerSchema, got %v", err) + } +} diff --git a/nore/internal/storage/migrate/registry.go b/nore/internal/storage/migrate/registry.go new file mode 100644 index 0000000..cab89c9 --- /dev/null +++ b/nore/internal/storage/migrate/registry.go @@ -0,0 +1,50 @@ +package migrate + +import ( + "context" + "database/sql" +) + +// KVStoreMigrations returns the initial migrations for the kv_store schema. +func KVStoreMigrations(drv Driver) []Migration { + switch drv { + case DriverSQLite: + return []Migration{ + { + Version: 1, + Description: "create kv_store table", + Up: func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv_store ( + key TEXT PRIMARY KEY, + value BLOB NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + )`) + return err + }, + }, + } + case DriverPostgres: + return []Migration{ + { + Version: 1, + Description: "create kv_store table", + Up: func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `CREATE TABLE IF NOT EXISTS kv_store ( + key TEXT PRIMARY KEY, + value BYTEA NOT NULL, + created_at TIMESTAMP DEFAULT NOW(), + updated_at TIMESTAMP DEFAULT NOW() + )`) + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS idx_kv_store_key ON kv_store (key)`) + return err + }, + }, + } + default: + return nil + } +} diff --git a/nore/internal/storage/parity_test.go b/nore/internal/storage/parity_test.go new file mode 100644 index 0000000..0017fe1 --- /dev/null +++ b/nore/internal/storage/parity_test.go @@ -0,0 +1,82 @@ +package storage_test + +import ( + "context" + "os" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/postgres" + "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" +) + +func TestCrossBackendParitySQLitePostgres(t *testing.T) { + // SQLite + sqliteDir := t.TempDir() + sqliteStore, err := sqlite.New(context.Background(), storage.Config{DataDir: sqliteDir}) + if err != nil { + t.Fatalf("sqlite new: %v", err) + } + defer sqliteStore.Close() + + // Postgres if available + pgDSN := os.Getenv("NEURON_TEST_POSTGRES_DSN") + if pgDSN == "" { + pgDSN = os.Getenv("NEURON_DATABASE_URL") + } + if pgDSN == "" { + t.Skipf("postgres DSN not set; skipping cross-backend parity") + } + pgStore, err := postgres.New(context.Background(), storage.Config{DSN: pgDSN}) + if err != nil { + t.Fatalf("postgres new: %v", err) + } + defer pgStore.Close() + + ctx := context.Background() + key := mustKeyParity(t, "parity", "k1") + val := []byte("hello") + for _, s := range []storage.Store{sqliteStore, pgStore} { + if err := s.Put(ctx, key, val); err != nil { + t.Fatalf("put: %v", err) + } + got, err := s.Get(ctx, key) + if err != nil { + t.Fatalf("get: %v", err) + } + if string(got) != string(val) { + t.Fatalf("mismatch") + } + keys, err := s.List(ctx, mustPrefixParity(t, "parity")) + if err != nil { + t.Fatalf("list: %v", err) + } + if len(keys) != 1 { + t.Fatalf("list len") + } + if err := s.Delete(ctx, key); err != nil { + t.Fatalf("del: %v", err) + } + if _, err := s.Get(ctx, key); err == nil { + t.Fatalf("expected err after delete") + } + } +} + +func mustKeyParity(t *testing.T, parts ...string) string { + t.Helper() + key, err := storage.Key(parts...) + if err != nil { + t.Fatalf("key: %v", err) + } + return key +} + +func mustPrefixParity(t *testing.T, parts ...string) string { + t.Helper() + p, err := storage.Prefix(parts...) + if err != nil { + t.Fatalf("prefix: %v", err) + } + return p +} diff --git a/nore/internal/storage/postgres/contract_test.go b/nore/internal/storage/postgres/contract_test.go new file mode 100644 index 0000000..620fe51 --- /dev/null +++ b/nore/internal/storage/postgres/contract_test.go @@ -0,0 +1,32 @@ +package postgres + +import ( + "context" + "os" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/storagetest" +) + +func TestContract(t *testing.T) { + dsn := os.Getenv("NEURON_TEST_POSTGRES_DSN") + if dsn == "" { + dsn = os.Getenv("NEURON_DATABASE_URL") + } + if dsn == "" { + t.Skipf("postgres DSN not set (NEURON_TEST_POSTGRES_DSN or NEURON_DATABASE_URL); skipping contract tests") + } + storagetest.Run(t, func(t *testing.T) storage.Store { + t.Helper() + ctx := context.Background() + store, err := New(ctx, storage.Config{DSN: dsn}) + if err != nil { + t.Fatalf("open postgres: %v", err) + } + t.Cleanup(func() { + _ = store.Close() + }) + return store + }) +} diff --git a/nore/internal/storage/postgres/postgres.go b/nore/internal/storage/postgres/postgres.go new file mode 100644 index 0000000..3931de6 --- /dev/null +++ b/nore/internal/storage/postgres/postgres.go @@ -0,0 +1,306 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + + _ "github.com/lib/pq" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/migrate" +) + +func init() { + storage.RegisterDriver("postgres", New) + storage.RegisterDriver("postgresql", New) +} + +type Store struct { + mu sync.Mutex + db *sql.DB + txActive atomic.Bool + closed atomic.Bool +} + +func New(ctx context.Context, cfg storage.Config) (storage.Store, error) { + if ctx == nil { + ctx = context.Background() + } + dsn := cfg.DSN + if dsn == "" { + dsn = os.Getenv("NEURON_DATABASE_URL") + } + if dsn == "" { + return nil, fmt.Errorf("postgres: dsn is required") + } + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres: %w", err) + } + + db.SetMaxOpenConns(25) + db.SetMaxIdleConns(25) + db.SetConnMaxLifetime(0) + + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping postgres: %w", err) + } + + s := &Store{db: db} + migs := migrate.KVStoreMigrations(migrate.DriverPostgres) + maxv := int64(0) + for _, m := range migs { + if m.Version > maxv { + maxv = m.Version + } + } + if err := migrate.Migrate(ctx, migrate.Config{DB: db, Driver: migrate.DriverPostgres, Migrations: migs, MaxVersion: maxv}); err != nil { + _ = db.Close() + return nil, fmt.Errorf("migrate: %w", err) + } + return s, nil +} + +func (s *Store) guard() error { + if s.closed.Load() { + return storage.ErrClosed + } + return nil +} + +func (s *Store) Get(ctx context.Context, key string) ([]byte, error) { + if err := s.guard(); err != nil { + return nil, err + } + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + var data []byte + err := s.db.QueryRowContext(ctx, "SELECT value FROM kv_store WHERE key = $1", key).Scan(&data) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", storage.ErrNotFound, key) + } + if err != nil { + return nil, fmt.Errorf("postgres get %q: %w", key, err) + } + return data, nil +} + +func (s *Store) Put(ctx context.Context, key string, data []byte) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + _, err := s.db.ExecContext(ctx, "INSERT INTO kv_store (key, value, updated_at) VALUES ($1, $2, NOW()) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()", key, data) + if err != nil { + return fmt.Errorf("postgres put %q: %w", key, err) + } + return nil +} + +func (s *Store) Delete(ctx context.Context, key string) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + _, err := s.db.ExecContext(ctx, "DELETE FROM kv_store WHERE key = $1", key) + if err != nil { + return fmt.Errorf("postgres delete %q: %w", key, err) + } + return nil +} + +func escapeLike(s string) string { + s = strings.ReplaceAll(s, "\\", "\\\\") + s = strings.ReplaceAll(s, "%", "\\%") + s = strings.ReplaceAll(s, "_", "\\_") + return s +} + +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + if err := s.guard(); err != nil { + return nil, err + } + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + pattern := escapeLike(prefix) + "%" + rows, err := s.db.QueryContext(ctx, "SELECT key FROM kv_store WHERE key LIKE $1 ESCAPE '\\' ORDER BY key", pattern) + if err != nil { + return nil, fmt.Errorf("postgres list %q: %w", prefix, err) + } + defer rows.Close() + var keys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + return nil, fmt.Errorf("postgres list scan: %w", err) + } + keys = append(keys, k) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres list: %w", err) + } + return keys, nil +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + if err := s.db.PingContext(ctx); err != nil { + return fmt.Errorf("postgres ping: %w", err) + } + return nil +} + +func (s *Store) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + if err := s.guard(); err != nil { + return err + } + if fn == nil { + return fmt.Errorf("transaction callback is required") + } + if s.txActive.Load() { + return storage.ErrNestedTransaction + } + s.mu.Lock() + if s.txActive.Load() { + s.mu.Unlock() + return storage.ErrNestedTransaction + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + s.mu.Unlock() + return fmt.Errorf("postgres begin: %w", err) + } + s.txActive.Store(true) + s.mu.Unlock() + + committed := false + defer func() { + s.txActive.Store(false) + if !committed { + _ = tx.Rollback() + } + }() + + t := &txOps{tx: tx} + if err := fn(t); err != nil { + return err + } + if ctx.Err() != nil { + return ctx.Err() + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("postgres commit: %w", err) + } + committed = true + return nil +} + +type txOps struct { + tx *sql.Tx +} + +func (t *txOps) Get(ctx context.Context, key string) ([]byte, error) { + var data []byte + err := t.tx.QueryRowContext(ctx, "SELECT value FROM kv_store WHERE key = $1", key).Scan(&data) + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: %s", storage.ErrNotFound, key) + } + if err != nil { + return nil, fmt.Errorf("postgres tx get %q: %w", key, err) + } + return data, nil +} + +func (t *txOps) Put(ctx context.Context, key string, data []byte) error { + _, err := t.tx.ExecContext(ctx, "INSERT INTO kv_store (key, value, updated_at) VALUES ($1, $2, NOW()) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, updated_at = NOW()", key, data) + if err != nil { + return fmt.Errorf("postgres tx put %q: %w", key, err) + } + return nil +} + +func (t *txOps) Delete(ctx context.Context, key string) error { + _, err := t.tx.ExecContext(ctx, "DELETE FROM kv_store WHERE key = $1", key) + if err != nil { + return fmt.Errorf("postgres tx delete %q: %w", key, err) + } + return nil +} + +func (t *txOps) List(ctx context.Context, prefix string) ([]string, error) { + pattern := escapeLike(prefix) + "%" + rows, err := t.tx.QueryContext(ctx, "SELECT key FROM kv_store WHERE key LIKE $1 ESCAPE '\\' ORDER BY key", pattern) + if err != nil { + return nil, fmt.Errorf("postgres tx list %q: %w", prefix, err) + } + defer rows.Close() + var keys []string + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + return nil, fmt.Errorf("postgres tx list scan: %w", err) + } + keys = append(keys, k) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("postgres tx list: %w", err) + } + return keys, nil +} + +func (s *Store) Close() error { + if !s.closed.CompareAndSwap(false, true) { + return nil + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + if err := s.db.Close(); err != nil { + return fmt.Errorf("postgres close: %w", err) + } + return nil +} diff --git a/nore/internal/storage/postgres/postgres.go | head -80 b/nore/internal/storage/postgres/postgres.go | head -80 new file mode 100644 index 0000000..469082b --- /dev/null +++ b/nore/internal/storage/postgres/postgres.go | head -80 @@ -0,0 +1,67 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "strings" + "sync" + "sync/atomic" + + _ "github.com/lib/pq" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/migrate" +) + +func init() { + storage.RegisterDriver("postgres", New) + storage.RegisterDriver("postgresql", New) +} + +type Store struct { + mu sync.Mutex + db *sql.DB + txActive atomic.Bool + closed atomic.Bool +} + +func New(ctx context.Context, cfg storage.Config) (storage.Store, error) { + dsn := cfg.DSN + if dsn == "" { + dsn = os.Getenv("NEURON_DATABASE_URL") + } + if dsn == "" { + return nil, fmt.Errorf("postgres: dsn is required") + } + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, fmt.Errorf("open postgres: %w", err) + } + + db.SetMaxOpenConns(25) + db.SetMaxIdleConns(25) + db.SetConnMaxLifetime(0) + + if err := db.PingContext(ctx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("ping postgres: %w", err) + } + + s := &Store{db: db} + migs := migrate.KVStoreMigrations(migrate.DriverPostgres) + maxv := int64(0) + for _, m := range migs { + if m.Version > maxv { + maxv = m.Version + } + } + if err := migrate.Migrate(ctx, migrate.Config{DB: db, Driver: migrate.DriverPostgres, Migrations: migs, MaxVersion: maxv}); err != nil { + _ = db.Close() + return nil, fmt.Errorf("migrate: %w", err) + } + return s, nil +} diff --git a/nore/internal/storage/provider.go b/nore/internal/storage/provider.go index 07d0b31..cc9282f 100644 --- a/nore/internal/storage/provider.go +++ b/nore/internal/storage/provider.go @@ -6,17 +6,82 @@ import ( ) var ( - ErrNotFound = errors.New("key not found") + // ErrNotFound reports that a key does not exist. Callers must use errors.Is + // rather than matching an error string so a backend can wrap the sentinel + // with its own context. + ErrNotFound = errors.New("storage: key not found") + + // ErrConflict reports that a compare-and-swap write lost to another writer. + // The caller must reload the record before retrying. + ErrConflict = errors.New("storage: write conflict") + + // ErrAlreadyExists reports that an insert-only write found an existing + // record. + ErrAlreadyExists = errors.New("storage: key already exists") + + // ErrNestedTransaction reports a WithTx call made from inside another + // transaction. Transactions do not compose implicitly; the caller already + // holds the transaction and must pass its Tx to nested work. + ErrNestedTransaction = errors.New("storage: nested transaction") + + // ErrTransactionActive reports a plain Store operation attempted while a + // WithTx callback holds the store's transaction. Use the callback's Tx so + // the operation participates in the same atomic unit. + ErrTransactionActive = errors.New("storage: transaction already active") + + // ErrUnknownDriver reports a storage driver that no backend registered. + ErrUnknownDriver = errors.New("storage: unknown driver") + + // ErrInvalidKey reports a key segment that cannot be represented safely. + ErrInvalidKey = errors.New("storage: invalid key") + + // ErrClosed reports an operation on a store that has been closed. + ErrClosed = errors.New("storage: closed") ) -type Store interface { +// KV is the key/value surface shared by a Store and a transaction. Keys are +// opaque strings; callers construct them with Key or Prefix. +type KV interface { Get(ctx context.Context, key string) ([]byte, error) Put(ctx context.Context, key string, data []byte) error Delete(ctx context.Context, key string) error List(ctx context.Context, prefix string) ([]string, error) +} + +// Tx is a KV bound to one transaction. Every operation on a Tx participates in +// the transaction opened by Store.WithTx. +type Tx interface { + KV +} + +// Store is the only storage type runtime components receive. It exposes plain +// reads and writes, health checking, explicit transactions, and lifecycle +// closure. Backend construction is owned by Open; components must not construct +// a backend themselves. +type Store interface { + KV + + // Ping verifies that the backend is reachable. + Ping(ctx context.Context) error + + // WithTx runs fn inside one transaction. Returning an error from fn rolls + // the transaction back. Nested WithTx calls return ErrNestedTransaction; + // plain Store operations during fn return ErrTransactionActive. + WithTx(ctx context.Context, fn func(Tx) error) error + + // Close releases the backend. Close during an active transaction returns + // ErrTransactionActive. Close() error } +// Config selects and configures a storage backend. type Config struct { + // Driver names the registered backend. Empty selects DefaultDriver. + Driver string + // DataDir is the backend's root directory (legacy). DataDir string + // Path is the SQLite database file path or root location depending on backend. + Path string + // DSN is the PostgreSQL connection string. It must not be logged verbatim. + DSN string } diff --git a/nore/internal/storage/sqlite/contract_test.go b/nore/internal/storage/sqlite/contract_test.go new file mode 100644 index 0000000..6268874 --- /dev/null +++ b/nore/internal/storage/sqlite/contract_test.go @@ -0,0 +1,22 @@ +package sqlite_test + +import ( + "context" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" + "github.com/neuron-runtime/neuron/nore/internal/storage/storagetest" +) + +func TestSQLiteStorageContract(t *testing.T) { + storagetest.Run(t, func(t *testing.T) storage.Store { + t.Helper() + store, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + return store + }) +} diff --git a/nore/internal/storage/sqlite/failure_test.go b/nore/internal/storage/sqlite/failure_test.go new file mode 100644 index 0000000..12b0b8f --- /dev/null +++ b/nore/internal/storage/sqlite/failure_test.go @@ -0,0 +1,53 @@ +package sqlite + +import ( + "context" + "errors" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" +) + +// TestCommitFailureIsReported proves a failed COMMIT is not converted into +// success. The callback rolls the transaction back underneath WithTx, so the +// subsequent Commit fails after the transaction has already written. +func TestCommitFailureIsReported(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + store, err := New(ctx, storage.Config{DataDir: dir}) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + key, err := storage.Key("failure", "commit") + if err != nil { + t.Fatalf("key: %v", err) + } + + err = store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(ctx, key, []byte("value")); err != nil { + return err + } + ops, ok := tx.(txOps) + if !ok { + return errors.New("unexpected transaction type") + } + if err := ops.tx.Rollback(); err != nil { + return err + } + return nil + }) + if err == nil { + t.Fatal("commit failure was reported as success") + } + + reopened, err := New(ctx, storage.Config{DataDir: dir}) + if err != nil { + t.Fatalf("reopen sqlite: %v", err) + } + t.Cleanup(func() { _ = reopened.Close() }) + if _, err := reopened.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("key from failed commit = %v, want ErrNotFound", err) + } +} diff --git a/nore/internal/storage/sqlite/open_test.go b/nore/internal/storage/sqlite/open_test.go new file mode 100644 index 0000000..d176eed --- /dev/null +++ b/nore/internal/storage/sqlite/open_test.go @@ -0,0 +1,38 @@ +package sqlite_test + +import ( + "context" + "errors" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" + _ "github.com/neuron-runtime/neuron/nore/internal/storage/sqlite" +) + +func TestOpenSelectsRegisteredDriver(t *testing.T) { + for _, driver := range []string{"sqlite", "SQLite", ""} { + t.Run("driver="+driver, func(t *testing.T) { + store, err := storage.Open(context.Background(), storage.Config{ + Driver: driver, + DataDir: t.TempDir(), + }) + if err != nil { + t.Fatalf("open driver %q: %v", driver, err) + } + defer store.Close() + if err := store.Ping(context.Background()); err != nil { + t.Fatalf("ping opened store: %v", err) + } + }) + } +} + +func TestOpenRejectsUnknownDriver(t *testing.T) { + _, err := storage.Open(context.Background(), storage.Config{ + Driver: "does-not-exist", + DataDir: t.TempDir(), + }) + if !errors.Is(err, storage.ErrUnknownDriver) { + t.Fatalf("open unknown driver = %v, want ErrUnknownDriver", err) + } +} diff --git a/nore/internal/storage/sqlite/sqlite.go b/nore/internal/storage/sqlite/sqlite.go index b04d544..191d953 100644 --- a/nore/internal/storage/sqlite/sqlite.go +++ b/nore/internal/storage/sqlite/sqlite.go @@ -4,27 +4,55 @@ import ( "context" "database/sql" "fmt" + "os" "path/filepath" - "strings" "sync" + "sync/atomic" "github.com/neuron-runtime/neuron/nore/internal/storage" + "github.com/neuron-runtime/neuron/nore/internal/storage/migrate" _ "modernc.org/sqlite" ) +func init() { + storage.RegisterDriver("sqlite", func(ctx context.Context, cfg storage.Config) (storage.Store, error) { + return New(ctx, cfg) + }) +} + +// Store is the SQLite storage backend. It serializes access with one connection +// so that a transaction can safely own the connection for its whole callback. +// txActive is checked atomically before taking mu, which lets an operation +// attempted during a transaction report ErrTransactionActive instead of blocking +// until the transaction commits. type Store struct { - mu sync.RWMutex - db *sql.DB - root string + mu sync.Mutex + db *sql.DB + txActive atomic.Bool + closed atomic.Bool } -func New(cfg storage.Config) (*Store, error) { - if cfg.DataDir == "" { +// New opens the SQLite database, initializes its schema, and verifies that it +// responds. A failure closes the database before returning so a partially +// initialized store is never observed. +func New(ctx context.Context, cfg storage.Config) (*Store, error) { + if ctx == nil { + ctx = context.Background() + } + dataDir := cfg.DataDir + if dataDir == "" { + dataDir = cfg.Path + } + if dataDir == "" { return nil, fmt.Errorf("data directory is required") } - dbPath := filepath.Join(cfg.DataDir, "nore.db") + if err := os.MkdirAll(dataDir, 0o755); err != nil { + return nil, fmt.Errorf("create data directory %s: %w", dataDir, err) + } + + dbPath := filepath.Join(dataDir, "nore.db") db, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL&_busy_timeout=5000&_synchronous=NORMAL") if err != nil { @@ -33,13 +61,20 @@ func New(cfg storage.Config) (*Store, error) { db.SetMaxOpenConns(1) - if err := db.Ping(); err != nil { + if err := db.PingContext(ctx); err != nil { _ = db.Close() return nil, fmt.Errorf("ping sqlite: %w", err) } - s := &Store{db: db, root: cfg.DataDir} - if err := s.migrate(); err != nil { + s := &Store{db: db} + migs := migrate.KVStoreMigrations(migrate.DriverSQLite) + maxv := int64(0) + for _, m := range migs { + if m.Version > maxv { + maxv = m.Version + } + } + if err := migrate.Migrate(ctx, migrate.Config{DB: db, Driver: migrate.DriverSQLite, Migrations: migs, MaxVersion: maxv}); err != nil { _ = db.Close() return nil, fmt.Errorf("migrate: %w", err) } @@ -47,38 +82,187 @@ func New(cfg storage.Config) (*Store, error) { return s, nil } -func (s *Store) migrate() error { - query := `CREATE TABLE IF NOT EXISTS kv_store ( - key TEXT PRIMARY KEY, - value BLOB NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - )` - _, err := s.db.Exec(query) - return err +func (s *Store) Get(ctx context.Context, key string) ([]byte, error) { + if err := s.guard(); err != nil { + return nil, err + } + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + return get(ctx, s.db, key) +} + +func (s *Store) Put(ctx context.Context, key string, data []byte) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + return put(ctx, s.db, key, data) } -func (s *Store) Get(ctx context.Context, key string) ([]byte, error) { - s.mu.RLock() - defer s.mu.RUnlock() +func (s *Store) Delete(ctx context.Context, key string) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + return del(ctx, s.db, key) +} + +func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { + if err := s.guard(); err != nil { + return nil, err + } + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return nil, storage.ErrTransactionActive + } + return list(ctx, s.db, prefix) +} + +func (s *Store) Ping(ctx context.Context) error { + if err := s.guard(); err != nil { + return err + } + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + return s.db.PingContext(ctx) +} + +func (s *Store) WithTx(ctx context.Context, fn func(storage.Tx) error) error { + if err := s.guard(); err != nil { + return err + } + if fn == nil { + return fmt.Errorf("transaction callback is required") + } + if s.txActive.Load() { + return storage.ErrNestedTransaction + } + + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrNestedTransaction + } + + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("sqlite begin: %w", err) + } + s.txActive.Store(true) + defer s.txActive.Store(false) + committed := false + defer func() { + if !committed { + _ = tx.Rollback() + } + }() + + if err := fn(txOps{tx: tx}); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("sqlite commit: %w", err) + } + committed = true + return nil +} + +func (s *Store) Close() error { + if s.txActive.Load() { + return storage.ErrTransactionActive + } + s.mu.Lock() + defer s.mu.Unlock() + if s.txActive.Load() { + return storage.ErrTransactionActive + } + if s.closed.Swap(true) { + return nil + } + return s.db.Close() +} + +func (s *Store) guard() error { + if s.closed.Load() { + return storage.ErrClosed + } + return nil +} + +type txOps struct { + tx *sql.Tx +} + +func (t txOps) Get(ctx context.Context, key string) ([]byte, error) { + return get(ctx, t.tx, key) +} + +func (t txOps) Put(ctx context.Context, key string, data []byte) error { + return put(ctx, t.tx, key, data) +} + +func (t txOps) Delete(ctx context.Context, key string) error { + return del(ctx, t.tx, key) +} + +func (t txOps) List(ctx context.Context, prefix string) ([]string, error) { + return list(ctx, t.tx, prefix) +} + +type querier interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +func get(ctx context.Context, q querier, key string) ([]byte, error) { var value []byte - err := s.db.QueryRowContext(ctx, `SELECT value FROM kv_store WHERE key = ?`, key).Scan(&value) + err := q.QueryRowContext(ctx, `SELECT value FROM kv_store WHERE key = ?`, key).Scan(&value) if err == sql.ErrNoRows { return nil, fmt.Errorf("%w: %s", storage.ErrNotFound, key) } if err != nil { return nil, fmt.Errorf("sqlite get %q: %w", key, err) } - return value, nil } -func (s *Store) Put(ctx context.Context, key string, data []byte) error { - s.mu.Lock() - defer s.mu.Unlock() - - _, err := s.db.ExecContext(ctx, +func put(ctx context.Context, q querier, key string, data []byte) error { + _, err := q.ExecContext(ctx, `INSERT INTO kv_store (key, value, created_at, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = CURRENT_TIMESTAMP`, key, data, @@ -86,27 +270,22 @@ func (s *Store) Put(ctx context.Context, key string, data []byte) error { if err != nil { return fmt.Errorf("sqlite put %q: %w", key, err) } - return nil } -func (s *Store) Delete(ctx context.Context, key string) error { - s.mu.Lock() - defer s.mu.Unlock() - - _, err := s.db.ExecContext(ctx, `DELETE FROM kv_store WHERE key = ?`, key) +func del(ctx context.Context, q querier, key string) error { + _, err := q.ExecContext(ctx, `DELETE FROM kv_store WHERE key = ?`, key) if err != nil { return fmt.Errorf("sqlite delete %q: %w", key, err) } - return nil } -func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { - s.mu.RLock() - defer s.mu.RUnlock() - - rows, err := s.db.QueryContext(ctx, `SELECT key FROM kv_store WHERE key LIKE ? ORDER BY key`, prefix+"%") +func list(ctx context.Context, q querier, prefix string) ([]string, error) { + rows, err := q.QueryContext(ctx, + `SELECT key FROM kv_store WHERE key LIKE ? ESCAPE '\' ORDER BY key`, + escapeLike(prefix)+"%", + ) if err != nil { return nil, fmt.Errorf("sqlite list %q: %w", prefix, err) } @@ -127,14 +306,14 @@ func (s *Store) List(ctx context.Context, prefix string) ([]string, error) { return keys, nil } -func (s *Store) Close() error { - return s.db.Close() -} - -func (s *Store) Root() string { - return s.root -} - -func SanitizeKey(parts ...string) string { - return strings.Join(parts, "/") +func escapeLike(value string) string { + var b []byte + for i := 0; i < len(value); i++ { + switch value[i] { + case '\\', '%', '_': + b = append(b, '\\') + } + b = append(b, value[i]) + } + return string(b) } diff --git a/nore/internal/storage/storagetest/contract.go b/nore/internal/storage/storagetest/contract.go new file mode 100644 index 0000000..f0b7bea --- /dev/null +++ b/nore/internal/storage/storagetest/contract.go @@ -0,0 +1,283 @@ +// Package storagetest defines the backend-neutral storage contract as an +// executable suite. A backend runs Run with its own constructor so every +// backend proves the same transaction and error semantics. +package storagetest + +import ( + "context" + "errors" + "testing" + + "github.com/neuron-runtime/neuron/nore/internal/storage" +) + +// Run exercises the storage contract against one backend. open must return a +// fresh, empty store for each call and must arrange its own cleanup. +func Run(t *testing.T, open func(t *testing.T) storage.Store) { + t.Helper() + + t.Run("key value basics", func(t *testing.T) { + store := open(t) + ctx := context.Background() + key := mustKey(t, "contract", "kv") + other := mustKey(t, "contractor") + + if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("get missing key = %v, want ErrNotFound", err) + } + if err := store.Put(ctx, key, []byte("one")); err != nil { + t.Fatalf("put key: %v", err) + } + if err := store.Put(ctx, other, []byte("outside")); err != nil { + t.Fatalf("put sibling key: %v", err) + } + if got, err := store.Get(ctx, key); err != nil || string(got) != "one" { + t.Fatalf("get key = %q, %v; want one", got, err) + } + if err := store.Put(ctx, key, []byte("two")); err != nil { + t.Fatalf("overwrite key: %v", err) + } + if got, err := store.Get(ctx, key); err != nil || string(got) != "two" { + t.Fatalf("get overwritten key = %q, %v; want two", got, err) + } + + keys, err := store.List(ctx, mustPrefix(t, "contract")) + if err != nil { + t.Fatalf("list prefix: %v", err) + } + if len(keys) != 1 || keys[0] != key { + t.Fatalf("list prefix = %v, want [%s]", keys, key) + } + + if err := store.Delete(ctx, key); err != nil { + t.Fatalf("delete key: %v", err) + } + if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("get deleted key = %v, want ErrNotFound", err) + } + }) + + t.Run("transaction commits", func(t *testing.T) { + store := open(t) + ctx := context.Background() + first := mustKey(t, "tx", "first") + second := mustKey(t, "tx", "second") + + err := store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(ctx, first, []byte("a")); err != nil { + return err + } + return tx.Put(ctx, second, []byte("b")) + }) + if err != nil { + t.Fatalf("commit transaction: %v", err) + } + for key, want := range map[string]string{first: "a", second: "b"} { + if got, err := store.Get(ctx, key); err != nil || string(got) != want { + t.Fatalf("get committed %s = %q, %v; want %q", key, got, err, want) + } + } + }) + + t.Run("transaction rolls back", func(t *testing.T) { + store := open(t) + ctx := context.Background() + existing := mustKey(t, "tx", "existing") + created := mustKey(t, "tx", "created") + abort := errors.New("abort transaction") + + if err := store.Put(ctx, existing, []byte("before")); err != nil { + t.Fatalf("seed key: %v", err) + } + err := store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(ctx, existing, []byte("after")); err != nil { + return err + } + if err := tx.Put(ctx, created, []byte("value")); err != nil { + return err + } + return abort + }) + if !errors.Is(err, abort) { + t.Fatalf("transaction error = %v, want %v", err, abort) + } + if got, err := store.Get(ctx, existing); err != nil || string(got) != "before" { + t.Fatalf("rolled-back existing = %q, %v; want before", got, err) + } + if _, err := store.Get(ctx, created); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("rolled-back created key = %v, want ErrNotFound", err) + } + }) + + t.Run("transaction callback panic rolls back", func(t *testing.T) { + store := open(t) + ctx := context.Background() + key := mustKey(t, "tx", "panic") + + func() { + defer func() { + if recover() == nil { + t.Fatal("panic in transaction callback was swallowed") + } + }() + _ = store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(ctx, key, []byte("value")); err != nil { + t.Fatalf("put in panic transaction: %v", err) + } + panic("boom") + }) + }() + + if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("key written by panicking transaction = %v, want ErrNotFound", err) + } + }) + + t.Run("nested and re-entrant operations are explicit", func(t *testing.T) { + store := open(t) + ctx := context.Background() + key := mustKey(t, "tx", "nested") + + err := store.WithTx(ctx, func(tx storage.Tx) error { + if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrTransactionActive) { + t.Fatalf("plain Get during transaction = %v, want ErrTransactionActive", err) + } + if err := store.Put(ctx, key, []byte("blocked")); !errors.Is(err, storage.ErrTransactionActive) { + t.Fatalf("plain Put during transaction = %v, want ErrTransactionActive", err) + } + if err := store.WithTx(ctx, func(storage.Tx) error { return nil }); !errors.Is(err, storage.ErrNestedTransaction) { + t.Fatalf("nested WithTx = %v, want ErrNestedTransaction", err) + } + return tx.Put(ctx, key, []byte("committed")) + }) + if err != nil { + t.Fatalf("outer transaction: %v", err) + } + if got, err := store.Get(ctx, key); err != nil || string(got) != "committed" { + t.Fatalf("outer transaction write = %q, %v; want committed", got, err) + } + }) + + t.Run("cancelled context aborts", func(t *testing.T) { + store := open(t) + key := mustKey(t, "tx", "cancelled") + + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + called := false + err := store.WithTx(cancelled, func(tx storage.Tx) error { + called = true + return tx.Put(cancelled, key, []byte("value")) + }) + if err == nil { + t.Fatal("WithTx with cancelled context succeeded") + } + if called { + t.Fatal("WithTx invoked callback with an already-cancelled context") + } + if _, err := store.Get(context.Background(), key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("key after cancelled transaction = %v, want ErrNotFound", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + err = store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(context.Background(), key, []byte("value")); err != nil { + return err + } + cancel() + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("WithTx cancellation error = %v, want context.Canceled", err) + } + if _, err := store.Get(context.Background(), key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("key after mid-transaction cancellation = %v, want ErrNotFound", err) + } + + putCtx, putCancel := context.WithCancel(context.Background()) + putCancel() + if err := store.Put(putCtx, key, []byte("value")); err == nil { + t.Fatal("Put with cancelled context succeeded") + } + if _, err := store.Get(context.Background(), key); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("key after cancelled Put = %v, want ErrNotFound", err) + } + }) + + t.Run("lifecycle", func(t *testing.T) { + store := open(t) + ctx := context.Background() + if err := store.Ping(ctx); err != nil { + t.Fatalf("ping: %v", err) + } + if err := store.Close(); err != nil { + t.Fatalf("close: %v", err) + } + key := mustKey(t, "lifecycle") + if err := store.Put(ctx, key, []byte("value")); !errors.Is(err, storage.ErrClosed) { + t.Fatalf("put after close = %v, want ErrClosed", err) + } + if _, err := store.Get(ctx, key); !errors.Is(err, storage.ErrClosed) { + t.Fatalf("get after close = %v, want ErrClosed", err) + } + if _, err := store.List(ctx, mustPrefix(t, "lifecycle")); !errors.Is(err, storage.ErrClosed) { + t.Fatalf("list after close = %v, want ErrClosed", err) + } + if err := store.Ping(ctx); !errors.Is(err, storage.ErrClosed) { + t.Fatalf("ping after close = %v, want ErrClosed", err) + } + }) + + t.Run("transaction failure injection", func(t *testing.T) { + store := open(t) + ctx := context.Background() + existing := mustKey(t, "tx", "existing-fi") + created := mustKey(t, "tx", "created-fi") + if err := store.Put(ctx, existing, []byte("before")); err != nil { + t.Fatalf("seed key: %v", err) + } + boom := errors.New("boom") + err := store.WithTx(ctx, func(tx storage.Tx) error { + if err := tx.Put(ctx, existing, []byte("after")); err != nil { + return err + } + if err := tx.Delete(ctx, created); err != nil { + // Delete may be no-op; ignore + } + if err := tx.Put(ctx, created, []byte("temp")); err != nil { + return err + } + if _, err := tx.Get(ctx, existing); err != nil { + return err + } + return boom + }) + if !errors.Is(err, boom) { + t.Fatalf("transaction error = %v, want %v", err, boom) + } + if got, err := store.Get(ctx, existing); err != nil || string(got) != "before" { + t.Fatalf("rolled-back existing = %q, %v; want before", got, err) + } + if _, err := store.Get(ctx, created); !errors.Is(err, storage.ErrNotFound) { + t.Fatalf("rolled-back created key = %v, want ErrNotFound", err) + } + }) +} + +func mustKey(t *testing.T, parts ...string) string { + t.Helper() + key, err := storage.Key(parts...) + if err != nil { + t.Fatalf("storage.Key(%v): %v", parts, err) + } + return key +} + +func mustPrefix(t *testing.T, parts ...string) string { + t.Helper() + prefix, err := storage.Prefix(parts...) + if err != nil { + t.Fatalf("storage.Prefix(%v): %v", parts, err) + } + return prefix +} diff --git a/nore/internal/stream/stream_test.go b/nore/internal/stream/stream_test.go index c0f6958..d19ab84 100644 --- a/nore/internal/stream/stream_test.go +++ b/nore/internal/stream/stream_test.go @@ -13,7 +13,7 @@ import ( func newTestStore(t *testing.T) *event.Store { t.Helper() - s, err := sqlite.New(storage.Config{DataDir: t.TempDir()}) + s, err := sqlite.New(context.Background(), storage.Config{DataDir: t.TempDir()}) if err != nil { t.Fatal(err) } diff --git a/nore/internal/types/blueprint.go b/nore/internal/types/blueprint.go index 1bc70fa..c949ecd 100644 --- a/nore/internal/types/blueprint.go +++ b/nore/internal/types/blueprint.go @@ -1,6 +1,11 @@ package types import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "sort" + "github.com/neuron-runtime/neuron/nore/internal/resolver" shared "github.com/neuron-runtime/neuron/shared/types/core" ) @@ -35,7 +40,7 @@ type ExecutionBlueprint struct { } // ExecutionResult is the compiled Assembly result contract and field mappings. -// It mirrors core.AssemblyResult but is defined here to avoid circular imports +// It mirrors shared.AssemblyResult but is defined here to avoid circular imports // and to be the planner's canonical representation. type ExecutionResult struct { Schema []shared.Port @@ -57,9 +62,28 @@ type ExecutionNode struct { // for each Capability execution. Configurations resolver.ConfigurationProgram + // Predecessors is the compiled set of incoming edges that must be satisfied + // before this capability may run. It is ordered by binding ID so readiness + // and merged inputs never depend on map iteration, declaration order, or + // completion timing. + Predecessors []ExecutionPredecessor + + // JoinPolicy is the effective policy that decides when this capability's + // predecessors have all been satisfied. The planner resolves it from the + // incoming bindings, applying the default for an undeclared policy. + JoinPolicy shared.JoinPolicy + Next []ExecutionTransition } +// ExecutionPredecessor identifies one incoming edge of an ExecutionNode. The +// binding ID is the stable identity of the edge; the capability ID names the +// source whose completion satisfies it. +type ExecutionPredecessor struct { + BindingID shared.ID + CapabilityID shared.ID +} + // CompiledMapping is a structurally resolved binding mapping: the planner // resolves a mapping's authored source into a canonical ValueRef, and the // scheduler reads the referenced value out of the transition environment at @@ -82,3 +106,51 @@ type ExecutionTransition struct { Mappings []CompiledMapping Validations []CompiledValidation } + +// Hash returns a deterministic content hash for this ExecutionBlueprint. +// The hash includes each node's capability identity, predecessors (incoming +// edges), and join policy. Two blueprints with the same hash have identical +// structure including join policies and incoming edges. This enables graph +// identity comparison for durable execution, fan-in verification, and recovery. +func (b *ExecutionBlueprint) Hash() (string, error) { + nodes := make([]normalizedBlueprintNode, 0, len(b.Nodes)) + for id, node := range b.Nodes { + predecessors := make([]normalizedPredecessor, 0, len(node.Predecessors)) + for _, pred := range node.Predecessors { + predecessors = append(predecessors, normalizedPredecessor{ + BindingID: pred.BindingID, + CapabilityID: pred.CapabilityID, + }) + } + cap := node.Capability + nodes = append(nodes, normalizedBlueprintNode{ + CapabilityID: string(id), + Type: string(cap.Type), + Params: append([]shared.Port(nil), cap.Params...), + Results: append([]shared.Port(nil), cap.Results...), + Predecessors: predecessors, + JoinPolicy: node.JoinPolicy, + }) + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].CapabilityID < nodes[j].CapabilityID }) + data, err := json.Marshal(nodes) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +type normalizedBlueprintNode struct { + CapabilityID string `json:"capabilityID"` + Type string `json:"type"` + Params []shared.Port `json:"params,omitempty"` + Results []shared.Port `json:"results,omitempty"` + Predecessors []normalizedPredecessor `json:"predecessors,omitempty"` + JoinPolicy shared.JoinPolicy `json:"joinPolicy"` +} + +type normalizedPredecessor struct { + BindingID shared.ID `json:"bindingID"` + CapabilityID shared.ID `json:"capabilityID"` +} diff --git a/packages/assembly-sdks/typescript/README.md b/packages/assembly-sdks/typescript/README.md index a2a12cb..3912c94 100644 --- a/packages/assembly-sdks/typescript/README.md +++ b/packages/assembly-sdks/typescript/README.md @@ -63,6 +63,9 @@ const validateOrder = Capability("order.validate") > [!NOTE] > Capability runtime names follow the `owner:capability:sub` convention (for example `neuron:core:set`, `example:echo`, `github:read`), so a capability without an explicit `.runtime()` defaults to the in-process built-in `neuron:core:set` and runs without any registry or installation. To run a capability through an external module — a signed process or a WebAssembly worker — set `.runtime({ name: ":", version, registry })` explicitly; the resolved artifact is verified and frozen at `neuron build` time. See [docs/MODULES.md](../../../docs/MODULES.md). +> [!NOTE] +> A built-in capability runtime runs in-process, so it is never resolved and never installed. It therefore carries no version and no registry: the manifest writes `"capabilityRuntime": { "name": "neuron:core:set" }` and nothing more. A version and a registry on a built-in would state that a package that does not exist has a version. + ### Runtime configuration A capability runtime declares **what** runs a capability. A `runtimeConfig` declares **how the runtime should drive** it: execution mode, timeout, retry behavior, and (reserved) resource constraints. @@ -363,7 +366,23 @@ Assembly() ]); ``` -A capability that reads from **multiple** capabilities is fan-in and is rejected at `toManifest()` time — execution does not support it. Route such inputs through an aggregation capability instead. +A capability that reads from **multiple** capabilities is **fan-in** and is now supported: +each source gets its own binding with only that source's mappings; the execution +engine merges inputs from all predecessors before dispatch. Route such inputs +directly — no aggregation capability is needed. + +```ts +const target = Capability("target").paramsSchema({ x: string(), y: string() }); +const sys = Assembly().run([ + a.withParams({}), + b.withParams({}), + target.withParams({ x: a.result.x, y: b.result.y }), +]); +// toManifest() succeeds; manifest carries two bindings from a and b respectively. +``` + +Fan-out still works as before — multiple capabilities reading from the same source +run in parallel. --- @@ -429,13 +448,13 @@ export default manifest; "capabilities": [ { "name": "order.validate", - "capabilityRuntime": { "name": "neuron:core:set", "version": "latest", "registry": "local" }, + "capabilityRuntime": { "name": "neuron:core:set" }, "params": [], "results": [] }, { "name": "payment.authorize", - "capabilityRuntime": { "name": "neuron:core:set", "version": "latest", "registry": "local" }, + "capabilityRuntime": { "name": "neuron:core:set" }, "params": [], "results": [] } @@ -508,6 +527,52 @@ The entry file default-exporting the manifest (`Assembly(...).toManifest()`) is | `AssemblyManifest`, `CapabilityManifest`, `BindingManifest` | Manifest structure types | | `Infer`, `Schema`, `SchemaField` | Schema type inference | +The `@neuronruntime/sdk/core` entry point exports the built-in capability runtimes: + +| Export | Purpose | +| --- | --- | +| `set` | Declare a capability that materializes a value set | +| `http` | Declare a capability that performs a single HTTP request | +| `command` | Declare a capability that executes a local program | +| `filesystem` | Declare a capability that reads files and directories | + +--- + +## Built-in capability runtimes + +The runtime provides four capability runtimes. They run in-process, they are never installed from a registry, and each one takes the name of the capability it runs. + +```ts +import { http, command, filesystem } from "@neuronruntime/sdk/core"; + +const fetchUser = http("users.get"); +const listFiles = filesystem("files.list"); +const runTool = command("tools.build"); +``` + +Each export returns a capability definition with the contract of the built-in already applied, so the params and results are typed. + +| Built-in | Accepts | Produces | +| --- | --- | --- | +| `set` | The capability's own configuration and params | The merged value set | +| `http` | `url`, `method`, `headers`, `query`, `body`, `contentType`, `auth`, `timeout`, `format`, `followRedirects`, `retry` | `status_code`, `headers`, `body`, `content_type`, `duration_ms` | +| `command` | `command`, `args`, `env`, `cwd`, `stdin`, `timeout`, `format`, `failOnNonZeroExit`, `maxOutputBytes` | `exit_code`, `stdout`, `stderr`, `success`, `timed_out`, `duration_ms` | +| `filesystem` | `operation`, `path`, `recursive`, `format` | `path`, `content`, `size`, `entries`, `exists`, `type` | + +The runtime enforces the boundary of each built-in on the assembly's behalf, and the project configuration may only narrow it. `neuron builtins` reports the contract the runtime publishes, including those boundaries. + +```bash +neuron builtins # every built-in +neuron builtins http # one built-in +neuron builtins http -v # with the full descriptions +``` + +### Migrating from the earlier core exports + +- `delay` is gone. It only waited, which is a sequencing concern rather than a capability; use an explicit dependency between the capabilities that must run in order instead. +- The exports are factories now. `http` was a capability definition named `neuron:core:http`; it is now `http("name")`, so each capability that uses the built-in has a name of its own. +- The `http` contract matches what the runtime publishes: the query parameters, the content type, the credentials, and the retry policy are accepted, and the response headers, content type, and duration are produced. + --- ## Development diff --git a/packages/assembly-sdks/typescript/src/assembly.ts b/packages/assembly-sdks/typescript/src/assembly.ts index fa7072b..93b9c08 100644 --- a/packages/assembly-sdks/typescript/src/assembly.ts +++ b/packages/assembly-sdks/typescript/src/assembly.ts @@ -302,29 +302,47 @@ function collectBindings( const bindings: BindingManifest[] = []; const seen = new Set(); + // R3-10: Track target param paths per capability to reject duplicate + // target field mappings. Two incoming bindings must not write to the same + // target param field path; if they do, compilation fails with a clear error. + const assignedPaths: Map> = new Map(); + for (const node of nodes) { const uniqueSources = Array.from(new Set(node.sources.filter(Boolean))); const sourceRefs = uniqueSources.length > 0 ? uniqueSources : node.anchor ? [node.anchor] : []; if (sourceRefs.length === 0) continue; - if (sourceRefs.length > 1) { - throw new Error( - `Capability "${node.capabilityRef}" reads inputs from multiple sources (${sourceRefs.join(", ")}). ` + - `Execution does not support fan-in; route the inputs through an aggregation capability instead.` - ); - } + // R3-10: Check for duplicate target field mappings across bindings to the + // same capability. Each source emits one binding; if two sources map to the + // same target path, compilation fails. + for (const from of sourceRefs) { + const key = `${from}->${node.capabilityRef}`; + if (seen.has(key)) continue; + seen.add(key); + + // R3-10: Validate that this binding's target paths do not conflict with + // previously assigned paths for the same target capability. + const targetCap = node.capabilityRef; + const existingPaths = assignedPaths.get(targetCap) || new Set(); + const mappings = bindingMappings(from, node, capabilities); + for (const mapping of mappings) { + const targetPath = mapping.target; + if (existingPaths.has(targetPath)) { + throw new Error( + `capability ${targetCap} has duplicate target path ${targetPath} ` + + `from bindings ${from} and ${Array.from(assignedPaths.keys()).find(k => assignedPaths.get(k)!.has(targetPath)) || 'unknown'}`, + ); + } + existingPaths.add(targetPath); + } + assignedPaths.set(targetCap, existingPaths); - const from = sourceRefs[0]!; - const key = `${from}->${node.capabilityRef}`; - if (seen.has(key)) continue; - seen.add(key); - - const mappings = bindingMappings(from, node, capabilities); - const validations = node.incomingConditions.map((c) => ({ - expression: c.expression, - message: c.message ?? "", - })); - bindings.push({ from, to: node.capabilityRef, mappings, validations }); + const validations = node.incomingConditions.map((c) => ({ + expression: c.expression, + message: c.message ?? "", + })); + bindings.push({ from, to: node.capabilityRef, mappings, validations }); + } } return bindings; @@ -335,10 +353,16 @@ function bindingMappings( target: CapabilityTarget, capabilities: Map> ): BindingManifest["mappings"] { - const explicit = Object.entries(target.bindings).map(([name, source]) => ({ - target: name, - source: qualifySource(source, fromRef, name, target.capabilityRef), - })); + const explicit = Object.entries(target.bindings).map(([name, source]) => { + const mapped = qualifySource(source, fromRef, name, target.capabilityRef); + // For capabilityResult/capabilityParams, only include mappings that reference + // this binding's from capability. This enables fan-in: each source emits a + // binding with only its own mappings. Other kinds (e.g. assemblyParams) are + // always included. + const isCapRef = mapped.kind === "capabilityResult" || mapped.kind === "capabilityParams"; + if (isCapRef && mapped.capability !== fromRef) return null; + return { target: name, source: mapped }; + }).filter((m): m is NonNullable => m !== null); if (explicit.length > 0) return explicit; const from = capabilities.get(fromRef)?.toManifest(); diff --git a/packages/assembly-sdks/typescript/src/capability.ts b/packages/assembly-sdks/typescript/src/capability.ts index f05194a..ae183f1 100644 --- a/packages/assembly-sdks/typescript/src/capability.ts +++ b/packages/assembly-sdks/typescript/src/capability.ts @@ -89,6 +89,30 @@ export interface RuntimeDeclaration { runtimeConfig?: RuntimeConfig; } +/** + * The core namespace holds the capability runtimes N.O.R.E. provides + * in-process. They are never resolved and never installed, so they have no + * version and no registry: a declaration that names one carries the name + * alone. + */ +const CORE_RUNTIME_NAMESPACE = "neuron:core:"; + +/** + * The bare names of the built-in capability runtimes. They are the legacy + * form of the core namespace and the runtime still accepts them. + */ +const CORE_RUNTIME_NAMES = new Set(["set", "http", "command", "filesystem"]); + +/** + * Reports whether a runtime name refers to a built-in capability runtime. + * + * A built-in runs inside the runtime process. It is not a package, so it is + * never resolved against a registry and it has no version to resolve. + */ +function isCoreRuntime(name: string): boolean { + return name.startsWith(CORE_RUNTIME_NAMESPACE) || CORE_RUNTIME_NAMES.has(name); +} + export type Condition = { when: Expression; message?: string; @@ -265,7 +289,7 @@ export class CapabilityDefinition) { this.ref = ref; this._state = { - runtime: state?.runtime ?? { name: "neuron:core:set", version: "latest", registry: "local" }, + runtime: state?.runtime ?? { name: "neuron:core:set" }, paramPorts: state?.paramPorts ?? [], resultPorts: state?.resultPorts ?? [], paramRules: state?.paramRules ?? {}, @@ -414,9 +438,14 @@ function cloneRuntimeConfig(config: RuntimeConfig): RuntimeConfig { function resolveRuntimeDeclaration(declaration: RuntimeDeclaration): CapabilityRuntimeManifest { const runtime: CapabilityRuntimeManifest = { name: declaration.name, - version: declaration.version ?? "latest", - registry: declaration.registry ?? "local", }; + // A built-in capability runtime runs in-process. It is never resolved and + // never installed, so it carries no version and no registry: writing them + // would state that a package that does not exist has a version. + if (!isCoreRuntime(declaration.name)) { + runtime.version = declaration.version ?? "latest"; + runtime.registry = declaration.registry ?? "local"; + } if (declaration.runtimeConfig !== undefined) { runtime.runtimeConfig = cloneRuntimeConfig(declaration.runtimeConfig); } @@ -431,7 +460,7 @@ export function Capability { return new CapabilityDefinition(name, { - runtime: { name: "neuron:core:set", version: "latest", registry: "local" }, + runtime: { name: "neuron:core:set" }, }); } diff --git a/packages/assembly-sdks/typescript/src/core.ts b/packages/assembly-sdks/typescript/src/core.ts index ea7e78e..922f5aa 100644 --- a/packages/assembly-sdks/typescript/src/core.ts +++ b/packages/assembly-sdks/typescript/src/core.ts @@ -1,6 +1,17 @@ export { - delay, + set, http, + command, + filesystem, + type SetParamsType, + type SetResultType, type HttpParamsType, - type HttpResultType -} from './core/capabilities'; \ No newline at end of file + type HttpResultType, + type HttpAuthType, + type HttpRetryType, + type CommandParamsType, + type CommandResultType, + type FilesystemParamsType, + type FilesystemResultType, + type FilesystemEntryType, +} from './core/capabilities'; diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/command.ts b/packages/assembly-sdks/typescript/src/core/capabilities/command.ts new file mode 100644 index 0000000..d040946 --- /dev/null +++ b/packages/assembly-sdks/typescript/src/core/capabilities/command.ts @@ -0,0 +1,66 @@ +import { Capability, type CapabilityDefinition } from "../../capability.js"; + +export type CommandParamsType = { + /** + * The program to run. A single executable, never a shell command line. + * + * The program is run without a shell, so an argument can never become a + * command. + */ + command: string; + /** The program's arguments. */ + args?: string[]; + /** Environment variables added to the program's environment. */ + env?: Record; + /** The working directory, resolved against the allowed root. */ + cwd?: string; + /** Text written to the program's standard input. */ + stdin?: string; + /** How long the program may run, such as "30s". */ + timeout?: string; + /** How standard output is represented: "json" or "text". */ + format?: "json" | "text"; + /** + * Whether a non-zero exit fails the capability. Defaults to true. Set it + * to false to inspect the result of a failing program instead. + */ + failOnNonZeroExit?: boolean; + /** The most output to retain per stream. The runtime's ceiling applies when this is absent. */ + maxOutputBytes?: number; +}; + +export type CommandResultType = { + /** The program's exit code. */ + exit_code: number; + /** The program's standard output: parsed JSON in the json format, a plain string in the text format. */ + stdout: unknown; + /** The program's standard error. */ + stderr: string; + /** Whether the program exited with code zero. */ + success: boolean; + /** Whether the program was terminated by its deadline. */ + timed_out: boolean; + /** How long the program ran, in milliseconds. */ + duration_ms: number; + /** Present when the standard output was shortened to the output limit. */ + stdout_truncated?: boolean; + /** Present when the standard error was shortened to the output limit. */ + stderr_truncated?: boolean; +}; + +/** + * Declares a capability that executes a local program. + * + * The program runs with an allowlisted environment, in a working directory + * confined to the allowed roots, and with a deadline. The whole process group + * is stopped when the run ends, so a child the program started does not + * outlive it. + * + * @param name the capability's logical identity in the assembly. + */ +export function command(name: string): CapabilityDefinition { + return Capability(name) + .runtime({ name: "neuron:core:command" }) + .paramsSchema() + .resultSchema(); +} diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/delay.ts b/packages/assembly-sdks/typescript/src/core/capabilities/delay.ts deleted file mode 100644 index c83180b..0000000 --- a/packages/assembly-sdks/typescript/src/core/capabilities/delay.ts +++ /dev/null @@ -1,11 +0,0 @@ -import {Capability} from "@/capability"; -import {string} from "@/schema"; - -export const delay = Capability("neuron:core:delay").paramsSchema({ - duration: string(), -}).resultSchema({ - delayed_for: string() -}).runtime({ - name: "neuron:core:delay", - version: "1.0.0" -}) \ No newline at end of file diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/filesystem.ts b/packages/assembly-sdks/typescript/src/core/capabilities/filesystem.ts new file mode 100644 index 0000000..9b075ed --- /dev/null +++ b/packages/assembly-sdks/typescript/src/core/capabilities/filesystem.ts @@ -0,0 +1,56 @@ +import { Capability, type CapabilityDefinition } from "../../capability.js"; + +export type FilesystemParamsType = { + /** What to do with the path: "read", "list", or "stat". */ + operation: "read" | "list" | "stat"; + /** The path to read, list, or describe. A relative path is resolved against the project root. */ + path: string; + /** Whether a listing descends into subdirectories. */ + recursive?: boolean; + /** How the content of a read file is represented: "json" or "text". */ + format?: "json" | "text"; +}; + +export type FilesystemEntryType = { + /** The name of the entry within its directory. */ + name: string; + /** The absolute path of the entry. */ + path: string; + /** What the entry is: "file", "directory", "link", or "other". */ + type: string; + /** The size of the entry in bytes. */ + size: number; +}; + +export type FilesystemResultType = { + /** The absolute path the operation applied to. */ + path: string; + /** The content of the file, for the read operation. */ + content?: unknown; + /** The size of the file in bytes, for the read and stat operations. */ + size?: number; + /** The entries of the directory, for the list operation. */ + entries?: FilesystemEntryType[]; + /** Whether the path exists, for the stat operation. */ + exists?: boolean; + /** What the path is, for the stat operation. */ + type?: string; + /** Present when the listing was shortened to the entry limit. */ + entries_truncated?: boolean; +}; + +/** + * Declares a capability that reads files and directories. + * + * It is the read-only view of the machine the assembly runs on: the files the + * assembly was written to work with, and nothing else. Writing, deleting, and + * moving files are not part of it. + * + * @param name the capability's logical identity in the assembly. + */ +export function filesystem(name: string): CapabilityDefinition { + return Capability(name) + .runtime({ name: "neuron:core:filesystem" }) + .paramsSchema() + .resultSchema(); +} diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/http.ts b/packages/assembly-sdks/typescript/src/core/capabilities/http.ts index 772902e..cf6e660 100644 --- a/packages/assembly-sdks/typescript/src/core/capabilities/http.ts +++ b/packages/assembly-sdks/typescript/src/core/capabilities/http.ts @@ -1,20 +1,84 @@ -import {Capability} from "@/capability"; +import { Capability, type CapabilityDefinition } from "../../capability.js"; + +/** + * The credentials a request carries. + * + * The credentials are applied to the request itself, never to the URL, so they + * do not end up in a log line or in an error that names the URL. + */ +export type HttpAuthType = + | { type: "bearer"; token: string } + | { type: "basic"; username: string; password: string } + | { type: "apiKey"; name: string; value: string; in?: "header" | "query" }; + +/** + * How a failed request is retried. + * + * Retries are off by default. A method that is not named in `methods` is never + * retried, because a request that was not designed to be repeated may not be + * safe to repeat. + */ +export type HttpRetryType = { + /** How many attempts are made in total. At least 1. */ + attempts?: number; + /** How long the runtime waits between attempts, such as "200ms". */ + backoff?: string; + /** The methods that may be retried. Defaults to the idempotent methods. */ + methods?: string[]; +}; export type HttpParamsType = { + /** The absolute URL to request. */ url: string; - method: 'GET' | 'POST' | 'PUT' | 'DELETE'; - headers?: Record; - body?: Record; -} + /** The HTTP method. Defaults to GET. */ + method?: string; + /** Request headers. */ + headers?: Record; + /** Query string parameters, appended to the URL. */ + query?: Record; + /** The request body, encoded as JSON. */ + body?: unknown; + /** The request content type. Only used when a body is present. */ + contentType?: string; + /** Credentials for the request. */ + auth?: HttpAuthType; + /** How long the request may take, such as "30s". */ + timeout?: string; + /** How the response body is represented: "json" or "text". */ + format?: "json" | "text"; + /** Whether redirects are followed. Each hop is checked against the address policy. */ + followRedirects?: boolean; + /** Retry behavior. Off by default. */ + retry?: HttpRetryType; +}; -export type HttpResultType = { +export type HttpResultType = { + /** The HTTP response status code. */ status_code: number; - body?: T; -} + /** The response headers. */ + headers: Record; + /** The response body: parsed JSON in the json format, a plain string in the text format. */ + body: TBody | string; + /** The response content type. */ + content_type: string; + /** How long the request took, in milliseconds. */ + duration_ms: number; +}; -export const http = Capability("neuron:core:http").paramsSchema() -.resultSchema() -.runtime({ - name: 'neuron:core:http', - version: '1.0.0', -}) \ No newline at end of file +/** + * Declares a capability that performs a single HTTP request. + * + * The request is bounded by the runtime's policy: it does not reach a host that + * is not publicly routable unless the project permits it, it does not follow + * redirects unless asked to, and it does not retry unless asked to. + * + * @param name the capability's logical identity in the assembly. + */ +export function http( + name: string +): CapabilityDefinition> { + return Capability(name) + .runtime({ name: "neuron:core:http" }) + .paramsSchema() + .resultSchema>(); +} diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/index.ts b/packages/assembly-sdks/typescript/src/core/capabilities/index.ts index 3952d0f..90947ed 100644 --- a/packages/assembly-sdks/typescript/src/core/capabilities/index.ts +++ b/packages/assembly-sdks/typescript/src/core/capabilities/index.ts @@ -1,2 +1,4 @@ +export * from './set'; export * from './http'; -export * from './delay'; \ No newline at end of file +export * from './command'; +export * from './filesystem'; diff --git a/packages/assembly-sdks/typescript/src/core/capabilities/set.ts b/packages/assembly-sdks/typescript/src/core/capabilities/set.ts new file mode 100644 index 0000000..777fd75 --- /dev/null +++ b/packages/assembly-sdks/typescript/src/core/capabilities/set.ts @@ -0,0 +1,31 @@ +import { Capability, type CapabilityDefinition } from "../../capability.js"; + +/** + * The value set a capability materializes. + * + * The names of the set are the names of the capability's configuration and + * params, so they are not known ahead of time. + */ +export type SetParamsType = Record; + +/** + * The merged value set the capability returns: its configuration with the + * params overlaid on it. + */ +export type SetResultType = Record; + +/** + * Declares a capability that materializes a value set. + * + * `set` is the default runtime for a capability that declares none, so most + * assemblies never name it. It is exported for the assembly that wants to + * declare the runtime explicitly, or to bind to a value set by name. + * + * @param name the capability's logical identity in the assembly. + */ +export function set(name: string): CapabilityDefinition { + return Capability(name) + .runtime({ name: "neuron:core:set" }) + .paramsSchema() + .resultSchema(); +} diff --git a/packages/assembly-sdks/typescript/src/manifest.ts b/packages/assembly-sdks/typescript/src/manifest.ts index 863a279..612501c 100644 --- a/packages/assembly-sdks/typescript/src/manifest.ts +++ b/packages/assembly-sdks/typescript/src/manifest.ts @@ -53,11 +53,15 @@ export interface CapabilityManifest { * installation deduplicate on, so several capabilities may share one resolved * artifact. `runtimeConfig` is not part of that identity: it stays attached to * the capability whose invocation it governs. + * + * A built-in capability runtime (the `neuron:core:` namespace) runs in-process + * and is never resolved or installed, so it carries no version and no + * registry. Those two are absent rather than empty for such a runtime. */ export interface CapabilityRuntimeManifest { name: string; - version: string; - registry: string; + version?: string; + registry?: string; /** * How N.O.R.E. executes this capability through this runtime. Absent means * the author declared nothing and N.O.R.E. supplies every default. It is diff --git a/packages/assembly-sdks/typescript/src/type-tests.test-d.ts b/packages/assembly-sdks/typescript/src/type-tests.test-d.ts index fdacf02..fef1689 100644 --- a/packages/assembly-sdks/typescript/src/type-tests.test-d.ts +++ b/packages/assembly-sdks/typescript/src/type-tests.test-d.ts @@ -174,3 +174,41 @@ const sharedRuntime = { name: "acme:http", version: "1.2.0", registry: "github" Capability("sync-call").runtime({ ...sharedRuntime, runtimeConfig: { execution: { mode: "wait" } } }); Capability("fire-and-forget").runtime({ ...sharedRuntime, runtimeConfig: { execution: { mode: "detach" } } }); + +// --------------------------------------------------------------------------- +// Built-in capability runtimes +// --------------------------------------------------------------------------- + +import { set, http, command, filesystem } from "./core.js"; + +// The built-ins are declared with the name the author gives them, and they run +// through the built-in runtime. +const setValues = set("values"); +const httpCall = http("call"); +const commandRun = command("run"); +const filesystemRead = filesystem("read"); + +setValues.withParams({ orderId: "o-1" }); +httpCall.withParams({ url: "https://example.com/api" }); +commandRun.withParams({ command: "git", args: ["status"] }); +filesystemRead.withParams({ operation: "read", path: "data.json" }); + +// @ts-expect-error the http capability needs a url. +httpCall.withParams({ method: "GET" }); + +// @ts-expect-error the command capability needs the program to run. +commandRun.withParams({ args: ["status"] }); + +// @ts-expect-error the filesystem capability needs an operation and a path. +filesystemRead.withParams({ operation: "read" }); + +// The results are typed. +httpCall.result.status_code satisfies Expression; +httpCall.result.content_type satisfies Expression; +commandRun.result.exit_code satisfies Expression; +commandRun.result.timed_out satisfies Expression; +filesystemRead.result.exists satisfies Expression; + +// The body of a response can be narrowed by the author. +const typedCall = http<{ id: string }>("typed-call"); +typedCall.result.body satisfies { readonly id: Expression } | Expression; diff --git a/packages/assembly-sdks/typescript/test/assembly.test.ts b/packages/assembly-sdks/typescript/test/assembly.test.ts index bbd671d..586d3a5 100644 --- a/packages/assembly-sdks/typescript/test/assembly.test.ts +++ b/packages/assembly-sdks/typescript/test/assembly.test.ts @@ -139,7 +139,7 @@ describe("Assembly", () => { expect(manifest.bindings).toHaveLength(0); }); - it("rejects fan-in when a capability reads from multiple sources", () => { + it("supports fan-in: a capability reads from multiple sources", () => { const a = Capability("a").resultSchema({ x: string() }); const b = Capability("b").resultSchema({ y: string() }); const target = Capability("target").paramsSchema({ x: string(), y: string() }); @@ -150,9 +150,20 @@ describe("Assembly", () => { target.withParams({ x: a.result.x, y: b.result.y }), ]); - expect(() => sys.toManifest()).toThrow( - /Capability "target" reads inputs from multiple sources \(a, b\)/ + expect(() => sys.toManifest()).not.toThrow(); + const manifest = sys.toManifest(); + const fanInBinding = manifest.bindings.find( + (b) => b.from === "a" || b.from === "b" ); + expect(fanInBinding).toBeDefined(); + // Each source should have its own binding with only its mappings + const aBinding = manifest.bindings.find((b) => b.from === "a"); + const bBinding = manifest.bindings.find((b) => b.from === "b"); + expect(aBinding).toBeDefined(); + expect(bBinding).toBeDefined(); + // aBinding should have mapping for x only, bBinding for y only + expect(aBinding?.mappings).toHaveLength(1); + expect(bBinding?.mappings).toHaveLength(1); }); it("deduplicates capabilities referenced multiple times", () => { diff --git a/packages/assembly-sdks/typescript/test/capability.test.ts b/packages/assembly-sdks/typescript/test/capability.test.ts index dcf8571..aac5187 100644 --- a/packages/assembly-sdks/typescript/test/capability.test.ts +++ b/packages/assembly-sdks/typescript/test/capability.test.ts @@ -9,18 +9,19 @@ describe("Capability", () => { expect(svc.ref).toBe("validate-order"); expect(svc.toManifest()).toEqual({ name: "validate-order", - capabilityRuntime: { name: "neuron:core:set", version: "latest", registry: "local" }, + capabilityRuntime: { name: "neuron:core:set" }, params: [], results: [], }); }); - it("defaults the capability runtime version to latest", () => { + it("gives the built-in runtime no version and no registry", () => { + // A built-in capability runtime runs in-process. It is never resolved + // against a registry, so a version and a registry would state that a + // package that does not exist has a version. const svc = Capability("my-capability"); expect(svc.toManifest().capabilityRuntime).toEqual({ name: "neuron:core:set", - version: "latest", - registry: "local", }); }); }); @@ -49,6 +50,26 @@ describe("Capability.runtime()", () => { }); }); + it("gives a built-in runtime no version and no registry", () => { + const svc = Capability("set-values").runtime({ name: "neuron:core:set" }); + expect(svc.toManifest().capabilityRuntime).toEqual({ + name: "neuron:core:set", + }); + }); + + it("drops a version and a registry declared for a built-in runtime", () => { + // A built-in is never resolved and never installed, so the version and + // the registry are not part of its identity and are not carried. + const svc = Capability("set-values").runtime({ + name: "neuron:core:set", + version: "1.0.0", + registry: "github", + }); + expect(svc.toManifest().capabilityRuntime).toEqual({ + name: "neuron:core:set", + }); + }); + it("emits no runtimeConfig key when none is declared", () => { // An author who declares nothing must produce no key at all, so N.O.R.E. // supplies every default rather than the SDK manufacturing an empty one. diff --git a/packages/assembly-sdks/typescript/test/core.test.ts b/packages/assembly-sdks/typescript/test/core.test.ts new file mode 100644 index 0000000..05d83ef --- /dev/null +++ b/packages/assembly-sdks/typescript/test/core.test.ts @@ -0,0 +1,80 @@ +import { describe, it, expect } from "vitest"; +import { set, http, command, filesystem } from "../src/core.js"; + +describe("built-in capability runtimes", () => { + it("declares every built-in with the core namespace and no version", () => { + const runtimes = [set("values"), http("call"), command("run"), filesystem("read")]; + + for (const capability of runtimes) { + const runtime = capability.toManifest().capabilityRuntime; + expect(runtime.name.startsWith("neuron:core:")).toBe(true); + expect(runtime.version).toBeUndefined(); + expect(runtime.registry).toBeUndefined(); + } + }); + + it("declares the built-ins the runtime provides", () => { + const runtimes = [set("values"), http("call"), command("run"), filesystem("read")]; + + expect(runtimes.map((c) => c.toManifest().capabilityRuntime.name)).toEqual([ + "neuron:core:set", + "neuron:core:http", + "neuron:core:command", + "neuron:core:filesystem", + ]); + }); + + it("names the capability the author declared", () => { + expect(set("values").ref).toBe("values"); + expect(http("call").ref).toBe("call"); + expect(command("run").ref).toBe("run"); + expect(filesystem("read").ref).toBe("read"); + }); + + it("declares no ports of its own", () => { + // The contract of a built-in is published by the runtime, not authored per + // capability, so the manifest carries no ports for it. The runtime + // validates the input against the contract it publishes. + for (const capability of [set("values"), http("call"), command("run"), filesystem("read")]) { + const manifest = capability.toManifest(); + expect(manifest.params).toEqual([]); + expect(manifest.results).toEqual([]); + } + }); + + it("binds the http params", () => { + const node = http("call").withParams({ url: "https://example.com/api" }); + + expect(node.bindings).toEqual({ + url: { kind: "literal", value: "https://example.com/api" }, + }); + }); + + it("binds the command args", () => { + const node = command("run").withParams({ command: "git", args: ["status"] }); + + expect(node.bindings).toEqual({ + command: { kind: "literal", value: "git" }, + args: { kind: "literal", value: ["status"] }, + }); + }); + + it("binds the filesystem operation", () => { + const node = filesystem("read").withParams({ operation: "read", path: "data.json" }); + + expect(node.bindings).toEqual({ + operation: { kind: "literal", value: "read" }, + path: { kind: "literal", value: "data.json" }, + }); + }); + + it("binds the http credentials and the retry policy", () => { + const node = http("call").withParams({ + url: "https://example.com/api", + auth: { type: "bearer", token: "secret" }, + retry: { attempts: 3, backoff: "200ms", methods: ["GET"] }, + }); + + expect(Object.keys(node.bindings)).toEqual(["url", "auth", "retry"]); + }); +}); diff --git a/scripts/script.sh b/scripts/script.sh index 9644e18..844b3ae 100755 --- a/scripts/script.sh +++ b/scripts/script.sh @@ -40,6 +40,9 @@ for mod in "${go_modules[@]}"; do scratch="$(mktemp -d)" go vet ./... \ && go test ./... \ + && gofmt -l . 2>/dev/null | while IFS= read -r f; do \ + echo "FAILED: gofmt needs $f" >&2; \ + done \ && if grep -rq '^package main' --include='*.go' .; then go build -o "$scratch" ./... else diff --git a/shared/types/builtins/policy.go b/shared/types/builtins/policy.go new file mode 100644 index 0000000..5ac8b5d --- /dev/null +++ b/shared/types/builtins/policy.go @@ -0,0 +1,308 @@ +// Package builtins defines the security policy that bounds the built-in +// capability runtimes N.O.R.E. provides. +// +// The built-ins reach outside the runtime: they open sockets, run programs, +// and touch the filesystem. A capability runtime that can do that without +// limits is a capability runtime whose limits are whatever the author of the +// assembly decides, so the limits live here, in one place, and are enforced +// by the runtime rather than by convention. +// +// The policy is split in two. +// +// The ceilings are compiled into the runtime. They are the strongest limits +// that exist, and no configuration can raise them: a project configuration +// file is part of the assembly's own input, and an input must not be able to +// widen the blast radius of a capability. +// +// The policy is what a project configures on top of those ceilings. It can +// only narrow them. Effective resolves the two into the limits the runtime +// actually applies, and refuses a configuration that asks for more than the +// ceilings allow. +package builtins + +import ( + "fmt" + "strings" + "time" +) + +// Policy bounds what the built-in capability runtimes are allowed to do. +// +// A zero field means "not configured", which is why the numeric limits have +// no meaningful zero value: a limit of zero bytes would disable a built-in +// rather than restrict it, and a configuration that wants that should say so +// by not using the built-in. +type Policy struct { + // ProjectRoot is the directory the assembly lives in. + // + // It is the root the built-ins are confined to: it is where a command + // runs by default, and it is always inside the reachable set. A runtime + // that runs a project sets it from the project it was started for, so + // the reach of the built-ins follows the assembly rather than the + // directory the runtime happened to be launched from. + // + // It is carried through the resolution unchanged, like + // AllowPrivateNetworks: it is the context of the run rather than a limit + // on it. + ProjectRoot string `json:"projectRoot,omitempty"` + + // FilesystemRoots are the roots the filesystem and command built-ins may + // reach in addition to the project root. + // + // The project root is always permitted: it is where the assembly lives. + // These roots extend that reach to directories the assembly is expected + // to work with. A path that resolves outside every permitted root is + // refused. + FilesystemRoots []string `json:"filesystemRoots,omitempty"` + + // AllowedExecutables restricts the programs the command built-in may run. + // + // When it is empty, any program resolvable on the program path may run. + // When it is not empty, only the listed programs may run. A listed + // program is matched by its resolved absolute path, so the match does not + // depend on the working directory or on the order of the program path. + AllowedExecutables []string `json:"allowedExecutables,omitempty"` + + // AllowedHTTPHosts restricts the http built-in to a set of hosts. + // + // When it is empty, the built-in may contact any public host and refuses + // loopback, private, link-local, multicast, unspecified, and cloud + // metadata addresses. + // + // When it is not empty, the built-in may contact only the listed hosts. A + // listed host is permitted even when it resolves to a private address, + // because listing it is the explicit and auditable decision to let a + // capability reach an internal service. + AllowedHTTPHosts []string `json:"allowedHTTPHosts,omitempty"` + + // AllowPrivateNetworks lifts the restriction on contacting addresses that + // are not publicly routable. + // + // The restriction exists because a capability's URL may be derived from + // untrusted input, and a capability that can reach the metadata service + // or an internal network turns an assembly into a pivot. A request the + // operator authored themselves is not that case, so the paths that run a + // capability outside an assembly set this. + // + // It is deliberately not configurable from a project configuration file: a + // project configuration is part of the assembly's own input, and an input + // must not be able to widen what a capability reaches. + AllowPrivateNetworks bool `json:"allowPrivateNetworks,omitempty"` + + // MaxCommandOutputBytes bounds the output retained from one stream of a + // single command. + MaxCommandOutputBytes int64 `json:"maxCommandOutputBytes,omitempty"` + + // MaxHTTPRequestBodyBytes bounds the request body a single http request + // may send. + MaxHTTPRequestBodyBytes int64 `json:"maxHTTPRequestBodyBytes,omitempty"` + + // MaxHTTPResponseBytes bounds the response body a single http request + // may read. + MaxHTTPResponseBytes int64 `json:"maxHTTPResponseBytes,omitempty"` + + // MaxFileReadBytes bounds the file content a single filesystem read may + // return. + MaxFileReadBytes int64 `json:"maxFileReadBytes,omitempty"` + + // MaxDirectoryEntries bounds the entries a single filesystem listing may + // return. + MaxDirectoryEntries int `json:"maxDirectoryEntries,omitempty"` + + // DefaultTimeout bounds a single built-in operation when the capability + // does not set a timeout of its own. + // + // It is written as a duration string such as "60s" so that the value + // survives the JSON round trip between the compiler, the manifest, and + // the runtime without losing its meaning. + DefaultTimeout string `json:"defaultTimeout,omitempty"` +} + +// DefaultPolicy returns the compiled-in ceilings for the built-in capability +// runtimes. +// +// These are the strongest limits that exist. A project may lower them +// through configuration but can never raise them. +func DefaultPolicy() Policy { + return Policy{ + MaxCommandOutputBytes: 1 << 20, // 1 MiB + MaxHTTPRequestBodyBytes: 1 << 20, // 1 MiB + MaxHTTPResponseBytes: 8 << 20, // 8 MiB + MaxFileReadBytes: 8 << 20, // 8 MiB + MaxDirectoryEntries: 1000, + DefaultTimeout: "60s", + } +} + +// Effective resolves the limits the runtime applies from the compiled-in +// ceilings and the policy a project configured. +// +// A configured limit replaces the ceiling when it is lower, and the +// resolution fails when it is higher: the ceiling exists precisely so that a +// project's own configuration cannot widen what a capability can reach. +// +// The resolution is idempotent. Applying it to an already effective policy +// returns the same policy, which lets the runtime re-apply it to a policy it +// received over a transport without the transport being trusted. +// +// AllowPrivateNetworks is not a limit and is carried through unchanged. It +// states the trust context of the request rather than a bound on it, and it +// is set by the paths that know the request was authored by the operator. +func Effective(configured Policy) (Policy, error) { + ceilings := DefaultPolicy() + effective := ceilings + + limits := []struct { + name string + configured int64 + ceiling int64 + apply func(*Policy, int64) + }{ + {"maxCommandOutputBytes", configured.MaxCommandOutputBytes, ceilings.MaxCommandOutputBytes, func(p *Policy, v int64) { + p.MaxCommandOutputBytes = v + }}, + {"maxHTTPRequestBodyBytes", configured.MaxHTTPRequestBodyBytes, ceilings.MaxHTTPRequestBodyBytes, func(p *Policy, v int64) { + p.MaxHTTPRequestBodyBytes = v + }}, + {"maxHTTPResponseBytes", configured.MaxHTTPResponseBytes, ceilings.MaxHTTPResponseBytes, func(p *Policy, v int64) { + p.MaxHTTPResponseBytes = v + }}, + {"maxFileReadBytes", configured.MaxFileReadBytes, ceilings.MaxFileReadBytes, func(p *Policy, v int64) { + p.MaxFileReadBytes = v + }}, + } + + for _, limit := range limits { + if limit.configured == 0 { + continue + } + if limit.configured < 0 { + return Policy{}, fmt.Errorf("built-in policy: %s must not be negative, got %d", limit.name, limit.configured) + } + if limit.configured > limit.ceiling { + return Policy{}, fmt.Errorf("built-in policy: %s is %d, which exceeds the built-in ceiling of %d", limit.name, limit.configured, limit.ceiling) + } + limit.apply(&effective, limit.configured) + } + + if configured.MaxDirectoryEntries != 0 { + if configured.MaxDirectoryEntries < 0 { + return Policy{}, fmt.Errorf("built-in policy: maxDirectoryEntries must not be negative, got %d", configured.MaxDirectoryEntries) + } + if configured.MaxDirectoryEntries > ceilings.MaxDirectoryEntries { + return Policy{}, fmt.Errorf("built-in policy: maxDirectoryEntries is %d, which exceeds the built-in ceiling of %d", + configured.MaxDirectoryEntries, ceilings.MaxDirectoryEntries) + } + effective.MaxDirectoryEntries = configured.MaxDirectoryEntries + } + + if timeout := strings.TrimSpace(configured.DefaultTimeout); timeout != "" { + ceiling, err := parseDuration(ceilings.DefaultTimeout) + if err != nil { + return Policy{}, fmt.Errorf("built-in policy: the built-in ceiling for defaultTimeout is invalid: %w", err) + } + requested, err := parseDuration(timeout) + if err != nil { + return Policy{}, fmt.Errorf("built-in policy: defaultTimeout must be a duration string such as \"60s\": %w", err) + } + if requested <= 0 { + return Policy{}, fmt.Errorf("built-in policy: defaultTimeout must be positive, got %q", timeout) + } + if requested > ceiling { + return Policy{}, fmt.Errorf("built-in policy: defaultTimeout is %s, which exceeds the built-in ceiling of %s", timeout, ceilings.DefaultTimeout) + } + effective.DefaultTimeout = timeout + } + + effective.FilesystemRoots = normalizeRoots(configured.FilesystemRoots) + effective.AllowedExecutables = normalizeList(configured.AllowedExecutables) + effective.AllowedHTTPHosts = normalizeList(configured.AllowedHTTPHosts) + effective.AllowPrivateNetworks = configured.AllowPrivateNetworks + effective.ProjectRoot = strings.TrimSpace(configured.ProjectRoot) + + return effective, nil +} + +// Timeout returns the policy's default timeout as a duration. +func (p Policy) Timeout() (time.Duration, error) { + timeout := strings.TrimSpace(p.DefaultTimeout) + if timeout == "" { + timeout = DefaultPolicy().DefaultTimeout + } + return parseDuration(timeout) +} + +// Validate reports whether the policy is internally consistent. +// +// It is applied to a policy that has already been resolved through Effective +// when the runtime receives one, so it checks the values themselves rather +// than comparing them with the ceilings. +func (p Policy) Validate() error { + if p.MaxCommandOutputBytes <= 0 { + return fmt.Errorf("built-in policy: maxCommandOutputBytes must be positive, got %d", p.MaxCommandOutputBytes) + } + if p.MaxHTTPRequestBodyBytes <= 0 { + return fmt.Errorf("built-in policy: maxHTTPRequestBodyBytes must be positive, got %d", p.MaxHTTPRequestBodyBytes) + } + if p.MaxHTTPResponseBytes <= 0 { + return fmt.Errorf("built-in policy: maxHTTPResponseBytes must be positive, got %d", p.MaxHTTPResponseBytes) + } + if p.MaxFileReadBytes <= 0 { + return fmt.Errorf("built-in policy: maxFileReadBytes must be positive, got %d", p.MaxFileReadBytes) + } + if p.MaxDirectoryEntries <= 0 { + return fmt.Errorf("built-in policy: maxDirectoryEntries must be positive, got %d", p.MaxDirectoryEntries) + } + if _, err := p.Timeout(); err != nil { + return fmt.Errorf("built-in policy: %w", err) + } + return nil +} + +// normalizeRoots cleans the configured filesystem roots. +// +// A root is kept as written apart from surrounding whitespace, because a root +// is resolved against the filesystem rather than used as an identifier: the +// runtime resolves it to an absolute path before comparing it against a +// target path. +func normalizeRoots(roots []string) []string { + if len(roots) == 0 { + return nil + } + normalized := make([]string, 0, len(roots)) + for _, root := range roots { + trimmed := strings.TrimSpace(root) + if trimmed == "" { + continue + } + normalized = append(normalized, trimmed) + } + if len(normalized) == 0 { + return nil + } + return normalized +} + +// normalizeList cleans a configured list of names. +func normalizeList(values []string) []string { + if len(values) == 0 { + return nil + } + normalized := make([]string, 0, len(values)) + for _, value := range values { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + continue + } + normalized = append(normalized, trimmed) + } + if len(normalized) == 0 { + return nil + } + return normalized +} + +// parseDuration parses a duration string. +func parseDuration(value string) (time.Duration, error) { + return time.ParseDuration(strings.TrimSpace(value)) +} diff --git a/shared/types/builtins/policy_test.go b/shared/types/builtins/policy_test.go new file mode 100644 index 0000000..91d1abb --- /dev/null +++ b/shared/types/builtins/policy_test.go @@ -0,0 +1,192 @@ +package builtins + +import ( + "reflect" + "strings" + "testing" + "time" +) + +func TestDefaultPolicyIsWithinItsOwnCeilings(t *testing.T) { + effective, err := Effective(Policy{}) + if err != nil { + t.Fatalf("Effective: %v", err) + } + if err := effective.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + + ceilings := DefaultPolicy() + if effective.MaxCommandOutputBytes != ceilings.MaxCommandOutputBytes { + t.Errorf("MaxCommandOutputBytes = %d, want %d", effective.MaxCommandOutputBytes, ceilings.MaxCommandOutputBytes) + } + if effective.MaxHTTPResponseBytes != ceilings.MaxHTTPResponseBytes { + t.Errorf("MaxHTTPResponseBytes = %d, want %d", effective.MaxHTTPResponseBytes, ceilings.MaxHTTPResponseBytes) + } + if effective.MaxFileReadBytes != ceilings.MaxFileReadBytes { + t.Errorf("MaxFileReadBytes = %d, want %d", effective.MaxFileReadBytes, ceilings.MaxFileReadBytes) + } + if effective.MaxDirectoryEntries != ceilings.MaxDirectoryEntries { + t.Errorf("MaxDirectoryEntries = %d, want %d", effective.MaxDirectoryEntries, ceilings.MaxDirectoryEntries) + } +} + +func TestEffectiveNarrowsTheCeilings(t *testing.T) { + effective, err := Effective(Policy{ + MaxCommandOutputBytes: 1024, + MaxHTTPRequestBodyBytes: 2048, + MaxHTTPResponseBytes: 4096, + MaxFileReadBytes: 8192, + MaxDirectoryEntries: 16, + DefaultTimeout: "5s", + }) + if err != nil { + t.Fatalf("Effective: %v", err) + } + + if effective.MaxCommandOutputBytes != 1024 { + t.Errorf("MaxCommandOutputBytes = %d, want 1024", effective.MaxCommandOutputBytes) + } + if effective.MaxHTTPRequestBodyBytes != 2048 { + t.Errorf("MaxHTTPRequestBodyBytes = %d, want 2048", effective.MaxHTTPRequestBodyBytes) + } + if effective.MaxHTTPResponseBytes != 4096 { + t.Errorf("MaxHTTPResponseBytes = %d, want 4096", effective.MaxHTTPResponseBytes) + } + if effective.MaxFileReadBytes != 8192 { + t.Errorf("MaxFileReadBytes = %d, want 8192", effective.MaxFileReadBytes) + } + if effective.MaxDirectoryEntries != 16 { + t.Errorf("MaxDirectoryEntries = %d, want 16", effective.MaxDirectoryEntries) + } + + timeout, err := effective.Timeout() + if err != nil { + t.Fatalf("Timeout: %v", err) + } + if timeout != 5*time.Second { + t.Errorf("Timeout = %s, want 5s", timeout) + } +} + +func TestEffectiveRefusesLimitsAboveTheCeiling(t *testing.T) { + ceilings := DefaultPolicy() + + cases := []struct { + name string + policy Policy + }{ + {"maxCommandOutputBytes", Policy{MaxCommandOutputBytes: ceilings.MaxCommandOutputBytes + 1}}, + {"maxHTTPRequestBodyBytes", Policy{MaxHTTPRequestBodyBytes: ceilings.MaxHTTPRequestBodyBytes + 1}}, + {"maxHTTPResponseBytes", Policy{MaxHTTPResponseBytes: ceilings.MaxHTTPResponseBytes + 1}}, + {"maxFileReadBytes", Policy{MaxFileReadBytes: ceilings.MaxFileReadBytes + 1}}, + {"maxDirectoryEntries", Policy{MaxDirectoryEntries: ceilings.MaxDirectoryEntries + 1}}, + {"defaultTimeout", Policy{DefaultTimeout: "10m"}}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + _, err := Effective(testCase.policy) + if err == nil { + t.Fatalf("Effective(%+v) succeeded, want an error", testCase.policy) + } + if !strings.Contains(err.Error(), "exceeds the built-in ceiling") { + t.Errorf("error %q does not explain that the ceiling was exceeded", err) + } + if !strings.Contains(err.Error(), testCase.name) { + t.Errorf("error %q does not name the limit", err) + } + }) + } +} + +func TestEffectiveRefusesNegativeLimits(t *testing.T) { + if _, err := Effective(Policy{MaxFileReadBytes: -1}); err == nil { + t.Fatal("Effective accepted a negative limit") + } + if _, err := Effective(Policy{MaxDirectoryEntries: -1}); err == nil { + t.Fatal("Effective accepted a negative limit") + } +} + +func TestEffectiveRefusesAnUnparseableTimeout(t *testing.T) { + _, err := Effective(Policy{DefaultTimeout: "soon"}) + if err == nil { + t.Fatal("Effective accepted an unparseable timeout") + } + if !strings.Contains(err.Error(), "defaultTimeout") { + t.Errorf("error %q does not name the limit", err) + } +} + +func TestEffectiveIsIdempotent(t *testing.T) { + configured := Policy{ + MaxFileReadBytes: 4096, + DefaultTimeout: "15s", + FilesystemRoots: []string{"/data"}, + } + + once, err := Effective(configured) + if err != nil { + t.Fatalf("Effective: %v", err) + } + twice, err := Effective(once) + if err != nil { + t.Fatalf("Effective(Effective(p)): %v", err) + } + + if !reflect.DeepEqual(once, twice) { + t.Errorf("Effective is not idempotent: %+v then %+v", once, twice) + } +} + +func TestEffectiveNormalizesTheConfiguredLists(t *testing.T) { + effective, err := Effective(Policy{ + FilesystemRoots: []string{" /data ", "", " "}, + AllowedExecutables: []string{" git ", ""}, + AllowedHTTPHosts: []string{" api.example.com "}, + }) + if err != nil { + t.Fatalf("Effective: %v", err) + } + + if len(effective.FilesystemRoots) != 1 || effective.FilesystemRoots[0] != "/data" { + t.Errorf("FilesystemRoots = %v, want [/data]", effective.FilesystemRoots) + } + if len(effective.AllowedExecutables) != 1 || effective.AllowedExecutables[0] != "git" { + t.Errorf("AllowedExecutables = %v, want [git]", effective.AllowedExecutables) + } + if len(effective.AllowedHTTPHosts) != 1 || effective.AllowedHTTPHosts[0] != "api.example.com" { + t.Errorf("AllowedHTTPHosts = %v, want [api.example.com]", effective.AllowedHTTPHosts) + } +} + +func TestValidateRejectsAnUnresolvedPolicy(t *testing.T) { + if err := (Policy{}).Validate(); err == nil { + t.Fatal("Validate accepted a policy with no limits") + } +} + +// The private-network allowance states the trust context of a request rather +// than a bound on it, so it survives the resolution unchanged. +func TestEffectiveCarriesThePrivateNetworkAllowance(t *testing.T) { + effective, err := Effective(Policy{AllowPrivateNetworks: true}) + if err != nil { + t.Fatalf("Effective: %v", err) + } + if !effective.AllowPrivateNetworks { + t.Error("Effective dropped the private-network allowance") + } +} + +// The project root is the context of the run rather than a limit on it, so it +// survives the resolution unchanged. +func TestEffectiveCarriesTheProjectRoot(t *testing.T) { + effective, err := Effective(Policy{ProjectRoot: " /srv/project "}) + if err != nil { + t.Fatalf("Effective: %v", err) + } + if effective.ProjectRoot != "/srv/project" { + t.Errorf("ProjectRoot = %q, want %q", effective.ProjectRoot, "/srv/project") + } +} diff --git a/shared/types/core/binding.go b/shared/types/core/binding.go index 609e028..88dd1bc 100644 --- a/shared/types/core/binding.go +++ b/shared/types/core/binding.go @@ -74,6 +74,11 @@ type Binding struct { From Endpoint To Endpoint + // JoinPolicy selects when To becomes ready when several bindings converge + // on it. An empty policy means JoinPolicyAll. Every incoming binding for one + // target must declare the same effective policy. + JoinPolicy JoinPolicy `json:"joinPolicy,omitempty"` + Mappings []MappingRule Validations []ValidationRule } diff --git a/shared/types/core/builtins.go b/shared/types/core/builtins.go new file mode 100644 index 0000000..e173841 --- /dev/null +++ b/shared/types/core/builtins.go @@ -0,0 +1,593 @@ +package core + +import ( + "fmt" + "sort" + "strings" +) + +// Built-in capability runtimes are the capabilities N.O.R.E. provides without +// installation. They are declared here so the runtime, the compiler, and the +// CLI describe the same set from one authoritative list. +// +// A built-in qualifies only when it provides a meaningful execution or +// integration boundary — access to a system resource — rather than a generic +// transformation an author could implement in a capability of their own. +// +// The declarations below are the published contract of each built-in. They +// describe the inputs a built-in accepts, the outputs it produces, and the +// boundaries the runtime enforces on its behalf. + +// BuiltinParameter describes one input or output of a built-in capability +// runtime. +type BuiltinParameter struct { + // Name is the parameter or result name. + Name string + + // Type describes the accepted value in prose, for display purposes. + Type string + + // Required reports whether the value must be supplied. + Required bool + + // Default describes the value used when the parameter is absent. An empty + // string means the parameter has no default and is required. + Default string + + // Description explains what the parameter controls. + Description string +} + +// BuiltinDescriptor describes one built-in capability runtime. +type BuiltinDescriptor struct { + // Name is the built-in's name within the core namespace, for example + // "http" for neuron:core:http. + Name string + + // Summary is a one-line description of the built-in. + Summary string + + // Description explains the built-in's purpose and its boundary. + Description string + + // Parameters are the accepted inputs, in the order they are documented. + Parameters []BuiltinParameter + + // Results are the produced outputs, in the order they are documented. + Results []BuiltinParameter + + // Constraints are the security boundaries the runtime enforces. + Constraints []string +} + +// Type returns the canonical capability runtime type of the built-in. +func (d BuiltinDescriptor) Type() CapabilityRuntimeType { + return CoreName(d.Name) +} + +// builtinDescriptors is the authoritative list of built-in capability +// runtimes. +// +// It is a single source of truth: the runtime registers exactly these +// built-ins, the compiler rejects a reference to any other name in the core +// namespace, and the CLI reports exactly these contracts. +var builtinDescriptors = []BuiltinDescriptor{ + { + Name: "set", + Summary: "Materializes a capability's static value set", + Description: "set combines the capability's own configuration with the params bound to the " + + "invocation and returns the combination, with params overriding configuration. It is the " + + "default runtime for a capability that declares none, which makes it the way an assembly " + + "supplies a value set to a downstream capability without writing a capability runtime.", + Parameters: []BuiltinParameter{ + { + Name: "format", + Type: "string", + Default: "json", + Description: "How the values of the set are represented: \"json\" or \"text\".", + }, + }, + Results: []BuiltinParameter{ + { + Name: "value", + Type: "object", + Required: true, + Description: "The merged value set: the capability's configuration with the params overlaid on it. The result of the capability carries the names of the merged set.", + }, + }, + Constraints: []string{ + "Reads only the capability's own input; it reaches nothing outside the runtime.", + "The format is an instruction to the built-in rather than a value of the set, so it is not returned as one.", + }, + }, + { + Name: "http", + Summary: "Makes an HTTP request", + Description: "http performs a single HTTP request and returns the status, the response headers, " + + "and the body. It is the general-purpose integration boundary: an author who needs to talk " + + "to an HTTP API should not have to write a capability runtime for it.", + Parameters: []BuiltinParameter{ + { + Name: "url", + Type: "string", + Required: true, + Description: "The absolute URL to request.", + }, + { + Name: "method", + Type: "string", + Default: "GET", + Description: "The HTTP method.", + }, + { + Name: "headers", + Type: "object of strings", + Description: "Request headers.", + }, + { + Name: "query", + Type: "object of strings", + Description: "Query string parameters, appended to the URL.", + }, + { + Name: "body", + Type: "any", + Description: "The request body, encoded as JSON.", + }, + { + Name: "contentType", + Type: "string", + Default: "application/json", + Description: "The request content type. Only used when a body is present.", + }, + { + Name: "auth", + Type: "object", + Description: "Credentials for the request: {\"type\": \"bearer\" | \"basic\" | \"apiKey\", ...}.", + }, + { + Name: "timeout", + Type: "duration string", + Description: "How long the request may take, such as \"30s\".", + }, + { + Name: "format", + Type: "string", + Default: "json", + Description: "How the response body is represented: \"json\" or \"text\".", + }, + { + Name: "followRedirects", + Type: "boolean", + Default: "false", + Description: "Whether redirects are followed. Each hop is checked against the address policy.", + }, + { + Name: "retry", + Type: "object", + Description: "Retry behavior: {\"attempts\": n, \"backoff\": \"200ms\", \"methods\": [...]}. Off by default.", + }, + }, + Results: []BuiltinParameter{ + { + Name: "status_code", + Type: "number", + Required: true, + Description: "The HTTP response status code.", + }, + { + Name: "headers", + Type: "object of strings", + Required: true, + Description: "The response headers.", + }, + { + Name: "body", + Type: "any or string", + Required: true, + Description: "The response body: parsed JSON in the json format, a plain string in the text format.", + }, + { + Name: "content_type", + Type: "string", + Required: true, + Description: "The response content type.", + }, + { + Name: "duration_ms", + Type: "number", + Required: true, + Description: "How long the request took, in milliseconds.", + }, + }, + Constraints: []string{ + "Refuses loopback, private, link-local, and cloud-metadata addresses, so a request cannot be aimed at the host or an internal service.", + "Does not follow redirects unless asked to; when it does, every hop is checked against the same address policy.", + "Does not retry unless asked to, and never retries a method that was not opted in.", + "Bounds the request body and the response body.", + }, + }, + { + Name: "command", + Summary: "Runs a local program", + Description: "command executes a local program with structured arguments and returns its exit code, " + + "standard output, and standard error. It is the boundary between an assembly and the programs " + + "already installed on the machine.", + Parameters: []BuiltinParameter{ + { + Name: "command", + Type: "string", + Required: true, + Description: "The program to run. A single executable, never a shell command line.", + }, + { + Name: "args", + Type: "array of strings", + Description: "The program's arguments.", + }, + { + Name: "env", + Type: "object of strings", + Description: "Environment variables added to the program's environment.", + }, + { + Name: "cwd", + Type: "string", + Description: "The working directory, resolved against the allowed root.", + }, + { + Name: "stdin", + Type: "string", + Description: "Text written to the program's standard input.", + }, + { + Name: "timeout", + Type: "duration string", + Description: "How long the program may run, such as \"30s\".", + }, + { + Name: "format", + Type: "string", + Default: "text", + Description: "How standard output is represented: \"json\" or \"text\".", + }, + { + Name: "failOnNonZeroExit", + Type: "boolean", + Default: "true", + Description: "Whether a non-zero exit fails the capability. Set it to false to inspect the result instead.", + }, + { + Name: "maxOutputBytes", + Type: "number", + Description: "The most output to retain per stream. The runtime's ceiling applies when this is absent.", + }, + }, + Results: []BuiltinParameter{ + { + Name: "exit_code", + Type: "number", + Required: true, + Description: "The program's exit code.", + }, + { + Name: "stdout", + Type: "any or string", + Required: true, + Description: "The program's standard output: parsed JSON in the json format, a plain string in the text format.", + }, + { + Name: "stderr", + Type: "string", + Required: true, + Description: "The program's standard error.", + }, + { + Name: "success", + Type: "boolean", + Required: true, + Description: "Whether the program exited with code zero.", + }, + { + Name: "timed_out", + Type: "boolean", + Required: true, + Description: "Whether the program was terminated by its deadline.", + }, + { + Name: "duration_ms", + Type: "number", + Required: true, + Description: "How long the program ran, in milliseconds.", + }, + }, + Constraints: []string{ + "Never runs a shell: the program and its arguments are passed to the operating system separately, so an argument can never become a command.", + "Runs only the programs the policy permits, when the policy names any.", + "Runs the program with an allowlisted environment, never the runtime's own environment.", + "Confines the working directory to an allowed root.", + "Bounds the retained output and reports truncation rather than hiding it.", + "Terminates the program's whole process group on cancellation or deadline, so no child survives the capability.", + }, + }, + { + Name: "filesystem", + Summary: "Reads files and directories", + Description: "filesystem reads a file, lists a directory, and reports what a path is. It is the " + + "read-only view of the machine the assembly runs on: the files the assembly was written to work " + + "with, and nothing else. Writing, deleting, and moving files are not part of it.", + Parameters: []BuiltinParameter{ + { + Name: "operation", + Type: "string", + Required: true, + Description: "What to do with the path: \"read\", \"list\", or \"stat\".", + }, + { + Name: "path", + Type: "string", + Required: true, + Description: "The path to read, list, or describe. A relative path is resolved against the project root.", + }, + { + Name: "recursive", + Type: "boolean", + Default: "false", + Description: "Whether a listing descends into subdirectories.", + }, + { + Name: "format", + Type: "string", + Default: "json", + Description: "How the content of a read file is represented: \"json\" or \"text\".", + }, + }, + Results: []BuiltinParameter{ + { + Name: "path", + Type: "string", + Required: true, + Description: "The absolute path the operation applied to.", + }, + { + Name: "content", + Type: "any or string", + Description: "The content of the file, for the read operation: parsed JSON in the json format, a plain string in the text format.", + }, + { + Name: "size", + Type: "number", + Description: "The size of the file in bytes, for the read and stat operations.", + }, + { + Name: "entries", + Type: "array of objects", + Description: "The entries of the directory, for the list operation.", + }, + { + Name: "exists", + Type: "boolean", + Description: "Whether the path exists, for the stat operation.", + }, + { + Name: "type", + Type: "string", + Description: "What the path is: \"file\", \"directory\", \"link\", or \"other\".", + }, + }, + Constraints: []string{ + "Refuses a path that resolves outside the permitted roots, and a symbolic link cannot be used to leave them.", + "Reads only: no operation writes, deletes, or moves anything.", + "Bounds the content a single read returns and the entries a single listing returns.", + }, + }, +} + +// BuiltinDescriptors returns the authoritative list of built-in capability +// runtimes. +// +// The returned slice is a copy: a caller that mutates it cannot change what +// the runtime registers. +func BuiltinDescriptors() []BuiltinDescriptor { + descriptors := make([]BuiltinDescriptor, len(builtinDescriptors)) + copy(descriptors, builtinDescriptors) + return descriptors +} + +// LookupBuiltin returns the descriptor of the built-in named by t. +// +// t may be the canonical namespaced form (neuron:core:http) or the legacy +// bare form (http). The second result reports whether a built-in was found. +func LookupBuiltin(t CapabilityRuntimeType) (BuiltinDescriptor, bool) { + name := builtinName(t) + if name == "" { + return BuiltinDescriptor{}, false + } + for _, descriptor := range builtinDescriptors { + if descriptor.Name == name { + return descriptor, true + } + } + return BuiltinDescriptor{}, false +} + +// IsBuiltinRuntimeType reports whether t names a built-in capability runtime +// that exists. +// +// This is a different question from IsCoreRuntimeType. The core namespace +// answers "is this capability runtime provided in-process by the runtime, +// and therefore never resolved or installed"; this answers "is there a +// built-in with this name". A name in the core namespace that is not a +// built-in is a reference to something that does not exist, and it must fail +// where the reference is made rather than where the capability runs. +func IsBuiltinRuntimeType(t CapabilityRuntimeType) bool { + _, ok := LookupBuiltin(t) + return ok +} + +// BuiltinTypeNames returns the names of every built-in capability runtime, +// sorted. +// +// The names are the bare forms. The canonical namespaced form of a built-in +// is neuron:core:. +func BuiltinTypeNames() []string { + names := make([]string, 0, len(builtinDescriptors)) + for _, descriptor := range builtinDescriptors { + names = append(names, descriptor.Name) + } + sort.Strings(names) + return names +} + +// BuiltinTypes returns the canonical type of every built-in capability +// runtime, sorted. +// +// The canonical type is the name an assembly refers to a built-in by. +func BuiltinTypes() []CapabilityRuntimeType { + names := BuiltinTypeNames() + types := make([]CapabilityRuntimeType, 0, len(names)) + for _, name := range names { + types = append(types, CoreName(name)) + } + return types +} + +// builtinTypeNames returns the canonical type of every built-in as text, for +// an error message that names the built-ins an author may refer to. +func builtinTypeNames() []string { + types := BuiltinTypes() + names := make([]string, 0, len(types)) + for _, builtinType := range types { + names = append(names, string(builtinType)) + } + return names +} + +// builtinName extracts the built-in name from a capability runtime type. +// +// It accepts the canonical namespaced form and the legacy bare form, and +// returns an empty string for anything else, including a name in the core +// namespace that is not a built-in. +func builtinName(t CapabilityRuntimeType) string { + s := string(t) + if s == "" { + return "" + } + parts := strings.Split(s, ":") + if len(parts) == 3 && parts[0] == CoreOwner && parts[1] == CoreNamespace { + return parts[2] + } + if len(parts) == 1 { + return s + } + return "" +} + +// RemovedBuiltin records a built-in capability runtime that no longer exists. +// +// The record exists for exactly one reason. An assembly authored before the +// removal still references the name, and a reference to something that no +// longer exists has to fail with an explanation of what happened and what to +// use instead — not with a bare "not found". Nothing is registered, resolved, +// or executed under these names. +type RemovedBuiltin struct { + // Name is the built-in's former name within the core namespace. + Name string + + // Reason explains why the built-in was removed. + Reason string + + // Replacement names what supersedes the built-in. + Replacement string +} + +// removedBuiltins records the built-in capability runtimes that were removed, +// with the reason each was removed and what supersedes it. +var removedBuiltins = []RemovedBuiltin{ + { + Name: "ai", + Reason: "it returned a canned string rather than calling a provider, so it was a mock rather than a capability", + Replacement: "a capability runtime published for the AI provider you use, referenced from the " + + "assembly like any other external capability runtime", + }, + { + Name: "delay", + Reason: "it only waited, which is a sequencing concern rather than a capability", + Replacement: "an explicit dependency between the capabilities that must run in order", + }, + { + Name: "log", + Reason: "the runtime already records what a capability received, so the built-in only duplicated it", + Replacement: "the runtime's own execution logs and the capability's stderr", + }, +} + +// RemovedBuiltins returns the built-in capability runtimes that were removed. +func RemovedBuiltins() []RemovedBuiltin { + removed := make([]RemovedBuiltin, len(removedBuiltins)) + copy(removed, removedBuiltins) + return removed +} + +// LookupRemovedBuiltin returns the removal record of the built-in named by t. +// +// t may be the canonical namespaced form or the legacy bare form. The second +// result reports whether a removal was found. +func LookupRemovedBuiltin(t CapabilityRuntimeType) (RemovedBuiltin, bool) { + name := builtinName(t) + if name == "" { + return RemovedBuiltin{}, false + } + for _, removed := range removedBuiltins { + if removed.Name == name { + return removed, true + } + } + return RemovedBuiltin{}, false +} + +// IsRemovedBuiltinType reports whether t names a built-in capability runtime +// that was removed. +func IsRemovedBuiltinType(t CapabilityRuntimeType) bool { + _, ok := LookupRemovedBuiltin(t) + return ok +} + +// UnknownBuiltinError describes a reference to a built-in capability runtime +// that does not exist. +// +// The error names the available built-ins so the author can correct the +// reference without consulting the documentation, and it explains a removal +// when the reference names a built-in that used to exist. +// +// It returns nil for a reference to a built-in that exists, and for a +// reference that names an external capability runtime package, because such a +// reference is resolved by the registry instead. +func UnknownBuiltinError(t CapabilityRuntimeType) error { + if IsBuiltinRuntimeType(t) { + return nil + } + + available := "available built-in capability runtimes are: " + strings.Join(builtinTypeNames(), ", ") + + if removed, wasRemoved := LookupRemovedBuiltin(t); wasRemoved { + return fmt.Errorf("capability runtime %q does not exist: it was removed because %s; use %s instead. %s", + t, removed.Reason, removed.Replacement, available) + } + if IsCoreNamespaceType(t) { + return fmt.Errorf("capability runtime %q does not exist. %s", t, available) + } + return nil +} + +// IsCoreNamespaceType reports whether t names the core namespace, whether or +// not a built-in with that name exists. +// +// It answers "was this meant to be a built-in?", which is a different +// question from IsBuiltinRuntimeType. A name in the core namespace that is +// not a built-in is a mistake in the assembly, and it must be reported as +// such rather than being sent to the package registry. +func IsCoreNamespaceType(t CapabilityRuntimeType) bool { + parts := strings.Split(string(t), ":") + return len(parts) == 3 && parts[0] == CoreOwner && parts[1] == CoreNamespace +} diff --git a/shared/types/core/builtins_test.go b/shared/types/core/builtins_test.go new file mode 100644 index 0000000..d58a717 --- /dev/null +++ b/shared/types/core/builtins_test.go @@ -0,0 +1,156 @@ +package core + +import ( + "strings" + "testing" +) + +func TestBuiltinDescriptorsAreTheSourceOfCoreTypes(t *testing.T) { + descriptors := BuiltinDescriptors() + if len(descriptors) == 0 { + t.Fatal("BuiltinDescriptors returned no descriptors") + } + + for _, descriptor := range descriptors { + if descriptor.Name == "" { + t.Error("a descriptor has no name") + } + if descriptor.Summary == "" { + t.Errorf("built-in %q has no summary", descriptor.Name) + } + if descriptor.Description == "" { + t.Errorf("built-in %q has no description", descriptor.Name) + } + } + + want := []string{"command", "filesystem", "http", "set"} + got := BuiltinTypeNames() + if len(got) != len(want) { + t.Fatalf("BuiltinTypeNames() = %v, want %v", got, want) + } + for i, name := range want { + if got[i] != name { + t.Errorf("BuiltinTypeNames()[%d] = %q, want %q", i, got[i], name) + } + } +} + +func TestCoreTypesMatchesTheDeclaredBuiltins(t *testing.T) { + names := BuiltinTypeNames() + if len(CoreTypes) != len(names) { + t.Fatalf("CoreTypes = %v, want %v", CoreTypes, names) + } + for i, name := range names { + if CoreTypes[i] != name { + t.Errorf("CoreTypes[%d] = %q, want %q", i, CoreTypes[i], name) + } + } +} + +func TestLookupBuiltinAcceptsBothForms(t *testing.T) { + descriptor, ok := LookupBuiltin(CoreName("http")) + if !ok { + t.Fatal("LookupBuiltin(neuron:core:http) did not find the built-in") + } + if descriptor.Name != "http" { + t.Errorf("descriptor name = %q, want %q", descriptor.Name, "http") + } + + if _, ok := LookupBuiltin(CapabilityRuntimeType("http")); !ok { + t.Error("LookupBuiltin(http) did not find the built-in") + } +} + +func TestLookupBuiltinRejectsUnknownNames(t *testing.T) { + for _, name := range []string{"delay", "neuron:core:delay", "neuron:other:http", "example:http", "neuron:core:http:extra"} { + if _, ok := LookupBuiltin(CapabilityRuntimeType(name)); ok { + t.Errorf("LookupBuiltin(%q) found a built-in", name) + } + } +} + +func TestIsBuiltinRuntimeType(t *testing.T) { + for _, name := range []string{"set", "http", "command", "filesystem", "neuron:core:set", "neuron:core:filesystem"} { + if !IsBuiltinRuntimeType(CapabilityRuntimeType(name)) { + t.Errorf("IsBuiltinRuntimeType(%q) = false, want true", name) + } + } + for _, name := range []string{"ai", "delay", "log", "neuron:core:delay", "neuron:core:ai", "neuron:core:log", "neuron:core:storage"} { + if IsBuiltinRuntimeType(CapabilityRuntimeType(name)) { + t.Errorf("IsBuiltinRuntimeType(%q) = true, want false", name) + } + } +} + +func TestUnknownBuiltinErrorNamesTheAvailableBuiltins(t *testing.T) { + err := UnknownBuiltinError(CoreName("delay")) + if err == nil { + t.Fatal("UnknownBuiltinError(neuron:core:delay) returned no error") + } + if !strings.Contains(err.Error(), "neuron:core:delay") { + t.Errorf("error %q does not name the unknown runtime", err) + } + for _, name := range BuiltinTypeNames() { + if !strings.Contains(err.Error(), name) { + t.Errorf("error %q does not list the available built-in %q", err, name) + } + } +} + +func TestUnknownBuiltinErrorExplainsARemoval(t *testing.T) { + err := UnknownBuiltinError(CoreName("log")) + if err == nil { + t.Fatal("UnknownBuiltinError(neuron:core:log) returned no error") + } + if !strings.Contains(err.Error(), "was removed") { + t.Errorf("error %q does not report the removal", err) + } + if !strings.Contains(err.Error(), "instead") { + t.Errorf("error %q does not name a replacement", err) + } +} + +func TestUnknownBuiltinErrorIgnoresExternalTypes(t *testing.T) { + if err := UnknownBuiltinError("example:http"); err != nil { + t.Errorf("UnknownBuiltinError(example:http) = %v, want nil", err) + } +} + +func TestIsCoreNamespaceType(t *testing.T) { + if !IsCoreNamespaceType(CoreName("http")) { + t.Error("IsCoreNamespaceType(neuron:core:http) = false, want true") + } + if IsCoreNamespaceType("example:http") { + t.Error("IsCoreNamespaceType(example:http) = true, want false") + } + if IsCoreNamespaceType("http") { + t.Error("IsCoreNamespaceType(http) = true, want false") + } +} + +func TestRemovedBuiltinsAreRecordedWithAnExplanation(t *testing.T) { + removed := RemovedBuiltins() + if len(removed) == 0 { + t.Fatal("RemovedBuiltins returned no records") + } + for _, record := range removed { + if record.Name == "" { + t.Error("a removal record has no name") + } + if record.Reason == "" { + t.Errorf("removal record %q has no reason", record.Name) + } + if record.Replacement == "" { + t.Errorf("removal record %q has no replacement", record.Name) + } + } +} + +func TestBuiltinDescriptorsAreCopied(t *testing.T) { + descriptors := BuiltinDescriptors() + descriptors[0].Name = "mutated" + + if BuiltinDescriptors()[0].Name == "mutated" { + t.Error("BuiltinDescriptors returned a view of the declared descriptors") + } +} diff --git a/shared/types/core/join.go b/shared/types/core/join.go new file mode 100644 index 0000000..1703042 --- /dev/null +++ b/shared/types/core/join.go @@ -0,0 +1,41 @@ +package core + +import "fmt" + +// JoinPolicy selects when a Capability with several incoming bindings becomes +// ready to execute. +// +// It is a property of the join at the target Capability. The canonical manifest +// attaches it to each incoming binding so the graph needs no second edge +// structure; every binding that converges on one target must declare the same +// effective policy. +type JoinPolicy string + +const ( + // JoinPolicyAll waits for every incoming binding to be satisfied before the + // target executes. It is the only policy supported today and is the default + // when a binding declares none. + JoinPolicyAll JoinPolicy = "all" +) + +// Effective returns the policy with its default applied. An unspecified policy +// means JoinPolicyAll, so an author never has to declare the only policy that +// exists. +func (p JoinPolicy) Effective() JoinPolicy { + if p == "" { + return JoinPolicyAll + } + return p +} + +// Validate reports whether the policy is one the runtime can honour. A policy +// that is unsupported must fail compilation rather than silently behaving like +// the default. +func (p JoinPolicy) Validate() error { + switch p.Effective() { + case JoinPolicyAll: + return nil + default: + return fmt.Errorf("unknown join policy %q", p) + } +} diff --git a/shared/types/core/join_test.go b/shared/types/core/join_test.go new file mode 100644 index 0000000..450ec21 --- /dev/null +++ b/shared/types/core/join_test.go @@ -0,0 +1,24 @@ +package core + +import "testing" + +func TestJoinPolicyEffectiveDefaultsToAll(t *testing.T) { + if got := JoinPolicy("").Effective(); got != JoinPolicyAll { + t.Fatalf("empty policy effective = %q, want %q", got, JoinPolicyAll) + } + if got := JoinPolicyAll.Effective(); got != JoinPolicyAll { + t.Fatalf("all policy effective = %q, want %q", got, JoinPolicyAll) + } +} + +func TestJoinPolicyValidateRejectsUnsupported(t *testing.T) { + if err := JoinPolicyAll.Validate(); err != nil { + t.Fatalf("all policy must be valid: %v", err) + } + if err := JoinPolicy("").Validate(); err != nil { + t.Fatalf("an unspecified policy must be valid and mean all: %v", err) + } + if err := JoinPolicy("any").Validate(); err == nil { + t.Fatal("an unsupported join policy must be rejected") + } +} diff --git a/shared/types/core/runtimes.go b/shared/types/core/runtimes.go index 16f411e..fafe56d 100644 --- a/shared/types/core/runtimes.go +++ b/shared/types/core/runtimes.go @@ -19,7 +19,14 @@ type CapabilityRuntimeType string // CoreTypes lists the core capability runtimes N.O.R.E. registers in-process // at startup. -var CoreTypes = []string{"set", "ai", "command", "delay", "http", "log"} +// +// It is derived from the built-in descriptors so the runtime registers +// exactly the built-ins the contract declares, and so there is one +// authoritative list rather than a name list that can drift from it. +// +// The names are the bare forms; the canonical namespaced form of a core +// capability runtime is neuron:core:. +var CoreTypes = BuiltinTypeNames() // CoreName returns the canonical namespaced name for a core capability // runtime, e.g. CoreName("set") == "neuron:core:set". diff --git a/shared/types/core/runtimes_test.go b/shared/types/core/runtimes_test.go index 7346b18..7921d25 100644 --- a/shared/types/core/runtimes_test.go +++ b/shared/types/core/runtimes_test.go @@ -4,9 +4,9 @@ import "testing" func TestCoreName(t *testing.T) { cases := map[string]CapabilityRuntimeType{ - "set": "neuron:core:set", - "ai": "neuron:core:ai", - "command": "neuron:core:command", + "set": "neuron:core:set", + "command": "neuron:core:command", + "filesystem": "neuron:core:filesystem", } for in, want := range cases { if got := CoreName(in); got != want { @@ -18,15 +18,12 @@ func TestCoreName(t *testing.T) { func TestIsCoreRuntimeType(t *testing.T) { core := []CapabilityRuntimeType{ "neuron:core:set", - "neuron:core:ai", - "neuron:core:log", - "neuron:core:command", "neuron:core:http", - "neuron:core:delay", + "neuron:core:command", "neuron:core:anything-new", "set", - "ai", "command", + "http", } for _, tt := range core { if !IsCoreRuntimeType(tt) { @@ -36,7 +33,7 @@ func TestIsCoreRuntimeType(t *testing.T) { notCore := []CapabilityRuntimeType{ "neuron:set", - "neuron:ai", + "neuron:http", "github:read", "neuron-runtime:github:read", "example:echo", @@ -52,12 +49,22 @@ func TestIsCoreRuntimeType(t *testing.T) { } } +// A bare name of a retired built-in is not a core runtime type any more. It +// was never an external package either, so a reference to it is reported as +// a mistake in the assembly rather than sent to the package resolver. +func TestIsCoreRuntimeTypeRejectsRetiredBareNames(t *testing.T) { + for _, name := range []CapabilityRuntimeType{"ai", "delay", "log"} { + if IsCoreRuntimeType(name) { + t.Errorf("IsCoreRuntimeType(%q) = true, want false", name) + } + } +} + func TestCanonicalCoreType(t *testing.T) { cases := map[CapabilityRuntimeType]CapabilityRuntimeType{ "set": "neuron:core:set", "command": "neuron:core:command", "neuron:core:set": "neuron:core:set", - "neuron:core:ai": "neuron:core:ai", } for in, want := range cases { got := CanonicalCoreType(in) diff --git a/shared/types/protocol/hash.go b/shared/types/protocol/hash.go index b199bae..796b6d4 100644 --- a/shared/types/protocol/hash.go +++ b/shared/types/protocol/hash.go @@ -101,12 +101,37 @@ func declaredRuntimeConfig(config *core.RuntimeConfig) *core.RuntimeConfig { } type normalizedBinding struct { - From string `json:"from"` - To string `json:"to"` + From string `json:"from"` + To string `json:"to"` + // JoinPolicy is normalized to its effective value so an omitted policy and + // an explicit JoinPolicyAll contribute identically: they mean the same thing + // at execution time and must not invalidate an otherwise identical assembly. + JoinPolicy core.JoinPolicy `json:"joinPolicy"` Mappings []core.MappingRule `json:"mappings,omitempty"` Validations []core.ValidationRule `json:"validations,omitempty"` } +// bindingSortKey renders a normalized binding in its deterministic JSON form. +// encoding/json sorts map keys, so the key is stable for every field the hash +// includes, including structured mapping sources and inline literal values. +func bindingSortKey(binding normalizedBinding) string { + data, err := json.Marshal(binding) + if err != nil { + return binding.From + "\x00" + binding.To + "\x00" + string(binding.JoinPolicy) + } + return string(data) +} + +// mappingSortKey is the deterministic tie-breaker for two mappings that target +// the same path. +func mappingSortKey(mapping core.MappingRule) string { + data, err := json.Marshal(mapping) + if err != nil { + return mapping.TargetPath + } + return string(data) +} + func normalizeAssembly(assembly core.Assembly) normalizedAssembly { m := assembly.Metadata capabilities := make([]normalizedCapability, 0, len(assembly.Specification.Capabilities)+len(assembly.Specification.Triggers)) @@ -135,21 +160,45 @@ func normalizeAssembly(assembly core.Assembly) normalizedAssembly { bindings := make([]normalizedBinding, 0, len(assembly.Specification.Bindings)) for _, b := range assembly.Specification.Bindings { mappings := append([]core.MappingRule(nil), b.Mappings...) - sort.Slice(mappings, func(i, j int) bool { return mappings[i].TargetPath < mappings[j].TargetPath }) + sort.Slice(mappings, func(i, j int) bool { + if mappings[i].TargetPath != mappings[j].TargetPath { + return mappings[i].TargetPath < mappings[j].TargetPath + } + // Two rules may share a target path in an artifact the compiler did + // not produce. Their structured source is the tie-breaker so the + // hash never depends on the order they happened to be declared in. + return mappingSortKey(mappings[i]) < mappingSortKey(mappings[j]) + }) validations := append([]core.ValidationRule(nil), b.Validations...) - sort.Slice(validations, func(i, j int) bool { return validations[i].Expression < validations[j].Expression }) + sort.Slice(validations, func(i, j int) bool { + if validations[i].Expression != validations[j].Expression { + return validations[i].Expression < validations[j].Expression + } + return validations[i].Message < validations[j].Message + }) bindings = append(bindings, normalizedBinding{ From: string(b.From.CapabilityID), To: string(b.To.CapabilityID), + JoinPolicy: b.JoinPolicy.Effective(), Mappings: mappings, Validations: validations, }) } + // Bindings are sorted by identity and then by every remaining contributing + // field. From/To alone is not a total order: parallel edges may share both + // endpoints while differing in mappings, validations, or join policy, and + // their relative order must not depend on declaration order. sort.Slice(bindings, func(i, j int) bool { if bindings[i].From != bindings[j].From { return bindings[i].From < bindings[j].From } - return bindings[i].To < bindings[j].To + if bindings[i].To != bindings[j].To { + return bindings[i].To < bindings[j].To + } + if bindings[i].JoinPolicy != bindings[j].JoinPolicy { + return bindings[i].JoinPolicy < bindings[j].JoinPolicy + } + return bindingSortKey(bindings[i]) < bindingSortKey(bindings[j]) }) return normalizedAssembly{ diff --git a/shared/types/protocol/hash_join_test.go b/shared/types/protocol/hash_join_test.go new file mode 100644 index 0000000..0e58190 --- /dev/null +++ b/shared/types/protocol/hash_join_test.go @@ -0,0 +1,116 @@ +package protocol + +import ( + "testing" + + "github.com/neuron-runtime/neuron/shared/types/core" +) + +func bindingAssembly(bindings ...core.Binding) core.Assembly { + return core.Assembly{ + Metadata: core.Metadata{Name: "assembly", Version: "1.0.0"}, + Specification: core.AssemblySpec{ + Capabilities: []core.Capability{capability("a"), capability("b")}, + Bindings: bindings, + }, + } +} + +func TestHashAssemblyNormalizesJoinPolicy(t *testing.T) { + omitted, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + explicit, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + JoinPolicy: core.JoinPolicyAll, + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + if omitted != explicit { + t.Fatalf("an omitted join policy and explicit all must hash identically: %s != %s", omitted, explicit) + } +} + +func TestHashAssemblyIncludesJoinPolicy(t *testing.T) { + all, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + JoinPolicy: core.JoinPolicyAll, + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + other, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + JoinPolicy: core.JoinPolicy("any"), + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + if all == other { + t.Fatal("a different join policy must change the assembly hash") + } +} + +func TestHashAssemblyBindingOrderIndependentWithTieBreakers(t *testing.T) { + first := core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + Mappings: []core.MappingRule{ + {TargetPath: "left", Source: &core.ValueRef{Kind: core.ValueRefCapabilityResult, Capability: "a", Path: []string{"left"}}}, + }, + Validations: []core.ValidationRule{{Expression: "source.result.left != null", Message: "left required"}}, + } + second := core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + Mappings: []core.MappingRule{ + {TargetPath: "right", Source: &core.ValueRef{Kind: core.ValueRefCapabilityResult, Capability: "a", Path: []string{"right"}}}, + }, + Validations: []core.ValidationRule{{Expression: "source.result.right != null", Message: "right required"}}, + } + + forward, err := HashAssembly(bindingAssembly(first, second)) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + reversed, err := HashAssembly(bindingAssembly(second, first)) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + if forward != reversed { + t.Fatalf("hash must not depend on parallel-edge declaration order: %s != %s", forward, reversed) + } +} + +func TestHashAssemblyMappingOrderIndependentWithTieBreakers(t *testing.T) { + left := core.MappingRule{TargetPath: "value", Source: &core.ValueRef{Kind: core.ValueRefCapabilityResult, Capability: "a", Path: []string{"left"}}} + right := core.MappingRule{TargetPath: "value", Source: &core.ValueRef{Kind: core.ValueRefCapabilityResult, Capability: "a", Path: []string{"right"}}} + + forward, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + Mappings: []core.MappingRule{left, right}, + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + reversed, err := HashAssembly(bindingAssembly(core.Binding{ + From: core.Endpoint{CapabilityID: "a"}, + To: core.Endpoint{CapabilityID: "b"}, + Mappings: []core.MappingRule{right, left}, + })) + if err != nil { + t.Fatalf("hash assembly: %v", err) + } + if forward != reversed { + t.Fatalf("hash must not depend on same-target mapping order: %s != %s", forward, reversed) + } +}