Fix compose precedence

shared/services/config/images.go

@@ -9,15 +9,17 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/alessio/shellescape"
 	"github.com/rocket-pool/smartnode/shared/services/rocketpool/assets"
 	"github.com/rocket-pool/smartnode/shared/types/config"
 )
 
 const (
-	ImagesMainnetFile = "mainnet.env"
-	ImagesTestnetFile = "testnet.env"
-	ImagesDevnetFile  = "devnet.env"
-	ComposeMainFile   = "compose.yml"
+	ImagesMainnetFile   = "mainnet.env"
+	ImagesTestnetFile   = "testnet.env"
+	ImagesDevnetFile    = "devnet.env"
+	ComposeMainFile     = "compose.yml"
+	ComposeImageEnvFile = "image-tags.env"
 
 	ImageSmartnode = "RP_IMAGE_SMARTNODE"
 
@@ -403,8 +405,7 @@ func (cfg *RocketPoolConfig) DevnetOnly() string {
 	return "#"
 }
 
-// ImagesEnvFiles is the on-disk catalog list (mainnet.env, then {network}.env).
-// Compose interpolates runtime/image-tags.env, which is generated from these.
+// OverlayNetworkNames lists custom networks that may have image overlays.
 func (cfg *RocketPoolConfig) OverlayNetworkNames() []string {
 	if cfg.networks == nil {
 		return nil
@@ -421,6 +422,8 @@ func (cfg *RocketPoolConfig) OverlayNetworkNames() []string {
 	return names
 }
 
+// ImagesEnvFiles returns mainnet.env followed by the selected network's env
+// file, if present. A network without an overlay inherits the mainnet pins.
 func (cfg *RocketPoolConfig) ImagesEnvFiles() []string {
 	files := []string{ImagesMainnetFile}
 	if name := cfg.overlayFileName(); name != "" {
@@ -636,28 +639,46 @@ func (cfg *RocketPoolConfig) tuiOverride(param *config.Parameter) (string, bool)
 	return value, true
 }
 
-// ComposeImageDefaults maps service-level *_IMAGE_TAG_DEFAULT vars to the
-// catalog pin for the selected client/network.
-func (cfg *RocketPoolConfig) ComposeImageDefaults() map[string]string {
+// composeImageKeys maps each service to its selected client's catalog key.
+func (cfg *RocketPoolConfig) composeImageKeys() map[string]string {
 	out := map[string]string{
-		SmartnodeImageTagDefault:    cfg.ResolvedImage(ImageSmartnode),
-		PrometheusImageTagDefault:   cfg.ResolvedImage(ImagePrometheus),
-		GrafanaImageTagDefault:      cfg.ResolvedImage(ImageGrafana),
-		ExporterImageTagDefault:     cfg.ResolvedImage(ImageExporter),
-		AlertmanagerImageTagDefault: cfg.ResolvedImage(ImageAlertmanager),
-		GWWImageTagDefault:          cfg.ResolvedImage(ImageGWW),
-		CurlImageTagDefault:         cfg.ResolvedImage(ImageCurl),
-		MevBoostImageTagDefault:     cfg.ResolvedImage(ImageMevBoost),
-		CommitBoostImageTagDefault:  cfg.ResolvedImage(ImageCommitBoost),
+		SmartnodeImageTagDefault:    ImageSmartnode,
+		PrometheusImageTagDefault:   ImagePrometheus,
+		GrafanaImageTagDefault:      ImageGrafana,
+		ExporterImageTagDefault:     ImageExporter,
+		AlertmanagerImageTagDefault: ImageAlertmanager,
+		GWWImageTagDefault:          ImageGWW,
+		CurlImageTagDefault:         ImageCurl,
+		MevBoostImageTagDefault:     ImageMevBoost,
+		CommitBoostImageTagDefault:  ImageCommitBoost,
 	}
 	if key, err := cfg.selectedECCatalogKey(); err == nil {
-		out[ECImageTagDefault] = cfg.ResolvedImage(key)
+		out[ECImageTagDefault] = key
 	}
 	if key, err := cfg.selectedBNCatalogKey(); err == nil {
-		out[BNImageTagDefault] = cfg.ResolvedImage(key)
+		out[BNImageTagDefault] = key
 	}
 	if key, err := cfg.selectedVCCatalogKey(); err == nil {
-		out[VCImageTagDefault] = cfg.ResolvedImage(key)
+		out[VCImageTagDefault] = key
+	}
+	return out
+}
+
+// ComposeImageDefaultRefs generates dotenv aliases, evaluated by Compose after
+// loading mainnet.env and the selected network overlay.
+func (cfg *RocketPoolConfig) ComposeImageDefaultRefs() map[string]string {
+	out := cfg.composeImageKeys()
+	for name, key := range out {
+		out[name] = EnvRef(key)
+	}
+	return out
+}
+
+// ComposeImageDefaults resolves the same aliases for Go callers.
+func (cfg *RocketPoolConfig) ComposeImageDefaults() map[string]string {
+	out := cfg.composeImageKeys()
+	for name, key := range out {
+		out[name] = cfg.ResolvedImage(key)
 	}
 	return out
 }
@@ -733,13 +754,15 @@ func (cfg *RocketPoolConfig) ComposeImageEnv() (map[string]string, error) {
 	return out, nil
 }
 
-// ComposeEnvAssignments returns KEY=value strings for Compose interpolation:
-// *_IMAGE_TAG_DEFAULT from the catalog and *_IMAGE_TAG_OVERRIDE when the TUI
-// customized a tag. Keys are sorted so command logs and diffs are reproducible.
+// ComposeEnvAssignments returns shell-quoted TUI overrides. Defaults come from
+// Compose's env_file list so network overlays can replace mainnet values.
+// Keys are sorted so command logs and diffs are reproducible.
 func (cfg *RocketPoolConfig) ComposeEnvAssignments() ([]string, error) {
-	env, err := cfg.ComposeImageEnv()
-	if err != nil {
-		return nil, err
+	env := cfg.ComposeEnvOverrides()
+	for key, value := range env {
+		if err := validateEnvAssignment(key, value); err != nil {
+			return nil, err
+		}
 	}
 	keys := make([]string, 0, len(env))
 	for key := range env {
@@ -748,7 +771,7 @@ func (cfg *RocketPoolConfig) ComposeEnvAssignments() ([]string, error) {
 	sort.Strings(keys)
 	out := make([]string, 0, len(keys))
 	for _, key := range keys {
-		out = append(out, key+"="+env[key])
+		out = append(out, key+"="+shellescape.Quote(env[key]))
 	}
 	return out, nil
 }

shared/services/rocketpool/assets/install/mainnet.env

@@ -4,7 +4,7 @@
 # plataberget.env, ...): only keys present there replace these defaults.
 # TUI container-tag changes are set in the Compose process environment and win over both.
 
-RP_IMAGE_SMARTNODE=rocketpool/smartnode:v1.24.1@sha256:707f1e11c4822f6aacee0d96dc453ee0d47a4f750981095b7af2328053612252
+RP_IMAGE_SMARTNODE=rocketpool/smartnode:v1.24.2@sha256:0e94df463f2fb461e16f9b45e8eaae38caa639e714167d6f28b2c8f6570bc7a5
 
 RP_IMAGE_GETH=ethereum/client-go:v1.17.5@sha256:523d3ba26623a619e912019068dc2784f02934070ac46bdae4d5b9df0d917814
 RP_IMAGE_NETHERMIND=nethermind/nethermind:1.39.3@sha256:1b6b01419de4ff75ed3d61995904bccc2fdcc2865fee6dae07d88c14a0758e40

shared/services/rocketpool/assets/install/templates/compose.tmpl

@@ -1,16 +1,13 @@
 # Autogenerated - DO NOT MODIFY THIS FILE DIRECTLY
-# Global overrides belong in override/compose.yml (merged after this file).
-# Network overlay env files are commented out unless that network is selected.
+# Merge all generated files and overrides in order, as with ordered -f flags.
+# Keep one include: separate includes do not preserve override precedence.
 include:
-{{- range .Includes }}
-  - path: {{.Path}}
-{{- if .WithEnv }}
-    env_file:
-      - {{$.MainnetEnv}}
-      {{$.TestnetOnly}}- {{$.TestnetEnv}}
-      {{$.DevnetOnly}}- {{$.DevnetEnv}}
-{{- range $.ExtraNetworks }}
-      {{$.CommentUnless .}}- {{$.EnvFile .}}
-{{- end }}
+  - path:
+{{- range .Paths }}
+      - {{printf "%q" .}}
 {{- end }}
+    project_directory: {{printf "%q" .ProjectDir}}
+    env_file:
+{{- range .EnvFiles }}
+      - {{printf "%q" .}}
 {{- end }}

shared/services/rocketpool/client.go

@@ -1149,6 +1149,10 @@ func (c *Client) compose(composeFiles []string, args string) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	expandedConfigPath, err = filepath.Abs(expandedConfigPath)
+	if err != nil {
+		return "", err
+	}
 
 	// Load config
 	cfg, isNew, err := c.LoadConfig()
@@ -1178,35 +1182,13 @@ func (c *Client) compose(composeFiles []string, args string) (string, error) {
 		return "", fmt.Errorf("error deploying Docker templates: %w", err)
 	}
 
-	if err := ensureImageEnvFiles(expandedConfigPath); err != nil {
-		return "", err
-	}
-
-	composePaths := template.ComposePaths{
-		RuntimePath:  filepath.Join(expandedConfigPath, runtimeDir),
-		TemplatePath: filepath.Join(expandedConfigPath, templatesDir),
-		OverridePath: filepath.Join(expandedConfigPath, overrideDir),
-	}
-	composePair, err := composePaths.File("compose").Write(composeTemplateData{
-		Includes:      composeIncludes(expandedConfigPath, deployedContainers),
-		ProjectDir:    expandedConfigPath,
-		Network:       string(cfg.GetNetwork()),
-		ExtraNetworks: cfg.OverlayNetworkNames(),
-	})
+	composePath, err := writeComposeFile(cfg, expandedConfigPath, deployedContainers, composeFiles)
 	if err != nil {
 		return "", fmt.Errorf("error writing compose.yml: %w", err)
 	}
 
-	composeFileFlags := []string{}
-	for _, path := range composePair {
-		composeFileFlags = append(composeFileFlags, fmt.Sprintf("-f %s", shellescape.Quote(path)))
-	}
-	for _, container := range composeFiles {
-		composeFileFlags = append(composeFileFlags, fmt.Sprintf("-f %s", shellescape.Quote(container)))
-	}
-
 	envPrefix := []string{
-		fmt.Sprintf("COMPOSE_PROJECT_NAME=%s", cfg.Smartnode.ProjectName.Value.(string)),
+		"COMPOSE_PROJECT_NAME=" + shellescape.Quote(cfg.Smartnode.ProjectName.Value.(string)),
 	}
 	assignments, err := cfg.ComposeEnvAssignments()
 	if err != nil {
@@ -1216,7 +1198,7 @@ func (c *Client) compose(composeFiles []string, args string) (string, error) {
 
 	parts := append(envPrefix, "docker compose",
 		"--project-directory", shellescape.Quote(expandedConfigPath),
-		strings.Join(composeFileFlags, " "),
+		"-f", shellescape.Quote(composePath),
 		args,
 	)
 	return strings.Join(parts, " "), nil
@@ -1247,66 +1229,48 @@ func ensureImageEnvFiles(dir string) error {
 	return nil
 }
 
-type composeInclude struct {
-	Path    string
-	WithEnv bool
-}
-
 type composeTemplateData struct {
-	Includes      []composeInclude
-	ProjectDir    string
-	Network       string
-	ExtraNetworks []string
+	Paths      []string
+	ProjectDir string
+	EnvFiles   []string
 }
 
-func (d composeTemplateData) TestnetOnly() string {
-	if d.Network == "testnet" {
-		return ""
+// Merge the entire application in one include path list. Splitting it into
+// includes changes precedence for services and shared networks/volumes.
+func writeComposeFile(cfg *config.RocketPoolConfig, projectDir string, deployed, extra []string) (string, error) {
+	if err := ensureImageEnvFiles(projectDir); err != nil {
+		return "", err
 	}
-	return "#"
-}
-
-func (d composeTemplateData) DevnetOnly() string {
-	if d.Network == "devnet" {
-		return ""
+	data := composeTemplateData{
+		Paths:      append(slices.Clone(deployed), filepath.Join(projectDir, overrideDir, config.ComposeMainFile)),
+		ProjectDir: projectDir,
 	}
-	return "#"
-}
-
-func (d composeTemplateData) CommentUnless(network string) string {
-	if d.Network == network {
-		return ""
-	}
-	return "#"
-}
-
-func (d composeTemplateData) MainnetEnv() string {
-	return filepath.Join(d.ProjectDir, config.ImagesMainnetFile)
-}
-
-func (d composeTemplateData) TestnetEnv() string {
-	return filepath.Join(d.ProjectDir, config.ImagesTestnetFile)
-}
-
-func (d composeTemplateData) DevnetEnv() string {
-	return filepath.Join(d.ProjectDir, config.ImagesDevnetFile)
-}
-
-func (d composeTemplateData) EnvFile(network string) string {
-	return filepath.Join(d.ProjectDir, network+".env")
-}
-
-func composeIncludes(projectDir string, deployed []string) []composeInclude {
-	out := make([]composeInclude, 0, len(deployed))
-	for _, path := range deployed {
-		entry := composeInclude{Path: path, WithEnv: true}
-		rel, err := filepath.Rel(projectDir, path)
-		if err == nil && (rel == overrideDir || strings.HasPrefix(rel, overrideDir+string(filepath.Separator))) {
-			entry.WithEnv = false
+	for _, path := range extra {
+		// CLI -f paths are relative to the working directory, not the include.
+		absolute, err := filepath.Abs(path)
+		if err != nil {
+			return "", err
 		}
-		out = append(out, entry)
+		data.Paths = append(data.Paths, absolute)
+	}
+	for _, name := range cfg.ImagesEnvFiles() {
+		data.EnvFiles = append(data.EnvFiles, filepath.Join(projectDir, name))
+	}
+	// Resolve the selected client aliases after mainnet and the network overlay
+	// have been loaded by Compose. Only TUI overrides go in the shell environment.
+	imageEnv := filepath.Join(projectDir, runtimeDir, config.ComposeImageEnvFile)
+	if err := config.WriteEnvFile(imageEnv, cfg.ComposeImageDefaultRefs()); err != nil {
+		return "", fmt.Errorf("could not write image aliases: %w", err)
+	}
+	data.EnvFiles = append(data.EnvFiles, imageEnv)
+	tmpl := template.Template{
+		Src: filepath.Join(projectDir, templatesDir, "compose.tmpl"),
+		Dst: filepath.Join(projectDir, runtimeDir, config.ComposeMainFile),
+	}
+	if err := tmpl.Write(data); err != nil {
+		return "", err
 	}
-	return out
+	return tmpl.Dst, nil
 }
 
 // Deploys all of the appropriate docker compose template files and provisions them based on the provided configuration