diff --git a/.github/workflows/labctl.yml b/.github/workflows/labctl.yml new file mode 100644 index 0000000..6bb1a9f --- /dev/null +++ b/.github/workflows/labctl.yml @@ -0,0 +1,45 @@ +name: labctl + +on: + push: + branches: [master] + paths: + - 'tools/labctl/**' + pull_request: + paths: + - 'tools/labctl/**' + +jobs: + lint-and-test: + runs-on: ubuntu-latest + defaults: + run: + working-directory: tools/labctl + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 + with: + go-version-file: tools/labctl/go.mod + cache-dependency-path: tools/labctl/go.sum + + - name: Format check + run: | + if [ -n "$(gofmt -l .)" ]; then + echo "Files not formatted:" + gofmt -l . + exit 1 + fi + + - name: Vet + run: go vet ./... + + - name: Lint + uses: golangci/golangci-lint-action@v7 + with: + version: v2.6.0 + working-directory: tools/labctl + + - name: Test + run: go test -race ./... diff --git a/tools/labctl/.gitignore b/tools/labctl/.gitignore new file mode 100644 index 0000000..6dd29b7 --- /dev/null +++ b/tools/labctl/.gitignore @@ -0,0 +1 @@ +bin/ \ No newline at end of file diff --git a/tools/labctl/.golangci.yml b/tools/labctl/.golangci.yml new file mode 100644 index 0000000..6ccb30c --- /dev/null +++ b/tools/labctl/.golangci.yml @@ -0,0 +1,73 @@ +run: + timeout: 5m + +version: "2" + +linters: + default: none + enable: + - govet + - staticcheck + - revive + - errcheck + - ineffassign + - unused + - gosec + - gocritic + - misspell + - sqlclosecheck + - bidichk + + settings: + gocritic: + enabled-tags: + - diagnostic + - experimental + - opinionated + - performance + - style + disabled-checks: + - whyNoLint + - wrapperFunc + - hugeParam + - rangeValCopy + + revive: + rules: + - name: exported + - name: unused-parameter + - name: unreachable-code + - name: redefines-builtin-id + - name: defer + - name: context-as-argument + - name: context-keys-type + - name: error-return + - name: error-naming + - name: if-return + - name: increment-decrement + - name: var-naming + - name: var-declaration + - name: package-comments + - name: range + - name: receiver-naming + - name: time-naming + - name: unexported-return + - name: indent-error-flow + - name: errorf + - name: empty-block + - name: superfluous-else + - name: unused-receiver + - name: get-return + - name: string-of-int + - name: early-return + - name: unconditional-recursion + - name: identical-branches + - name: waitgroup-by-value + - name: atomic + + exclusions: + presets: [] + +issues: + max-issues-per-linter: 0 + max-same-issues: 0 diff --git a/tools/labctl/cmd/images/list.go b/tools/labctl/cmd/images/list.go new file mode 100644 index 0000000..d41317f --- /dev/null +++ b/tools/labctl/cmd/images/list.go @@ -0,0 +1,30 @@ +package images + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var listCmd = &cobra.Command{ + Use: "list", + Short: "List images stored in e2", + Long: "List all images currently stored in the e2 bucket with their metadata.", + RunE: runList, +} + +var ( + listCredentials string + listSOPSAgeKeyFile string +) + +func init() { + listCmd.Flags().StringVar(&listCredentials, "credentials", "", "Path to SOPS-encrypted credentials file") + listCmd.Flags().StringVar(&listSOPSAgeKeyFile, "sops-age-key-file", "", "Path to age private key") +} + +func runList(_ *cobra.Command, _ []string) error { + // TODO(HOM-20): Implement list command + fmt.Println("list command not yet implemented") + return nil +} diff --git a/tools/labctl/cmd/images/prune.go b/tools/labctl/cmd/images/prune.go new file mode 100644 index 0000000..8740c0a --- /dev/null +++ b/tools/labctl/cmd/images/prune.go @@ -0,0 +1,38 @@ +package images + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var pruneCmd = &cobra.Command{ + Use: "prune", + Short: "Remove orphaned images from e2", + Long: `Remove images from e2 that are not in the manifest. + +The prune command compares images in e2 storage against the manifest and +removes any that are no longer referenced. This is a manual-only operation +and is not run automatically.`, + RunE: runPrune, +} + +var ( + pruneManifest string + pruneCredentials string + pruneSOPSAgeKeyFile string + pruneDryRun bool +) + +func init() { + pruneCmd.Flags().StringVar(&pruneManifest, "manifest", "./images/images.yaml", "Path to images.yaml") + pruneCmd.Flags().StringVar(&pruneCredentials, "credentials", "", "Path to SOPS-encrypted credentials file") + pruneCmd.Flags().StringVar(&pruneSOPSAgeKeyFile, "sops-age-key-file", "", "Path to age private key") + pruneCmd.Flags().BoolVar(&pruneDryRun, "dry-run", false, "Show what would be removed") +} + +func runPrune(_ *cobra.Command, _ []string) error { + // TODO(HOM-20): Implement prune command + fmt.Println("prune command not yet implemented") + return nil +} diff --git a/tools/labctl/cmd/images/root.go b/tools/labctl/cmd/images/root.go new file mode 100644 index 0000000..5d5434e --- /dev/null +++ b/tools/labctl/cmd/images/root.go @@ -0,0 +1,21 @@ +// Package images provides CLI commands for managing lab images. +package images + +import ( + "github.com/spf13/cobra" +) + +// Cmd is the images subcommand. +var Cmd = &cobra.Command{ + Use: "images", + Short: "Manage lab images", + Long: "Commands for syncing, validating, listing, pruning, and uploading lab images to e2 storage.", +} + +func init() { + Cmd.AddCommand(syncCmd) + Cmd.AddCommand(validateCmd) + Cmd.AddCommand(listCmd) + Cmd.AddCommand(pruneCmd) + Cmd.AddCommand(uploadCmd) +} diff --git a/tools/labctl/cmd/images/sync.go b/tools/labctl/cmd/images/sync.go new file mode 100644 index 0000000..8720a67 --- /dev/null +++ b/tools/labctl/cmd/images/sync.go @@ -0,0 +1,40 @@ +package images + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var syncCmd = &cobra.Command{ + Use: "sync", + Short: "Sync images to e2 storage", + Long: `Download source images, upload to e2, update files, and create PR if needed. + +The sync command reads the image manifest, downloads any new or updated images, +uploads them to e2 storage, and optionally updates file references to trigger +downstream builds.`, + RunE: runSync, +} + +var ( + syncManifest string + syncCredentials string + syncSOPSAgeKeyFile string + syncDryRun bool + syncForce bool +) + +func init() { + syncCmd.Flags().StringVar(&syncManifest, "manifest", "./images/images.yaml", "Path to images.yaml") + syncCmd.Flags().StringVar(&syncCredentials, "credentials", "", "Path to SOPS-encrypted credentials file") + syncCmd.Flags().StringVar(&syncSOPSAgeKeyFile, "sops-age-key-file", "", "Path to age private key for SOPS decryption") + syncCmd.Flags().BoolVar(&syncDryRun, "dry-run", false, "Show what would be done without executing") + syncCmd.Flags().BoolVar(&syncForce, "force", false, "Force re-upload even if checksums match") +} + +func runSync(_ *cobra.Command, _ []string) error { + // TODO(HOM-20): Implement sync command + fmt.Println("sync command not yet implemented") + return nil +} diff --git a/tools/labctl/cmd/images/upload.go b/tools/labctl/cmd/images/upload.go new file mode 100644 index 0000000..6cbf883 --- /dev/null +++ b/tools/labctl/cmd/images/upload.go @@ -0,0 +1,43 @@ +package images + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var uploadCmd = &cobra.Command{ + Use: "upload", + Short: "Upload a local file to e2", + Long: `Upload a local file to e2 storage. + +The upload command is used by Packer workflows to upload built images. +It computes the SHA256 checksum and writes metadata JSON in the same +format as the sync command.`, + RunE: runUpload, +} + +var ( + uploadSource string + uploadDestination string + uploadCredentials string + uploadSOPSAgeKeyFile string + uploadName string +) + +func init() { + uploadCmd.Flags().StringVar(&uploadSource, "source", "", "Path to local file to upload (required)") + uploadCmd.Flags().StringVar(&uploadDestination, "destination", "", "Destination path in e2 bucket (required)") + uploadCmd.Flags().StringVar(&uploadCredentials, "credentials", "", "Path to SOPS-encrypted credentials file") + uploadCmd.Flags().StringVar(&uploadSOPSAgeKeyFile, "sops-age-key-file", "", "Path to age private key") + uploadCmd.Flags().StringVar(&uploadName, "name", "", "Image name for metadata (defaults to destination filename)") + + _ = uploadCmd.MarkFlagRequired("source") + _ = uploadCmd.MarkFlagRequired("destination") +} + +func runUpload(_ *cobra.Command, _ []string) error { + // TODO(HOM-20): Implement upload command + fmt.Println("upload command not yet implemented") + return nil +} diff --git a/tools/labctl/cmd/images/validate.go b/tools/labctl/cmd/images/validate.go new file mode 100644 index 0000000..e20b71d --- /dev/null +++ b/tools/labctl/cmd/images/validate.go @@ -0,0 +1,30 @@ +package images + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate the image manifest", + Long: `Validate manifest syntax, check source URLs, and verify updateFile regex patterns. + +The validate command performs a dry-run validation of the image manifest, +checking that all URLs are reachable (via HEAD requests) and that regex +patterns in updateFile sections compile successfully.`, + RunE: runValidate, +} + +var validateManifest string + +func init() { + validateCmd.Flags().StringVar(&validateManifest, "manifest", "./images/images.yaml", "Path to images.yaml") +} + +func runValidate(_ *cobra.Command, _ []string) error { + // TODO(HOM-20): Implement validate command + fmt.Println("validate command not yet implemented") + return nil +} diff --git a/tools/labctl/cmd/root.go b/tools/labctl/cmd/root.go new file mode 100644 index 0000000..c16ae33 --- /dev/null +++ b/tools/labctl/cmd/root.go @@ -0,0 +1,23 @@ +// Package cmd provides the CLI commands for labctl. +package cmd + +import ( + "github.com/spf13/cobra" + + "github.com/GilmanLab/lab/tools/labctl/cmd/images" +) + +var rootCmd = &cobra.Command{ + Use: "labctl", + Short: "Lab control CLI for managing infrastructure", + Long: "labctl is a CLI tool for managing lab infrastructure including images, configurations, and deployments.", +} + +func init() { + rootCmd.AddCommand(images.Cmd) +} + +// Execute runs the root command. +func Execute() error { + return rootCmd.Execute() +} diff --git a/tools/labctl/go.mod b/tools/labctl/go.mod new file mode 100644 index 0000000..6847068 --- /dev/null +++ b/tools/labctl/go.mod @@ -0,0 +1,35 @@ +module github.com/GilmanLab/lab/tools/labctl + +go 1.23.0 + +require ( + github.com/aws/aws-sdk-go-v2 v1.41.0 + github.com/aws/aws-sdk-go-v2/config v1.32.6 + github.com/aws/aws-sdk-go-v2/credentials v1.19.6 + github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.11.1 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 // indirect + github.com/aws/smithy-go v1.24.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/tools/labctl/go.sum b/tools/labctl/go.sum new file mode 100644 index 0000000..91190ee --- /dev/null +++ b/tools/labctl/go.sum @@ -0,0 +1,56 @@ +github.com/aws/aws-sdk-go-v2 v1.41.0 h1:tNvqh1s+v0vFYdA1xq0aOJH+Y5cRyZ5upu6roPgPKd4= +github.com/aws/aws-sdk-go-v2 v1.41.0/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= +github.com/aws/aws-sdk-go-v2/config v1.32.6 h1:hFLBGUKjmLAekvi1evLi5hVvFQtSo3GYwi+Bx4lpJf8= +github.com/aws/aws-sdk-go-v2/config v1.32.6/go.mod h1:lcUL/gcd8WyjCrMnxez5OXkO3/rwcNmvfno62tnXNcI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6 h1:F9vWao2TwjV2MyiyVS+duza0NIRtAslgLUM0vTA1ZaE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.6/go.mod h1:SgHzKjEVsdQr6Opor0ihgWtkWdfRAIwxYzSJ8O85VHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16 h1:80+uETIWS1BqjnN9uJ0dBUaETh+P1XwFy5vwHwK5r9k= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.16/go.mod h1:wOOsYuxYuB/7FlnVtzeBYRcjSRtQpAW0hCP7tIULMwo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16 h1:rgGwPzb82iBYSvHMHXc8h9mRoOUBZIGFgKb9qniaZZc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.16/go.mod h1:L/UxsGeKpGoIj6DxfhOWHWQ/kGKcd4I1VncE4++IyKA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16 h1:1jtGzuV7c82xnqOVfx2F0xmJcOw5374L7N6juGW6x6U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.16/go.mod h1:M2E5OQf+XLe+SZGmmpaI2yy+J326aFf6/+54PoxSANc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16 h1:CjMzUs78RDDv4ROu3JnJn/Ig1r6ZD7/T2DXLLRpejic= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.16/go.mod h1:uVW4OLBqbJXSHJYA9svT9BluSvvwbzLQ2Crf6UPzR3c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7 h1:DIBqIrJ7hv+e4CmIk2z3pyKT+3B6qVMgRsawHiR3qso= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.7/go.mod h1:vLm00xmBke75UmpNvOcZQ/Q30ZFjbczeLFqGx5urmGo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16 h1:oHjJHeUy0ImIV0bsrX0X91GkV5nJAyv1l1CC9lnO0TI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.16/go.mod h1:iRSNGgOYmiYwSCXxXaKb9HfOEj40+oTKn8pTxMlYkRM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16 h1:NSbvS17MlI2lurYgXnCOLvCFX38sBW4eiVER7+kkgsU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.16/go.mod h1:SwT8Tmqd4sA6G1qaGdzWCJN99bUmPGHfRwwq3G5Qb+A= +github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0 h1:SWTxh/EcUCDVqi/0s26V6pVUq0BBG7kx0tDTmF/hCgA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.94.0/go.mod h1:79S2BdqCJpScXZA2y+cpZuocWsjGjJINyXnOsf5DTz8= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4 h1:HpI7aMmJ+mm1wkSHIA2t5EaFFv5EFYXePW30p1EIrbQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.4/go.mod h1:C5RdGMYGlfM0gYq/tifqgn4EbyX99V15P2V3R+VHbQU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8 h1:aM/Q24rIlS3bRAhTyFurowU8A0SMyGDtEOY/l/s/1Uw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.8/go.mod h1:+fWt2UHSb4kS7Pu8y+BMBvJF0EWx+4H0hzNwtDNRTrg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12 h1:AHDr0DaHIAo8c9t1emrzAlVDFp+iMMKnPdYy6XO4MCE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.12/go.mod h1:GQ73XawFFiWxyWXMHWfhiomvP3tXtdNar/fi8z18sx0= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5 h1:SciGFVNZ4mHdm7gpD1dgZYnCuVdX1s+lFTg4+4DOy70= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.5/go.mod h1:iW40X4QBmUxdP+fZNOpfmkdMZqsovezbAeO+Ubiv2pk= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/labctl/internal/config/manifest.go b/tools/labctl/internal/config/manifest.go new file mode 100644 index 0000000..9c51892 --- /dev/null +++ b/tools/labctl/internal/config/manifest.go @@ -0,0 +1,193 @@ +// Package config provides configuration parsing for the image pipeline. +package config + +import ( + "fmt" + "os" + "regexp" + "strings" + + "gopkg.in/yaml.v3" +) + +// SupportedAPIVersion is the supported API version for the image manifest. +const SupportedAPIVersion = "images.lab.gilman.io/v1alpha1" + +// ImageManifest represents the top-level image manifest configuration. +type ImageManifest struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata Metadata `yaml:"metadata"` + Spec Spec `yaml:"spec"` +} + +// Metadata contains manifest metadata. +type Metadata struct { + Name string `yaml:"name"` +} + +// Spec contains the list of images to manage. +type Spec struct { + Images []Image `yaml:"images"` +} + +// Image represents a single image configuration. +type Image struct { + Name string `yaml:"name"` + Source Source `yaml:"source"` + Destination string `yaml:"destination"` + Validation *Validation `yaml:"validation,omitempty"` + UpdateFile *UpdateFile `yaml:"updateFile,omitempty"` +} + +// Source defines where to download the image from. +type Source struct { + URL string `yaml:"url"` + Checksum string `yaml:"checksum"` + Decompress string `yaml:"decompress,omitempty"` // xz, gzip, zstd +} + +// Validation defines post-processing validation rules. +type Validation struct { + Algorithm string `yaml:"algorithm"` // sha256, sha512 + Expected string `yaml:"expected"` +} + +// UpdateFile defines file updates to trigger downstream builds. +type UpdateFile struct { + Path string `yaml:"path"` + Replacements []Replacement `yaml:"replacements"` +} + +// Replacement defines a regex-based replacement in a file. +type Replacement struct { + Pattern string `yaml:"pattern"` // Regex pattern + Value string `yaml:"value"` // Template: {{ .Source.URL }}, {{ .Source.Checksum }} +} + +// EffectiveChecksum returns the checksum to use for idempotency checks. +// If validation.expected is set, use that; otherwise use source.checksum. +func (i *Image) EffectiveChecksum() string { + if i.Validation != nil && i.Validation.Expected != "" { + return i.Validation.Expected + } + return i.Source.Checksum +} + +// LoadManifest reads and parses an image manifest from a file. +func LoadManifest(path string) (*ImageManifest, error) { + data, err := os.ReadFile(path) //nolint:gosec // G304: Path is provided by user + if err != nil { + return nil, fmt.Errorf("read manifest file: %w", err) + } + + return ParseManifest(data) +} + +// ParseManifest parses an image manifest from YAML data. +func ParseManifest(data []byte) (*ImageManifest, error) { + var manifest ImageManifest + if err := yaml.Unmarshal(data, &manifest); err != nil { + return nil, fmt.Errorf("parse manifest YAML: %w", err) + } + + if err := manifest.Validate(); err != nil { + return nil, fmt.Errorf("validate manifest: %w", err) + } + + return &manifest, nil +} + +// Validate checks that the manifest is well-formed. +func (m *ImageManifest) Validate() error { + if m.APIVersion != SupportedAPIVersion { + return fmt.Errorf("unsupported apiVersion %q, expected %q", m.APIVersion, SupportedAPIVersion) + } + + if m.Kind != "ImageManifest" { + return fmt.Errorf("unsupported kind %q, expected %q", m.Kind, "ImageManifest") + } + + if m.Metadata.Name == "" { + return fmt.Errorf("metadata.name is required") + } + + for i, img := range m.Spec.Images { + if err := img.Validate(); err != nil { + return fmt.Errorf("image[%d] %q: %w", i, img.Name, err) + } + } + + return nil +} + +// Validate checks that the image configuration is valid. +func (i *Image) Validate() error { + if i.Name == "" { + return fmt.Errorf("name is required") + } + + if i.Source.URL == "" { + return fmt.Errorf("source.url is required") + } + + if !strings.HasPrefix(i.Source.URL, "https://") { + return fmt.Errorf("source.url must use HTTPS") + } + + if i.Source.Checksum == "" { + return fmt.Errorf("source.checksum is required") + } + + if i.Destination == "" { + return fmt.Errorf("destination is required") + } + + // Validate decompress option + if i.Source.Decompress != "" { + switch i.Source.Decompress { + case "xz", "gzip", "zstd": + // valid + default: + return fmt.Errorf("unsupported decompress format %q, must be xz, gzip, or zstd", i.Source.Decompress) + } + + // validation.expected is required when decompress is used + if i.Validation == nil || i.Validation.Expected == "" { + return fmt.Errorf("validation.expected is required when decompress is used") + } + } + + // Validate algorithm if validation is specified + if i.Validation != nil { + switch i.Validation.Algorithm { + case "sha256", "sha512": + // valid + default: + return fmt.Errorf("unsupported validation algorithm %q, must be sha256 or sha512", i.Validation.Algorithm) + } + } + + // Validate updateFile regex patterns compile + if i.UpdateFile != nil { + if i.UpdateFile.Path == "" { + return fmt.Errorf("updateFile.path is required") + } + + for j, r := range i.UpdateFile.Replacements { + if r.Pattern == "" { + return fmt.Errorf("updateFile.replacements[%d].pattern is required", j) + } + + if _, err := regexp.Compile(r.Pattern); err != nil { + return fmt.Errorf("updateFile.replacements[%d].pattern is invalid: %w", j, err) + } + + if r.Value == "" { + return fmt.Errorf("updateFile.replacements[%d].value is required", j) + } + } + } + + return nil +} diff --git a/tools/labctl/internal/config/manifest_test.go b/tools/labctl/internal/config/manifest_test.go new file mode 100644 index 0000000..078b81a --- /dev/null +++ b/tools/labctl/internal/config/manifest_test.go @@ -0,0 +1,362 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseManifest(t *testing.T) { + tests := []struct { + name string + yaml string + wantErr string + }{ + { + name: "valid manifest with simple image", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: talos-1.9.1 + source: + url: https://factory.talos.dev/image/metal-amd64.raw.xz + checksum: sha256:abc123 + destination: talos/talos-1.9.1-amd64.raw +`, + }, + { + name: "valid manifest with decompression", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: talos-1.9.1 + source: + url: https://factory.talos.dev/image/metal-amd64.raw.xz + checksum: sha256:abc123 + decompress: xz + destination: talos/talos-1.9.1-amd64.raw + validation: + algorithm: sha256 + expected: sha256:def456 +`, + }, + { + name: "valid manifest with updateFile", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: vyos-iso + source: + url: https://github.com/vyos/vyos-rolling-nightly-builds/releases/download/1.5/vyos-1.5.iso + checksum: sha256:abc123 + destination: vyos/vyos-1.5.iso + updateFile: + path: infrastructure/network/vyos/packer/source.auto.pkrvars.hcl + replacements: + - pattern: 'vyos_iso_url\s*=\s*"[^"]*"' + value: 'vyos_iso_url = "{{ .Source.URL }}"' +`, + }, + { + name: "invalid apiVersion", + yaml: `apiVersion: images.lab.gilman.io/v2 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: [] +`, + wantErr: `unsupported apiVersion "images.lab.gilman.io/v2"`, + }, + { + name: "invalid kind", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: SomethingElse +metadata: + name: lab-images +spec: + images: [] +`, + wantErr: `unsupported kind "SomethingElse"`, + }, + { + name: "missing metadata name", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: "" +spec: + images: [] +`, + wantErr: "metadata.name is required", + }, + { + name: "missing image name", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - source: + url: https://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso +`, + wantErr: `image[0] "": name is required`, + }, + { + name: "missing source url", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + checksum: sha256:abc123 + destination: images/image.iso +`, + wantErr: "source.url is required", + }, + { + name: "http url rejected", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: http://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso +`, + wantErr: "source.url must use HTTPS", + }, + { + name: "missing checksum", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + destination: images/image.iso +`, + wantErr: "source.checksum is required", + }, + { + name: "missing destination", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 +`, + wantErr: "destination is required", + }, + { + name: "invalid decompress format", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 + decompress: zip + destination: images/image.iso +`, + wantErr: "unsupported decompress format", + }, + { + name: "decompress without validation.expected", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.raw.xz + checksum: sha256:abc123 + decompress: xz + destination: images/image.raw +`, + wantErr: "validation.expected is required when decompress is used", + }, + { + name: "invalid validation algorithm", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso + validation: + algorithm: md5 + expected: md5:xyz +`, + wantErr: "unsupported validation algorithm", + }, + { + name: "invalid regex pattern", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso + updateFile: + path: some/file.txt + replacements: + - pattern: '[invalid(regex' + value: 'replacement' +`, + wantErr: "pattern is invalid", + }, + { + name: "missing updateFile path", + yaml: `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: lab-images +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso + updateFile: + replacements: + - pattern: 'foo' + value: 'bar' +`, + wantErr: "updateFile.path is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + manifest, err := ParseManifest([]byte(tt.yaml)) + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + + require.NoError(t, err) + assert.NotNil(t, manifest) + }) + } +} + +func TestLoadManifest(t *testing.T) { + t.Run("file exists", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "images.yaml") + + content := `apiVersion: images.lab.gilman.io/v1alpha1 +kind: ImageManifest +metadata: + name: test +spec: + images: + - name: test-image + source: + url: https://example.com/image.iso + checksum: sha256:abc123 + destination: images/image.iso +` + err := os.WriteFile(path, []byte(content), 0o600) + require.NoError(t, err) + + manifest, err := LoadManifest(path) + require.NoError(t, err) + assert.Equal(t, "test", manifest.Metadata.Name) + assert.Len(t, manifest.Spec.Images, 1) + }) + + t.Run("file not found", func(t *testing.T) { + _, err := LoadManifest("/nonexistent/path/images.yaml") + require.Error(t, err) + assert.Contains(t, err.Error(), "read manifest file") + }) +} + +func TestImage_EffectiveChecksum(t *testing.T) { + tests := []struct { + name string + image Image + expected string + }{ + { + name: "uses source checksum when no validation", + image: Image{ + Source: Source{Checksum: "sha256:source"}, + }, + expected: "sha256:source", + }, + { + name: "uses source checksum when validation.expected is empty", + image: Image{ + Source: Source{Checksum: "sha256:source"}, + Validation: &Validation{Algorithm: "sha256", Expected: ""}, + }, + expected: "sha256:source", + }, + { + name: "uses validation.expected when set", + image: Image{ + Source: Source{Checksum: "sha256:source"}, + Validation: &Validation{Algorithm: "sha256", Expected: "sha256:validated"}, + }, + expected: "sha256:validated", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, tt.image.EffectiveChecksum()) + }) + } +} diff --git a/tools/labctl/internal/credentials/credentials.go b/tools/labctl/internal/credentials/credentials.go new file mode 100644 index 0000000..8e9a602 --- /dev/null +++ b/tools/labctl/internal/credentials/credentials.go @@ -0,0 +1,60 @@ +// Package credentials provides credential resolution for e2 storage. +package credentials + +import "fmt" + +// E2Credentials holds credentials for iDrive e2 storage. +type E2Credentials struct { + AccessKey string `yaml:"access_key"` + SecretKey string `yaml:"secret_key"` + Endpoint string `yaml:"endpoint"` + Bucket string `yaml:"bucket"` +} + +// Validate checks that all required fields are present. +func (c *E2Credentials) Validate() error { + if c.AccessKey == "" { + return fmt.Errorf("access_key is required") + } + if c.SecretKey == "" { + return fmt.Errorf("secret_key is required") + } + if c.Endpoint == "" { + return fmt.Errorf("endpoint is required") + } + if c.Bucket == "" { + return fmt.Errorf("bucket is required") + } + return nil +} + +// ResolveOptions configures credential resolution. +type ResolveOptions struct { + // SOPSFile is the path to a SOPS-encrypted credentials file. + SOPSFile string + + // AgeKeyFile is the path to the age private key for SOPS decryption. + AgeKeyFile string +} + +// Resolve attempts to resolve e2 credentials using the following order: +// 1. Environment variables (if all are present) +// 2. SOPS file (if specified in options) +func Resolve(opts ResolveOptions) (*E2Credentials, error) { + // Try environment variables first + creds, err := FromEnv() + if err == nil { + return creds, nil + } + + // Try SOPS file if specified + if opts.SOPSFile != "" { + creds, err := FromSOPS(opts.SOPSFile, opts.AgeKeyFile) + if err != nil { + return nil, fmt.Errorf("resolve from SOPS: %w", err) + } + return creds, nil + } + + return nil, fmt.Errorf("no credentials found: environment variables incomplete and no SOPS file specified") +} diff --git a/tools/labctl/internal/credentials/credentials_test.go b/tools/labctl/internal/credentials/credentials_test.go new file mode 100644 index 0000000..512ce35 --- /dev/null +++ b/tools/labctl/internal/credentials/credentials_test.go @@ -0,0 +1,116 @@ +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestE2Credentials_Validate(t *testing.T) { + tests := []struct { + name string + creds E2Credentials + wantErr string + }{ + { + name: "valid credentials", + creds: E2Credentials{ + AccessKey: "access123", + SecretKey: "secret456", + Endpoint: "https://e2.example.com", + Bucket: "my-bucket", + }, + }, + { + name: "missing access key", + creds: E2Credentials{ + SecretKey: "secret456", + Endpoint: "https://e2.example.com", + Bucket: "my-bucket", + }, + wantErr: "access_key is required", + }, + { + name: "missing secret key", + creds: E2Credentials{ + AccessKey: "access123", + Endpoint: "https://e2.example.com", + Bucket: "my-bucket", + }, + wantErr: "secret_key is required", + }, + { + name: "missing endpoint", + creds: E2Credentials{ + AccessKey: "access123", + SecretKey: "secret456", + Bucket: "my-bucket", + }, + wantErr: "endpoint is required", + }, + { + name: "missing bucket", + creds: E2Credentials{ + AccessKey: "access123", + SecretKey: "secret456", + Endpoint: "https://e2.example.com", + }, + wantErr: "bucket is required", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.creds.Validate() + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + + require.NoError(t, err) + }) + } +} + +func TestResolve(t *testing.T) { + t.Run("resolves from environment variables", func(t *testing.T) { + t.Setenv("E2_ACCESS_KEY", "access123") + t.Setenv("E2_SECRET_KEY", "secret456") + t.Setenv("E2_ENDPOINT", "https://e2.example.com") + t.Setenv("E2_BUCKET", "my-bucket") + + creds, err := Resolve(ResolveOptions{}) + require.NoError(t, err) + assert.Equal(t, "access123", creds.AccessKey) + assert.Equal(t, "secret456", creds.SecretKey) + assert.Equal(t, "https://e2.example.com", creds.Endpoint) + assert.Equal(t, "my-bucket", creds.Bucket) + }) + + t.Run("returns error when no credentials available", func(t *testing.T) { + // Clear all env vars + for _, key := range []string{EnvAccessKey, EnvSecretKey, EnvEndpoint, EnvBucket} { + t.Setenv(key, "") + } + + _, err := Resolve(ResolveOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "no credentials found") + }) + + t.Run("returns error when SOPS file not found", func(t *testing.T) { + // Clear all env vars + for _, key := range []string{EnvAccessKey, EnvSecretKey, EnvEndpoint, EnvBucket} { + t.Setenv(key, "") + } + + _, err := Resolve(ResolveOptions{ + SOPSFile: "/nonexistent/path/credentials.sops.yaml", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "SOPS file not found") + }) +} diff --git a/tools/labctl/internal/credentials/env.go b/tools/labctl/internal/credentials/env.go new file mode 100644 index 0000000..aa1c465 --- /dev/null +++ b/tools/labctl/internal/credentials/env.go @@ -0,0 +1,49 @@ +package credentials + +import ( + "fmt" + "os" +) + +// Environment variable names for e2 credentials. +const ( + EnvAccessKey = "E2_ACCESS_KEY" + EnvSecretKey = "E2_SECRET_KEY" //nolint:gosec // G101: This is the name of the environment variable + EnvEndpoint = "E2_ENDPOINT" + EnvBucket = "E2_BUCKET" +) + +// FromEnv resolves e2 credentials from environment variables. +// All four variables must be set: E2_ACCESS_KEY, E2_SECRET_KEY, E2_ENDPOINT, E2_BUCKET. +func FromEnv() (*E2Credentials, error) { + accessKey := os.Getenv(EnvAccessKey) + secretKey := os.Getenv(EnvSecretKey) + endpoint := os.Getenv(EnvEndpoint) + bucket := os.Getenv(EnvBucket) + + // Check if any are missing + var missing []string + if accessKey == "" { + missing = append(missing, EnvAccessKey) + } + if secretKey == "" { + missing = append(missing, EnvSecretKey) + } + if endpoint == "" { + missing = append(missing, EnvEndpoint) + } + if bucket == "" { + missing = append(missing, EnvBucket) + } + + if len(missing) > 0 { + return nil, fmt.Errorf("missing environment variables: %v", missing) + } + + return &E2Credentials{ + AccessKey: accessKey, + SecretKey: secretKey, + Endpoint: endpoint, + Bucket: bucket, + }, nil +} diff --git a/tools/labctl/internal/credentials/env_test.go b/tools/labctl/internal/credentials/env_test.go new file mode 100644 index 0000000..9dcb5e4 --- /dev/null +++ b/tools/labctl/internal/credentials/env_test.go @@ -0,0 +1,99 @@ +package credentials + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFromEnv(t *testing.T) { + tests := []struct { + name string + envVars map[string]string + want *E2Credentials + wantErr string + }{ + { + name: "all variables set", + envVars: map[string]string{ + "E2_ACCESS_KEY": "access123", + "E2_SECRET_KEY": "secret456", + "E2_ENDPOINT": "https://e2.example.com", + "E2_BUCKET": "my-bucket", + }, + want: &E2Credentials{ + AccessKey: "access123", + SecretKey: "secret456", + Endpoint: "https://e2.example.com", + Bucket: "my-bucket", + }, + }, + { + name: "missing access key", + envVars: map[string]string{ + "E2_SECRET_KEY": "secret456", + "E2_ENDPOINT": "https://e2.example.com", + "E2_BUCKET": "my-bucket", + }, + wantErr: "E2_ACCESS_KEY", + }, + { + name: "missing secret key", + envVars: map[string]string{ + "E2_ACCESS_KEY": "access123", + "E2_ENDPOINT": "https://e2.example.com", + "E2_BUCKET": "my-bucket", + }, + wantErr: "E2_SECRET_KEY", + }, + { + name: "missing endpoint", + envVars: map[string]string{ + "E2_ACCESS_KEY": "access123", + "E2_SECRET_KEY": "secret456", + "E2_BUCKET": "my-bucket", + }, + wantErr: "E2_ENDPOINT", + }, + { + name: "missing bucket", + envVars: map[string]string{ + "E2_ACCESS_KEY": "access123", + "E2_SECRET_KEY": "secret456", + "E2_ENDPOINT": "https://e2.example.com", + }, + wantErr: "E2_BUCKET", + }, + { + name: "all missing", + envVars: map[string]string{}, + wantErr: "missing environment variables", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Clear all env vars first + for _, key := range []string{EnvAccessKey, EnvSecretKey, EnvEndpoint, EnvBucket} { + t.Setenv(key, "") + } + + // Set test env vars + for key, value := range tt.envVars { + t.Setenv(key, value) + } + + creds, err := FromEnv() + + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + + require.NoError(t, err) + assert.Equal(t, tt.want, creds) + }) + } +} diff --git a/tools/labctl/internal/credentials/sops.go b/tools/labctl/internal/credentials/sops.go new file mode 100644 index 0000000..da9877b --- /dev/null +++ b/tools/labctl/internal/credentials/sops.go @@ -0,0 +1,59 @@ +package credentials + +import ( + "bytes" + "fmt" + "os" + "os/exec" + + "gopkg.in/yaml.v3" +) + +// commandRunner executes a command and returns stdout, stderr, and error. +// This is a variable to allow mocking in tests. +var commandRunner = func(name string, args []string, env []string) (stdout, stderr []byte, err error) { + cmd := exec.Command(name, args...) //nolint:gosec // G204: sops execution is intended behavior + if len(env) > 0 { + cmd.Env = env + } + + var outBuf, errBuf bytes.Buffer + cmd.Stdout = &outBuf + cmd.Stderr = &errBuf + + err = cmd.Run() + return outBuf.Bytes(), errBuf.Bytes(), err +} + +// FromSOPS decrypts a SOPS-encrypted YAML file and parses e2 credentials. +// If ageKeyFile is provided, it sets SOPS_AGE_KEY_FILE for the sops command. +func FromSOPS(sopsFile, ageKeyFile string) (*E2Credentials, error) { + // Verify the SOPS file exists + if _, err := os.Stat(sopsFile); err != nil { + return nil, fmt.Errorf("SOPS file not found: %w", err) + } + + // Build sops command + args := []string{"--decrypt", sopsFile} + var env []string + if ageKeyFile != "" { + env = append(os.Environ(), fmt.Sprintf("SOPS_AGE_KEY_FILE=%s", ageKeyFile)) + } + + stdout, stderr, err := commandRunner("sops", args, env) + if err != nil { + return nil, fmt.Errorf("sops decrypt failed: %w: %s", err, string(stderr)) + } + + // Parse decrypted YAML + var creds E2Credentials + if err := yaml.Unmarshal(stdout, &creds); err != nil { + return nil, fmt.Errorf("parse decrypted credentials: %w", err) + } + + if err := creds.Validate(); err != nil { + return nil, fmt.Errorf("validate credentials: %w", err) + } + + return &creds, nil +} diff --git a/tools/labctl/internal/credentials/sops_test.go b/tools/labctl/internal/credentials/sops_test.go new file mode 100644 index 0000000..50ba5c1 --- /dev/null +++ b/tools/labctl/internal/credentials/sops_test.go @@ -0,0 +1,139 @@ +package credentials + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFromSOPS(t *testing.T) { + // Save original command runner and restore after tests + originalRunner := commandRunner + t.Cleanup(func() { + commandRunner = originalRunner + }) + + t.Run("successful decryption", func(t *testing.T) { + // Create a temporary file to satisfy the file existence check + dir := t.TempDir() + sopsFile := filepath.Join(dir, "credentials.sops.yaml") + err := os.WriteFile(sopsFile, []byte("encrypted content"), 0o600) + require.NoError(t, err) + + // Mock the command runner to return valid credentials + commandRunner = func(name string, args []string, _ []string) ([]byte, []byte, error) { + assert.Equal(t, "sops", name) + assert.Equal(t, []string{"--decrypt", sopsFile}, args) + + yamlOutput := `access_key: test-access-key +secret_key: test-secret-key +endpoint: https://e2.example.com +bucket: test-bucket +` + return []byte(yamlOutput), nil, nil + } + + creds, err := FromSOPS(sopsFile, "") + require.NoError(t, err) + assert.Equal(t, "test-access-key", creds.AccessKey) + assert.Equal(t, "test-secret-key", creds.SecretKey) + assert.Equal(t, "https://e2.example.com", creds.Endpoint) + assert.Equal(t, "test-bucket", creds.Bucket) + }) + + t.Run("successful decryption with age key file", func(t *testing.T) { + dir := t.TempDir() + sopsFile := filepath.Join(dir, "credentials.sops.yaml") + err := os.WriteFile(sopsFile, []byte("encrypted content"), 0o600) + require.NoError(t, err) + + ageKeyFile := "/path/to/age-key.txt" + + commandRunner = func(_ string, _ []string, env []string) ([]byte, []byte, error) { + // Verify age key file is in environment + found := false + for _, e := range env { + if strings.HasPrefix(e, "SOPS_AGE_KEY_FILE=") { + assert.Equal(t, "SOPS_AGE_KEY_FILE="+ageKeyFile, e) + found = true + break + } + } + assert.True(t, found, "SOPS_AGE_KEY_FILE should be set in environment") + + yamlOutput := `access_key: test-access-key +secret_key: test-secret-key +endpoint: https://e2.example.com +bucket: test-bucket +` + return []byte(yamlOutput), nil, nil + } + + creds, err := FromSOPS(sopsFile, ageKeyFile) + require.NoError(t, err) + assert.Equal(t, "test-access-key", creds.AccessKey) + }) + + t.Run("file not found", func(t *testing.T) { + _, err := FromSOPS("/nonexistent/path/credentials.sops.yaml", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "SOPS file not found") + }) + + t.Run("sops command fails", func(t *testing.T) { + dir := t.TempDir() + sopsFile := filepath.Join(dir, "credentials.sops.yaml") + err := os.WriteFile(sopsFile, []byte("encrypted content"), 0o600) + require.NoError(t, err) + + commandRunner = func(_ string, _ []string, _ []string) ([]byte, []byte, error) { + return nil, []byte("error: could not decrypt"), errors.New("exit status 1") + } + + _, err = FromSOPS(sopsFile, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "sops decrypt failed") + assert.Contains(t, err.Error(), "could not decrypt") + }) + + t.Run("invalid yaml output", func(t *testing.T) { + dir := t.TempDir() + sopsFile := filepath.Join(dir, "credentials.sops.yaml") + err := os.WriteFile(sopsFile, []byte("encrypted content"), 0o600) + require.NoError(t, err) + + commandRunner = func(_ string, _ []string, _ []string) ([]byte, []byte, error) { + return []byte("invalid: yaml: content: ["), nil, nil + } + + _, err = FromSOPS(sopsFile, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "parse decrypted credentials") + }) + + t.Run("missing required fields in credentials", func(t *testing.T) { + dir := t.TempDir() + sopsFile := filepath.Join(dir, "credentials.sops.yaml") + err := os.WriteFile(sopsFile, []byte("encrypted content"), 0o600) + require.NoError(t, err) + + commandRunner = func(_ string, _ []string, _ []string) ([]byte, []byte, error) { + // Missing access_key + yamlOutput := `secret_key: test-secret-key +endpoint: https://e2.example.com +bucket: test-bucket +` + return []byte(yamlOutput), nil, nil + } + + _, err = FromSOPS(sopsFile, "") + require.Error(t, err) + assert.Contains(t, err.Error(), "validate credentials") + assert.Contains(t, err.Error(), "access_key is required") + }) +} diff --git a/tools/labctl/internal/store/s3.go b/tools/labctl/internal/store/s3.go new file mode 100644 index 0000000..93e635f --- /dev/null +++ b/tools/labctl/internal/store/s3.go @@ -0,0 +1,280 @@ +// Package store provides storage operations for the image pipeline. +package store + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "path" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + labcreds "github.com/GilmanLab/lab/tools/labctl/internal/credentials" +) + +// ImageMetadata represents metadata stored alongside each image. +type ImageMetadata struct { + Name string `json:"name"` + Checksum string `json:"checksum"` + Size int64 `json:"size"` + UploadedAt time.Time `json:"uploadedAt"` + Source SourceMetadata `json:"source"` +} + +// SourceMetadata describes the origin of an image. +type SourceMetadata struct { + // Type is "http" for downloaded images or "local" for uploaded files. + Type string `json:"type,omitempty"` + // URL is set for HTTP sources. + URL string `json:"url,omitempty"` + // Path is set for local file uploads. + Path string `json:"path,omitempty"` +} + +// s3API defines the S3 operations used by S3Client. +// This interface enables mocking for unit tests. +type s3API interface { + PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) +} + +// S3Client wraps the AWS S3 client for image storage operations. +type S3Client struct { + api s3API + bucket string +} + +// S3Option configures the S3 client. +type S3Option func(*s3ClientConfig) + +type s3ClientConfig struct { + ctx context.Context +} + +// WithContext sets the context for S3 client initialization. +func WithContext(ctx context.Context) S3Option { + return func(c *s3ClientConfig) { + c.ctx = ctx + } +} + +// NewS3Client creates a new S3 client from e2 credentials. +func NewS3Client(creds *labcreds.E2Credentials, opts ...S3Option) (*S3Client, error) { + cfg := &s3ClientConfig{ + ctx: context.Background(), + } + for _, opt := range opts { + opt(cfg) + } + + // Load AWS config with custom credentials + awsCfg, err := config.LoadDefaultConfig(cfg.ctx, + config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(creds.AccessKey, creds.SecretKey, ""), + ), + config.WithRegion("us-east-1"), + ) + if err != nil { + return nil, fmt.Errorf("load AWS config: %w", err) + } + + // Create S3 client with custom endpoint for e2 + client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(creds.Endpoint) + o.UsePathStyle = true // Required for S3-compatible services + }) + + return &S3Client{ + api: client, + bucket: creds.Bucket, + }, nil +} + +// newS3ClientWithAPI creates an S3Client with a custom API implementation (for testing). +func newS3ClientWithAPI(api s3API, bucket string) *S3Client { + return &S3Client{ + api: api, + bucket: bucket, + } +} + +// Upload uploads a file to the S3 bucket. +func (c *S3Client) Upload(ctx context.Context, key string, body io.Reader, size int64) error { + _, err := c.api.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + Body: body, + ContentLength: aws.Int64(size), + }) + if err != nil { + return fmt.Errorf("upload to s3://%s/%s: %w", c.bucket, key, err) + } + return nil +} + +// Download downloads a file from the S3 bucket. +func (c *S3Client) Download(ctx context.Context, key string) (io.ReadCloser, error) { + output, err := c.api.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, fmt.Errorf("download from s3://%s/%s: %w", c.bucket, key, err) + } + return output.Body, nil +} + +// Exists checks if an object exists in the S3 bucket. +func (c *S3Client) Exists(ctx context.Context, key string) (bool, error) { + _, err := c.api.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err != nil { + // Check if it's a "not found" error + // The AWS SDK v2 doesn't have a typed NotFound error, so we check the error message + if isNotFoundError(err) { + return false, nil + } + return false, fmt.Errorf("check existence of s3://%s/%s: %w", c.bucket, key, err) + } + return true, nil +} + +// isNotFoundError checks if the error is an S3 not found error. +func isNotFoundError(err error) bool { + // AWS SDK v2 returns errors that can be checked via their error code + // For S3 HeadObject, a missing object returns a 404 status + return err != nil && ( + // Check for common not found patterns + contains(err.Error(), "NotFound") || + contains(err.Error(), "404") || + contains(err.Error(), "NoSuchKey")) +} + +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || s != "" && containsImpl(s, substr)) +} + +func containsImpl(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} + +// List lists all objects in the bucket with the given prefix. +func (c *S3Client) List(ctx context.Context, prefix string) ([]string, error) { + var keys []string + var continuationToken *string + + for { + output, err := c.api.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + Bucket: aws.String(c.bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, fmt.Errorf("list objects in s3://%s/%s: %w", c.bucket, prefix, err) + } + + for _, obj := range output.Contents { + keys = append(keys, aws.ToString(obj.Key)) + } + + if !aws.ToBool(output.IsTruncated) { + break + } + continuationToken = output.NextContinuationToken + } + + return keys, nil +} + +// Delete deletes an object from the S3 bucket. +func (c *S3Client) Delete(ctx context.Context, key string) error { + _, err := c.api.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(c.bucket), + Key: aws.String(key), + }) + if err != nil { + return fmt.Errorf("delete s3://%s/%s: %w", c.bucket, key, err) + } + return nil +} + +// MetadataKey returns the metadata key for a given image path. +// Example: "vyos/vyos-1.5.iso" -> "metadata/vyos/vyos-1.5.iso.json" +func MetadataKey(imagePath string) string { + return path.Join("metadata", imagePath+".json") +} + +// ImageKey returns the image key for a given destination path. +// Example: "vyos/vyos-1.5.iso" -> "images/vyos/vyos-1.5.iso" +func ImageKey(destination string) string { + return path.Join("images", destination) +} + +// GetMetadata retrieves metadata for an image. +func (c *S3Client) GetMetadata(ctx context.Context, imagePath string) (*ImageMetadata, error) { + key := MetadataKey(imagePath) + + body, err := c.Download(ctx, key) + if err != nil { + return nil, err + } + defer func() { _ = body.Close() }() + + data, err := io.ReadAll(body) + if err != nil { + return nil, fmt.Errorf("read metadata: %w", err) + } + + var metadata ImageMetadata + if err := json.Unmarshal(data, &metadata); err != nil { + return nil, fmt.Errorf("parse metadata: %w", err) + } + + return &metadata, nil +} + +// PutMetadata stores metadata for an image. +func (c *S3Client) PutMetadata(ctx context.Context, imagePath string, metadata *ImageMetadata) error { + key := MetadataKey(imagePath) + + data, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + + return c.Upload(ctx, key, bytes.NewReader(data), int64(len(data))) +} + +// ChecksumMatches checks if the stored metadata checksum matches the expected checksum. +func (c *S3Client) ChecksumMatches(ctx context.Context, imagePath, expectedChecksum string) (bool, error) { + exists, err := c.Exists(ctx, MetadataKey(imagePath)) + if err != nil { + return false, err + } + if !exists { + return false, nil + } + + metadata, err := c.GetMetadata(ctx, imagePath) + if err != nil { + return false, err + } + + return metadata.Checksum == expectedChecksum, nil +} diff --git a/tools/labctl/internal/store/s3_test.go b/tools/labctl/internal/store/s3_test.go new file mode 100644 index 0000000..23070bf --- /dev/null +++ b/tools/labctl/internal/store/s3_test.go @@ -0,0 +1,586 @@ +package store + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockS3API is a mock implementation of s3API for testing. +type mockS3API struct { + putObjectFunc func(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) + getObjectFunc func(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) + headObjectFunc func(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + deleteObjectFunc func(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) + listObjectsV2Func func(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) +} + +func (m *mockS3API) PutObject(ctx context.Context, params *s3.PutObjectInput, optFns ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + if m.putObjectFunc != nil { + return m.putObjectFunc(ctx, params, optFns...) + } + return &s3.PutObjectOutput{}, nil +} + +func (m *mockS3API) GetObject(ctx context.Context, params *s3.GetObjectInput, optFns ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + if m.getObjectFunc != nil { + return m.getObjectFunc(ctx, params, optFns...) + } + return &s3.GetObjectOutput{}, nil +} + +func (m *mockS3API) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + if m.headObjectFunc != nil { + return m.headObjectFunc(ctx, params, optFns...) + } + return &s3.HeadObjectOutput{}, nil +} + +func (m *mockS3API) DeleteObject(ctx context.Context, params *s3.DeleteObjectInput, optFns ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + if m.deleteObjectFunc != nil { + return m.deleteObjectFunc(ctx, params, optFns...) + } + return &s3.DeleteObjectOutput{}, nil +} + +func (m *mockS3API) ListObjectsV2(ctx context.Context, params *s3.ListObjectsV2Input, optFns ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + if m.listObjectsV2Func != nil { + return m.listObjectsV2Func(ctx, params, optFns...) + } + return &s3.ListObjectsV2Output{}, nil +} + +// nopCloser wraps an io.Reader to implement io.ReadCloser. +type nopCloser struct { + io.Reader +} + +func (nopCloser) Close() error { return nil } + +func TestS3Client_Upload(t *testing.T) { + t.Run("successful upload", func(t *testing.T) { + mock := &mockS3API{ + putObjectFunc: func(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + assert.Equal(t, "test-bucket", aws.ToString(params.Bucket)) + assert.Equal(t, "images/test.iso", aws.ToString(params.Key)) + assert.Equal(t, int64(100), aws.ToInt64(params.ContentLength)) + return &s3.PutObjectOutput{}, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.Upload(context.Background(), "images/test.iso", bytes.NewReader(make([]byte, 100)), 100) + require.NoError(t, err) + }) + + t.Run("upload error", func(t *testing.T) { + mock := &mockS3API{ + putObjectFunc: func(_ context.Context, _ *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return nil, errors.New("network error") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.Upload(context.Background(), "images/test.iso", bytes.NewReader(nil), 0) + require.Error(t, err) + assert.Contains(t, err.Error(), "upload to s3://test-bucket/images/test.iso") + assert.Contains(t, err.Error(), "network error") + }) +} + +func TestS3Client_Download(t *testing.T) { + t.Run("successful download", func(t *testing.T) { + expectedData := []byte("file contents") + mock := &mockS3API{ + getObjectFunc: func(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + assert.Equal(t, "test-bucket", aws.ToString(params.Bucket)) + assert.Equal(t, "images/test.iso", aws.ToString(params.Key)) + return &s3.GetObjectOutput{ + Body: nopCloser{bytes.NewReader(expectedData)}, + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + body, err := client.Download(context.Background(), "images/test.iso") + require.NoError(t, err) + defer func() { _ = body.Close() }() + + data, err := io.ReadAll(body) + require.NoError(t, err) + assert.Equal(t, expectedData, data) + }) + + t.Run("download error", func(t *testing.T) { + mock := &mockS3API{ + getObjectFunc: func(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return nil, errors.New("not found") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + _, err := client.Download(context.Background(), "images/missing.iso") + require.Error(t, err) + assert.Contains(t, err.Error(), "download from s3://test-bucket/images/missing.iso") + }) +} + +func TestS3Client_Exists(t *testing.T) { + t.Run("object exists", func(t *testing.T) { + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, params *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + assert.Equal(t, "test-bucket", aws.ToString(params.Bucket)) + assert.Equal(t, "images/test.iso", aws.ToString(params.Key)) + return &s3.HeadObjectOutput{}, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + exists, err := client.Exists(context.Background(), "images/test.iso") + require.NoError(t, err) + assert.True(t, exists) + }) + + t.Run("object not found", func(t *testing.T) { + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, _ *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return nil, errors.New("NotFound: object does not exist") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + exists, err := client.Exists(context.Background(), "images/missing.iso") + require.NoError(t, err) + assert.False(t, exists) + }) + + t.Run("other error", func(t *testing.T) { + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, _ *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return nil, errors.New("permission denied") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + _, err := client.Exists(context.Background(), "images/test.iso") + require.Error(t, err) + assert.Contains(t, err.Error(), "check existence of s3://test-bucket/images/test.iso") + }) +} + +func TestS3Client_List(t *testing.T) { + t.Run("list objects", func(t *testing.T) { + mock := &mockS3API{ + listObjectsV2Func: func(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + assert.Equal(t, "test-bucket", aws.ToString(params.Bucket)) + assert.Equal(t, "images/", aws.ToString(params.Prefix)) + return &s3.ListObjectsV2Output{ + Contents: []types.Object{ + {Key: aws.String("images/a.iso")}, + {Key: aws.String("images/b.iso")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + keys, err := client.List(context.Background(), "images/") + require.NoError(t, err) + assert.Equal(t, []string{"images/a.iso", "images/b.iso"}, keys) + }) + + t.Run("list with pagination", func(t *testing.T) { + callCount := 0 + mock := &mockS3API{ + listObjectsV2Func: func(_ context.Context, params *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + callCount++ + if callCount == 1 { + return &s3.ListObjectsV2Output{ + Contents: []types.Object{ + {Key: aws.String("images/a.iso")}, + }, + IsTruncated: aws.Bool(true), + NextContinuationToken: aws.String("token1"), + }, nil + } + assert.Equal(t, "token1", aws.ToString(params.ContinuationToken)) + return &s3.ListObjectsV2Output{ + Contents: []types.Object{ + {Key: aws.String("images/b.iso")}, + }, + IsTruncated: aws.Bool(false), + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + keys, err := client.List(context.Background(), "images/") + require.NoError(t, err) + assert.Equal(t, []string{"images/a.iso", "images/b.iso"}, keys) + assert.Equal(t, 2, callCount) + }) + + t.Run("list error", func(t *testing.T) { + mock := &mockS3API{ + listObjectsV2Func: func(_ context.Context, _ *s3.ListObjectsV2Input, _ ...func(*s3.Options)) (*s3.ListObjectsV2Output, error) { + return nil, errors.New("access denied") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + _, err := client.List(context.Background(), "images/") + require.Error(t, err) + assert.Contains(t, err.Error(), "list objects in s3://test-bucket/images/") + }) +} + +func TestS3Client_Delete(t *testing.T) { + t.Run("successful delete", func(t *testing.T) { + mock := &mockS3API{ + deleteObjectFunc: func(_ context.Context, params *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + assert.Equal(t, "test-bucket", aws.ToString(params.Bucket)) + assert.Equal(t, "images/test.iso", aws.ToString(params.Key)) + return &s3.DeleteObjectOutput{}, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.Delete(context.Background(), "images/test.iso") + require.NoError(t, err) + }) + + t.Run("delete error", func(t *testing.T) { + mock := &mockS3API{ + deleteObjectFunc: func(_ context.Context, _ *s3.DeleteObjectInput, _ ...func(*s3.Options)) (*s3.DeleteObjectOutput, error) { + return nil, errors.New("access denied") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.Delete(context.Background(), "images/test.iso") + require.Error(t, err) + assert.Contains(t, err.Error(), "delete s3://test-bucket/images/test.iso") + }) +} + +func TestS3Client_GetMetadata(t *testing.T) { + t.Run("successful get metadata", func(t *testing.T) { + metadata := ImageMetadata{ + Name: "test-image", + Checksum: "sha256:abc123", + Size: 1024, + UploadedAt: time.Date(2024, 12, 20, 10, 0, 0, 0, time.UTC), + Source: SourceMetadata{Type: "http", URL: "https://example.com/image.iso"}, + } + metadataJSON, _ := json.Marshal(metadata) + + mock := &mockS3API{ + getObjectFunc: func(_ context.Context, params *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + assert.Equal(t, "metadata/images/test.iso.json", aws.ToString(params.Key)) + return &s3.GetObjectOutput{ + Body: nopCloser{bytes.NewReader(metadataJSON)}, + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + result, err := client.GetMetadata(context.Background(), "images/test.iso") + require.NoError(t, err) + assert.Equal(t, "test-image", result.Name) + assert.Equal(t, "sha256:abc123", result.Checksum) + assert.Equal(t, int64(1024), result.Size) + }) + + t.Run("metadata not found", func(t *testing.T) { + mock := &mockS3API{ + getObjectFunc: func(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return nil, errors.New("NotFound") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + _, err := client.GetMetadata(context.Background(), "images/missing.iso") + require.Error(t, err) + }) + + t.Run("invalid json", func(t *testing.T) { + mock := &mockS3API{ + getObjectFunc: func(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return &s3.GetObjectOutput{ + Body: nopCloser{bytes.NewReader([]byte("invalid json"))}, + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + _, err := client.GetMetadata(context.Background(), "images/test.iso") + require.Error(t, err) + assert.Contains(t, err.Error(), "parse metadata") + }) +} + +func TestS3Client_PutMetadata(t *testing.T) { + t.Run("successful put metadata", func(t *testing.T) { + metadata := &ImageMetadata{ + Name: "test-image", + Checksum: "sha256:abc123", + Size: 1024, + UploadedAt: time.Date(2024, 12, 20, 10, 0, 0, 0, time.UTC), + Source: SourceMetadata{Type: "http", URL: "https://example.com/image.iso"}, + } + + var uploadedData []byte + mock := &mockS3API{ + putObjectFunc: func(_ context.Context, params *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + assert.Equal(t, "metadata/images/test.iso.json", aws.ToString(params.Key)) + uploadedData, _ = io.ReadAll(params.Body) + return &s3.PutObjectOutput{}, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.PutMetadata(context.Background(), "images/test.iso", metadata) + require.NoError(t, err) + + // Verify the uploaded JSON + var decoded ImageMetadata + err = json.Unmarshal(uploadedData, &decoded) + require.NoError(t, err) + assert.Equal(t, metadata.Name, decoded.Name) + assert.Equal(t, metadata.Checksum, decoded.Checksum) + }) + + t.Run("put metadata error", func(t *testing.T) { + mock := &mockS3API{ + putObjectFunc: func(_ context.Context, _ *s3.PutObjectInput, _ ...func(*s3.Options)) (*s3.PutObjectOutput, error) { + return nil, errors.New("access denied") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + err := client.PutMetadata(context.Background(), "images/test.iso", &ImageMetadata{}) + require.Error(t, err) + }) +} + +func TestS3Client_ChecksumMatches(t *testing.T) { + t.Run("checksum matches", func(t *testing.T) { + metadata := ImageMetadata{Checksum: "sha256:abc123"} + metadataJSON, _ := json.Marshal(metadata) + + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, _ *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return &s3.HeadObjectOutput{}, nil + }, + getObjectFunc: func(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return &s3.GetObjectOutput{ + Body: nopCloser{bytes.NewReader(metadataJSON)}, + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + matches, err := client.ChecksumMatches(context.Background(), "images/test.iso", "sha256:abc123") + require.NoError(t, err) + assert.True(t, matches) + }) + + t.Run("checksum does not match", func(t *testing.T) { + metadata := ImageMetadata{Checksum: "sha256:abc123"} + metadataJSON, _ := json.Marshal(metadata) + + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, _ *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return &s3.HeadObjectOutput{}, nil + }, + getObjectFunc: func(_ context.Context, _ *s3.GetObjectInput, _ ...func(*s3.Options)) (*s3.GetObjectOutput, error) { + return &s3.GetObjectOutput{ + Body: nopCloser{bytes.NewReader(metadataJSON)}, + }, nil + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + matches, err := client.ChecksumMatches(context.Background(), "images/test.iso", "sha256:different") + require.NoError(t, err) + assert.False(t, matches) + }) + + t.Run("metadata does not exist", func(t *testing.T) { + mock := &mockS3API{ + headObjectFunc: func(_ context.Context, _ *s3.HeadObjectInput, _ ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return nil, errors.New("NotFound") + }, + } + + client := newS3ClientWithAPI(mock, "test-bucket") + matches, err := client.ChecksumMatches(context.Background(), "images/test.iso", "sha256:abc123") + require.NoError(t, err) + assert.False(t, matches) + }) +} + +func TestMetadataKey(t *testing.T) { + tests := []struct { + name string + imagePath string + want string + }{ + { + name: "simple path", + imagePath: "image.iso", + want: "metadata/image.iso.json", + }, + { + name: "nested path", + imagePath: "vyos/vyos-1.5.iso", + want: "metadata/vyos/vyos-1.5.iso.json", + }, + { + name: "deeply nested path", + imagePath: "talos/v1.9.1/metal-amd64.raw", + want: "metadata/talos/v1.9.1/metal-amd64.raw.json", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, MetadataKey(tt.imagePath)) + }) + } +} + +func TestImageKey(t *testing.T) { + tests := []struct { + name string + destination string + want string + }{ + { + name: "simple path", + destination: "image.iso", + want: "images/image.iso", + }, + { + name: "nested path", + destination: "vyos/vyos-1.5.iso", + want: "images/vyos/vyos-1.5.iso", + }, + { + name: "deeply nested path", + destination: "talos/v1.9.1/metal-amd64.raw", + want: "images/talos/v1.9.1/metal-amd64.raw", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, ImageKey(tt.destination)) + }) + } +} + +func TestImageMetadata_JSON(t *testing.T) { + t.Run("marshal and unmarshal HTTP source", func(t *testing.T) { + metadata := ImageMetadata{ + Name: "talos-1.9.1", + Checksum: "sha256:abc123", + Size: 1234567890, + UploadedAt: time.Date(2024, 12, 20, 10, 0, 0, 0, time.UTC), + Source: SourceMetadata{ + Type: "http", + URL: "https://factory.talos.dev/image.raw", + }, + } + + data, err := json.Marshal(metadata) + require.NoError(t, err) + + var decoded ImageMetadata + err = json.Unmarshal(data, &decoded) + require.NoError(t, err) + + assert.Equal(t, metadata.Name, decoded.Name) + assert.Equal(t, metadata.Checksum, decoded.Checksum) + assert.Equal(t, metadata.Size, decoded.Size) + assert.Equal(t, metadata.Source.Type, decoded.Source.Type) + assert.Equal(t, metadata.Source.URL, decoded.Source.URL) + }) + + t.Run("marshal and unmarshal local source", func(t *testing.T) { + metadata := ImageMetadata{ + Name: "vyos-gateway", + Checksum: "sha256:def456", + Size: 8589934592, + UploadedAt: time.Date(2024, 12, 20, 12, 0, 0, 0, time.UTC), + Source: SourceMetadata{ + Type: "local", + Path: "infrastructure/network/vyos/packer/output/vyos-lab.raw", + }, + } + + data, err := json.Marshal(metadata) + require.NoError(t, err) + + var decoded ImageMetadata + err = json.Unmarshal(data, &decoded) + require.NoError(t, err) + + assert.Equal(t, metadata.Name, decoded.Name) + assert.Equal(t, metadata.Checksum, decoded.Checksum) + assert.Equal(t, metadata.Size, decoded.Size) + assert.Equal(t, metadata.Source.Type, decoded.Source.Type) + assert.Equal(t, metadata.Source.Path, decoded.Source.Path) + }) +} + +func TestIsNotFoundError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil error", + err: nil, + want: false, + }, + { + name: "NotFound error", + err: errors.New("NotFound: key does not exist"), + want: true, + }, + { + name: "404 error", + err: errors.New("operation failed with status 404"), + want: true, + }, + { + name: "NoSuchKey error", + err: errors.New("NoSuchKey: The specified key does not exist"), + want: true, + }, + { + name: "other error", + err: errors.New("connection refused"), + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isNotFoundError(tt.err)) + }) + } +} diff --git a/tools/labctl/justfile b/tools/labctl/justfile new file mode 100644 index 0000000..140dc7c --- /dev/null +++ b/tools/labctl/justfile @@ -0,0 +1,17 @@ +# Default recipe +default: all + +# Run formatting checks and auto-fix +fmt: + go fmt ./... + +# Run linters +lint: + golangci-lint run ./... + +# Run tests +test: + go test -v ./... + +# Run formatting, linting, and testing +all: fmt lint test diff --git a/tools/labctl/main.go b/tools/labctl/main.go new file mode 100644 index 0000000..c4a2a2d --- /dev/null +++ b/tools/labctl/main.go @@ -0,0 +1,14 @@ +// Package main provides the entry point for the labctl CLI tool. +package main + +import ( + "os" + + "github.com/GilmanLab/lab/tools/labctl/cmd" +) + +func main() { + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +}