Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 80 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ jobs:
needs: [agent, cli, containers]
runs-on: blacksmith-2vcpu-ubuntu-2404
steps:
- name: Checkout
uses: actions/checkout@v6

- name: Download agent artifacts
uses: actions/download-artifact@v8
with:
Expand All @@ -132,11 +135,71 @@ jobs:
pattern: tc-*
merge-multiple: true

- name: Download image digests
uses: actions/download-artifact@v8
with:
path: image-digests
pattern: release-digest-*
merge-multiple: true

- name: Generate SHA256 checksums
run: |
cd binaries
sha256sum agent-* tc-* tc_* > checksums.txt

- name: Generate release manifest
env:
RELEASE_VERSION: ${{ github.ref_name }}
RELEASE_COMMIT: ${{ github.sha }}
run: |
sha256_pattern='^[0-9a-f]{64}$'
digest_pattern='^sha256:[0-9a-f]{64}$'

agent_amd64=$(sha256sum binaries/agent-linux-amd64 | awk '{print $1}')
agent_arm64=$(sha256sum binaries/agent-linux-arm64 | awk '{print $1}')
compose_production=$(sha256sum deployment/compose.production.yml | awk '{print $1}')
compose_postgres=$(sha256sum deployment/compose.postgres.yml | awk '{print $1}')
web_digest=$(cat image-digests/web)
registry_digest=$(cat image-digests/registry)
updater_digest=$(cat image-digests/updater)

for checksum in "$agent_amd64" "$agent_arm64" "$compose_production" "$compose_postgres"; do
[[ "$checksum" =~ $sha256_pattern ]]
done
for digest in "$web_digest" "$registry_digest" "$updater_digest"; do
[[ "$digest" =~ $digest_pattern ]]
done
[[ "$RELEASE_VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]]
[[ "$RELEASE_COMMIT" =~ ^[0-9a-f]{40}$ ]]

jq -n \
--arg version "$RELEASE_VERSION" \
--arg commit "$RELEASE_COMMIT" \
--arg agent_amd64 "$agent_amd64" \
--arg agent_arm64 "$agent_arm64" \
--arg compose_production "$compose_production" \
--arg compose_postgres "$compose_postgres" \
--arg web_digest "$web_digest" \
--arg registry_digest "$registry_digest" \
--arg updater_digest "$updater_digest" \
'{
version: $version,
commit: $commit,
binaries: {
"agent-linux-amd64": $agent_amd64,
"agent-linux-arm64": $agent_arm64
},
composeFiles: {
"deployment/compose.production.yml": $compose_production,
"deployment/compose.postgres.yml": $compose_postgres
},
images: {
web: $web_digest,
registry: $registry_digest,
updater: $updater_digest
}
}' > binaries/release-manifest.json

- name: Create GitHub release
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand All @@ -145,7 +208,7 @@ jobs:
--repo ${{ github.repository }} \
--title "${{ github.ref_name }}" \
--generate-notes \
binaries/agent-* binaries/tc-* binaries/tc_* binaries/checksums.txt
binaries/agent-* binaries/tc-* binaries/tc_* binaries/checksums.txt binaries/release-manifest.json

homebrew:
needs: release
Expand Down Expand Up @@ -359,3 +422,19 @@ jobs:
--tag "$image:${{ github.ref_name }}" \
--tag "$image:tip" \
$(printf "$image@sha256:%s " *)

- name: Export multi-platform manifest digest
run: |
image="ghcr.io/${{ github.repository }}/${{ matrix.image }}:${{ github.ref_name }}"
mkdir -p /tmp/release-digests
docker buildx imagetools inspect "$image" --format '{{json .Manifest}}' \
| jq -er '.digest | select(test("^sha256:[0-9a-f]{64}$"))' \
> "/tmp/release-digests/${{ matrix.image }}"

- name: Upload multi-platform manifest digest
uses: actions/upload-artifact@v7
with:
name: release-digest-${{ matrix.image }}
path: /tmp/release-digests/${{ matrix.image }}
if-no-files-found: error
retention-days: 1
11 changes: 8 additions & 3 deletions cli/internal/cli/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,7 @@ func (a *App) linkCommand() *cobra.Command {
HealthCheck *manifest.HealthCheck `json:"healthCheck"`
StartCommand *string `json:"startCommand"`
Resources *manifest.Resources `json:"resources"`
Crons []manifest.Cron `json:"crons"`
} `json:"current"`
Management *serviceManagement `json:"management"`
}
Expand Down Expand Up @@ -467,7 +468,7 @@ func (a *App) linkCommand() *cobra.Command {
}
}
}
m := manifest.Manifest{APIVersion: "v1", Target: &manifest.Target{ServiceID: service.ID}, Service: manifest.Service{Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, Placement: placement, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources}}
m := manifest.Manifest{APIVersion: "v1", Target: &manifest.Target{ServiceID: service.ID}, Service: manifest.Service{Name: service.Name, Source: service.Source, Hostname: cfg.Current.Hostname, Ports: ports, Replicas: cfg.Current.Replicas, Placement: placement, HealthCheck: cfg.Current.HealthCheck, StartCommand: cfg.Current.StartCommand, Resources: cfg.Current.Resources, Crons: cfg.Current.Crons}}
if existing != nil {
if existing.Manifest.Linked() && existing.Manifest.Target.ServiceID != service.ID {
return fmt.Errorf("manifest is linked to service %s; remove target.serviceId before relinking", existing.Manifest.Target.ServiceID)
Expand Down Expand Up @@ -516,7 +517,11 @@ func (a *App) applyCommand() *cobra.Command {
if placement == nil {
return errors.New("service.placement is required")
}
body := map[string]any{"name": loaded.Manifest.Service.Name, "source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand, "resources": loaded.Manifest.Service.Resources}
crons := loaded.Manifest.Service.Crons
if crons == nil {
crons = []manifest.Cron{}
}
body := map[string]any{"name": loaded.Manifest.Service.Name, "source": sourcePatch(loaded.Manifest.Service.Source), "hostname": loaded.Manifest.Service.Hostname, "ports": loaded.Manifest.Service.Ports, "healthCheck": loaded.Manifest.Service.HealthCheck, "startCommand": loaded.Manifest.Service.StartCommand, "resources": loaded.Manifest.Service.Resources, "crons": crons}
if placement.Mode == "automatic" {
body["placement"] = map[string]any{"mode": "automatic", "replicas": loaded.Manifest.Service.Replicas}
} else {
Expand Down Expand Up @@ -636,7 +641,7 @@ func (a *App) finishLink(path string, existing *manifest.Loaded, service service
}
return a.printLinked(path, target)
}
m := manifest.Manifest{APIVersion: "v1", Target: &manifest.Target{ServiceID: serviceID}, Service: manifest.Service{Name: service.Name, Source: service.Source, Hostname: service.Hostname, Ports: service.Ports, Replicas: service.Replicas, Placement: service.Placement, HealthCheck: service.HealthCheck, StartCommand: service.StartCommand, Resources: service.Resources}}
m := manifest.Manifest{APIVersion: "v1", Target: &manifest.Target{ServiceID: serviceID}, Service: manifest.Service{Name: service.Name, Source: service.Source, Hostname: service.Hostname, Ports: service.Ports, Replicas: service.Replicas, Placement: service.Placement, HealthCheck: service.HealthCheck, StartCommand: service.StartCommand, Resources: service.Resources, Crons: service.Crons}}
if err := manifest.Save(path, m); err != nil {
return err
}
Expand Down
34 changes: 33 additions & 1 deletion cli/internal/cli/app_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -335,9 +335,12 @@ func TestApplyExactNestedPatchForSources(t *testing.T) {
}
source := body["source"].(map[string]any)
placement := body["placement"].(map[string]any)
if source["type"] != tc.sourceType || placement["mode"] != "automatic" || placement["replicas"] != float64(2) || len(body) != 8 {
if source["type"] != tc.sourceType || placement["mode"] != "automatic" || placement["replicas"] != float64(2) || len(body) != 9 {
t.Fatalf("body=%#v", body)
}
if crons, ok := body["crons"].([]any); !ok || len(crons) != 0 {
t.Fatalf("omitted crons must be sent as an empty replacement: %#v", body["crons"])
}
if tc.name == "github_clear_root" {
rootDir, present := source["rootDir"]
if !present || rootDir != nil {
Expand All @@ -348,6 +351,35 @@ func TestApplyExactNestedPatchForSources(t *testing.T) {
}
}

func TestLinkAndApplyRoundTripCrons(t *testing.T) {
d := t.TempDir()
var applied map[string]any
s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
w.Write([]byte(`{"target":{"project":{"slug":"app"},"environment":{"name":"prod"},"service":{"id":"s","name":"web"}},"service":{"id":"s","name":"web","source":{"type":"image","image":"nginx"},"ports":[],"replicas":1,"placement":{"mode":"automatic"},"hostname":"web","healthCheck":null,"startCommand":null,"resources":null,"crons":[{"path":"/jobs/nightly","schedule":"0 5 * * *"}]},"management":{"patchable":true,"blockers":[]}}`))
return
}
json.NewDecoder(r.Body).Decode(&applied)
w.Write([]byte(`{"action":"noop","currentVersion":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","desiredVersion":"sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","changes":[]}`))
}))
defer s.Close()
writeConfig(t, s.URL)
app, _ := testApp(t, d, s.Client())
if err := execute(app, "link", "--service", "s"); err != nil {
t.Fatal(err)
}
loaded, err := manifest.Load(d)
if err != nil || !reflect.DeepEqual(loaded.Manifest.Service.Crons, []manifest.Cron{{Path: "/jobs/nightly", Schedule: "0 5 * * *"}}) {
t.Fatalf("crons=%#v err=%v", loaded.Manifest.Service.Crons, err)
}
if err := execute(app, "apply", "--yes"); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(applied["crons"], []any{map[string]any{"path": "/jobs/nightly", "schedule": "0 5 * * *"}}) {
t.Fatalf("applied crons=%#v", applied["crons"])
}
}

func TestApplyPlacementPayloads(t *testing.T) {
for _, tc := range []struct {
name, yaml string
Expand Down
1 change: 1 addition & 0 deletions cli/internal/cli/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ type serviceItem struct {
HealthCheck *manifest.HealthCheck `json:"healthCheck"`
StartCommand *string `json:"startCommand"`
Resources *manifest.Resources `json:"resources"`
Crons []manifest.Cron `json:"crons"`
}
type targetProject struct {
ID string `json:"id"`
Expand Down
46 changes: 46 additions & 0 deletions cli/internal/manifest/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ type Service struct {
HealthCheck *HealthCheck `json:"healthCheck" yaml:"healthCheck"`
StartCommand *string `json:"startCommand" yaml:"startCommand"`
Resources *Resources `json:"resources,omitempty" yaml:"resources,omitempty"`
Crons []Cron `json:"crons,omitempty" yaml:"crons,omitempty"`
}
type Cron struct {
Path string `json:"path" yaml:"path"`
Schedule string `json:"schedule" yaml:"schedule"`
}
type Placement struct {
Mode string `json:"mode" yaml:"mode"`
Expand Down Expand Up @@ -141,6 +146,10 @@ func ApplyDefaults(m *Manifest) {
if m.Service.Ports == nil {
m.Service.Ports = []Port{}
}
for i := range m.Service.Crons {
m.Service.Crons[i].Path = strings.TrimSpace(m.Service.Crons[i].Path)
m.Service.Crons[i].Schedule = strings.TrimSpace(m.Service.Crons[i].Schedule)
}
if m.Service.Replicas == 0 {
m.Service.Replicas = 1
}
Expand Down Expand Up @@ -307,6 +316,43 @@ func Validate(m Manifest) error {
return errors.New("service.resources.memoryMb must be between 64 and 65536")
}
}
seenCronPaths := make(map[string]struct{}, len(m.Service.Crons))
for i, cron := range m.Service.Crons {
if err := validateCronPath(cron.Path); err != nil {
return fmt.Errorf("service.crons[%d].path: %w", i, err)
}
if _, exists := seenCronPaths[cron.Path]; exists {
return fmt.Errorf("service.crons[%d].path must be unique", i)
}
seenCronPaths[cron.Path] = struct{}{}
if len(cron.Schedule) > 255 || len(strings.Fields(cron.Schedule)) != 5 {
return fmt.Errorf("service.crons[%d].schedule must be a five-field cron expression", i)
}
}
return nil
}

func validateCronPath(value string) error {
if value == "" || len(value) > 2048 || !strings.HasPrefix(value, "/") || strings.HasPrefix(value, "//") {
return errors.New("must begin with exactly one slash and be at most 2048 characters")
}
decoded, err := url.PathUnescape(value)
if err != nil {
return errors.New("must contain valid URL escapes")
}
if !strings.HasPrefix(decoded, "/") || strings.HasPrefix(decoded, "//") || strings.ContainsAny(decoded, "?#\\") {
return errors.New("cannot contain a host override, query, fragment, or backslash")
}
for _, r := range decoded {
if r < 0x20 || r == 0x7f {
return errors.New("cannot contain control characters")
}
}
for _, segment := range strings.Split(decoded, "/") {
if segment == "." || segment == ".." {
return errors.New("cannot contain '.' or '..' segments")
}
}
return nil
}
func (m Manifest) Linked() bool {
Expand Down
23 changes: 22 additions & 1 deletion cli/internal/manifest/manifest_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,37 @@ func base() Manifest {
func TestDefaultsAndRoundTrip(t *testing.T) {
m := base()
m.Service.Replicas = 0
m.Service.Crons = []Cron{{Path: " /api/cron/digest ", Schedule: " 0 5 * * * "}}
b, e := Marshal(m)
if e != nil {
t.Fatal(e)
}
got, e := Parse(b)
if e != nil || got.Service.Replicas != 1 || got.Service.Ports == nil {
if e != nil || got.Service.Replicas != 1 || got.Service.Ports == nil || got.Service.Crons[0] != (Cron{Path: "/api/cron/digest", Schedule: "0 5 * * *"}) {
t.Fatalf("got=%#v err=%v", got, e)
}
}

func TestCronValidation(t *testing.T) {
invalidPaths := []string{"//host/job", "/job?x=1", "/job#part", `/job\\next`, "/a/../b", "/a/%2e%2e/b", "/%2fhost", "/bad%zz", "/line%0Abreak"}
for _, path := range invalidPaths {
m := base()
m.Service.Crons = []Cron{{Path: path, Schedule: "0 5 * * *"}}
if err := Validate(m); err == nil {
t.Fatalf("invalid cron path %q accepted", path)
}
}
m := base()
m.Service.Crons = []Cron{{Path: "/jobs/nightly", Schedule: "0 5 * * *"}, {Path: "/jobs/nightly", Schedule: "0 6 * * *"}}
if err := Validate(m); err == nil || !strings.Contains(err.Error(), "unique") {
t.Fatalf("duplicate path error = %v", err)
}
m.Service.Crons = []Cron{{Path: "/jobs/nightly", Schedule: "0 5 * *"}}
if err := Validate(m); err == nil || !strings.Contains(err.Error(), "five-field") {
t.Fatalf("schedule error = %v", err)
}
}

func TestHostnameValidationAndSlugify(t *testing.T) {
for _, value := range []string{"", "Upper", "two words", "-leading", "trailing-", "two--hyphens", strings.Repeat("a", 64)} {
m := base()
Expand Down
9 changes: 8 additions & 1 deletion deployment/.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,14 @@ INNGEST_EVENT_KEY=xxx
# Control plane deployment
# Use compose.production.yml for external PostgreSQL.
COMPOSE_FILE=compose.production.yml
TECHULUS_CLOUD_VERSION=v0.0.0
# Omit TECHULUS_CLOUD_VERSION to install GitHub's latest release.
# Set it to a specific vX.Y.Z release to install that release.
# TECHULUS_CLOUD_VERSION=vX.Y.Z
# The installer and updater populate immutable release image references.
# When these image references are omitted, Compose uses TECHULUS_CLOUD_VERSION.
# TECHULUS_CLOUD_WEB_IMAGE=ghcr.io/techulus/cloud/web@sha256:...
# TECHULUS_CLOUD_REGISTRY_IMAGE=ghcr.io/techulus/cloud/registry@sha256:...
# TECHULUS_CLOUD_UPDATER_IMAGE=ghcr.io/techulus/cloud/updater@sha256:...
CONTROL_PLANE_UPDATER_TOKEN=generate-with-openssl-rand-hex-32

# Server error tracking (optional)
Expand Down
12 changes: 6 additions & 6 deletions deployment/compose.postgres.yml
Original file line number Diff line number Diff line change
Expand Up @@ -70,11 +70,11 @@ services:
restart: unless-stopped

migrate:
image: ghcr.io/techulus/cloud/web:${TECHULUS_CLOUD_VERSION:-tip}
image: "${TECHULUS_CLOUD_WEB_IMAGE:-ghcr.io/techulus/cloud/web:${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}}"
env_file:
- ./.env
environment:
- TECHULUS_CLOUD_VERSION=${TECHULUS_CLOUD_VERSION:-tip}
- TECHULUS_CLOUD_VERSION=${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}
- DATABASE_URL=${DATABASE_URL}
- BETTER_AUTH_URL=https://${ROOT_DOMAIN}
- APP_URL=https://${ROOT_DOMAIN}
Expand All @@ -96,12 +96,12 @@ services:
restart: on-failure

web:
image: ghcr.io/techulus/cloud/web:${TECHULUS_CLOUD_VERSION:-tip}
image: "${TECHULUS_CLOUD_WEB_IMAGE:-ghcr.io/techulus/cloud/web:${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}}"
scale: ${WEB_REPLICAS:-1}
env_file:
- ./.env
environment:
- TECHULUS_CLOUD_VERSION=${TECHULUS_CLOUD_VERSION:-tip}
- TECHULUS_CLOUD_VERSION=${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}
- DATABASE_URL=${DATABASE_URL}
- BETTER_AUTH_URL=https://${ROOT_DOMAIN}
- APP_URL=https://${ROOT_DOMAIN}
Expand Down Expand Up @@ -147,7 +147,7 @@ services:
restart: unless-stopped

control-plane-updater:
image: ghcr.io/techulus/cloud/updater:${TECHULUS_CLOUD_VERSION:-tip}
image: "${TECHULUS_CLOUD_UPDATER_IMAGE:-ghcr.io/techulus/cloud/updater:${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}}"
environment:
- DEPLOY_DIR=/opt/techulus-cloud
- CONTROL_PLANE_UPDATER_TOKEN=${CONTROL_PLANE_UPDATER_TOKEN}
Expand All @@ -163,7 +163,7 @@ services:
restart: unless-stopped

registry:
image: ghcr.io/techulus/cloud/registry:${TECHULUS_CLOUD_VERSION:-tip}
image: "${TECHULUS_CLOUD_REGISTRY_IMAGE:-ghcr.io/techulus/cloud/registry:${TECHULUS_CLOUD_VERSION:?TECHULUS_CLOUD_VERSION is required}}"
env_file:
- ./.env
volumes:
Expand Down
Loading
Loading