Skip to content

Commit eaa2d96

Browse files
committed
feat: add browser-managed backup connections and repository scopes
Create and connect multiple GitHub Apps through the manifest flow. Configure per-installation repository selection and DigitalOcean Spaces in the protected setup page. Preserve excluded backups, isolate failed connections, and retain manual configuration compatibility. Release-As: 1.3.0
1 parent 86de63b commit eaa2d96

21 files changed

Lines changed: 1984 additions & 407 deletions

‎.env.example‎

Lines changed: 29 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,15 @@
11
# Deployment target: Ubuntu 24.04+ x86_64; compose.yaml fixes linux/amd64.
22
# Required values must be available to Coolify Compose build-time interpolation AND runtime.
3-
# REQUIRED: fill these six values. They intentionally have no fake working defaults.
4-
# GitHub Settings > Developer settings > GitHub Apps > your App > General: App ID.
5-
GITHUB_APP_ID=
6-
# Account/org login names from github.com/NAME; install the App on All repositories for each.
7-
GITHUB_APP_OWNERS=
8-
# App General > Private keys > Generate a private key; download the PEM.
9-
# Encode locally: base64 < app-private-key.pem | tr -d '\n'
10-
# Use that single-line output below; treat it as a secret.
11-
GITHUB_APP_PRIVATE_KEY_BASE64=
12-
# DigitalOcean > Spaces Object Storage > Access Keys > Create Access Key.
13-
# AWS_ names are boto3/S3 conventions: enter DIGITALOCEAN values; no AWS account needed.
14-
# Access key ID (not friendly name), then the matching secret saved at creation.
15-
# Neither value is a DigitalOcean personal API token.
16-
AWS_ACCESS_KEY_ID=
17-
AWS_SECRET_ACCESS_KEY=
18-
# DigitalOcean > Spaces Object Storage > Buckets: copy the destination bucket name only.
19-
# It may contain other applications' files outside SPACES_PREFIX/.
20-
SPACES_BUCKET=
21-
22-
# OPTIONAL: working defaults; get the region from your bucket's Spaces settings/origin hostname.
23-
# company-files.ams3.digitaloceanspaces.com means bucket=company-files, region=ams3.
24-
SPACES_REGION=fra1
25-
# Reserve this folder exclusively for this worker. Nested prefixes are allowed.
26-
SPACES_PREFIX=github-backup
3+
# Automatic GitHub App setup (default): no App ID, PEM, or Base64 to copy.
4+
# REQUIRED for browser setup: the exact HTTPS origin assigned to github-setup in Coolify.
5+
# Example: https://github-backup.example.com (no path, no trailing slash).
6+
GITHUB_SETUP_URL=
7+
# REQUIRED for browser setup: create a unique random password with: openssl rand -hex 32
8+
# Login username is admin. Keep this value masked and independent of GitHub credentials.
9+
GITHUB_SETUP_PASSWORD=
10+
# Everything else needed for GitHub/Spaces is entered in the protected setup page.
11+
# Setup defaults: Spaces region=fra1, root folder=github-backup.
12+
# OPTIONAL: scheduler, retention, capacity and monitoring defaults.
2713
BACKUP_INTERVAL_HOURS=1
2814
# Recent changed versions plus older UTC calendar checkpoints; deduplicated.
2915
BACKUP_RETENTION=3
@@ -33,7 +19,7 @@ BACKUP_KEEP_OLDEST=true
3319
# 0 disables the age floor to keep the above count policy bounded.
3420
BACKUP_MIN_AGE_DAYS=0
3521
BACKUP_MANIFEST_RETENTION=14
36-
# Budget counts only live objects inside SPACES_PREFIX/, not the shared bucket.
22+
# Budget counts only live objects inside the configured backup root, not the shared bucket.
3723
BACKUP_MAX_STORAGE_GIB=100
3824
BACKUP_WORKERS=1
3925
BACKUP_TIMEOUT_HOURS=12
@@ -49,3 +35,21 @@ UPTIME_KUMA_ALLOW_HTTP=false
4935
UPTIME_KUMA_TIMEOUT_SECONDS=5
5036
# 0 = automatic: 2 * BACKUP_INTERVAL_HOURS + 1 (3 hours with default hourly checks).
5137
UPTIME_KUMA_MAX_SUCCESS_AGE_HOURS=0
38+
39+
# OPTIONAL manual alternative: leave all three empty for browser setup.
40+
# GitHub App General > App ID; install on All repositories for each listed login.
41+
# Private keys > Generate a private key; base64 < app-private-key.pem | tr -d '\n'
42+
# Manual settings take precedence over saved browser credentials; use one mode.
43+
GITHUB_APP_ID=
44+
GITHUB_APP_OWNERS=
45+
GITHUB_APP_PRIVATE_KEY_BASE64=
46+
47+
# OPTIONAL legacy Spaces environment alternative (saved browser settings take precedence).
48+
# DigitalOcean > Spaces Object Storage > Access Keys: access key ID and matching secret.
49+
# These boto3 variable names take DigitalOcean credentials, not AWS credentials.
50+
AWS_ACCESS_KEY_ID=
51+
AWS_SECRET_ACCESS_KEY=
52+
# DigitalOcean > Spaces > Buckets: bucket name alone and its origin region slug.
53+
SPACES_BUCKET=
54+
SPACES_REGION=fra1
55+
SPACES_PREFIX=github-backup

‎.github/workflows/ci.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,21 @@ jobs:
7373
github-backup:ci --version
7474
docker run --rm --platform linux/amd64 --read-only --cap-drop=ALL --security-opt=no-new-privileges \
7575
github-backup:ci help run
76+
- name: Smoke-test protected setup server
77+
run: |
78+
docker run --detach --name github-backup-setup-ci --platform linux/amd64 \
79+
--read-only --cap-drop=ALL --security-opt=no-new-privileges \
80+
--tmpfs /tmp:rw,noexec,nosuid,size=16m \
81+
--entrypoint gunicorn github-backup:ci \
82+
--bind 0.0.0.0:8080 --workers 1 --threads 2 --worker-tmp-dir /tmp \
83+
github_backup.onboarding:application
84+
trap 'docker rm --force github-backup-setup-ci' EXIT
85+
for attempt in $(seq 1 20); do
86+
if docker exec github-backup-setup-ci python -c \
87+
"import urllib.request; assert urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=2).read() == b'OK'"; then
88+
exit 0
89+
fi
90+
sleep 1
91+
done
92+
docker logs github-backup-setup-ci
93+
exit 1

‎README.md‎

Lines changed: 78 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,9 @@ Spaces, so you can recover when GitHub is unavailable or an account is compromis
99
or newer supported Ubuntu server with an x86_64 (Intel/AMD 64-bit) CPU.**
1010
Compose explicitly builds and runs `linux/amd64`; no Raspberry Pi OS or ARM hardware is required.
1111

12-
Install one GitHub App with **All repositories** on each account and organization. New
13-
repositories are discovered automatically. The worker checks hourly and uploads a new
12+
Connect one or more GitHub Apps to personal accounts and organizations and choose **all**,
13+
**selected**, or **all except selected** repositories per installation. All modes that cover
14+
future repositories discover them automatically. The worker checks hourly and uploads a new
1415
self-contained backup only when Git refs change. It never runs the backed-up code.
1516

1617
## What you get by default
@@ -31,114 +32,86 @@ issues, Actions artifacts, secrets, or submodule contents from other repositorie
3132

3233
## Install with Coolify
3334

34-
### 1. Create and install your GitHub App
35+
### 1. Deploy in Coolify
3536

36-
Follow [GitHub App setup](docs/GITHUB_APP.md): grant repository **Contents: Read-only**
37-
(Metadata read is automatic), disable webhooks, and install it with **All repositories**
38-
on each user and organization to back up. Record its **App ID** and download a private-key PEM.
39-
A personal access token is not required. Being a member of an organization is insufficient:
40-
the App must be installed on that organization with its approval.
41-
42-
Encode the key as a single line on your own computer:
43-
44-
```bash
45-
base64 < app-private-key.pem | tr -d '\n'
46-
```
47-
48-
Treat this output as a secret; base64 is not encryption.
49-
50-
### 2. Prepare DigitalOcean Spaces
51-
52-
Use an existing shared **unversioned** bucket or create one. Keep the backup prefix private.
53-
Create a Spaces key that can list, read, write, and delete this backup data, and read bucket
54-
versioning status. The worker does not change bucket policies or other applications' files.
55-
Record the bucket name and its region, e.g. `fra1` (Frankfurt) or `ams3` (Amsterdam).
56-
57-
### 3. Configure the Coolify application
58-
59-
1. In the target Coolify project/environment, add a resource from this Git repository.
60-
For public source, use `https://github.com/drhdev/github-backup`.
61-
2. Choose the **Docker Compose** build pack / Compose build strategy. Set Base Directory to `/`
62-
and Docker Compose Location to `/compose.yaml` (the file at the repository root).
63-
3. Use a reviewed source revision containing this feature. The Compose file builds the image;
64-
old published images do not contain this scheduler. Keep **one replica** and the named cache volume.
65-
4. Leave domain/port settings empty: this is a background worker with no inbound HTTP listener.
66-
5. **Disable automatic deployments, preview deployments, and overlapping/rolling replacements.**
67-
A compromised GitHub account must not be able to deploy replacement worker code with Spaces secrets.
68-
6. In **Environment Variables**, set the six required values below in the production environment.
69-
For the current Compose file, enable **Buildtime and Runtime** on all six required values
70-
so Coolify can interpolate Compose during its build command. Keep secret values masked. See the [complete deployment guide](docs/COOLIFY.md) for UI details.
71-
7. Save and deploy. Do not add a Coolify scheduled task: the container includes its own scheduler.
72-
73-
### 4. Set the environment
74-
75-
These six values must be supplied; placeholders will not work:
37+
1. Add `https://github.com/drhdev/github-backup` using the **Docker Compose** build strategy.
38+
Set Base Directory to `/` and Docker Compose Location to `/compose.yaml`.
39+
2. Keep one replica of each service and the persistent `backup-cache` volume.
40+
3. Assign an **HTTPS domain to `github-setup`, target port 8080**. If Coolify encodes
41+
the target port in its domain field, use `https://github-backup.example.com:8080`.
42+
The external URL remains `https://github-backup.example.com` on port 443.
43+
Leave `github-backup` without a domain or published port.
44+
4. **Disable automatic deployments, previews, and overlapping/rolling replacements.**
45+
5. Set just these two required environment variables, keep the password masked, and deploy:
7646

7747
```dotenv
78-
GITHUB_APP_ID=123456
79-
GITHUB_APP_OWNERS=alice,example-team,alice-work
80-
GITHUB_APP_PRIVATE_KEY_BASE64=REPLACE_WITH_SINGLE_LINE_BASE64_PEM
81-
AWS_ACCESS_KEY_ID=REPLACE_WITH_SPACES_KEY
82-
AWS_SECRET_ACCESS_KEY=REPLACE_WITH_SPACES_SECRET
83-
SPACES_BUCKET=company-files
48+
GITHUB_SETUP_URL=https://github-backup.example.com
49+
GITHUB_SETUP_PASSWORD=REPLACE_WITH_OUTPUT_OF_OPENSSL_RAND_HEX_32
8450
```
8551

86-
`GITHUB_APP_OWNERS` is a comma-separated list of accounts/organizations where the App is installed.
87-
The Spaces credentials are a Spaces access-key pair, not a DigitalOcean API token.
88-
`SPACES_BUCKET` is just the bucket name, without a URL or folder.
89-
90-
### Where to find each required value
91-
92-
| Environment variable | Value to enter | Where to find it |
93-
|---|---|---|
94-
| `GITHUB_APP_ID` | Numeric App ID | GitHub **Settings → Developer settings → GitHub Apps → your App → General**; copy **App ID**, not Client ID or installation ID |
95-
| `GITHUB_APP_OWNERS` | User/org login names, comma-separated | The account and organization names in GitHub URLs, e.g. `github.com/alice` and `github.com/example-team`; install the App on each first |
96-
| `GITHUB_APP_PRIVATE_KEY_BASE64` | Single-line base64 of your downloaded PEM | Your App's **General → Private keys → Generate a private key**, then run the base64 command above locally |
97-
| `AWS_ACCESS_KEY_ID` | **DigitalOcean Spaces access key ID** | DigitalOcean **Spaces Object Storage → Access Keys → Create Access Key** |
98-
| `AWS_SECRET_ACCESS_KEY` | **Secret belonging to that same Spaces key** | Copy the secret when creating the Spaces key; if lost, create a replacement pair and update both values |
99-
| `SPACES_BUCKET` | Existing bucket name, e.g. `company-files` | DigitalOcean **Spaces Object Storage → Buckets**; open the destination bucket |
100-
101-
**Why the `AWS_` names?** Spaces implements the S3 API, and this project uses boto3's standard
102-
credential environment variables. Put **DigitalOcean** credentials in them: no AWS account,
103-
AWS IAM key, or AWS bill is involved. `SPACES_BUCKET` and `SPACES_REGION` configure this
104-
application; they are not credential-provider variable names. Keep the exact names shown—
105-
renaming them to `SPACES_KEY`/`SPACES_SECRET` is not supported by this configuration.
106-
See [DigitalOcean's S3 SDK guide](https://docs.digitalocean.com/products/spaces/reference/aws-sdks/)
107-
and [Spaces key creation](https://docs.digitalocean.com/products/spaces/how-to/manage-access/).
108-
109-
Use a separate Spaces key for this worker with access to the intended bucket and the required
110-
read/write/delete operations. Do not paste a DigitalOcean personal API token, the key's display
111-
name, an endpoint URL, or the GitHub App key into either Spaces credential field.
112-
For example, the bucket origin `company-files.ams3.digitaloceanspaces.com` means
113-
`SPACES_BUCKET=company-files` and `SPACES_REGION=ams3`. Keep `SPACES_PREFIX=github-backup`
114-
unless you want a different folder; that prefix is your choice, not a secret to obtain elsewhere.
115-
116-
Optional values already have working defaults. **Change the region if your bucket is not in Frankfurt:**
117-
118-
```dotenv
119-
SPACES_REGION=fra1
120-
SPACES_PREFIX=github-backup
121-
BACKUP_INTERVAL_HOURS=1
122-
BACKUP_RETENTION=3
123-
BACKUP_KEEP_WEEKLY=2
124-
BACKUP_KEEP_MONTHLY=2
125-
BACKUP_KEEP_OLDEST=true
126-
BACKUP_MAX_STORAGE_GIB=100
127-
```
128-
129-
[`.env.example`](.env.example) lists **every configurable variable and its default**.
130-
In Coolify, configure these values in the application; do not commit a real `.env` or PEM file.
131-
For ordinary Docker Compose outside Coolify, copy `.env.example` to `.env`, fill the required
132-
values, then run `docker compose up -d --build`.
133-
134-
### 5. Verify the first backup
135-
136-
Open the Coolify container logs. A completed healthy run emits `backup_finished` with
137-
`exit_code: 0`. An unchanged repository is still a successful check and does not create a new bundle.
138-
139-
Check `github-backup/_system/manifests/latest.json` in Spaces: all expected repositories
140-
should be represented, `missing_resources` empty, and failure/partial counts zero. Download
141-
a bundle and its repository catalog, verify the SHA-256, then test cloning and `git fsck --full`.
52+
Generate the password with `openssl rand -hex 32`. The URL must be the exact external HTTPS
53+
origin without a path or internal `:8080` suffix. Enable Buildtime and Runtime for supplied
54+
values in Coolify. No Spaces credentials, bucket settings, GitHub App IDs or PEMs need to
55+
be entered in the environment. See [the deployment guide](docs/COOLIFY.md).
56+
57+
### 2. Configure DigitalOcean Spaces in the setup page
58+
59+
Open the setup URL and log in as **admin** with your setup password. Enter:
60+
61+
| Setup field | Where to get it |
62+
|---|---|
63+
| Bucket name | DigitalOcean **Spaces Object Storage → Buckets**; use a shared unversioned bucket or create one |
64+
| Region | Bucket origin hostname: `company-files.ams3.digitaloceanspaces.com` means `ams3` |
65+
| Root folder | Choose an exclusive prefix; default `github-backup` |
66+
| Access key ID and secret | DigitalOcean **Spaces Object Storage → Access Keys → Create Access Key**; use the Spaces key pair, not a personal API token |
67+
68+
Keep the backup prefix private. The Spaces key needs bucket listing and versioning-status
69+
read access, plus object read/write/delete permissions. Click **Test and save Spaces**.
70+
The check writes, reads back and deletes a random temporary object inside your prefix.
71+
Credentials are saved privately on the persistent volume and are never shown in the page.
72+
73+
### 3. Connect GitHub Apps and choose repositories
74+
75+
Click **Connect GitHub / create another App**, then approve creation on GitHub. Use the
76+
installation link to install it on a personal account or organization. GitHub lets you grant
77+
**All repositories** or **Only select repositories**. Return to setup, enter the account's
78+
login and click **Verify and connect account**, then **Choose repositories / pause backups**.
79+
New connections start paused until you choose and save a scope:
80+
81+
| Backup scope | Behavior |
82+
|---|---|
83+
| All | Back up every repository, including future ones; requires All repositories on GitHub |
84+
| Only selected | Back up checked repositories; supports GitHub selected-repository installations |
85+
| All except selected | Back up everything except checked repositories, including future ones; requires All repositories on GitHub |
86+
| Pause | Create no backups for this account; preserve existing copies |
87+
88+
Repeat **create another App** for additional GitHub accounts or organizations, switching
89+
GitHub logins when necessary. You can register an App under an organization by entering
90+
its login in the optional creation field. One App may also have several installations;
91+
multiple Apps can cover the same account, with the union of their selections backed up once.
92+
Organization approval may be required. Every App requests read-only repository Contents
93+
with automatic metadata access. No PEM download, Base64 conversion or personal token is needed.
94+
95+
Once Spaces and at least one installation are configured, the scheduler starts within
96+
30 seconds. Changes apply on the next scheduled run without restarting containers. Excluded
97+
repositories keep all existing backups without pruning; selected repositories that become
98+
inaccessible still alert. Selections use repository IDs, so renames preserve the selection.
99+
See [GitHub setup and recovery](docs/GITHUB_APP.md).
100+
101+
Optional scheduling, capacity, retention and monitoring settings retain the defaults in
102+
[.env.example](.env.example). Existing manual environment deployments remain supported;
103+
saved browser Spaces settings take precedence over their legacy Spaces variables.
104+
105+
### 4. Verify the first backup
106+
107+
Open the Coolify logs. A successful run emits `backup_finished` with `exit_code: 0`.
108+
Check `github-backup/_system/manifests/latest.json` in Spaces: expected selected repositories
109+
must be present, `missing_resources` and `discovery_errors` empty, and failure/partial counts zero.
110+
`excluded_resources` lists previously archived repositories intentionally excluded or paused.
111+
An unchanged repository is a successful check without a duplicate bundle. An explicitly
112+
paused/empty selection produces no code backups, even if the check succeeds.
113+
114+
Download a bundle and its catalog, verify SHA-256, then test cloning and `git fsck --full`.
142115
**A green container alone does not prove recovery works.** See [restore verification](docs/COOLIFY.md#validate-recovery).
143116

144117
## Storage layout and retention

‎compose.yaml‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,13 @@ services:
2222
volumes:
2323
- backup-cache:/cache
2424
environment:
25-
GITHUB_APP_ID: ${GITHUB_APP_ID:?Set GitHub App ID}
26-
GITHUB_APP_OWNERS: ${GITHUB_APP_OWNERS:?Set comma-separated account and organization names}
27-
GITHUB_APP_PRIVATE_KEY_BASE64: ${GITHUB_APP_PRIVATE_KEY_BASE64:?Set base64 PEM private key}
28-
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:?Set Spaces key}
29-
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:?Set Spaces secret}
25+
GITHUB_APP_ID: ${GITHUB_APP_ID:-}
26+
GITHUB_APP_OWNERS: ${GITHUB_APP_OWNERS:-}
27+
GITHUB_APP_PRIVATE_KEY_BASE64: ${GITHUB_APP_PRIVATE_KEY_BASE64:-}
28+
AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
29+
AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
3030
AWS_EC2_METADATA_DISABLED: "true"
31-
SPACES_BUCKET: ${SPACES_BUCKET:?Set Spaces bucket name}
31+
SPACES_BUCKET: ${SPACES_BUCKET:-}
3232
SPACES_REGION: ${SPACES_REGION:-fra1}
3333
SPACES_PREFIX: ${SPACES_PREFIX:-github-backup}
3434
BACKUP_INTERVAL_HOURS: ${BACKUP_INTERVAL_HOURS:-1}
@@ -57,5 +57,41 @@ services:
5757
options:
5858
max-size: "10m"
5959
max-file: "3"
60+
github-setup:
61+
platform: linux/amd64
62+
build:
63+
context: .
64+
platforms: [linux/amd64]
65+
entrypoint: ["gunicorn"]
66+
command: ["--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "2",
67+
"--timeout", "120", "--graceful-timeout", "20",
68+
"--worker-tmp-dir", "/tmp", "github_backup.onboarding:application"]
69+
restart: unless-stopped
70+
init: true
71+
read_only: true
72+
cap_drop: [ALL]
73+
security_opt: [no-new-privileges:true]
74+
pids_limit: 32
75+
cpus: "0.25"
76+
mem_limit: 128m
77+
tmpfs:
78+
- /tmp:rw,noexec,nosuid,size=16m,mode=1777
79+
volumes:
80+
- backup-cache:/cache
81+
expose: ["8080"]
82+
environment:
83+
GITHUB_SETUP_URL: ${GITHUB_SETUP_URL:-}
84+
GITHUB_SETUP_PASSWORD: ${GITHUB_SETUP_PASSWORD:-}
85+
healthcheck:
86+
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health', timeout=5)"]
87+
interval: 60s
88+
timeout: 10s
89+
retries: 3
90+
start_period: 30s
91+
logging:
92+
driver: json-file
93+
options:
94+
max-size: "10m"
95+
max-file: "3"
6096
volumes:
6197
backup-cache:

0 commit comments

Comments
 (0)