diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index e29b496..0c650f2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,14 +1,7 @@ -# One runner, one image pull: the render diff and the offline check. name: check on: {pull_request: {}} permissions: {contents: read} jobs: check: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: katoptra/lib/.github/actions/toolbox@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 - with: {op: 'false'} - - run: task check - - run: task run -- task offline + uses: katoptra/lib/.github/workflows/check.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 + with: {offline: true} diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 41d1b2f..51d6f37 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -3,15 +3,18 @@ on: workflow_dispatch: inputs: vars: - description: 'KEY=value pairs for the pipeline; none are read today' + description: 'KEY=value pairs for all the tasks. LIST_FLOOR and OWNERS here change prune.' type: string default: '' permissions: contents: read - actions: write # the called workflow chains a run that left .run/chain; this one never does, but a called workflow cannot raise this + # The sync workflow of lib starts the next run if a run writes .run/chain. The runs of + # this mirror do not write that file. But the workflow of lib cannot have more + # permissions than this workflow gives it. + actions: write concurrency: {group: sync, cancel-in-progress: false} jobs: sync: - uses: katoptra/lib/.github/workflows/sync.yml@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0 + uses: katoptra/lib/.github/workflows/sync.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0 with: {vars: '${{ inputs.vars }}', timeout-minutes: 90} secrets: inherit diff --git a/Taskfile.yml b/Taskfile.yml index 56b1c42..a19fd93 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -1,21 +1,33 @@ -# github: a nightly mirror of every repository under OWNERS into Proton Drive. +# This mirror keeps a copy of each GitHub repository of the owners in OWNERS, in one +# Proton Drive folder. Each copy is one git bundle, and the mirror updates the copies +# daily. katoptra/lib supplies the toolbox and the proton engine, and this file includes +# the two at v2 with `flatten: true`. The README of lib gives the items that each of the +# two supplies. The other items in this file are only for this mirror: +# - OWNERS: the owners of the repositories +# - LIST_FLOOR: the minimum number of repositories in a listing +# - list: the verb that gets the repositories of each owner from the GitHub API +# - stage and prune: the two hooks that the engine lets a mirror write +# - report-mirror: the rows of the mirror in the run summary +# - offline: the check that two bundles of one repository have the same bytes, a +# condition that is necessary for this mirror # -# Each repository becomes one git bundle, /.bundle under the destination -# folder: a mirror clone's every ref in one file that `git clone name.bundle` restores. -# A bundle's bytes are the same on every night the repository did not change, so the -# proton engine's upload skips it, and a night it changed becomes a new revision in -# Proton's version history. Nothing is kept between runs but the engine's CLI session. +# op.env contains the names of the secrets. `op run` gets their values from the vault, +# and each secret goes into the image with its name. # -# The toolbox and the proton engine are katoptra/lib's, included below at v2 and -# flattened into this file. What is here is the mirror's own: which owners, the floor -# under a listing, the two hooks the engine leaves to a mirror (stage, prune), its rows -# of the report, and the offline check of the assumption everything rests on. +# A bundle is one file with all the refs of a mirror clone, at /.bundle in +# the destination folder. `git clone name.bundle` gets the repository back from it. If a +# repository does not change, its bundle has the same bytes in each run. Thus, the upload +# of the proton engine does not send it again. If a repository changes, its bundle +# becomes a new revision in the version history of Proton. Between runs, the mirror keeps +# only the CLI session of the engine. version: '3' vars: OWNERS: jshvn katoptra - # A listing under this many repositories is a broken token or a broken API, never an - # account that emptied itself; stage refuses it, so prune can never trash on one. + # If a listing has less than this number of repositories, a token or the API does not + # operate correctly. An account that removed its repositories is not the cause. stage + # rejects such a listing. Thus, prune cannot move bundles to the trash because of it. + # The vars input of the sync workflow can change this value. LIST_FLOOR: 20 API: https://api.github.com CURL: curl -fsS --connect-timeout 15 --max-time 120 --retry 3 --retry-connrefused @@ -25,11 +37,10 @@ includes: toolbox: taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml flatten: true - excludes: [report-engine, report-mirror] + excludes: [report-mirror] vars: NAME: github - DESC: a nightly mirror of every repository under jshvn and katoptra into Proton Drive - IMAGE: ghcr.io/katoptra/toolbox:proton-v2 + DESC: a daily mirror of every repository under {{.OWNERS}} into Proton Drive MENU: >- "" "${c}write -- changes Proton Drive${r}" @@ -50,8 +61,10 @@ tasks: set: [pipefail] cmds: - mkdir -p {{.RUN}} && rm -f {{.RUN}}/repos.txt && touch {{.RUN}}/repos.txt - # A user token lists that user's own repositories and an org token the org's; the - # endpoint follows the owner's kind. Each is read a page at a time until a short one. + # With a user token, the API gives the repositories of that user. With an + # organization token, the API gives the repositories of the organization. The verb + # selects the endpoint for the type of the owner. It reads the listing of each owner + # one page at a time, until a page has less than 100 repositories. - for: {var: OWNERS} cmd: | o={{.ITEM}}; tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')") || { echo "list: no token for $o in the environment" >&2; exit 1; } @@ -74,17 +87,18 @@ tasks: test "$n" -ge {{.LIST_FLOOR}} || { echo "list: $n repositories is under the floor of {{.LIST_FLOOR}}; refusing to take a short listing for the account" >&2; exit 1; } echo "list: $n repositories under {{.OWNERS}}" - # ---- the engine's hooks ---- + # ---- the hooks of the engine ---- stage: desc: list, then clone each repository as a mirror and bundle it into STAGING//.bundle cmds: - {task: list} - rm -rf {{.STAGING}} {{.RUN}}/work && mkdir -p {{.STAGING}} {{.RUN}}/work - # The token reaches git as a header through its environment, never on a command line, - # and git's stderr goes to a file: a clone error names the repository, and the log is - # public. A repository with no refs yet has nothing to bundle. Bundles are - # byte-for-byte reproducible for an unchanged repository (offline checks this), which - # is what lets the engine skip them. + # All persons can read the run log. Thus, the token goes to git as a header in the + # environment of git, not on a command line. The stderr of git goes to .run/git.err, + # not to the run log, because a clone error shows the name of the repository. stage + # makes no bundle for an empty repository, which has no refs. If a repository does + # not change, its bundle has the same bytes each time (offline examines this). Thus, + # the engine does not upload the bundle again. - | i=0 while IFS=/ read -r o name; do diff --git a/op.env b/op.env index d5ae549..618f0a8 100644 --- a/op.env +++ b/op.env @@ -1,5 +1,6 @@ -# op:// references only; `op run --env-file=op.env` resolves them at run time. -# One vault, addressed by UUID so a rename cannot break it; item github. See katoptra/lib, Secrets. +# This file contains only op:// references. `op run --env-file=op.env` gets their values +# at run time. The secrets are in one vault, addressed by UUID. Thus, the references stay +# correct when the vault gets a new name. The item is github. Refer to katoptra/lib, Secrets. MIRROR_GITHUB_TOKEN_JSHVN=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_jshvn MIRROR_GITHUB_TOKEN_KATOPTRA=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_katoptra MIRROR_PROTON_DESTINATION=op://y6y6b6l2zjbpv7szc5ym5sprne/github/proton/destination diff --git a/render.txt b/render.txt index 1230802..294c5e3 100644 --- a/render.txt +++ b/render.txt @@ -16,7 +16,7 @@ rm -f "$k"; exit $rc task: [session] tar -xf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json && chmod 600 /work/.run/session/*.json task: [session] rm -f /work/.run/session.tar /work/.run/session.tar.age task: [session] sha256sum /work/.run/session/auth-session.json > /work/.run/session.sha -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -106,7 +106,7 @@ done < /work/.run/repos.txt task: [stage] find /work/staging -type f -printf '%P\t%s\n' | sort > /work/.run/staged.txt task: [stage] echo "stage: $(wc -l < /work/.run/staged.txt) bundles, $(awk -F'\t' '{ s += $2 } END { printf "%.1f", s / 1e6 }' /work/.run/staged.txt) MB" task: [upload] test -n "$(ls -A /work/staging 2>/dev/null)" || { echo "upload: staging is empty; stage put nothing there" >&2; exit 1; } -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -130,8 +130,9 @@ task: [confirm] files=$(find /work/staging -type f | wc -l); folders=$(find /wor python3 - "$files" "$folders" /work/.run/upload.json /work/.run/confirm.txt <<'PY' import json, sys files, folders = int(sys.argv[1]), int(sys.argv[2]) -# The CLI writes progress and its summary as one JSON object per line; the last -# line carrying transferredItems is the summary. +# While the upload continues, the CLI writes one JSON object on each line. Its +# summary is also one JSON object on a line. The last line with transferredItems +# is the summary. summary = None for line in open(sys.argv[3]): try: @@ -150,7 +151,7 @@ print("confirm: " + verdict) sys.exit(0 if ok else 1) PY -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -170,7 +171,7 @@ task: [pd] rc=$(cat /work/.run/pd.rc) case " 0 " in *" $rc "*) exit 0 ;; esac echo "proton-drive exited $rc; its stderr, $(wc -l < /work/.run/pd.err) lines, is in .run/pd.err" >&2; exit $rc -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; } @@ -206,7 +207,7 @@ for owner in sys.argv[1:]: PY task: [prune] echo "prune: $(wc -l < /work/.run/prune.txt) bundles to trash" -task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem trash 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc +task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem trash 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc task: [session-push] echo "session: token rotated, sealing it back to the bucket" task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }