refactor(taskfile): adopt lib v2.6.0

.github/workflows/check.yml

@@ -1,14 +1,7 @@
-# One runner, one image pull: the render diff and the offline check.
 name: check
 on: {pull_request: {}}
 permissions: {contents: read}
 jobs:
   check:
-    runs-on: ubuntu-latest
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      - uses: katoptra/lib/.github/actions/toolbox@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0
-        with: {op: 'false'}
-      - run: task check
-      - run: task run -- task offline
+    uses: katoptra/lib/.github/workflows/check.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0
+    with: {offline: true}

.github/workflows/sync.yml

@@ -3,15 +3,18 @@ on:
   workflow_dispatch:
     inputs:
       vars:
-        description: 'KEY=value pairs for the pipeline; none are read today'
+        description: 'KEY=value pairs for all the tasks. LIST_FLOOR and OWNERS here change prune.'
         type: string
         default: ''
 permissions:
   contents: read
-  actions: write   # the called workflow chains a run that left .run/chain; this one never does, but a called workflow cannot raise this
+  # The sync workflow of lib starts the next run if a run writes .run/chain. The runs of
+  # this mirror do not write that file. But the workflow of lib cannot have more
+  # permissions than this workflow gives it.
+  actions: write
 concurrency: {group: sync, cancel-in-progress: false}
 jobs:
   sync:
-    uses: katoptra/lib/.github/workflows/sync.yml@fe8162386f612ddb4e5c886632fa7a8d74c54d9f # v2.2.0
+    uses: katoptra/lib/.github/workflows/sync.yml@36d75e6b21473400b5b15003025d9a2e81d4eeea # v2.6.0
     with: {vars: '${{ inputs.vars }}', timeout-minutes: 90}
     secrets: inherit

Taskfile.yml

@@ -1,21 +1,33 @@
-# github: a nightly mirror of every repository under OWNERS into Proton Drive.
+# This mirror keeps a copy of each GitHub repository of the owners in OWNERS, in one
+# Proton Drive folder. Each copy is one git bundle, and the mirror updates the copies
+# daily. katoptra/lib supplies the toolbox and the proton engine, and this file includes
+# the two at v2 with `flatten: true`. The README of lib gives the items that each of the
+# two supplies. The other items in this file are only for this mirror:
+#   - OWNERS: the owners of the repositories
+#   - LIST_FLOOR: the minimum number of repositories in a listing
+#   - list: the verb that gets the repositories of each owner from the GitHub API
+#   - stage and prune: the two hooks that the engine lets a mirror write
+#   - report-mirror: the rows of the mirror in the run summary
+#   - offline: the check that two bundles of one repository have the same bytes, a
+#     condition that is necessary for this mirror
 #
-# Each repository becomes one git bundle, <owner>/<name>.bundle under the destination
-# folder: a mirror clone's every ref in one file that `git clone name.bundle` restores.
-# A bundle's bytes are the same on every night the repository did not change, so the
-# proton engine's upload skips it, and a night it changed becomes a new revision in
-# Proton's version history. Nothing is kept between runs but the engine's CLI session.
+# op.env contains the names of the secrets. `op run` gets their values from the vault,
+# and each secret goes into the image with its name.
 #
-# The toolbox and the proton engine are katoptra/lib's, included below at v2 and
-# flattened into this file. What is here is the mirror's own: which owners, the floor
-# under a listing, the two hooks the engine leaves to a mirror (stage, prune), its rows
-# of the report, and the offline check of the assumption everything rests on.
+# A bundle is one file with all the refs of a mirror clone, at <owner>/<name>.bundle in
+# the destination folder. `git clone name.bundle` gets the repository back from it. If a
+# repository does not change, its bundle has the same bytes in each run. Thus, the upload
+# of the proton engine does not send it again. If a repository changes, its bundle
+# becomes a new revision in the version history of Proton. Between runs, the mirror keeps
+# only the CLI session of the engine.
 version: '3'
 
 vars:
   OWNERS: jshvn katoptra
-  # A listing under this many repositories is a broken token or a broken API, never an
-  # account that emptied itself; stage refuses it, so prune can never trash on one.
+  # If a listing has less than this number of repositories, a token or the API does not
+  # operate correctly. An account that removed its repositories is not the cause. stage
+  # rejects such a listing. Thus, prune cannot move bundles to the trash because of it.
+  # The vars input of the sync workflow can change this value.
   LIST_FLOOR: 20
   API: https://api.github.com
   CURL: curl -fsS --connect-timeout 15 --max-time 120 --retry 3 --retry-connrefused
@@ -25,11 +37,10 @@ includes:
   toolbox:
     taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml
     flatten: true
-    excludes: [report-engine, report-mirror]
+    excludes: [report-mirror]
     vars:
       NAME: github
-      DESC: a nightly mirror of every repository under jshvn and katoptra into Proton Drive
-      IMAGE: ghcr.io/katoptra/toolbox:proton-v2
+      DESC: a daily mirror of every repository under {{.OWNERS}} into Proton Drive
       MENU: >-
         ""
         "${c}write -- changes Proton Drive${r}"
@@ -50,8 +61,10 @@ tasks:
     set: [pipefail]
     cmds:
       - mkdir -p {{.RUN}} && rm -f {{.RUN}}/repos.txt && touch {{.RUN}}/repos.txt
-      # A user token lists that user's own repositories and an org token the org's; the
-      # endpoint follows the owner's kind. Each is read a page at a time until a short one.
+      # With a user token, the API gives the repositories of that user. With an
+      # organization token, the API gives the repositories of the organization. The verb
+      # selects the endpoint for the type of the owner. It reads the listing of each owner
+      # one page at a time, until a page has less than 100 repositories.
       - for: {var: OWNERS}
         cmd: |
           o={{.ITEM}}; tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')") || { echo "list: no token for $o in the environment" >&2; exit 1; }
@@ -74,17 +87,18 @@ tasks:
         test "$n" -ge {{.LIST_FLOOR}} || { echo "list: $n repositories is under the floor of {{.LIST_FLOOR}}; refusing to take a short listing for the account" >&2; exit 1; }
         echo "list: $n repositories under {{.OWNERS}}"
 
-  # ---- the engine's hooks ----
+  # ---- the hooks of the engine ----
   stage:
     desc: list, then clone each repository as a mirror and bundle it into STAGING/<owner>/<name>.bundle
     cmds:
       - {task: list}
       - rm -rf {{.STAGING}} {{.RUN}}/work && mkdir -p {{.STAGING}} {{.RUN}}/work
-      # The token reaches git as a header through its environment, never on a command line,
-      # and git's stderr goes to a file: a clone error names the repository, and the log is
-      # public. A repository with no refs yet has nothing to bundle. Bundles are
-      # byte-for-byte reproducible for an unchanged repository (offline checks this), which
-      # is what lets the engine skip them.
+      # All persons can read the run log. Thus, the token goes to git as a header in the
+      # environment of git, not on a command line. The stderr of git goes to .run/git.err,
+      # not to the run log, because a clone error shows the name of the repository. stage
+      # makes no bundle for an empty repository, which has no refs. If a repository does
+      # not change, its bundle has the same bytes each time (offline examines this). Thus,
+      # the engine does not upload the bundle again.
       - |
         i=0
         while IFS=/ read -r o name; do

op.env

@@ -1,5 +1,6 @@
-# op:// references only; `op run --env-file=op.env` resolves them at run time.
-# One vault, addressed by UUID so a rename cannot break it; item github. See katoptra/lib, Secrets.
+# This file contains only op:// references. `op run --env-file=op.env` gets their values
+# at run time. The secrets are in one vault, addressed by UUID. Thus, the references stay
+# correct when the vault gets a new name. The item is github. Refer to katoptra/lib, Secrets.
 MIRROR_GITHUB_TOKEN_JSHVN=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_jshvn
 MIRROR_GITHUB_TOKEN_KATOPTRA=op://y6y6b6l2zjbpv7szc5ym5sprne/github/github/token_katoptra
 MIRROR_PROTON_DESTINATION=op://y6y6b6l2zjbpv7szc5ym5sprne/github/proton/destination

render.txt

@@ -16,7 +16,7 @@ rm -f "$k"; exit $rc
 task: [session] tar -xf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json && chmod 600 /work/.run/session/*.json
 task: [session] rm -f /work/.run/session.tar /work/.run/session.tar.age
 task: [session] sha256sum /work/.run/session/auth-session.json > /work/.run/session.sha
-task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
+task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$(dirname "$MIRROR_PROTON_DESTINATION")" > /work/.run/parent.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
 task: [session-push] echo "session: token rotated, sealing it back to the bucket"
 task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json
 task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }
@@ -106,7 +106,7 @@ done < /work/.run/repos.txt
 task: [stage] find /work/staging -type f -printf '%P\t%s\n' | sort > /work/.run/staged.txt
 task: [stage] echo "stage: $(wc -l < /work/.run/staged.txt) bundles, $(awk -F'\t' '{ s += $2 } END { printf "%.1f", s / 1e6 }' /work/.run/staged.txt) MB"
 task: [upload] test -n "$(ls -A /work/staging 2>/dev/null)" || { echo "upload: staging is empty; stage put nothing there" >&2; exit 1; }
-task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
+task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem upload -f create-new-revision -d merge -t --json /work/staging/* "$MIRROR_PROTON_DESTINATION" > /work/.run/upload.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
 task: [session-push] echo "session: token rotated, sealing it back to the bucket"
 task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json
 task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }
@@ -130,8 +130,9 @@ task: [confirm] files=$(find /work/staging -type f | wc -l); folders=$(find /wor
 python3 - "$files" "$folders" /work/.run/upload.json /work/.run/confirm.txt <<'PY'
 import json, sys
 files, folders = int(sys.argv[1]), int(sys.argv[2])
-# The CLI writes progress and its summary as one JSON object per line; the last
-# line carrying transferredItems is the summary.
+# While the upload continues, the CLI writes one JSON object on each line. Its
+# summary is also one JSON object on a line. The last line with transferredItems
+# is the summary.
 summary = None
 for line in open(sys.argv[3]):
     try:
@@ -150,7 +151,7 @@ print("confirm: " + verdict)
 sys.exit(0 if ok else 1)
 PY
 
-task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
+task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/jshvn" > /work/.run/proton-jshvn.json || echo "[]" > /work/.run/proton-jshvn.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
 task: [session-push] echo "session: token rotated, sealing it back to the bucket"
 task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json
 task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }
@@ -170,7 +171,7 @@ task: [pd] rc=$(cat /work/.run/pd.rc)
 case " 0 " in *" $rc "*) exit 0 ;; esac
 echo "proton-drive exited $rc; its stderr, $(wc -l < /work/.run/pd.err) lines, is in .run/pd.err" >&2; exit $rc
 
-task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
+task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem list -j "$MIRROR_PROTON_DESTINATION/katoptra" > /work/.run/proton-katoptra.json || echo "[]" > /work/.run/proton-katoptra.json 2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
 task: [session-push] echo "session: token rotated, sealing it back to the bucket"
 task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json
 task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }
@@ -206,7 +207,7 @@ for owner in sys.argv[1:]:
 PY
 
 task: [prune] echo "prune: $(wc -l < /work/.run/prune.txt) bundles to trash"
-task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session PROTON_DRIVE_CREDENTIALS_STORE=unsafe_file proton-drive filesystem trash  2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
+task: [pd] rc=0; PROTON_DRIVE_CACHE_DIR=/work/.run/session proton-drive filesystem trash  2> /work/.run/pd.err || rc=$?; echo $rc > /work/.run/pd.rc
 task: [session-push] echo "session: token rotated, sealing it back to the bucket"
 task: [seal] mkdir -p /work/.run && tar -cf /work/.run/session.tar -C /work/.run/session auth-session.json clientUid.json
 task: [age] test -n "$MIRROR_AGE_IDENTITY" || { echo "MIRROR_AGE_IDENTITY is unset" >&2; exit 1; }