Taskfile.yml

# This mirror keeps a copy of each GitHub repository of the owners in OWNERS, in one
# Proton Drive folder. Each copy is one git bundle, and the mirror updates the copies
# daily. katoptra/lib supplies the toolbox and the proton engine, and this file includes
# the two at v2 with `flatten: true`. The README of lib gives the items that each of the
# two supplies. The other items in this file are only for this mirror:
#   - OWNERS: the owners of the repositories
#   - LIST_FLOOR: the minimum number of repositories in a listing
#   - list: the verb that gets the repositories of each owner from the GitHub API
#   - stage and prune: the two hooks that the engine lets a mirror write
#   - report-mirror: the rows of the mirror in the run summary
#   - offline: the check that two bundles of one repository have the same bytes, a
#     condition that is necessary for this mirror
#
# op.env contains the names of the secrets. `op run` gets their values from the vault,
# and each secret goes into the image with its name.
#
# A bundle is one file with all the refs of a mirror clone, at <owner>/<name>.bundle in
# the destination folder. `git clone name.bundle` gets the repository back from it. If a
# repository does not change, its bundle has the same bytes in each run. Thus, the upload
# of the proton engine does not send it again. If a repository changes, its bundle
# becomes a new revision in the version history of Proton. Between runs, the mirror keeps
# only the CLI session of the engine.
version: '3'

vars:
  OWNERS: jshvn katoptra
  # If a listing has less than this number of repositories, a token or the API does not
  # operate correctly. An account that removed its repositories is not the cause. stage
  # rejects such a listing. Thus, prune cannot move bundles to the trash because of it.
  # The vars input of the sync workflow can change this value.
  LIST_FLOOR: 20
  API: https://api.github.com
  CURL: curl -fsS --connect-timeout 15 --max-time 120 --retry 3 --retry-connrefused
  ACCEPT: 'Accept: application/vnd.github+json'

includes:
  toolbox:
    taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/toolbox.yml
    flatten: true
    excludes: [report-mirror]
    vars:
      NAME: github
      DESC: a daily mirror of every repository under {{.OWNERS}} into Proton Drive
      MENU: >-
        ""
        "${c}write -- changes Proton Drive${r}"
        "  task session-seal -- .run/pd   Encrypt a laptop Proton CLI session into R2"
        "  task empty-trash               Permanently delete Proton trash (never scheduled)"
        ""
        "${c}checks -- offline, no credentials${r}"
        "  task run -- task offline       Bundle a throwaway repository twice; the two must match"
  proton:
    taskfile: https://raw.githubusercontent.com/katoptra/lib/v2/engines/proton.yml
    flatten: true
    excludes: [stage, prune]

tasks:
  # ---- the upstream ----
  list:
    desc: Every repository under each of OWNERS, as owner/name lines in RUN/repos.txt
    set: [pipefail]
    cmds:
      - mkdir -p {{.RUN}} && rm -f {{.RUN}}/repos.txt && touch {{.RUN}}/repos.txt
      # With a user token, the API gives the repositories of that user. With an
      # organization token, the API gives the repositories of the organization. The verb
      # selects the endpoint for the type of the owner. It reads the listing of each owner
      # one page at a time, until a page has less than 100 repositories.
      - for: {var: OWNERS}
        cmd: |
          o={{.ITEM}}; tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')") || { echo "list: no token for $o in the environment" >&2; exit 1; }
          kind=$({{.CURL}} -H "Authorization: Bearer $tok" -H "{{.ACCEPT}}" {{.API}}/users/$o | python3 -c 'import json, sys; print(json.load(sys.stdin)["type"])')
          case $kind in
            User) url="{{.API}}/user/repos?affiliation=owner&per_page=100" ;;
            Organization) url="{{.API}}/orgs/$o/repos?type=all&per_page=100" ;;
            *) echo "list: $o is a $kind, neither a user nor an organization" >&2; exit 1 ;;
          esac
          page=1
          while :; do
            {{.CURL}} -H "Authorization: Bearer $tok" -H "{{.ACCEPT}}" "$url&page=$page" | python3 -c 'import json, sys; [print(r["full_name"]) for r in json.load(sys.stdin)]' > {{.RUN}}/page.txt
            if grep -qv "^$o/" {{.RUN}}/page.txt; then echo "list: the token for $o listed another owner's repositories; refusing the listing" >&2; exit 1; fi
            cat {{.RUN}}/page.txt >> {{.RUN}}/repos.txt
            test "$(wc -l < {{.RUN}}/page.txt)" -eq 100 || break
            page=$((page + 1))
          done
      - |
        sort -o {{.RUN}}/repos.txt {{.RUN}}/repos.txt; n=$(wc -l < {{.RUN}}/repos.txt)
        test "$n" -ge {{.LIST_FLOOR}} || { echo "list: $n repositories is under the floor of {{.LIST_FLOOR}}; refusing to take a short listing for the account" >&2; exit 1; }
        echo "list: $n repositories under {{.OWNERS}}"

  # ---- the hooks of the engine ----
  stage:
    desc: list, then clone each repository as a mirror and bundle it into STAGING/<owner>/<name>.bundle
    cmds:
      - {task: list}
      - rm -rf {{.STAGING}} {{.RUN}}/work && mkdir -p {{.STAGING}} {{.RUN}}/work
      # All persons can read the run log. Thus, the token goes to git as a header in the
      # environment of git, not on a command line. The stderr of git goes to .run/git.err,
      # not to the run log, because a clone error shows the name of the repository. stage
      # makes no bundle for an empty repository, which has no refs. If a repository does
      # not change, its bundle has the same bytes each time (offline examines this). Thus,
      # the engine does not upload the bundle again.
      - |
        i=0
        while IFS=/ read -r o name; do
          i=$((i + 1))
          tok=$(printenv "MIRROR_GITHUB_TOKEN_$(printf %s "$o" | tr 'a-z-' 'A-Z_')")
          auth="AUTHORIZATION: basic $(printf 'x-access-token:%s' "$tok" | base64 -w0)"
          w={{.RUN}}/work/$name.git
          GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=http.https://github.com/.extraheader GIT_CONFIG_VALUE_0="$auth" \
            git clone --quiet --mirror "https://github.com/$o/$name.git" "$w" 2> {{.RUN}}/git.err \
            || { echo "stage: the clone of repository $i of $(wc -l < {{.RUN}}/repos.txt) failed with exit $?; git's stderr is in .run/git.err" >&2; exit 1; }
          if [ -z "$(git -C "$w" for-each-ref)" ]; then echo "stage: repository $i has no refs, nothing to bundle"; rm -rf "$w"; continue; fi
          mkdir -p {{.STAGING}}/$o
          git -C "$w" bundle create --quiet {{.STAGING}}/$o/$name.bundle --all 2> {{.RUN}}/git.err || { echo "stage: bundling repository $i failed with exit $?" >&2; exit 1; }
          rm -rf "$w"
        done < {{.RUN}}/repos.txt
      - find {{.STAGING}} -type f -printf '%P\t%s\n' | sort > {{.RUN}}/staged.txt
      - 'echo "stage: $(wc -l < {{.RUN}}/staged.txt) bundles, $(awk -F''\t'' ''{ s += $2 } END { printf "%.1f", s / 1e6 }'' {{.RUN}}/staged.txt) MB"'

  prune:
    desc: Trash the bundles in Proton whose repository is no longer listed; a renamed one is trashed under its old name and uploaded under the new
    cmds:
      - for: {var: OWNERS}
        task: list-folder
        vars: {FOLDER: '$MIRROR_PROTON_DESTINATION/{{.ITEM}}', OUT: '{{.RUN}}/proton-{{.ITEM}}.json'}
      - |
        python3 - {{.OWNERS}} <<'PY' > {{.RUN}}/prune.txt
        import json, sys
        unwrap = lambda v: v.get("value") if isinstance(v, dict) else v
        staged = {line.split("\t")[0] for line in open("{{.RUN}}/staged.txt")}
        for owner in sys.argv[1:]:
            try:
                nodes = json.load(open(f"{{.RUN}}/proton-{owner}.json"))
            except (OSError, ValueError):
                nodes = []
            for node in nodes:
                name = str(unwrap(node.get("name")) or "") if isinstance(node, dict) else ""
                if name.endswith(".bundle") and f"{owner}/{name}" not in staged:
                    print(f"{owner}/{name}")
        PY
      - 'echo "prune: $(wc -l < {{.RUN}}/prune.txt) bundles to trash"'
      - task: trash
        vars:
          PATHS:
            sh: 'test -s {{.RUN}}/prune.txt && sed "s|^|$MIRROR_PROTON_DESTINATION/|" {{.RUN}}/prune.txt | tr "\n" " " || true'

  report-mirror:
    internal: true
    silent: true
    cmds:
      - |
        n() { test -f "$1" && wc -l < "$1" || echo 0; }
        cat >> "${GITHUB_STEP_SUMMARY:-/dev/stdout}" <<EOF
        | Repositories | $(n {{.RUN}}/repos.txt) listed under {{.OWNERS}}, $(n {{.RUN}}/staged.txt) bundled |
        | Pruned | $(n {{.RUN}}/prune.txt) bundles trashed |
        EOF

  # ---- checks ----
  offline:
    desc: Bundle a throwaway repository twice and compare; the engine skips a bundle only because this holds
    cmds:
      - rm -rf {{.RUN}}/fixture && git init -q {{.RUN}}/fixture
      - git -C {{.RUN}}/fixture -c user.name=offline -c user.email=offline@localhost commit -q --allow-empty -m one
      - git -C {{.RUN}}/fixture -c user.name=offline -c user.email=offline@localhost commit -q --allow-empty -m two
      - git -C {{.RUN}}/fixture bundle create --quiet {{.RUN}}/one.bundle --all
      - git -C {{.RUN}}/fixture bundle create --quiet {{.RUN}}/two.bundle --all
      - cmp {{.RUN}}/one.bundle {{.RUN}}/two.bundle
      - test "$(git -C {{.RUN}}/fixture bundle list-heads {{.RUN}}/one.bundle | wc -l)" -ge 2
      - 'echo "offline: two bundles of one repository are identical"'