From 36dbf462c2a8829dfada5e599ab79237f4ae4c51 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 18:31:01 -0700 Subject: [PATCH 1/2] feat(pages): footer from UPSTREAM --- CLAUDE.md | 7 +++---- README.md | 10 +++++----- Taskfile.yml | 14 ++++++++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 64c20f2..43a963d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,10 +34,9 @@ a run did the last reconcile. - **GNU's mirror guidelines are applicable to each page that the mirror serves.** Text must be "as short as possible, and strictly explanatory". Images and logos are not permitted. The only permitted link is a link for bug reports. -- **`PAGE_FOOT` is the only text that the mirror adds.** The pages in the bucket show a - change to `PAGE_FOOT` only after the engine makes all the pages again. To make all the - pages again, run `aws s3 rm s3://gnu/.state/indexed.txt.xz`. The next run then sends - approximately 2,600 PutObjects. +- **The footer is the only text that the mirror adds.** The engine makes it from `UPSTREAM` + and `HOST`. Do not set `PAGE_FOOT`. A change to `UPSTREAM` or `HOST` gets to each page at + the next run. That run sends approximately 2,600 PutObjects. - **The bug-report link is `mailto:gnu@katoptra.org`.** That address must receive e-mail. The mail records of the zone are not in this repository. They are in the same location as the zone rules. diff --git a/README.md b/README.md index ce271aa..98160cd 100644 --- a/README.md +++ b/README.md @@ -74,10 +74,9 @@ flowchart LR moves a file. If a listing does not have more than 40,000 lines (`LIST_FLOOR`), the run also stops. A short listing is not full, and the engine must not delete files because of it. -- **Directory pages.** When `INDEX` has a value, the engine makes a page for each - directory. `PAGE_FOOT` is the last line of each page. This line identifies the mirror and - tells how frequently the mirror gets an update. It also gives an address for problem - reports. +- **Directory pages.** When `INDEX` has a value, the engine makes a page for each directory + of upstream, also for a directory with no file. The footer of each page gives the same + directory on ftp.gnu.org (`UPSTREAM`) as text, and an address for problem reports. `INDEX_FILES` is `index.html index.htm`. In a directory with one of these files, the URL of the directory serves that file, as on ftp.gnu.org. - **The canary.** `CANARY` is `aspell/dict/0index.html`. This file has plain `http://` @@ -102,7 +101,8 @@ example: ### 1. Fork it 1. Fork [katoptra/gnu](https://github.com/katoptra/gnu). -2. Change `HOST`, `BUCKET` and the e-mail address in `PAGE_FOOT` to your values. +2. Change `HOST` and `BUCKET` to your values. The engine makes the e-mail address for + problem reports from `HOST`. If that address does not receive e-mail, set `CONTACT`. 3. Keep `SOURCE`, or set it to a secondary mirror. GNU recommends a secondary mirror, to decrease the load on `ftp.gnu.org`. Use an rsync address from [GNU's mirror page](https://www.gnu.org/server/mirror.html). That mirror must have the diff --git a/Taskfile.yml b/Taskfile.yml index 0cc0fe0..ca64f21 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -8,7 +8,7 @@ # The values in this file are for this mirror only: # - SOURCE, HOST and BUCKET identify the mirror. # - CEILING_GB and LIST_FLOOR are the limits for a run. -# - INDEX and PAGE_FOOT are for the directory pages. +# - INDEX and UPSTREAM are for the directory pages. # - CANARY and FRESH_KEY are for the checks at the end of each run. The engine also sends # the FRESH_KEY file to the bucket last. # - NAME and DESC are for the menu. @@ -21,7 +21,7 @@ vars: # SOURCE is ftp.gnu.org, the primary site of GNU. A secondary mirror gets a change # approximately 2 h after ftp.gnu.org. SOURCE: rsync://ftp.gnu.org/gnu/ - HOST: gnu.katoptra.org # A fork changes this line and the address in PAGE_FOOT. + HOST: gnu.katoptra.org # A fork changes this line. # BUCKET is the name of the bucket. The objects are at its root, and their keys are GNU's # paths. BUCKET: gnu @@ -38,10 +38,12 @@ vars: # INDEX_FILES are the upstream index files, in sequence. The page key of a directory with # such a file is a copy of that file. Thus, the URL of that directory serves the file. INDEX_FILES: index.html index.htm - # PAGE_FOOT is the last line of each directory page. In GNU's mirror guidelines, text - # must be short and "strictly explanatory". The only link that is permitted is a link for - # bug reports. That link is an email address, which must receive mail. - PAGE_FOOT: 'A mirror of https://ftp.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu@katoptra.org.' + # UPSTREAM is the URL of this tree on ftp.gnu.org. The footer of each directory page gives + # it as text, with the path of the directory. In GNU's mirror guidelines, text must be + # short and "strictly explanatory". The only link that is permitted is a link for bug + # reports. The engine makes that link from HOST: gnu@katoptra.org, which must receive + # mail. + UPSTREAM: https://ftp.gnu.org/gnu/ # CANARY is a file with 8 http:// links. If the zone uses an HTML rewriter, the rewriter # can change these links. CANARY: aspell/dict/0index.html From e02606d498a3227e69404aff3e8430873daa5f03 Mon Sep 17 00:00:00 2001 From: Josh Vaughen Date: Mon, 19 Oct 2026 18:45:02 -0700 Subject: [PATCH 2/2] chore(ci): lib v2.8.0 --- .github/workflows/check.yml | 2 +- .github/workflows/sync.yml | 2 +- render.txt | 94 ++++++++++++++++++++++++++++--------- 3 files changed, 73 insertions(+), 25 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index cfbf399..5a165fe 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -3,4 +3,4 @@ on: {pull_request: {}} permissions: {contents: read} jobs: check: - uses: katoptra/lib/.github/workflows/check.yml@e18ef7d810f0a239a7f6908f55060d8e0f40ce2c # v2.6.1 + uses: katoptra/lib/.github/workflows/check.yml@fbe4a0d42066b9f50a2ce5a4938889dd2c52d0f7 # v2.8.0 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index ead4372..b6dc79c 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -14,6 +14,6 @@ permissions: concurrency: {group: sync, cancel-in-progress: false} jobs: sync: - uses: katoptra/lib/.github/workflows/sync.yml@e18ef7d810f0a239a7f6908f55060d8e0f40ce2c # v2.6.1 + uses: katoptra/lib/.github/workflows/sync.yml@fbe4a0d42066b9f50a2ce5a4938889dd2c52d0f7 # v2.8.0 with: {vars: '${{ inputs.vars }}'} secrets: inherit diff --git a/render.txt b/render.txt index 9a75360..41a1fa0 100644 --- a/render.txt +++ b/render.txt @@ -23,7 +23,7 @@ echo "due: RECONCILE=$r RECONCILE_HOURS=$h, last reconcile $ago: $(test -f /work task: [list] rm -f /work/.run/publish.txt /work/.run/labeled.txt /work/.run/orphans.txt /work/.run/unlanded.txt /work/.run/fresh.txt task: [list] mkdir -p /work/.run /work/staging task: [retry] for i in 1 2 3 4 5; do - rc=0; ( rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://ftp.gnu.org/gnu/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. + rc=0; ( TZ=UTC rsync --timeout=300 --contimeout=60 --no-h -rL --list-only rsync://ftp.gnu.org/gnu/ > /work/.run/listing.txt ) || rc=$? # subshell: an exit in CMD cannot stop the loop. ||: the errexit of Task cannot stop it. case $rc in 0|23|24) exit 0;; 5|10|12|30|35) ;; *) exit $rc;; esac test $i -lt 5 || break s=$((15 * 2 ** i + RANDOM % (2 * 15 + 1))); echo "rsync exit $rc, retry $i in ${s}s" >&2; sleep $s @@ -35,6 +35,10 @@ task: [normalise] awk ' size = substr($0, 11, RSTART - 11); gsub(/[ ,]/, "", size) print path "\t" size + 0 "\t" mtime }' /work/.run/listing.txt \ | LC_ALL=C sort > /work/.run/upstream.txt +task: [normalise] awk ' + /^d/ && match($0, / [0-9][0-9][0-9][0-9]\/[0-9][0-9]\/[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9] /) { + path = substr($0, RSTART + RLENGTH); if (path != ".") print path "/" }' /work/.run/listing.txt \ +| LC_ALL=C sort > /work/.run/upstream-dirs.txt task: [normalise] test "$(wc -l < /work/.run/upstream.txt)" -gt 40000 task: [state] mkdir -p /work/.run task: [pull] rc=0; aws s3 ls s3://gnu/.state/applied.txt.xz > /dev/null || rc=$? @@ -256,8 +260,8 @@ task: [push] xz -T0 -c /work/.run/applied.new > /work/.run/applied.new.xz task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/.run/applied.new.xz s3://gnu/.state/applied.txt.xz task: [rebuild] mv /work/.run/applied.new /work/.run/applied.txt task: [rebuild] rm -f /work/.run/rebuild-now -task: [reconcile] awk -F'\t' '!("" in s) { s[""] = 1; print "" } - { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/.run/index-keys.txt +task: [reconcile] cat /work/.run/applied.txt /work/.run/upstream-dirs.txt | awk -F'\t' '!("" in s) { s[""] = 1; print "" } + { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | awk '$0 != "" { print substr($0, 1, length($0) - 1) }' | LC_ALL=C sort > /work/.run/index-keys.txt task: [reconcile] cut -f1 /work/.run/bucket.txt | grep -vE '^\.state/' | awk -v i="/katoptra.org.directory.index.html" '{ s = "/" $0 } substr(s, length(s) - length(i) + 1) != i' | LC_ALL=C comm -23 - /work/.run/index-keys.txt | LC_ALL=C comm -23 - /work/.run/paths.txt > /work/.run/orphans.txt task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/orphans.txt /work/.run/del- task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") @@ -280,11 +284,15 @@ case $rc in esac task: [pages] mkdir -p /work/staging /work/.run/slash && find /work/staging /work/.run/slash -mindepth 1 -delete && : > /work/.run/index-real.txt -task: [pages] LC_ALL=C comm -3 /work/.run/indexed.txt /work/.run/applied.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }' | awk -F'\t' '!("" in s) { s[""] = 1; print "" } +task: [pages] { cat /work/.run/applied.txt; awk '{ print $0 "\t\t" }' /work/.run/upstream-dirs.txt; } | LC_ALL=C sort > /work/.run/page-state.txt +task: [pages] printf '%s\n' 2 katoptra.org.directory.index.html 'index.html index.htm' '' https://ftp.gnu.org/gnu/ '' gnu.katoptra.org | sha256sum | awk '{ print "#pages " substr($1, 1, 16) }' > /work/.run/index-stamp.txt +task: [pages] awk 'NR == 1 && /^#pages / { next } { print }' /work/.run/indexed.txt > /work/.run/index-old.txt +task: [pages] { LC_ALL=C comm -3 /work/.run/index-old.txt /work/.run/page-state.txt | awk -F'\t' '{ print ($1 == "" ? $2 : $1) }'; head -1 /work/.run/indexed.txt | cmp -s - /work/.run/index-stamp.txt || cat /work/.run/page-state.txt; } | awk -F'\t' '!("" in s) { s[""] = 1; print "" } { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' | LC_ALL=C sort > /work/.run/index-dirty.txt task: [pages] awk -F'\t' '!("" in s) { s[""] = 1; print "" } - { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/applied.txt | LC_ALL=C sort > /work/.run/index-dirs.txt -task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' > /work/.run/index-gone.txt + { n = split($1, c, "/"); d = ""; for (i = 1; i < n; i++) { d = d c[i] "/"; if (!(d in s)) { s[d] = 1; print d } } }' /work/.run/page-state.txt | LC_ALL=C sort > /work/.run/index-dirs.txt +task: [pages] cat /work/.run/index-stamp.txt /work/.run/page-state.txt > /work/.run/indexed.new +task: [pages] LC_ALL=C comm -23 /work/.run/index-dirty.txt /work/.run/index-dirs.txt | awk -v i="katoptra.org.directory.index.html" '{ print $0 i } $0 != "" { print substr($0, 1, length($0) - 1) }' | awk -F'\t' 'NR == FNR { f[$1] = 1; next } !($0 in f)' /work/.run/page-state.txt - > /work/.run/index-gone.txt task: [pages] LC_ALL=C comm -12 /work/.run/index-dirty.txt /work/.run/index-dirs.txt > /work/.run/index-make.txt task: [pages] test ! -s /work/.run/index-make.txt || awk -v s="/work/staging/" '{ print s $0 }' /work/.run/index-make.txt | tr '\n' '\0' | xargs -0 mkdir -p task: [pages] test ! -s /work/.run/index-make.txt || awk -v r="/work/.run/slash/" '$0 != "" { n = split($0, c, "/"); d = "" @@ -293,27 +301,66 @@ task: [pages] test ! -s /work/.run/index-make.txt || awk -v r="/work/.run/slash/ task: [pages] test ! -s /work/.run/index-make.txt || awk -F'\t' -v OFS='\t' 'NR == FNR { want[$0] = 1; next } { n = split($1, c, "/"); d = "" for (i = 1; i < n; i++) { if ((d in want) && !((d c[i]) in seen)) { seen[d c[i]] = 1; print d, "d", c[i], "", "" }; d = d c[i] "/" } - if (d in want) print d, "f", c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/applied.txt \ -| LC_ALL=C sort | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v F='A mirror of https://ftp.gnu.org/gnu/, updated twice a day. Sizes in bytes, times as upstream lists them. Report a problem to gnu@katoptra.org.' \ - -v N='index.html index.htm' -v X="/work/.run/index-real.txt" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } + if (d in want) print d, (c[n] == "" ? "e" : "f"), c[n], $2, $3 }' /work/.run/index-make.txt /work/.run/page-state.txt \ +| LC_ALL=C sort -t "$(printf '\t')" -k1,1 -k2,2 -k3,3V | awk -F'\t' -v S="/work/staging/" -v R="/work/.run/slash/" -v I="katoptra.org.directory.index.html" -v F='' \ + -v N='index.html index.htm' -v X="/work/.run/index-real.txt" -v U=https://ftp.gnu.org/gnu/ \ + -v C='' -v H="gnu.katoptra.org" 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } - BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i } + BEGIN { n = split(N, nm, " "); for (i = n; i >= 1; i--) rank[nm[i]] = i + if (C == "" && (j = index(H, ".")) > 1) C = substr(H, 1, j - 1) "@" substr(H, j + 1) + css = ":root{color-scheme:light dark;--bg:#f6f5f2;--fg:#1c1c28;--mut:#6e6e7e;--line:#e2e0da;--hov:#edebe5;--a:#3b45d9}" + css = css "@media (prefers-color-scheme:dark){:root{--bg:#121218;--fg:#f0f0f5;--mut:#8b8b9c;--line:#262630;--hov:#1a1a22;--a:#8b93ff}}" + css = css "body{margin:0 auto;max-width:60rem;padding:2rem 1rem 3rem;background:var(--bg);color:var(--fg);font:15px/1.45 system-ui,-apple-system,sans-serif}" + css = css "h1{font-size:1.35rem;font-weight:650;margin:0;overflow-wrap:anywhere}p{color:var(--mut);margin:.2rem 0 1.25rem;font-size:.9rem}" + css = css "table{width:100%;border-collapse:collapse;font:13.5px/1.4 ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}" + css = css "th{font:600 .7rem/1 system-ui,sans-serif;letter-spacing:.06em;text-transform:uppercase;color:var(--mut);text-align:left;padding:0 .5rem .6rem;white-space:nowrap}" + css = css "td{padding:.4rem .5rem;border-top:1px solid var(--line);vertical-align:baseline}td:first-child{overflow-wrap:anywhere;width:100%}" + css = css "td+td{white-space:nowrap;color:var(--mut);font-variant-numeric:tabular-nums}th:last-child,td:last-child{text-align:right}" + css = css "tr:hover td{background:var(--hov)}a{color:var(--a);text-decoration:none}a:hover{text-decoration:underline}.s a{color:var(--mut)}" + css = css ".k{font-size:.72rem;margin-left:.6rem;padding:.05rem .4rem;border:1px solid var(--line);border-radius:.3rem;color:var(--mut)}" + css = css ".k:hover{color:var(--a);border-color:var(--a);text-decoration:none}" + css = css "footer{margin-top:2rem;padding-top:1rem;border-top:1px solid var(--line);color:var(--mut);font-size:.85rem;overflow-wrap:anywhere}" + css = css "footer a{color:inherit;text-decoration:underline}@media (max-width:36rem){thead{display:none}" + css = css "tr{display:grid;grid-template-columns:1fr auto;border-top:1px solid var(--line)}td{border:0;padding:.1rem .5rem}" + css = css "td:first-child{grid-column:1/-1;padding-top:.45rem}td+td{padding-bottom:.45rem;font-size:.8rem}}" } function esc(s) { gsub(/&/, "\\&", s); gsub(//, "\\>", s); gsub(/"/, "\\"", s); return s } - function open(d) { cur = d; live = 1; real = "" - p = "Index of /" esc(d) "\n" - p = p "\n" - p = p "\n" - p = p "

Index of /" esc(d) "

\n" - if (d != "") p = p "\n" } - function shut( f, b, k, e) { p = p "
NameLast modifiedSize
../
" (F == "" ? "" : "

" sprintf(F, enc(cur)) "

") "\n" - if (real == "") { f = S cur I; printf("%s", p) > f; close(f) } else print cur real "\t" cur I > X - if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/") - f = R k "/" b; printf("%s", p) > f; close(f) } } + function grp(n, s, o) { s = sprintf("%.0f", n); o = "" + while (length(s) > 3) { o = "," substr(s, length(s) - 2) o; s = substr(s, 1, length(s) - 3) } return s o } + function iec(n, u, i) { if (n < 1024) return sprintf("%.0f B", n) + split("KiB MiB GiB TiB", u, " "); n /= 1024; i = 1; while (n >= 1024 && i < 4) { n /= 1024; i++ } + return sprintf(n < 10 ? "%.1f %s" : "%.0f %s", n, u[i]) } + function ymd(t) { gsub(/\//, "-", t); return substr(t, 1, 16) } + function foot( t) { if (F != "") return "
" sprintf(F, enc(cur)) "
" + t = (U == "" ? "" : "Mirror of " esc(U) enc(cur) ".") + if (C != "") t = t (t == "" ? "" : " ") "Report a problem to " esc(C) "." + return (t == "" ? "" : "
" t "
") } + function open(d) { cur = d; live = 1; real = ""; nd = 0; nfl = 0; tot = 0; split("", has) } + function w(s) { if (real == "") printf("%s", s) > fa; if (cur != "") printf("%s", s) > fb } + function shut( i, r, x, c, e, k, b) { split("", fold); split("", links) + for (i = 1; i <= nfl; i++) { r = fname[i]; x = "" + while (match(r, /\.(sig|asc|sign|sha256|sha512)$/)) { r = substr(r, 1, RSTART - 1); if (r in has) x = r } + if (x != "") { fold[i] = 1 + links[x] = links[x] "" esc(substr(fname[i], length(x) + 1)) "" } } + c = (nd ? grp(nd) (nd == 1 ? " directory" : " directories") : "") + if (nfl) c = c (c == "" ? "" : ", ") grp(nfl) (nfl == 1 ? " file" : " files") ", " iec(tot) + if (c == "") c = "This directory is empty." + fa = S cur I; if (real != "") print cur real "\t" cur I > X + if (cur != "") { b = substr(cur, 1, length(cur) - 1); k = split(b, e, "/"); fb = R k "/" b } + w("\nIndex of /" esc(cur) "\n") + w("\n\n

Index of /" esc(cur) "

" c "

\n") + w("\n") + if (cur != "") w("\n") + for (i = 1; i <= nd; i++) w("\n") + for (i = 1; i <= nfl; i++) if (!(i in fold)) + w("\n") + w("
NameModified (UTC)Size
../
" esc(dname[i]) "/
" esc(fname[i]) "" links[fname[i]] "" ymd(ftime[i]) "" iec(fsize[i]) "
" foot() "\n") + if (real == "") close(fa); if (cur != "") close(fb) } !live || $1 != cur { if (live) shut(); open($1) } - $2 == "d" { p = p "" esc($3) "/\n"; next } + $2 == "e" { next } + $2 == "d" { dname[++nd] = $3; next } ($3 in rank) && (real == "" || rank[$3] < rank[real]) { real = $3 } - { p = p "" esc($3) "" $5 "" $4 "\n" } + { fname[++nfl] = $3; fsize[nfl] = $4; ftime[nfl] = $5; has[$3] = 1; tot += $4 } END { if (live) shut() }' task: [index] test ! -s /work/.run/index-make.txt || aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' /work/staging/ s3://gnu/ task: [index] for d in /work/.run/slash/*/; do test -d "$d" || continue; aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 --recursive --content-type 'text/html; charset=utf-8' "$d" s3://gnu/; done @@ -322,7 +369,8 @@ task: [remove] rm -f /work/.run/del-*; split -l 1000 -a 4 /work/.run/index-gone. task: [remove] for f in /work/.run/del-*; do awk 'BEGIN { printf "{\"Objects\":[" } { gsub(/\\/, "\\\\"); gsub(/"/, "\\\"") printf "%s{\"Key\":\"%s\"}", (NR > 1 ? "," : ""), $0 } END { print "],\"Quiet\":true}" }' "$f" \ > "$f.json" && aws s3api delete-objects --bucket gnu --delete "file://$f.json" | tee -a /work/.run/publish.txt | { test "$(grep -c Errors)" = 0; }; done -task: [index] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 s3://gnu/.state/applied.txt.xz s3://gnu/.state/indexed.txt.xz +task: [push] xz -T0 -c /work/.run/indexed.new > /work/.run/indexed.new.xz +task: [push] aws s3 cp --no-progress --cli-connect-timeout 60 --cli-read-timeout 300 /work/.run/indexed.new.xz s3://gnu/.state/indexed.txt.xz task: [smoke] { awk -F'\t' '!seen[$2]++' /work/.run/labeled.txt 2>/dev/null; } | while IFS="$(printf '\t')" read -r k t; do u=$(printf '%s\n' "$k" | awk 'BEGIN { for (i = 0; i < 256; i++) ord[sprintf("%c", i)] = i } function enc(s, out, j, c) { out = ""; for (j = 1; j <= length(s); j++) { c = substr(s, j, 1) out = out ((c ~ /[A-Za-z0-9\/._~-]/) ? c : sprintf("%%%02X", ord[c])) } return out } { print $0 "\t" enc($0) }' | cut -f2); want=$(awk -F'\t' -v k="$k" '$1 == k { print $2 }' /work/.run/upstream.txt); h=$(curl -fsS --connect-timeout 15 --max-time 60 --retry 6 --retry-connrefused --retry-max-time 600 -r 0-0 -D - -o /dev/null "https://gnu.katoptra.org/$u"); got=$(printf '%s\n' "$h" | awk 'tolower($1) == "content-range:" { split($3, a, "/"); r = a[2] }