From 8e6a76adb2599c91bd66a4b133bbaa4f92133663 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:43:16 -0700 Subject: [PATCH 01/11] chore(cli): bundle teploy v0.1.55 and pin corpus rev 21 Adds the backup-preflight, backup-head and backup-detach envelopes to the validator's expectation table. MANIFEST rev 21: "Dash simple backup setup". Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 2 +- Dockerfile.goreleaser | 2 +- ci/contracts.pin | 4 ++-- ci/validate-contracts.mjs | 5 +++++ 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f56e4ee..7f03e2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ RUN apk add --no-cache ca-certificates tzdata wget tini git rsync openssh-client # source build uses, so the image running in production was the one NOT # checking what it executed. ARG TARGETARCH -ARG TEPLOY_VERSION=v0.1.54 +ARG TEPLOY_VERSION=v0.1.55 RUN set -eux; \ archive="teploy_linux_${TARGETARCH:-amd64}.tar.gz"; \ release_url="https://github.com/useteploy/teploy-cli/releases/download/${TEPLOY_VERSION}"; \ diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index 2d64d98..16400cf 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -9,7 +9,7 @@ RUN apk add --no-cache ca-certificates tzdata wget tini git rsync openssh-client # Teploy CLI - used by dash to delegate deploy/rollback actions. BuildKit sets # TARGETARCH from the goreleaser build platform (amd64 or arm64). ARG TARGETARCH -ARG TEPLOY_VERSION=v0.1.54 +ARG TEPLOY_VERSION=v0.1.55 RUN set -eux; \ archive="teploy_linux_${TARGETARCH}.tar.gz"; \ release_url="https://github.com/useteploy/teploy-cli/releases/download/${TEPLOY_VERSION}"; \ diff --git a/ci/contracts.pin b/ci/contracts.pin index 60e4088..5a7c931 100644 --- a/ci/contracts.pin +++ b/ci/contracts.pin @@ -3,5 +3,5 @@ # this file records WHICH revision dash CI proves it can decode. Bump both # lines together, quoting the MANIFEST's revision table, in the same change # that updates .github/workflows/ci.yml expectations. -cli_rev=59e37d6ba9fe67708f9415ba73fad3a3abf1c526 -corpus_rev=20 +cli_rev=d4c9bb3736871edcbcc936ec04a4558bfe6dfc7d +corpus_rev=21 diff --git a/ci/validate-contracts.mjs b/ci/validate-contracts.mjs index 97cbd09..1471826 100644 --- a/ci/validate-contracts.mjs +++ b/ci/validate-contracts.mjs @@ -87,6 +87,11 @@ const artifacts = { "backup-status-envelope": { schema: "backup-status-envelope.schema.json", classes: { valid: "validate" } }, "backup-run-envelope": { schema: "backup-run-envelope.schema.json", classes: { valid: "validate" } }, "backup-policy-envelope": { schema: "backup-policy-envelope.schema.json", classes: { valid: "validate" } }, + // Simple backup setup (corpus rev 21): preflight, the ranged read the browser + // proves a recovery key against, and the detached run. + "backup-preflight-envelope": { schema: "backup-preflight-envelope.schema.json", classes: { valid: "validate" } }, + "backup-head-envelope": { schema: "backup-head-envelope.schema.json", classes: { valid: "validate" } }, + "backup-detach-envelope": { schema: "backup-detach-envelope.schema.json", classes: { valid: "validate" } }, // Git-source artifacts (corpus rev 16, plan D4): the connected-source // show envelope, credential provisioning receipts and the poll results. // Dash adopts these surfaces separately (D4); classified here so the From 153c4e8ffdf6e3c09c79e6eebf41ee5ec83d32ff Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:43:19 -0700 Subject: [PATCH 02/11] feat(frontend): vendor age-encryption 0.3.1 for browser recovery keys One IIFE exposing generate, derivePublic, decryptStart and parseIdentityFile, with forceWebCryptoOff(true) because Dash runs on plain HTTP where crypto.subtle is undefined. Tested with subtle deleted against files from the native age CLI. scripts/vendor-age/build.sh reproduces the bundle. Co-Authored-By: Claude Opus 5.5 --- .../frontend-test/age_vendor.test.mjs | 73 +++++++++++++++++ .../frontend-test/testdata/age/prefix-70k.age | Bin 0 -> 71680 bytes .../frontend-test/testdata/age/test-key.txt | 3 + .../frontend-test/testdata/age/whole.age | 6 ++ .../frontend/js/age.vendor.LICENSE | 27 +++++++ .../frontend/js/age.vendor.README.txt | 7 ++ cmd/teploy-dash/frontend/js/age.vendor.js | 1 + scripts/vendor-age/build.sh | 17 ++++ scripts/vendor-age/entry.mjs | 75 ++++++++++++++++++ 9 files changed, 209 insertions(+) create mode 100644 cmd/teploy-dash/frontend-test/age_vendor.test.mjs create mode 100644 cmd/teploy-dash/frontend-test/testdata/age/prefix-70k.age create mode 100644 cmd/teploy-dash/frontend-test/testdata/age/test-key.txt create mode 100644 cmd/teploy-dash/frontend-test/testdata/age/whole.age create mode 100644 cmd/teploy-dash/frontend/js/age.vendor.LICENSE create mode 100644 cmd/teploy-dash/frontend/js/age.vendor.README.txt create mode 100644 cmd/teploy-dash/frontend/js/age.vendor.js create mode 100644 scripts/vendor-age/build.sh create mode 100644 scripts/vendor-age/entry.mjs diff --git a/cmd/teploy-dash/frontend-test/age_vendor.test.mjs b/cmd/teploy-dash/frontend-test/age_vendor.test.mjs new file mode 100644 index 0000000..2e5b067 --- /dev/null +++ b/cmd/teploy-dash/frontend-test/age_vendor.test.mjs @@ -0,0 +1,73 @@ +// The vendored browser crypto, on what Dash really runs on: plain HTTP, where +// crypto.subtle is undefined. Fixtures come from the native age CLI, so this also +// proves the two agree. testdata/age/test-key.txt exists only for this test. +import { test, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; + +const here = dirname(fileURLToPath(import.meta.url)); +const fixture = name => new Uint8Array(readFileSync(join(here, 'testdata/age', name))); +const keyFile = readFileSync(join(here, 'testdata/age/test-key.txt'), 'utf8'); +let age; + +before(() => { + Object.defineProperty(globalThis.crypto, 'subtle', { value: undefined, configurable: true }); + assert.equal(globalThis.crypto.subtle, undefined); + new Function(readFileSync(join(here, '../frontend/js/age.vendor.js'), 'utf8'))(); + age = globalThis.TeployAge; +}); + +test('generates a key and derives the same public half without crypto.subtle', async () => { + const { identity, publicKey } = await age.generate(); + assert.match(identity, /^AGE-SECRET-KEY-1[0-9A-Z]+$/); + assert.match(publicKey, /^age1[0-9a-z]{58}$/); + assert.equal(await age.derivePublic(identity), publicKey); +}); + +test('the file parser skips comment lines and reads the notes', () => { + const text = '# Teploy backup recovery key.\n\n# bucket: my-bucket\n# endpoint: https://s3.us-west-004.backblazeb2.com\n# base: teploy\nAGE-SECRET-KEY-1ABC\n'; + const parsed = age.parseIdentityFile(text); + assert.equal(parsed.identity, 'AGE-SECRET-KEY-1ABC'); + assert.equal(parsed.notes.bucket, 'my-bucket'); + assert.equal(parsed.notes.base, 'teploy'); + assert.equal(age.parseIdentityFile('\r\n# c\r\nAGE-SECRET-KEY-1ABC\r\n').identity, 'AGE-SECRET-KEY-1ABC'); +}); + +test('the file parser refuses what is not one key', () => { + assert.throws(() => age.parseIdentityFile('# only comments\n'), /no recovery key/); + assert.throws(() => age.parseIdentityFile('AGE-SECRET-KEY-1ABC\nAGE-SECRET-KEY-1DEF\n'), /more than one/); + assert.throws(() => age.parseIdentityFile('hello\n'), /does not look like/); +}); + +test('reads a real age-keygen file and matches its public key', async () => { + const { identity } = age.parseIdentityFile(keyFile); + const expected = /public key: (age1[0-9a-z]+)/.exec(keyFile)[1]; + assert.equal(await age.derivePublic(identity), expected); +}); + +test('decrypts a whole file from the native age CLI', async () => { + const { identity } = age.parseIdentityFile(keyFile); + const out = await age.decryptStart(identity, fixture('whole.age')); + assert.equal(out.length, 'hello backups\n'.length); + assert.equal(out.truncated, false); +}); + +test('a truncated prefix gives its first chunk: 65536 bytes is the proof', async () => { + const { identity } = age.parseIdentityFile(keyFile); + const out = await age.decryptStart(identity, fixture('prefix-70k.age')); + assert.equal(out.length, 65536); + assert.equal(out.truncated, true); +}); + +test('a wrong key is refused before any byte comes out', async () => { + const { identity } = await age.generate(); + await assert.rejects(() => age.decryptStart(identity, fixture('prefix-70k.age'))); + await assert.rejects(() => age.decryptStart(identity, fixture('whole.age'))); +}); + +test('a prefix cut before the first chunk ends proves nothing', async () => { + const { identity } = age.parseIdentityFile(keyFile); + await assert.rejects(() => age.decryptStart(identity, fixture('prefix-70k.age').slice(0, 1000))); +}); diff --git a/cmd/teploy-dash/frontend-test/testdata/age/prefix-70k.age b/cmd/teploy-dash/frontend-test/testdata/age/prefix-70k.age new file mode 100644 index 0000000000000000000000000000000000000000..6c12f1248e09e8cb644270da447b548d6c9b3888 GIT binary patch literal 71680 zcmV(lK=i+1XJsvAZewzJaCB*JZZ2n2a!YPfT5D%zHdi@C zYk6pRMR;dOdNo=!c12HTdNFHJa8(K|EiE8(R&g;%QY&OmFK07ZH%Vz^P)2JwK}KmS zHE1(-Id(5bOEUl!yW<%fD<2Or^rd^5{2%pWlgXXODfZ?nIe|e=Z zZbtjn2Bjd+F}(gN8`$=47^p$$a}-ezW|Vh)W@5PY^rg(v@mhwVun)QQp(OlICw+Eb ztov}eQ~s|lvm{fJbcaXR$~S6Hg~eit7b>W)oH$SEKE$UQ2fm7(M5q-H&by@Y77A5H zGP{}tp&ml^pY>f9JPF;BBY+AaA_4}(SS>uyl>!g~AA{F9xHVlEh9E8Qo?E?D)(A75 zu;93hzUg-2PSs`XtKO-{Fk$i~vb+1dyc6%KY3i+BE%^kn#oo?UuS%TTFARQ#zvegc z@u6$j|FSD+JRa5Zz2tgUo|<&Uv;k5V8?H+1a$R&@O)Ik^0kc57bh_UJWSP^K2a8if z^m248cM5={Fbwz7H zntip;ttTacqiO^@xJ}Ac78S{^t|?;7zR}I#51H(w+1H4+U*)q}kh^+U#Me!Pu2q11 z#!^FaaYAsZ)BURUQZgG}jMv!Di&ovrdz;Ez%_wtj7cZS+M5F$E*}@^O@Qn9p8kawb z^s9o=d9w%a=DufNGN`F;wfqQ?E z)UFGi-?A-0lbphP77mzS$^J)RW^*1YtyLOka=^`xt1uknRe)PC_j8;2Z2WGjLE|kQ zFigRZW=w3pxo`g(B9xj$H}f9Q;wb7R#^AT4<;gx~i5S2=sG``!%8N~~Uiz#I1!U;c zohylPtrO3+KzU^r7XXwd10$X{;Y-S3323w{YF0_K&7L2>va|(=rizY^0$gNekk2jr zfC-(5H>IuXc~E6S7HbQ@O6s+K04MJ0`1ZpW>Bf%tl?i2im{K2}Vt}>Wm{Xf~C%XD= z?aKm}&i6g;o6~yLud#IY!6ZnTqGSdTzqeXt)3klTArc#W1a_NK;vW;P*CkI4e@DiX zSao*nfcpUTEnV)ZSb#(Fx6FLV8Aik^B)6wiacX$l@X}IeGqgrCk@M&7PmY_VE!U3- zqxVdf;NDA1fCuTww&GRNUUcPh%JrylYn}T{$GtcDB{ISDKpNd+3OR9rbO`XnUM?T$ zpZo>1&pdn&BNA?UBtlE9Tm3+=183omw3dW$3#7l1GXQyLnC!4+Wpm}wN`eokaf>uUwH z4K^uy@P=j%tf|=Z7oR&Vs=Ue`8VJuKHH}+khCDEyaeaLpa)sf|JZ1NRfy(~*LI+=7 zm}36khk$(2<3b8knLaO(e=O>@ziR8Ji4m?VSYbO37_}m!=8hs-&oy47=~c@5 z9heROzP*Ru>g|>DQaFJDHa(go7o)4KA3kDhyaI6~a4}$K#1{cBiU{3h9y5*;NriWq zs0n;<**}}2wFR3BR#@St&I8TkD|t2GqwVU;|6#fDk308r|016lWZq|XC6-B^Jjm$I zUY=(EY5Q2irge}!C2l@@ATPD^eY&)-JROgm25->w7L)&74E$JzJyVKrU+7=ztBgi&9aahj7jl@jTxGtZz zkPbdbQK=U#6~v;r&IDH6 zdIOZ3us$jU1AB?pV6KmPVKI-8!q1#t-tocGT{P8noZU^OqQj2d-IVz}DNOw&^#CRA z=qOr|kW0eq;jd(sOw%vmyHyDcXwzyj;Q94N^Y>(oq(J1#UNE=l4+*j3p78pdnA;KA zxqhI(C-)MvF~qrIIm5+hPQq<^-^ptVij*YM6Z=PGE`${7_y%NNnYGCYJ2)7C|2mQ( ztAMKjxuTIcXpd0VW&fGQ>hF$=-do{NG~X{v1=x6OP0bImPx^$r85H#lDf+2aw4ZI;9C*?NVC-OILxI@@M9q{Dh;YOg24h5=M3^7{6DfXcVhq@<^u3X4KTFO zJs4l9O~c@t%urv#EM&iZ5F9O(;2cSRs`PomLoqIhM6s@zJ)aPKua1!N%%sns-_b1K z^;YM>>zxtS09?=MWSH^&3!;#u@c6ZBLV-;T&gEn7D+M%rTN(~`_9)UmP+*jhWw?0e zh6E=UV|nXrzhf#%DQFXHibxwO1oZe#adTak2VRw8tF2kslR$e?F|P-;A{z@$w;BM_ z_RIBT{x(-eLDP*6?b8!9PFg;!;6FN!|7s1_m9p6Iqm{I1b!*wsf8*xphps-nV%U30 znEWJ~@4TcYBWS8Zt``N<^$+@-=G-oBS4w?!P4Dn2H z?0@+4S|aSl6}4x$QA=zYmLJ?tC=&XR>w~kuU710bR80jtI%Ha+?0lrBe2X49fW;Bo zSPD-2A4Lv{?-b}W586Fa;Ju?5^hDkXCewE}abcoE*D|}Qs8L{U94J4}syKFEzaFq( zE+l+Cwtjtp&@&zzu%s%L91O^bqq-xUKRk*}G}oiYU3gSC7&p4Ssz>*r6aT|t$O|N% zi-R{6@e-;);|})vTE>7(E_gPM0KIo+-{>11Z3zgl53Ahmmty2&A*C~rw?w+X;)YQZ z3d7U)o>&+VP#+C9gz$}{^SnSN;w9od%Y=d_&1=VJOKd`a!ObvMh49I_^B22y2EY#N zv>`{6$#Si~Rdkprg-K-qgiwf z4fqU{?et6JJ?w2mZ^&MtkRvfRr@YMI;uk6PL)P{FQ;vEzi!I4@dyz-#3`O7`v>fYy zd1(?Q+e4hlh5UOMprY|J<<>Xc{L-Gsq;kCpSre6Y1D!843n01LfEneVY;#dqXX!~a^o|~dhav4>k*ekfrB_q~>T7%zjOBWyzRJo*bxhv=C z{rHC4qC6*xClr3BL+Jc~-388Oa|BSI%?(<5$)T}UE!vj3G3^Pq!L`%~8j!e4`dfJF z7-DjTTBdTk{DGl!lfmlx+InzRAUNOGyTo{63#I8C%!Lt3s$gdQ* zHtl-DH6hL-DNc3+n6gIo4CXJX2`E=clq=N`p6t(0>Qpt3Z>AgzTu?2KFLeHxNL#9i z)7L}Y$_d0-Y1S9I8tL}--;=D-x+P+kOvjS(WO?RwMI7lq)I<&qc~&rnIX|Z_D{kEp zyoe3x#Pk^PD>#>ZO~qMAs0B~dU4ZCU`a+^F2(^1<+~|^06+Z^4rwO*l@!RD)IiGed zPG9ps-Fm@V;P}TF2keT{m#-9T0@lQR;#IxvCjy~>M5(%xN@w7V>~B{fUH~24lMO9o_^Goz6h1>=nnAoO99fbIpiOxid-E8gl$cZ-nbg&jp(Ui^vV*+qW{UrN4(RA zze;xOyF|y@_FG?{4hiEd z8l8u8pwyK4hLyOn7P`ZzAqdJAlGU7+y+vfjqY!WKvsvH?(i)Zn83n@X8Kf+nxfYU$j;$?svWrGJ-V1KQ4e>r4Ov^ zG~hxGa5Up6O`xsEi&SZkmNtQP3i-;5}*bk^{Xc0PZ48 z?;D={DsdsOwP3xFb~$eL+icGl6 z1D-nHdEMGn8)SAN-1uk zRL49jgYFu*YAYiy13PZffDb)`xqE4rbA9noMBI^45e%V@@7T9{kN@HoAkw;-na4M= zn{j6Y@O)PuWJ0TN4ifPO9nGLka@UDJAo20H0qhVg5j2vFN*`f?tgd@BDRdiqCE~^I z>W5=6=8y-3O}0aU$${b?D0SalV~Y*I4r*S1qbmd>N*`cseDsqDVspe})}3v8B~g7Q zjnnZ~CS`LK`5kK@&>zOkv?fm?bMb^{7+mMk(?6L3C91S*%Q%>6HDJr0_ar_7r9Jh(SOf zz)(IoqMUY=5J3>jKUT0;clgxoaf=7Ex@LM2D3(4&gKD*t)CD#l)RTu-NINrl09ef0 zcG=QqFg3PkR{bwJWpmX4X(qZeX-*%J*H}ADAUdL1S#a zg_dPf0PQL@6q%QzJbFPWj>4?I;R2PX4#)!aCP21?DySpVj)0@EVM11f7Wzv&um9ZR zEjAaqeG_|#u!%b5zoY9HCq17LJs{)82RQJyq{BCUKTAksqj8ep3AjDv2IdD9OZJJ~ z4ra~2F&P_>15{hjfY{<^@@#ijiIY`ok8#$R>P@N*(BUW5l*a1 zBcu%zE*0M7DtZ%b#%g6bPz z0xQuX_fz}hrQhIfoBqv!r`Hc06aTQV^2D0?9n^0T`9b8#ZQbGDt3Vo>q(~eNC1PfJ zUX_Jdx;jxY3pt4{&LM~U8?KIAsrFT%x#GY6zH;^peSDM(&f+4s+Z9&oG3!H65!F+6oUQ z9oi?i+5CpAcN3TrDTeHmu{Hj(YVXF_yvY}Ld|YpylCMQGIrCY#KQ3{dA4Iu(sL#Z$ z^f|ncZAT}cGaz&P0qW5`*qx@1IsxNGHHD4EDs^T#Ra_$Ct4-~S#1v=RZa?9Zgpj)5 z13Ii8?m|`gN8jA5#kYAoG<{dQEEnj<@f zaqt55_z8_%1GRuCkic^m7ZGw8QzA4u3t)zs;BMMCMRnQJT+|d# zs9;=BgL_b2-#nJ@@MaVQkqZ64bIf2Kb(C}5DnC8w?k(R^7_BL~cdSIDg8?kI*n2R< zBAN<6eL==pAD1^%^y$b+N<}MoB$FJMNgLB7Du<~gEl?Y~7y>IhVNi{3kG3< zD<~o$_dP-!Q7zgpz`{+rY+QXaaP3{a%Fhs5aY*rFCx*xezdHxaAZ}F1kjD<1ejN16 zgcsC*X}_#oL|qQVNhu}i&PhD3&wB@j;P7_S2CjtOsaTEI--4laJ>z1`0B2PKg!)fy z=AL5a9jks1#F!LU&JxIa+LF12q)9t>0*DE2+oh*YBvH-oDDQGz+{N?ZKmV(`i-#;h z(zvmP=l!l!yRJkI;vuV%E;kAojrTVR20^3AS2fHF+s*mPK3ZseJZyN|+voKq7=y37 z)M@%Y^-sbH*bA*wxa(|Z@-TEf3SAH`Q0*I?zv8VBe|MI9->vdOIe@{Vj99pUZU;w5 z5HPIhQmYR9`W~>LdY{c$*m7}q1i_&!amjjZOJ2yz)5ffuY) zqyDeHkG(;g6`+-pbo#Xn?Zu=ae339WYx1Ev#Gfh7_q74I+r3&FW(G^qIe zzDM2bm+l&e&XNKg+j5@KF1T7e%&qS!)bCnM)!a8&+S*P&YU$zP=28MMu#051f1Ub3 z@)+}(XE7!WeyLsB#wr0^D=CTBUccB8oWlj!ClFg^wh?`9nd&~XcdiUUyKv@VieHr6 zAuJ#u3-FNiQEj!b#W$UMM9{wWl5wgfz{qEynmmyViUuMm_cf6}9!2BOdxDhc6X{{Ps%$o=N+~L4KAdP7_vhmZ z%pW9pPPWBHNb(T!m#23$@yW4*f9T|JGKuzl1$`TBMf3(9_ydGre+)go9(v_K+3GziSx&FN`~g8ri@?4Ih=YB z^G2Bc<>ZBXxZkry`obIgwNV!Og7VeYbjl$Nga-ZHdDr68qh&i>>lV{rI>$CsU8cZ4l=Sac~a9|T*wz?JJ}Z{;D%z~Vbd|Nr@to**mgTD;YGmb`cJrq2ql zcmS|D*jDk58bQifsNas5z;+mVBQ5R1U-O&G;R^Z| zF5$)J5|Z-06BZ(?AkWqhs}( z26y%e#zRa*+WqyCVH0c(+|}ydBd~|thg3m&P;1_%f;}x3D0&CqUZ)hm=24(Z1*>h$ zC;!bO4Ed+HBZ$+74AY{~K_S)0fR>)ZtVG+oy1BLq)Q8g%7RPR$V#R75~BNH&$$ z8WvHOldbl5&s)ca+$y%xk^TId3V*psiph6|v3XN6jGEadF!+v`p_D`9d*Tq3cjVih zuZ-|FlSoe|cKhL-C4iBR2_k9;a%LZYvAR*09K5Y3>C{|@Eug1&2XB<3Yi*vnbC#qI z_ZNS?@%Yl68>6jo#O}aP9!S16Ri^DL=^{=Jz(@*d=-V zb|IiinXGZ#dk|#0mr-1JUm*~Ow^cgKr)7eF^dl||W0b7^ak#G4lMd=0n8lvZ0MuY^ z3)W(e$H7J&$E85SqiqC)N}NVKT@8w1DnXxGg6i%I`WIW;|bt zSlJqyK0C72JMV)eNxfUZFodj;E(SWbC0EE0Tw*~k``yqzpGwS0R@dfufg+Mo*0BRO+C%WtD}zk+TaF&Rciz6;SpnBiUlOj=S=AHKCS3lPlo3n!%+z-D z9*MW2H`0kw+S!<;5T_5t0XeQ=Fb#mId-e~S=F(DqvTI+n&*R{D@JAG6vw z$Zkh@%I6#ODZy;SvYcn>WZQ$?$P&%upa#;Pzy%JnxxS+ZNHDrtUn$-4?zT{8dDm@h z`8ujjD)EzPCP0cB1R_G$)2Wo3Q@EiBmQt7l`P3mWDcX5gW1;;}_|8!|AJYI!?MAvl zsY4}vjGkm{48LzrIVWi4FPQW4XO({~A%Q?K2h%4mNi{^( z-n;EOi1zQet)cxoyN~`CYzb0hm})y_wZaD0*!Sv5HHjR=$Q%8FNgeCEIGJ%aq?O9t z_=3k=K>mqdj2)~9+Z0q*WQvj9|40@h`^)--_yie)ZceSB``ebOoUR{i^Rme$XM>)* zm^p>_@~MmEq9vJu3{C)F8!DrgdyD_B=6Lh<3B2D8n~x8K2~88+_ZuS?p%W%|dH&th z<+PyN<}`-#QP~?7p{cEj8u<41DDg4m^7mcyjCRKZbGzM&MH^Ypo20||zcauPkI zD3Dk&a+m4S`@JPkE_#qUsR{)(*H*(-`rO}now>{0zBOSUjy5n6PGkm$_^3Gvh`uhr z?tw@ceak&+0Erz_*;h;%yV*h5Za4QQmQqP@RP?JK4IdtYCiZ+JoP^ z5-_3?IlCF_QXZ<>aFRyXL(LUg)Y-0}aIq#O;|&-=*^kjd>wRNkRHc2pvXxJ9MbKP{ z`Z^;pihrr+EOhe@u@D-Yl=FQTmJnVsg8{F2Jt5j3j^qGZC!7_ek3~7t`Z29_ zk_%$t9vvLqNYc4+s3t~#w%+slSv3hFLp?9PS>i_>cg*a=O-;w)I2kdJ(EPUj54>vb zoQmzLfJ-;y9ZS3oxR)3@9(9L7>kKvxb$KLlsFO|l^2s+;bEt%k-jI)~*nte%?xa^c zQ)2t6QzL$)G-6;Wm9&n3&iJ&+R>9j<=4Ps@RU3D_a&yiN4UWhJc$RvEVN(o)9|Jd( zuGX%%imE)BPr}eT;>;+nH(tZ#F+myrFd?fnEGlZzQ&Ha~T^mWnxq$8^8IjfXCMZ)P zRxA7+de(o?7-0G@_4f{Oi_?3oZ?~rOPz)yV_J%V0o=?_;2O34%u4>b3b|KTYv?J=^ zE^(<)HFCnfTGb+Cr%bw480@>lp%~8NsrE?#avHCQ+Qr$hD$j`Qwd6!<*i6qmm;NSU z4!^d*ppo9DDXJu|0+!xwyI@B*%Qx!`Fx%9Z5uFisFOoWaX%?fbQqhJE>s|2a*Oi*Z zF!P;tp@EW6_+Bl#n&x;Oxl4Xa#f(~nta=@k#U6I9w8WO*r*P**-c|js;EKGL;TA#U z(f~JgHVmTgEeGqA@Gwz!g)@(My0U5W`D)fNc7rbGHTv(_EKQMdq*7yeAfX6Qc$ugU zQq_k92c(jzWKug1gVgjfsY80AKkT?4)-;|&F~L38b8xK!<1p%S1r1=)&4B<-acN^$~UbyL_S)s0`QX71CFnXkO zYj#lfL8i&w2U4}H8B7wL`aeUk&+^3 z=THJa+f(}XI0O?4+Vpr5BZurrZ zaN2L_P{9H9v;_f5;-q$ij-dxk>(`v^um0T=M9FS#!-;@6F^l6a=NPUtUrQR_BOY{iX!!JJl5^U8@>igy zPiE$46MdRqF*3H}cehWrNULRDf+nF+hUEvZRmSRdlV)J}%Z?SY`60L@zwv8XZ==^o z91k4rdk+1`d;$od;^=ZkhYImzz~AsZ#Y$VH_<%02pi_E7JPic8`5&gAbKaSgtmAp4U+DI|k}^)U#RI;({5uJOG;Qlw3P)45Em zsTK^%dEpB?t;r)R0_?peAg%q^AFzBSOnYZWi-q}36&tj0oxDP9Os^G<_6Z7mOFJT& z(ylS8&(xJ_XgWIk-{0ch!);m+baxG3eU3>*7u$+M3f*}|nLgt*3w zg`-gL^?pJt@5|AkiBgHCbnYB{(IOrhC^2(nKaP;8(`MVD<`Y14TF@?bzqsv+-*mJd znR5D|^9AwUyyom3#_5PBx~QSqPzw#)vRn``P*rB8!jKlm2b^lLROtD! ztB#c81s1fekYHct>r?L$Ko`RoacfFjszR(Rwh@J;R}*v%{Ksg%nokb11);9^y>4Y zDsyg~keX2PsM`)D*}WP;&=)aKe=9U3$3CMl-8YwDqMT5Hqw+79NI0z{clRrn)o_ZzkMc(&D2)^yHCDdNHW4`^z#*SJVrr{Zgeg)#%USdw3+<5ChK-6gn-OK(54Y^VBOI&3RhR z06}vdUP#(goHSKH&|hg=NU$0hmC?jRM}=d7rUgjH86(dO>rT1_I&l8Eg_FgtvkkUkmI zz(gT#)clDZrvW2~BvP}r*p&Ss@BWyE#vurc3;&46n`?D3h;vBE5~NFqopNd%=X_Ex-~z=U4EK zDaD?y)nVCq__x_-Kpb;WgZ!o_BO#}S1QtFaEsHTOD|RryG#6|r9Q8digCwv6Q3wzI z5F97z)KfUlOGk7pPN;|LHq6c7BRo_&ROM01p2HfhDBLW}CsIUsJtvZVc|Vk2z=$WC zqiJrH*@s&%)0hM8aqd@iFsmv$=?L5OE{3KF`?y}4c&Ic##^1C8(s**BvE7fAdqCC^ zem5PWtR8u0f1@SHTA8*3*HNC%D`j}|kyAk+wiJh>KGH9(8LT&d)`ycsiM#t4ZXsGt z>aiDirKIPJdCAjIToTJCXVhsk5?z zAT%sSwvIk{3nxkP@8R*0!^dvR+ z$9nV6f@n?&4h%~iI)Nh&{qhh=Ki0#bj%VFHFo2A_l7giL5O%NNWl53PxIKC3nNrLf ztodtvXfF}!4Atg8cy1P>=IMsf016k6kCHK7a;hMQ(rnYyunnd1XLKMr?`j$7>mYf% zGH5W$WxJV1?)Si2an2t~e*ivfD1LQY%`ftnSK!5{$=!D&4zH>r45?YYI0ABR=VmWGM=H zdq3f2W>Km1m*}!)yW;N@=^Zb16Riqxv6Vk$2C7c}P=X6P*r}MngNa#HNCRXg%)5cy zynYec($%HC38a|RUw*u=qPNqNB}~9w_et@C*G$1ZC86t0g_MTiZsv|)>6E9!x6JP#0=2x0)|*)RWw^1!}#UbZ8CJhGhu-4J-Ak2R6s$ z`YuX(n?-k?Sb%l$BQgwX0v@7e&CmB`DfS>=nFsKjvcSP2Q0h#q^P(g-yW&|HPC|mF zj%FmJY(VR`#Lg}<+~@oyj&|$I+Y5w&p=jV8*8jeyY*Xtus)fV0-YV zWDhwFHTGu#WWVhL^ds33FO)7_9|}G7CC*8MzP7lZj_oW(@lEz5p0V6T7DV(t#X>cW z;v}XWDi&;)RO|!LE>?y5^>v?iIuEneR0Xp}B_by%q*sS{dU7p(U+>u!v28T3yq%bt zF#axdX$TL`pj^PIPWcq9vmmhEaNh}7<5`}w(P)m0eie^JYHD~E-zS`!ceuGQdFDas z@zKa!E%V`~=!lu~)H*g=?7Np=!JNL=Q25x@SyZA!(^dsL2C#R>`5NKUeRPf2%!9z_ zyHRjS_ImOrWQ^grD>c0f*QRfR*R2;hFmSj1BWl%GakoBN zd%c+(J2Fo0H;dXjjH^}_645aFTL0{#(*Hc$szWEfh8N@ikvp~W1>v{0ldPRq_@7T+ zGLb7}m8w&J>?o-!YNd9Mu!@$Mst9R*t9@=%cWRU0a4m@DsL3RHm-3st6I>zGHS7ww zjqlEEmSZPGmV?rX;iMt8%P(ldPGR=Vh)0Z=WcIk@|Nk$m>LcbZX6ru;_Dp|oU1cHb z(6zYY27`F;Y~!{yS<{f)_k-QxBy7l8SXnoe<1|BU%IGUsMs3>nm2=$r6s0e+wR|!P z_!N6}&=GHxk<{aj2@1Hy1gX-i+#GrtKjY?sj+|R_TVI?+{~#MOPz+A+ zq=l86NxDEqT2+j*yoEYZSw2U8(a%8&9W|4EHR8OM@Ftqm$qjF5IUHW-rJ4wD_TEKO z5TL+>L0Z%4N;HTuAUFvfYcXbHrkiDn(_2O;UIJ*V2){d6yyEJpJ`q!<% z1FOwxw>nCbnZUbg&W(1jI$ZqMdUNDL9*GKi2#)n*EymQC)aWfzOXNIE?kVfu%IjtI z+6>z-%~!70PvaO#5h0t}T1f7TnFe(yt{+&0{Ckgd+xwffrCLe@sjP~ApkM`uf&eSS z8C_qjFs4dUjaFH}cNipJ>QzsZeOXN{e}6EvfjpdapF*KO1lg{6C(TC-kxe&bwPAht z2G*R668uxvsIXx1<)3U9pFwSF`-=T; zBk}jCruw6y*mc#mrC)LIwr^_4`MqwioYPNM48?NJfF2M;3Ls93V-QDZcq{k;a$|BY zB2E}hq~zOHjIQ_|vgrXHXvL){4EL4aAGyM^{#!aMH^Ee*J{N`s5)@;X3{;P_)+^1s zhyVgX`LRv27S{(d#zD}o?$hs!6##}}PGHLG-C3d1nM*HAoH?Jk+i@{ySWK5U>b9F} z;D?5f31=cIG|DXKFlJ!5Mxk2|rv+c_xKH~{{!gseAXZvB_ek8*(O-nb3q1jJ#9zp3 zwra_v+Nw_@IFK1#$x-!EJ~7rQ=d>h+&>rKlvdA@BZx6L`xma=Qsjy4;Uv zrT2gikD>)Vw3@IlZ*Rb=t$qe0&%B5QQQF6Q^%7nDVSe%)93VFmQSn`dfcUHiezVG9 z`S1*~0+in7*5D$-cy)C`LN5FZ#YZMp97+g6{1scY6!jb8MJ0vZ82eRVp2B z+l9;gRYE0Hr1@Z3s=;e>RDCk@YMF~gzWn&uH1K1!qqeBW2J{80V*x*r;nxAmAR^(} z-?v>6dE?~5*EF+)WB+>Dj%h=N28V)6x}hZ@)3!fst-IHykT(Lg{ecYYhvkKE7F$EK zM7!FHKQ`;43{nY3<>*n6u@tr>*dUMYzWd!#I`v%>28}ftMf!IF%f`t$KU+81g+y44 z$=Z-6+21UDBdfglINL%(-!G1M3#99-lq1G54EWPGax89u0y0O@&Qjd3&_2-V@4vW4 z5xtZ5@PY;=_&1#s`Ny`bSGmZ(t^4lOlvnUrkh!z!$ll|`H-W0ll@Fkd6&zc%xk zJpjn2L_UiH3|2c@%u1+-PIN#d8yQ(rb8#Q#aeimqi6k$_`!p+kf{nYwFAizbAu_1F z1uzueJ8yYFf!rN7EESaLwW>QBkO6$j=cvigs2!{9jy3;P>#QV{Nn&SG^(iiVP~BS; zmOT^LRm%GhlqR6ShINZ&fhhZY@gg{`#0!9gi3X{-?j?Oyjn~jET*)RiowLZ-bnhdz zgMW0MqM;XNHILxAKU??+*lakdpyhSV{yQ64?fJMYBYM-fI*dMB!rYU80ztCG{f|CiHm(iDLpB|456I1v$JuHNsT@OwX| zA3{I16`tilNCt|LxVB@3mSd{z7LxnzdT;LLx}9pkm1UYOSRu0PXLke+E)-Hl(Fvtj z15mU}9$tes%Wy(Xo)Js5oNVf4fB~}J!Iy_Z=A=$2LwoYtr7R56HX%jHneFqAsO4we zy%Hprd3Q7dHTvnH-G1lG%h^@8#1WlM`8s1S@a<7BT076P*3NT7z}gf~-qAe(1179Y zcDvTMsi7L1z2`#H9Qb)Qilv`k`;#GZcyo7n2}d}g2-ALmFbHAxU9K6OM+e^znqS!y z$_vi;V5Exy;;yYY;;s;o0!z0hyr83Q6|4bGdb}bH(|-V4$X{^Wb8m|!odNPitrT41 z4r3sQ6vxFCDK^Fzz0NlF-~LcY=#kNhxOqvSkYF(HP5HF6^qnc%&`ivBmk#RcbqlpY z&z|LR4x5$>PK3y{PRH8;^)3HG_u=+=zGC5@dN!i$10n>b!3V)C*_WfVcy$taUW#TQ>iBd{M!f;FG0V&f{igbLQSR|QQ-KSm9gnq9i+8rQ1Fl79qJW|#P0;3c^12FT6!{Mpml7dQ^o%+zQ?aP2|9)R8J%Pns z24Zj+AFKm{jnyZmo-0uhmj|YK-4F%~84&)%+pge^t(=zM#GdOw01j4%+GIDp!Wpn} zEyNMsw{#`T2&{d)7)0H-*AZgAo)KNfz5o-PcC=F7-IRk0enOB?cE*|Xx00*poGDY# z`Jnfk3i+4+iw@^{{F zZ!yRo^^GPdrU#v`yKJyyH7n<+lbgsZ-216$F?lvn_$m$Fn7J_sR5NXtjLHZJ|9ZI^ z5;gu^F>!NO^w#UxvQZx3&sDfu3n6hS3j68T5a#@&rO<6Q{8dLGBUQt*f}cuKjNnsO z!B@%xA)Ld4$C;iqoZ(}}9n^_!sVK@3%;|&9&K!u8c)VP3bLe*@yZU-q{79lZidqeJ zPFK7@cPa9+--`&d+%rVxtw>!yIo(}dxDt3e$=_;CL3&tO{t4s;JmzInpSUEh@#Fzg ztj2KvwBXUSg5e8s5=6TO?0S@T)NC4QUb_&mGR~aR&497cx!shOtZ_&-`JI*|mV}PP zx@1jNP!8czedHQLf@weNcr#CbH}VdX4VP*0+<%-9@OTwi%ePbfgK-CaQ3j1Mjwwj2 zr(q|b)Rj+N2d+V4ywfkItyw`xJ;eE4%Qm#>cUBHfo=9o8+@gVJUnSMfBRS4FE!FmE zfHz2^i+wXY4PJc?4feY7WpvMj@tJ~E)I!NmcH45jK5Yf08YX#s2V{ir1;U;pDo zq@58QhOmYuv$l{4;JHi0D8rcJQv8n@WrwDv4iM5h4WiiEvw_lYV>5kOrJ`2akV;{J z%@(UJBr0WKFm?Cg7IfyjrOAyDnYvy{847BL-Wv5bwUpj6^KYt)!VA(P*J%l2b;rVga>G zASI`I>Q&siZ_Xo~^$}orcquADSCnz5pKQEE-n*|>MUQHGd2t6!H~i2D^MSb_7sY)T z^Xnnd`$|W-IGeTWo1Hviy6Vp!C5vWAjye5}S=P0Os@(UhYt_?wI0qm`=kmd_mgxz+`>JD9l^`cWv` zBgV^Rl6o-E2k9xNS1-p?f%%F*44gy<25+*JvjYZ$li(mpzdDl8!2K^jk)>5rYzH}eW4szkt%%+IYCRdaj2SULRha-dj)w&nh_CC)}*NNT!lKB z01@bmWt{FrrRU4Ra+10U&{6~C5Z*#yP}A#}*Z7sCYYYh$MqKR%f2^q{vbhf*#N?*< z3}}_C6iM)?3j3*jF=4eOW2_2Cz4{Rp2k#pb!Bj8cXY`*#(}R0hgAo)ht_H{vv@rc7 zaR`O_rfd%mXs@O26kV6rcTU?NOq9^cv|TZO5k4~2W%N?m?*VKYPV|6e2sp*qr&`TeoxXIC9>>jvC=j-GA1dMSBaNZ>N9W94 zc$R&5aRrsokUX&b_#TDbNKB2c_POvhIRKcaMUrQ$oVfITQUx*C(Cv?=#K2Q~@>u;B zyV0v^^45?@@ik%aX7y23fPyhw1e*drU4Y*TfWtSqhIEsX#+=OIn__f1Xd-dn6Lrce zU}TKA#S6xp`T$i`d!xg+q3D3boO=o_i4SJQ$iZ63o9pd3Bj{ha;n%HsGfy&yjy4`i zJqFc~lwDdcCfTP;K4OmD&#`N_vpn55wLLI7jS8#K`@SBF3CN*V&%?-yNdPcNhxT$v zUfc*Yv-8w^i<%UmqCJUWGg%HiUqYL|70%uE=Tz1ejwoP9aEh1fT!LdzU5jVmf|OkK zQaQ_L37`UKeGf{q8u;L^+d=^)F~akn z(5aQE8e5^AG%Km%A+Zg)d#SqcY(A{!tn;+uZ?nq@e%d~GEOIaWL1RF5)L1t>Hw8gr z*%A-W?ridNZ=V8low}IN&Pu%5Kaon$J2zRv)VO7XfO&S8n%!K=rl1{7tY6e-eJk{|v7X+{@w}$_ zj{sr`+?Mq0u-AVNEZ8dG88@{D6E9+?uwcX+W*;hlS`Kz9K(j~iX2oV(jhYUZC=5KC z3TZAdOOnalF#C2=+mVlzYc*+FhL)ewrEs}VW`W^?1IiN+qWV$XGYq(_>ONiW0?7!tNyV0m3c_CsC)E^HyW%m(uban zne9ZT`t@(!WWkASG&-#TONXBvhW!9EK+C@%k~Xkxi1a(pCw*2ULzJJ;g3cY!v3gnF z80zpll3`|>Xaru(wTRb~KF3@u7$mu#>;&s^$-kJc_DrU{>l(N1(q zFWlLq$h%kFUKL&*noq~4(MUNK>N5H}3g7@?=H$7x;KCNUBYX@4c&)XKuvTsQ8>ur$ zJ0`PIucE4`r*27|t^d$?7TEpz;duq}r}g?eeE`V{6&Rvz2_L#EFfPYPvd1(~?JROC z?s(LYOz`1s1$j;-o8g9y5KQiA!zl|bRF4BgAO;1oi_>)?ep=a8pnYgxK0~~l&z%Uz zTu;V`khzRF0xbT;hhvv|qRb@*SWx+p6KSPkY9$f~#6rW4OchJ%T_<=x+0XTA zj!(VMrc<%~ZwhK_r;45wLzBFaAnKj^E$|MDwxrZwdv@{hAkse`EkGxcWSZl>yNP%aUo5JUFOqyVW*AW8^Ya$E1T!qclk_swFc#}r18-6!? zI50Xc0JSlFYhGz`5&s}JZouRMx4 z^|t^04!l9Xv)r6bg1t@@I1LR*GDO;qd39h{LC68a^w)eOfEh?^)mkWMHe0zXDuRP< zOaE7qnYMYR!I%>UsQ|ff-NN|oD_ok6;SD=%mE(K(fftWB7BdX+=bmqn|`PdCkEMG;~ zh-(wM5~$Rgtsr+0|J1!jYQ#}J@|n;nZ#7($;LNGB!m5V@EtBz4OpuvD&0 znrepuHiP(BQiL^SkYI;I%X7#Un{J+zFYEpJKW*L)Mn8P}aHU-YoL()m*;ERcD@A-q zI@m>z+d)pw4O=|=T5H3^oMZp1VbYJY$lUPV9`ZvXkMx48k?g^IpU0Z%+|aMjj#82F={!R zXC-?rEdcZ5fho3>fF5bNJ_?G@?#q_RaLd-`C6>X#=Z(S8ksUe)8ViFbD+6pR><<7O zG-^SRhakN2zQ`?Cy z)<_|E4TBGbi(jgu^GrQPs4Y0C5_IJjwbz6DNVr7_^)jUk$*?e*>Yvm!V4I%uYp`WW zrO1M-c?vg>#r1Kp@6}@bczsw}ZIz&HS_la>zpk|$`lt$!v*U53`*{f}5(ZP6l!joN z>8rfZ8JIz({DP44SB?I+k zZ&|s8Cfh}*8J)DrByNlCHX6ea+}u*Ds2Cp*4yfS41DN2PJ^e!nsbovTo&+2dJ11Gj z1rjP()O()p0Ss)7=Hq%lgcg)SQ- zHJr61*nOr4HyJpp+Wy<%Mgki$I_XydZc_aTiQk-y4?wa3Z@UZ$Tk?u9r)ZjYc6I6g zJPxAxM582N;-7wZ_RMIf?eIZzkjn5OiWkZqIvXRZANWj+hfdPdVewV>EIm#_x{+AoT5N zB=#`0y^Tphrw0Y7Elc7VTKj`)${El+ z0Xf$O7i|RTMJ)!e#nDTN$Wlyouw<;(K$_GPLDSv7_&{#y_Vkt3ig@%hsVkd7`6drD z?efDEV=uc%WJLAEBu!GN(L0*p$n!0 z-BP|2ed+G01M`Y4d5lo9|y(F+wivTgofUG{_&Wf`e`~!ottZ4N5mX zJznNR{3DoXYm|w;p#d&~y)giSb0-}kj)Xn<42?sM+}pVqI_JWp;D^u^vqrX-7a`FQ zFt3y&_q-m4`B-_7)~qt__dyIC9Tx&>u-3r+dzYKHL{Rk`GZ=ig?VBn+(MFw4zSI}> zs-9+L%8eI^T7^M~u)zNL4xwd=xpzqi_7H4zo9K>ArthTa=gl3{i=P_YQ_JZIH6u!k zUCl9iwQYxh3aSLQ6&`cCtD*+n zpM(0vGwuP0<8Xei9#y|uFHFGB#yn{0ao2-e+0`T;?=yJNv}~+3ib%<0oM$0qV+bLU zoU#)3Fs_aOjboYdrX#z?K74PuunBPIN>otNNoRh?Ud`Gu_@SmbAr&D(<`I0~K`?`L z?Q4mZWgcekM6wDHRxwCuf(xv|p{X<+y%_K~ zSMWj`VoedmY39X(AP=79Gk*@BS1Io&64{N&`;Aje){bEcYX>B&2++O2e&$P(D;ExP zt{6Ma$A+p{YfxrkFV_GD>rNO`58LzoT{Q56H~QI=N!-S7iIV=MR{GObID^YUdX?QB z^Lu3k#9ueC^_MsvG&>}V(hxpAIpy?wsYHrv8q5|+<+;cjId^kxRo~fd?W5*us4Y?2 z@hFS=V*=Yrk<(6|gDDHcCP)$BhZHchy5}tEA6u&v)Zr>BUBb)~2SxFON||^=AmCYd zWl)VTX|CFO9dB%s*#_iv z;;Tkw6kfDN!YhcNfYw*LQvp;pd$x01`p?U5C;NFz1k7flR}&0JQNw&B5~#dMZo8

~KJt-yOjJ--uip0ypgFx9jt6|>yXAa7liNs(sluR@^8&KH%8>xL z3ZuuHn@KjJcf#zrSfrM2C)oPj+;jNu;lffe|HZE1*A8K8e(STG|F6Fjv)Jh67kxF` zqMJbut`KcwU#GCQT^XE9V!9gGc{ zXA)X@*83Yl0F&Xq#JCWg$jsWl4M7FWZb?g-eNQ&>A4!;wyHmbVMh#P^_s+&~ZeiO) z(SYk@-ef<*-w`zhpZUS%J2rHzwoZShkznsX3L!yQ;C!Ts!o)_MXMGGN{lb@G{54rN z_8LB11e$zPx0_N79f>ge&RQ9~I7i(4DMK`CKO|Rouc?s}E#tzw^)N28L{<9dAYgru z6DPa=w8^b~ZhyQJ_t1!xOM5v_RsESmx;DM*?PCBcqL{!20fKEu>u*Bg{GdYpZfw*- z3D+M0{h^>MMSMqyYoqR6agC389@V9i;uS%0w4KQHa6ssqD1-yn5V;g}g2z2K@KMB6 z3RMxYSUipz6soV-%kYyz(SPiGze_P<0h(ynM6W12bGhVbZ+&Hu$4iL{X}v$ zm&tCoM~Q7~44@sr!w&IbbNB;Z*MO$PVZ2Jk0=%J#{raR+kmre2e#R*urqnZmFO}7j z^Q1xJqf#Gpgw4@qqzc1FY2w7?|3u0JIKAN4Whp2jDR?_w16$Eds1R8X0tLJ*6gpCj zao=<3n47|Q9`zjN%KH;4Mcfrqj84gqX*y16L`@0Ft7Ht(ouR!F7wXh)yC9I zR1K^ewP+8#JqIKyw(6=yqH3p26}r9AHgf5fRf|Qc+v)&GtxpFKO->t#ynZppE8aC9 zIxx`t^T7iGmy*D8W*2(*^72fEW!9w5>TW&uwTg8b_V*r*SIRN7Q;8@#oNCTC>SlAo z=95SKo~f1fL%dOO*w)HQ7CMNx;{R(K@)Blj(M{rURG^6|n%TLuEO7=|1bBzCFS%~R zN+?0ZQsyJ&3AsOoDN+X23o*%vwG-tRMj2dF2J#K*;u)cJBZ|0y#gkCs>rN0KM^|v? z6|YZHj)cp3)eE^bRH(X7?Ks!@G><0Gip$I0urnc%6u~;aIhfLX*3$dFv#Gowri(I@s*^^_!6II| zofSp*bY`Xj!4B$G_YS$TCX^(KGcUD!>_nxoy1BA+dX8SZ3*2wwTzY+1Qbm_zWzPgfPvX6k<2fSH`RjBMAVR$Mx^=@TCco@>7WW3i5)X5g|vcW zNUat#4o`WK$tV4oagw*5>wpkLdyg|Y7!azfsv+X5H*|t??kCc8nvnwN{X8j+S6gVw zqcA8RYS|BBK&pC03(w9XlmQsh{TMF`uNB~a?ZqOxZE2+9h1movdKiwhBLwGKHm_7? z(8@xKY=*v7_yQLQKi2K~DSD&Gb^jrbK>c#{t zIrvuxh5OmdeCPeNBCcL9>Am>drD}G~$UNG0Wj-iEZdHqeWc?nfb`7P{+I}T0=Y0Tg zE$L)p+bNLzW-BBQo$6PK%^s|c&D66x?*k(A0aNz8G67N;V+1W(y_Lh+*6-({CF6~d zFPCYzGk0aR#UY)PMQ6sYiT{{7g3kY(WPdZ>iQG(l+-e`=qPUw@%ID}DoYYrwyc=v5 zTYu@63k^h{;i!46@F=)7Kqi08R8{|)b?IQn*L;^DrZBXnf_u7?kz@{DK<~V}?T-~K z)<~kFRe%wEf;;mV7(G9Xn9#yvCY;)3|=8Wz-~l%*2MGCUChES<;QFj z??2bg!x1@5eNBwxy`qFy4YEq)D5gMNRwe8G)?pa;13LaI^{w={=Zu0L>4-FFGxapS zd-Xvr&fwt*u6nnPG*TUuyor9!z>_N|wKnK4;${UP3d>S*SX(sn4IxbSV1h78YjfM11IQlRL|5Fcy<7X z?-Zo6MH2Zoc&dcT3ACu8>j#Y9z2eQIXLsQlTG;l*H4 zMh_U6(ew7t;ao5jv$RkWSM!A}CP*b@O=b#2YY~8wl&cO4k=?^TG@mJ5LdX5Lpe?ha zd$YjGHU#r{Z&fKfD9mpaNB*Mn5VkF=VXh z9?|!svy4DEGk)_4_*Msdl;1}!Tt2(FxK&#WBCoYZGO7#vsSg6I2%p8X>f-oS#TarH ztvlu26rl``l`fwLyO$hzItGV${8z8aJyLkhB*Nm;FheE9)>9a=x_g>|()_x?%g1QbrVEKd~OD-wT9w8ryrO}4)$ zf|19J77(ZsOUX}3ai9u)`Zi<(OE?i$!ZnoZ+2S8{Z?T5jjXW`1lW@?T4XzUmW>wb` z5mvORV%Lbtp2x!|vlO*3a2&&cm^n6iC}`SuXOMu|@Fm@>M9xfFvmvP@w7HQS&X8 z%tAeQ+}enLD3d`W=M9U(uCZbV%H0a?;h?>g?>gl5cARf+_B*>t0WmMKYz74T6w^{~ ztOD%KoSdGr=}3$C5uEsfGxvy>el_z-g_o07Zkb#0i>Fb#?(P z&%BlDH}>CYZky#i?gyc%rD34*&-t)!9g{bL1b#V2Ta*&@bGc2Ei*7IP?NJ-qH1Tg6 zAvIJCHcuofZ0Biz6lqCf^Y)JUAD!{@*RzDOi?kD3!bSPVo!7<4?3fEC{OCm0p)k-! zHqnccy2Ql<@vJJz{mP?R*t<*L?V201Rt zYU2Bqlwyx$|4X+-K8vKTvx1YHr#jta^QiLPDF30KJ1#N(M($?&?s%k_j96h~{X=Et z^=}fgA0%jbTwu^P3$dkUc7hO39|X z6(QcU!{rg^TgpE9nxKWaq2cY$bEwUrWUfGChMW3lw4+7v;rh2m><^={8U|n}Gy1YiKCh@`OWOaW&QSi9Dlv>v3-VdwhEf=UYg5@_d?&>m_SeS{{;Y^~sRN*H(?n#Mn{Q zPY~C<)dWKET_#)->&b3KN~YQJcivakQhP)7rU(}q=;8o(QtBNolr}~t_TCO=?QtCF z*q+&hX^67$)n_?N#V=vcPgUAe^WM(@ZfCfWIyFcQAEf@pQC%NFm`d{*oM~C~bopXQ zWnMbLzLeFQ9O?AE-SR6OI>6VJ@Z9osZlYWt;r#n$4&rK4NGl|irNt+udNP2B!R|ZN zhDGHw@Jw(bHS{kYf41r*QVHz1+B}fPM&(08D^U(S7Tp_M8Kgq-5x3vRdfP2D1!^{r zh$d*DYELI8hc1whlQU{*1Zxi#0I^izY>V(+nV_!4RB}%SNAH0CJ6@6?d>*!EPMlXs zxr-@xRi)Jmv^pz9;Tf^_svQz(P3O8fO1#ME-1^=$HGK|hpog7d}tG!0T{V)&FkT&AfSkwRJw32WCBx-5e=l{JLu-`Sd zMP9)7i|$~gJi&e|?Gb>0=yWwtWw#U6j6t-CEIV6x&oAY%L8iw9IZ;CTvc+PFfJAfH&`@O%fG zW|HHwd@tcF?LxNHd)`ae!b;)SK+pR;t-toZ!?)qJ#zJl%C%|b5KcIn#1SXr1R_z5u z3Q<|9e=32hvAY*zZ%dUJ&@wahN~e(2%3%avIX==NXS8%l=a5Vl9hM*oL%l8g<8gvh z^FlD~fm)yOOa2LC%L&_=6MRcvZV7uh{}Q*Q)ArqVKoP%l%M&Xt0qgJfrpmUBA@3+y z>O!0=cO_@GuMLO)aJ(O4)Y13%mmYV+u3*H!d*evd`mrmC%rJ7}Na5B6mAeGA6UB7+ zA5=cAx-tOd#u#Q*vZmVN!}EDWPudr9Y{NjM$hPZeO1HMmE$?GpD#Jn!gM-AKh{`{} zO=yVZ+HuNn2C6)=z4b9}NSq>F1#~2wN8wK-aXmB%jVd!9*aqLw<!9Y2U=iw>F$i~QW-)>&_J0#_qtfP)#SxSDPFOQxo_Tm0Z zKcmv8MnVITA!AN&rwb2Q{N6RY{46DZnpNVY4t^!p(@Hm`;ZlN5(4yLigV&^`#{T9B z20{9$RLv45_vyBbJcW0D?DoB=kJ4i9XpBjZX}~HR|)dwpqV|&VJKeY~GF^0;wGkSqhw= z!YN6X4`*+PID^242Rc+TU-NBG4@LxjNFbVpmZ@u>e}7fyqdoJLmhe~sDfo5hiexA^ zyi+96_dZMQY|kp^65dX(-%|fReCa__^_pb9Q&$ob1_9$T0*;*cZSK#lnE0Z0EvnOm zAr}j_EOCL|J-sA#mD|ms6UE33>$&xQPrpkQ*7`cAmiTRi ze`n^Ux*nKT3ZJ;(%Ci{xUZN|q-D$d0j*&1n7g|Biy@}Vo& zN)NzF@=-XSo3L91a^}#Qf)-IfbTJMHj-q+}Y2T@4;=b@Z@!&_f!reK9fC8L7Aq#jK zoc+1a`gsUI@WY%k8za-VKJ`e?Iot(=eYjbPli1)K1<8{f_tRR_clX5%o#V)p7H%>*r20W=xCj0db$Jv7l z%oUaw2*HHy7NW$!cBMWDOQ{aDw{No3OV9$^vs?a4^pLT__J{&fdvfQQ{BELg zEwci0Tgf=L|CGIhRvCC6#9?%s81dmb@UrwFC~RNlu&ldns7aqR-v(3Wgp~6y*(yo;v3M)rk|leLY?CSC4=u@>Qm>s3DA$uo z_Vqgb@I@_zD}Q!pWuXItuU-2E5kQ4gFPCcP(KVhsF0zQ+4H`ICxScMHo#dZBm)?lU zu;+j(?w2~uvsaubMO1>*CBp1xfAp-?@gwjKN)U*;cfyXJ{G9H8f0KgnN65BmTcvdk zDfnsacMk3CguVeBtZ&}9-}yBz4dCDjifD+2N_;q`0xSWIaGHspm!4zA8HzQxS6CL% zk82TwVBmFi;+46&9ICGfioQpiITMbf^H^8hz~&wx@pG{4u^G6lulc@{H@Cqy z+8_y=44G#RL&gJ3qngGZDhHuA!e6i9r$2LLXQJuM@Zu-bfw;U8bVOjTiyJAtwpE;#n3gBmDiGbZ(=vZVc*Ez7CI4?fi0P%}|}?t&O{Y5S^9 z2;?3Y(yTvV0lM;i32{0p)t2Fr$}m)v^n7ad>Mv>;(_J$uk*fPRn_k zsh#C(_rPNp7sy9OlMdX7Nzr)kh?^tPfw$G}L1-TPUy0$qWtQOM!qBr5yKwIw?8S6U z`F#4rEF+$4(tDA$(5bmQ=ie79R1I|>xw?;1;i4y|C6B@P`i{*tJ1P=G4`kN6x@?mG ztJ+_D?PeCG`W)iW{I7$+zEJRAVbHH#yYF5zF`h>R2AQ3@*Q=ouK3s5_HM0ye^I~C8 z#@R+G_86pLb@gCWiD;FU=I>OeAlzd>t|V{~-@^R-k_T-hf#}C82^rud6eJE2uk9*- z1TQGNu73=?U$7*H&<;j7CGtRu4KoqOmP{FJ>fYSMXH`kRT1=rzwx~L#cy4pRDA3nK zU?ksvPrISaioJBE(wITgHxaU4t-VgI9_9Mbx$E@}qRON?E9vi9Qlfp<~GkeV%{ZB-wgQTit}``R{8PMq5-m#3~~103MS}fA;=WKFUAK?9TT_p@E4sRa)FGMLp^RYwR$^0V#7@}J1Kl8FmWR{yGaOXPWY$( z4|s5X4;`mfbM3ateqEcQ^ZO1vFOHvROAU)8a1L_=EG+OY)M1sbbD@HZ7g@IdXtBkaxyaed9R7?dH+WFp=#P}aj* z?{(nJ=#qBLa}x*(L+>*<>BsThMAq!>QUtu!tVP@e`i%eUJ~nr{QmiJUzqJGe{drzc zaO&UCbUa0PJuVC2z)XBDDsyP{k8UN6$S9|lg=iISXd*RFv+fHh%5u2FeEd;gZ>`g; zFoha5=?x;60MR3Gh~`}`B|RpjfzriRbhJEJV&(VFnN6RjCKYr2Z{^QIeC?y)8ngsTJ%z`muvmf+X(wgI z$7dw*%}p3b1|{a7z|AEMVgDz>)&-#h#CzfUlQ74?*Ok+~Y89v@%o87bMY*6*b#a8X zT~dYbTu&FXHv>PgvH@w4Ozz01c-bn0!S?DBQL^Hm{wL?m;4}el{5{L0L&elETm0#^ zt=xAd1PFgs1Sxf1`hI~<6pNWNTmcJ905(1B!?Xfct?5RXr(~{OqZPBEXk#2^Jqd)Q zs{AC{xj#Fr)TXf}F>jd`YPHuClu9O&2EY_8^;Pr=uQlEDY}>Csp1bc4sA#9t?hhP| z-}faja%nw9J77pqf2DySywr6kipyuOvo)e64ReuAyoP6CE~P9UEUO)$f;@cdyM%>N z&N@j?BTc+y+#zCsW`>4C&wb~4Vn1yd*d6x-V`UNS!q7VLMEL((W;O?A9l(gLTOjlY zgKye{5fTtx&@W~jEsyLwHlv1>_GAidq}ZH9utze_58W!<_k87M$Jy(xw#-#~?-G2k z$mzR)8`r=?3y$tL#AD1~* z^ni2%pskl2WY7J$u>Q*MMh7LaU#BFf4H2R#a%bwEnyBh|tUlf)V$(M}$o67DN>qi$ zeK|ihh6N;ves-Pf+AoD#6(fxD=0or?$!d%7)TK~Dk_u~znVCb6^JKXUH;)#gU!kpt>5W$h%TN&U-?HXBFU@Q+|0Zp_ z@nMq_a_F7>S%4Fdao=v-SMFhUIh1Q9k^y^+i?P~g$Xj{}uki)Q`Ru)#QE!|e! zFAx&f=*osdx;oX|j_`Gldsq;urJJV~m*b4b%cK`wh@5$Hn5`ff|Jm90?!eF__`L+B zx}!C`#XmzKMl6HhE4B^dhN+(VSUfAqt@ONN{}XPHUJeN0|yn zQSNa;@c~oT&ugkY|LU8>jMGA)ecZ~s^QhUu_CsV}?+`x}7^`CKoo#3;3_wpZu zy7#681^2q^HzBo1`|^$Zbi6UFRKH{Caw;!4`UozUY>(v+5O~q6UY_Ul7g&)T_cg0w z2J}R3eW&d zVy6+xiSlzZz$vx z`y0p3KYcen)&@3!-(XNjLGWG~ZYF6!v|Vu*zi7s5DuJ3?(EnP)-P1WHoPYSWX$05~ z@o}5eU^OP>IPEGr-Hf)-!m;F`o8uR{a6O9gJuy!pX?yxCE=rm)bQq4DRbOomc_}}N@&4^&4#0C}4QU;)bmX@o89H{wcNqpUbBmRnq;LK(05&@Ak_CgV8z& zYn76WO-_;Y#{4B~*aL+}In_WL{k%Pwa_+sS66oV{_)=EAs7taI(BFWMTeldEFZ(Wx zo=E_4V-l`H8>wNpzPD1(=*PQ$S+}(kb#Eb%;w$ZHOQ)0);IKjJ1lZ(aawf{aDy$aK zC2CH9g~b8wl|j>n-H*7MPSi5N?cB0C8?l!b!sRj8v1L&A6o&WpZTT-(q^9`8^6_3L zZ@=qv<3~{f4O=-Jipx`%_BE<3-TG_%vW~wkMg|eW9DPw>4Rs_^SX0JHuRxf+CX;XK zD>+!2m64)0Rt&4Iwq;l>Kj!3to%$9biJ(agEUl*Vw!Q6dW+SHO)9siTmUfIv~}7^k8q`^2>B$X$4+SR(?6e#Bc6^n#Q=lK^#?i2UZ?&cx^qt5 zo?Ku&+i{b@lfk=o1A?F7)a5eZ+xG`|_5jT%2;5r4JWVjzmS#Q)m0#Kb6SN>CBDxSp z=sU2m;VW|yIwq&;3e1~Eelb~_W`ZRu)E|E@Gi>TXq2F){CQJG6(*4AKD&T%T2MY$8 za3wDe4$J)vzYGz=y?xi#iwHw>7|HJO#F0RCi%D-$kxffm)R?eH;7F}eM2`T)bydg* zp;***mE*S z>nQ|*Zhkg_IcB@2Y15RY3Tky;TP4Ef=F;_>XA?0)*VBLl397g_G=8urQ0&qGA`AC%SJ9>;ZZyz)$@fIrMhmruL6pX*a z&B_YVvqwsY4w5uBs~8pRt(qa*-n6mN0%gc znB@nF^zWk3&Gd?|@X?;IT+P#~lE-ILOnDKy_RTR23LAFBPF!}Lf9${#!v5^xb;BM% z#}7aOavwntu;=a{M5I}w&FzU$X3*YqC>u0b4HumvZ5|8z+og2Wd}O~aia>a#5SmCO<({ZIY{x>cNL_|QzkGP@LSl9G;yFy3rZ30eh+@bV_^ zsmmPHHK(_a9g+@r_mUwQj4616*io4~nV#tTw@Xc`PxUFnXKCvP=ApmznBS+=0xaa2 z>6{Srmh9BP@Hs9!KtC4UrA=KbI~|Q^=O@6t@TjDQZZX&Am>-nX$Ru%5Yv^C|>Tb?j zUB_U+;cx&m_h#cSsnMN04HhuM;O=+BX)2a^MA;t&G8)eho)iEq66*%vge*i!DKXsL z326f?D%1SQp`!x%_&mev`;i`=g%H# zBR3(bA9FhO&T~Lc-^>kYn7Gq2HW^va?<0UEC_f@$(#bA9VT_IUkb~+jI4}5#*2of) z&~qZ(J^N9jGxyM+VGyon@h;p~JOkJjx2JW#jnWv`n-)(ALDVF z8O(70AL{Wk6gM!7F;L1AFP%R#nRX)%UbkZ*+y!0N{)dlw1YLJz1{55~4c6NgC4RxA z9?Ze(-alBYfqHepOVIi@jnbYzW$lw51%RhYJA$Ip4o2#a(M5L25is=ZR|!KYMK?HF zK?hK#C3hpw04=$hI4)$oiWn`TK;UH?qY%6ef$|c^#Q5ZR5C^Om)CM8fz@GTR7G|*F z3ZCUkghQ1TJJS%Fd}5GUOuK{yf}yVRsDLp@lwSR#wo?Rchf}E|1{p5!cMfS9fOK(1 z=l0f<5FQ(_wNpvuyB>2O%g5;~#B7}`%b{(A!yVbN)(iIMepdpHW`fWvaw8S3+2=X) zgj}X#S%0rfE3Hf=nBXuw+_2+uz#v%7dnXh6PDG6oWWTwkhdb4nea~s)qGN+27YH%^6cG<3iITvbxVofXb)-p70Il@~7P|nm$?L zKtdMx^yLHz|9)XigKV}YINys^oOGwT=`3S*R(Ty?N;L7@(`i5e>|MFBJ~Dq^g3o5i zMlJCqeVj%MD8A?dqrz>TK|i`q019S>fSbmi3Z~nn-kZ6c^-z087gr&^W(!kjG$M}?TwCEbppd-_F^Fz9JYud(Xi6eR+Fm@tFbh-X>Wkt zfmO-HFD=e*w;MYmB&D(C-IeMs|9Dxc9b{qZAZd&&(Ibd$BKm5bNx?$OSs5`in(VzLvy2CO?NrjZ+Kh$i1O8>Y7+ zuQE5R8573w5I2QQ{smcWd%7nJf=6tf3*P=btX6zkx&6NH!q#hv)iy5Sof`>OL_R

YxA4eyX{8Cl zmuyQc`Hh*Pya2r)VgraM%|DAuv|E$=8gG3Mc0a{gJ8;Q+WW$HbWH7E~P`r{gk@l;^ zlj?TFOYJ6+Q?I=Z8axA1>&5q!sdY=W=x`00PXIyVhN@zJM=UfP=o0CP0vpXSKx_N8 zhIT0v_l==-)}3AJ4`pIvDU4hmXQ}l6X5S*2UrY2K;P8q8(gqtz{o_qVg&K?%Jxl)O zVtRgF(eWmUMgTm$Z+j9{pvQl9uv9~hA9#{VdDjgu?<0b+k>4gbxmNhN44hxK_85%k zi-eb5b=BA?rzqg`J>C+*H_s~kiTf-VJ>Jw{|J=&=8`yId{yOT2UMRhd^Jte2>JsPM zAuLWCgLSmx-RVte5D(pA`4i&v0_;WGkPZ3lEyn$OOCtJ^SjbXNj&nE?umLIg>zG=; zqU#;PjC&*6W%s7tnch{1Lf5NX6WC@G`F$->^0Ofyspq1Hq1F-w-uKykUDq32u)%-J z>9li{Fv&Ek)Rq{IH%|9%!IGU?kOoe%YuJoQX;0>+DqDeV<_FV0sTOnJv!}=kMV-LZ zkle_9SHs`RQRk#{fb22g2q^ZCtR92m#1rh=zU2rCW4FhGsZb)$K|Vy)UZb^gb)+?q zNzLuZxZvXtG6be#1^;r&l?!8)71b=shim9U#!DWR>p+JZepnhH{zy0vQ9WA%4~RyhT0j7ni%m1fhH7XeP`yu7VUDSGPk5*e zOavEnKKZ%Xm~jN*uW0#zTL3xqk;jQY;p7Ts+2D>!CMJ0t#4Ao%xUP{CM3=cOY=az= zWo=qB&`k+BSk=%MqEWyuk6@w5)7YuO@i4OM(LL)MSK;t44ANCG+uV~0i|_CK8)b@? zMJDxc%8p@`xs4e3Q>s=TuwU$9$RReTK6c4ua+z>+O}R`}mnw?AH_-B3Em9#~ z6^n0$xA_aMNyQb4=3C690b&uH%X zC0aUWK|RM012Yzrl|gh{!orX>5zW$WN>FE0(s+8xA!5|ySp^_-fbomSZ7z(YSs9vx zQu*gQTG+I=;O|D{yE9)O;Eze!zl?|)@lzY!@RHh|597%dEz2~f^t~!OUJNs_*cCV; z9}sW*=Js%io>oOp`!azO=IYcl!D2_=)6+%*UQgXca7kcQ9i_S*(os^)LX_~Z(#Nkkvor-x>8b!i5+%z8c8^skHsAmNTr>4P&Xa=Gr%R>bOvojm^Q@i5l!>EsM0nlu#pC-z z|8qZ}psPI&7F$*4pSwKfPj+BiA7aZtTKSNGJ741iH9l6sn zPbb+EKnBpTDz=y79;OJn_Fw+y-9$cs& zfqqY93`wbtV8Lws5>OFEeNv{nJLh}vYWHLxY1V4E@IY&!*tlP5r^*`vDzvAMv-*^- z$rb*=o?^1rjcb}mj6Ya><{6C0eb-r3i#h>(QTExWsSxvw?}CFCLiCgVGUS`J!e1Ay zTjoSD8tFf}o>-Hnkgg0#Dd$qrV^QD0s;HPPi^CSXa17A&0SB7Bs2wM+Z?yzN=+8lk zcM>~iz?id36jxup%Y&_KV>^dckZzn!(ymkU9B2RjdpT4ev~?~Ek?1BY8{?|n;K_tH z18xHD&+(dC9e~(;Bl$pgh2A>(2U;^Wa@l@|9;DA73kfv_@@sd)ccmZ+$%ba8OqZ(j z=ofX=z5NDwGe)J|h83e^4Uie$r+^EmCgyR8N+9V#48Nkv-5R`79gzKA!6;gP*-D<% zx|u{w1oU#xSwQPIhp#8H-ti&pc|^&G(y{O92VA!RJ3z$0qcPjf2Kz&)%lqGuBQ+Vz zWBagiC(@Lu`6)Zjw%CUtD66CsuxT+Oj_*ipfv0-i6FiSD>FuYlcdcoaT$eLuV(TYi z-NBuU&Mk9~-_?*eROaA%@*W@bci_Y=cYl=|p|KiFm}In#!UAmd29_}u>Fmb-*7x$6 z1a5s8!P{7JeT>~=)WtXD_xk>oB zVNjPWL%$GF-oI3(_w4kMu%bPmpMVfQXXP7#M*OWDCbs*hWlaQ4)K<2V_6|In63+no z|BRMZ8eFndcMcR7g^qaTD<`~F|6+4i+PxGvCmQc34JWMrV+FC8Wt12SvTiBo)cZQ zua^OJ3U~^C{PG(FQ&fCu?dfq3m^(cjS7W$ngBre=TKJtj}-9Y3L z7i?Alp zt}=TLAv{)K3{EdbQ;)L9o~x$5ltbCZYa{H`5snv zw&~07j2*BchGiX-gh&2qg3L5k_J@h`xj@I*BXcWp2=^T+!tyk0o=GMj)sz2){G)c5 z`X>)j8m{57rV?T&A0U@}RfUNYlsPT|jf0eTnlK_7@P7v=?3AZs+SBCkqzBof3~1iYgPnU8&R#IDc^M}Qk8N6*W^@)>yyd=0N{ z-viTMajLyNZ(GWV$1GWB-Ws4C+0zOb&Cd#_Q2*LCk$81t`eI<#I=3dsmc(xF2;oe- z{q1+AL+E`x0kQM{QBx~ZV(cZl2g%Mv)VO$6G=ESWz|-439pY z7R(jhd$K7yEk6R%yVDnLY$@#UF0V_o!B5ZZ!=9s9M@~FyT;oj{lFc4Nia5a+II>fz z#S1oh!-u=O2eyZ4JDZO?$KMy)_UUC)F^R~&CRtLrxdR@CYenKDPO{bvET}GeI)E!p zs@G057o`SNRFDp_HEB$d&G1Ga?$X}A{rP}`yClf&EiXNg_4dpQsly8&iiWxumF(OH ziHuQcy7HJO`#7B_i!FwaUIBYop$M|EN~mUl(m~wTGpl9#xLyOY0&TUH&z)boFlUJX zLnDUMw4lA>=2OsA@yMti`CJG|jj$^WY|~wIStr*KU*%h*FJ#PQQxbwWXH;hB@EmWWj7U4=K*Uyy zn!W-)aH5@-quqX4SDcyjW{YwDY)M_wHK>2vqj0Iq0=jhZcI16K3O2NahObm1zIgW$ zqW^fdq4|tm#mTNQ+p#n?%;{K~w^WQt24)E{Xh@p+MR81cIlP8{c^Kglbi+{RPdiSW zOhC82*&Fj1S+7?d!?kr(B!~8$JlAU2L!DFtO)Gl7C`n5MOTOLn@)?M4|D%*dKsF

Tb#d^FM^VnnvktW;^9C)N=ky&`46z?ApjAnGU6}iK1W$K^ zkrBZu7axP*mtQ?5k{qJrEQHc#AI{gF%u*jFl15Bd@)s zacTCrD}WP$ZFQ`+yu|kuP|=D6lJ@xq;@j_kCo_syv+bV>)^YO=)r?2-3xSSRGGe8~J@E^l4%ZFkvSN7_{$ zePSm?^8Mnm#-oaW?eO}g8TF+aA;r7e;wX=IcMNxBibov-LSh)Sjyia}yv;e>l6AMF ziQI?oF!gWCC8lytrL6o8%~!lq@{u|)(K}{6B`Q=Uh`FO_gm%I?oooiHZM~QEH2-56 zz-nTyhTuw+R%)H`3-{CF_Qpkc4a*p>58G-K=u0-8hWwEsD}7yjDHi&Y(GQ~*yAt6Z zKk#uG@J<%+V)038hrj3D5FJomPyqg~O5y#m7XE#{=&DvNM5f8)#igCotngQKDL~gW_Ah&Ms@>io5pIuhnxWza~%gyNSC&#uu_XuiY#l@er<@Grn#6r|bQRKf@@* z9403Zmui1ea9rZUcN$`S@+ylS%nz}D6@Q<-%$&8nPhKm@sA5rQ|46tqmW2o5`$8+F zdHDs(rPKEkqP_=J7UE1b7PKnGSy#;;b^j!H&3k**s{AP_bl)`N?vKsAjL4JP87+tu zyFO2HvFN6K@3OIp=I99EKoCVH2O&vIOd&fkkbFvxhOC7LCt8FLGj2@{2j`~KdCwuGas3S$;+GwbK3vNh=RtqEQX zj|B1vU$h!J!thce6rSU*H5$EBfm-L2M0z0W%;*B|1{{`XUv?(gY4=Lgp*~w975OeF z(V^EH5M_edy=jM!KKCs1>%Y@;xh4Wr?I4wv>2_Oc#*g+xkp<)^o)2|<{4)wA0g}R? z29f%q;Z9n@oua}hDN+Gzy%Q$9q`@b-Fyw|a{=z{t`XxIsfj)N~ZdELepL6s@i?w;+ zqkkoJ-T@X@?u|8^-o)p_J7!${wncw7%5B=y5z49&ywHM-u~80Y5i{|u8x zi7+*DABUs-gM>;o!)Yo7q5VAA9``~QY|k#d|2ICtT|+gKXjMYF*|;~4<+ICEOZG|@ zF1Fh|F>5Szt8N96j*gsQRZ-7{8caW@TcuBj;6~pEVi9!Fi*+$#xUO7NL~qhc1utI4 z-YozUSdZ2OChxEi2L~ILPOkXPP~>{B)pu_w(=X=MaviGFNy6Do4KmFkCYa?&tj;&U z=%i!KaVNoeD^aHNL;+w+5m(UmhY-;c_bPo8M*f7fd)ZIP zlx{EjrnyRXHaPPFWAj)AaE&L91%LVJ`!kmEP<&@*0Cw&76oR|0yl*}e!`BfSwEa5d zy1_y&Bo+7eFlXw(Itt9|zgKs+aVRETrDLcIDMZ63qbJD>Iu3Sm`^fX7L*qdu=j^%8 zroRKAk$5$LMhmU0I3e2cvbDwOmP9Jh5G+u)f2A+OC43^tX#pWE{OpWYB=e#CreZHejHw^k6nLH3}v&`m!|kW&1ukm*yS+ zvq5&3D=~VjDFib$wSCAVnwTMH@EBmb}>onS#s+ShUEtzEHC9#f3CI zI2sNI^$WV)g7FX^vTSI8H&FuPePt8vn@}q`BN9>r*gq&44i&X>tl=a$~wDoUGce71N5u5S=A%$|FVTUX!=Y&Bla?j>gHJKEwK)U*c21?3Go`^KNIs zsuC_f>hNojS5pp3_RAOT7J~e0bnaFXH{r+U23-vee+6|a&kK6o!eD5e)!073`yof*yspdM0 zuW*Pc_$Eitd%`f@Q8Ilspa37Gd{QM4*cq8UU(RAk1#blW!BXX%=gE% zyznGY=7_tjbWgfjIExeSJiW;ROMy1oV^9=y>M_XHJ@KABtt;p5+ch%qS9$}={si;` z>ALP367`U*U!3Kmr7mQr5MIO;SLx{cXvaYCu_nUD`VVx z8{P$DjH!}0#F9$lbxcJP3kF)y?!a4_;2WU@Vo?B?;3EAgRBR?j)eV6v)OhXyIy;p~E z$+ljKD%;)tRLQ@{RC<~k5C3)NBh>cAM3jwYO zrA(@#O@eov-L1iwFtYwz82>U(f~uVz&RAKtg_W_>_e5qSXJDZciE+z*6VB=2F|o5y z+NS%bTn2=3!TW}m(YMOWY|@Bt#YnvK&S;7h`)Ci!Ys94Qt=1Hn=HR7d(f9cV-LAl~ z#8A>(VoY@NOE~%KpONmTfy{rPT*=-X-c}!-Mm3cokhBOGIZ>GbpjGb(arM>6C!@iH zJI!Qt^!^R{6UCiRV|x}+h-%j=zQ4dsYdbcvI4WD?fjO-f`_==c&FRkMY~xo5_;`o5 zF$Bjaofx{6A!^&{&Y`*$S5iTJtTn&xXW@>*CU*+}IM_bdpQ)EKAc#IhkaV)W*D)b} zkej9Gy7LN$D-f+291;1~FOn>Ka{{jmTE~8thMRu5z`darL&^Ghqqw;;0O%-jN~8h1 zT*Q@(ZI_eJ>3mWSq|&}502Bn}o>8>DuzHYx@jWz7c5RwaUTMHO(3@OVA1i*9hBqb? zaJMZ9d&LD|+s&>#`$HHnLn#Ca~ymglH>($2q3m14T%1RV_g1w{~BeQ`%fLk|sjEO=<*aG^-u^2=);UW%i)I zoOYeymSJLd>0!psdXfh>>o|FrRP3^J__frif<1vD?ri-xpzw&y-($0coBY;h)%(=% zwDU}5=k?C9;I8C+cgot((iT`bgM(#!KFL!V)@Y?JtZVS?7nC_O9hoqzg$(IsjE16E z;Y9_T_=Ts$$=SGHiY$F3|BG0(mUdR1-kwhbL>QuyidSdnX6#M;@Sd)e`6`c+bnd2; zjZX*T?Vsv9D;%^9qrcpFWJ#gxmL^(2>aBAOl<9Jb8Ra2D>@0EFA!rY7u}y5TOHP;N zj<7&#Ym*v3YK&RIo1Uu{q)6smE;bak@iEeHG46`Zz<;rk_|WCF&?x2pRow8gDX8$| zkx+^mzNLFXEglI0@Dj(AJUiW)bD$OS*Bzup&|QJYMvM3$J8irTwMs{N?XveJa}i}e zjLomX%;rd{0{G#~WG5bs=#G{Ow@m3=JzuEaP0JE;b+_lcbgVx5C;@HQ+%G-~Uj?$% zF_4&q<|)6+aKgYbqd%;{m);-YbmSv*S=Ek|iLmN~s^!$@=gzSyRC4U`Z(N>DikObaxrxJ5)LDz99jeIv2pOOdyQymQvBivY@tiLNr4e0BL^&O zjF%_xsIphC(mnKh{ggMLr_;QPKW@@TH%K7g?oXc3LgDV8WR}R% zIJg!>`?MZjgcn+9VYp^3?tbf^{s^?AN_KLn93N3g+xyy>moEi)IjbVd(a3J`TOoSs zaymVAI8{L(k0yHCVwnJ7hrDD6SCDb+V@zYX67k(Vb#XsJJ6lSu2OrTgn{Wy!qF>*T ztuHI3$2|<&${YMNxGf6j5OYA?S~AGg>OLq@nWwaMe1(9}jjRz3BpvBsy{5 ziUK?+vUn$&!LS?|s*q)>FP)ujBf!0tUkmPRFmobANF&C74xm$1hnGFV=XDv%xN5;2fL(2;E7{hslkO@kcF_&6!mfO+Y7^gaV_S0uf0HSrG!W&){LT2lG zemwz^C+!RMQ<9jMD5UJr^F%;fI}3Vc8fee3A)vwUz<*>OLBi2$)+N7x5>L(dIpf)K z;4g&v>yRRV{w_j6XC6||#sMh_`ZH9S^9G{Kpg1U;BL1j z^|r2mn25IKJ3FfZx&pQvf8kaFra49eRwgRZw`*C%{)x2_t@oY)Z}(?%Zt$b9)7UWR zlr66Br{%t{z>cisijM0EK|#8V@hkxkLpC3*Pp)N^_z*ei-5Yw;M}Rp;@<1uGT3tCb zhefAOnmmWNzIttA3dZm-qHGKgvB=73c`79^#||(WS&X0wC=eVVne51fpm$1>}Qd2 zBH~uqnKIxom>kYOAT3DZns6O*YXmVZkm@Omi0aK);P{-jf*ws>G~7C*1#ngp4CyA0 z>O&}1oV>{+Yll{Z)gwNdbO3sFEW!&vc-XR3HHL`|6}k4@;v67n-ZWFi%e zXp1n43each-DkW>8@^w;Q7%KjNXt{~yk5;^?zwjDNXx(>4Tl~;SQ>%? zdyU!47bT1=IiTyyW0XW6CmGlVN+WZd-sV5NOwNVF!s9^9eN-f%KS1{$G3%%6wBP}e z|F%oA9dcb*Y<}#JWN=`RsFu?veGP_Qi?l>O=z1KalL+G>VrWbVTb3t^rmg_%^=C*n zKcmD++bbIj2-l0^l>CN!GjoEf)ff&#vc&Gr8uhjZ-|HelCJWKOqcfM85 zRU7c382qQ9nped(HdH>%a$f~+U|p8&BxK|HcQCX6*fL+W^6~=0;KPa~>zHl_G zVPyp>B$aTuKm{KEba>PN*rueIfq-V8tRw-qrLNxP} zP6P=lgz*Sy)cWom#685?ws4SQq6*FM1oGs7*Ah0fHExFai?_a)B;kCM4*TJ9 z!=ahT5;DoZ6U1Q+S`BMl!Oj})1v3!0!t-o^rEKYtdPC;^h@{W%#QWrfPQwQ!z!My~ zg_LwctB9s$caFzM(*)%g)h&j=adplwkXgmru$r*}xy;DC5|X=MHIJ|{6_N2D$8mG2 zZJ|m}88kOljZL}90CzJqMWQwO9P%H!<8WS*$WQScwMS#%=O-Mn>(}f0!4rivEA0IJ z9Vnw-AZEkq$@>1E{(`_u)3deOjww+>nntMfzGvs1hsdcniCxz^uDA5&GQc}Es&^Op zMmrDxjJV;5d;ByLucGVB%nsjsD-90gyXKiZd6u7YQ-v4cmBBr~d*{CVDF4 zY-mxyj6H4s?rJl^7>%;DEU|%92xa!48V&alc`@_@59zs`7A>H?Q%<)}-FIM*O?WQ` z#a%$d3uFuQOVr7Lk1R9Aq3L*&E;^Xz&Za<`S&Dov zA#b0+pWXOj*&RR7>PpsCB$T68T0d-gOvm|el35(!$>HP)napwOoBs||h*?=>ZIWC8 zLQUgyzk>vqo)c?f&=piR_5}w?-H8TO$nmhOL_|+1V6^^00U?Ux5Y4HYHXZYh0reXA zI9S_+CDyaHxSE{-e!uNDN5@?JRKy(=(F9lxA{0Cg5M6^>ClsXaaZbHj5@bBt{OaCr zJjjoQK{L061s*$k{M-|tFqRCevk7E>)Yh>K#GOg;{~ zl+#^v)$0j6aI&}%Qi{ErKl8Boj=Bxpu|10`YIjT5O#4Q?HI;)@1%{dK-Zr{T^-Lwz zSMuJoM1gUNa@Tl4!}t%a|E}T>2BwJ-hvvA8IYb|f?$pOr7tS{$=mn9h25g8uocai5 zdJ5to>-HVMBy6=|>7?4oD;SveOA)R>maQL<=3A$Bo*H#^1`U7E0@qc-+^&sm-LXYcXD1_90MtjlRd!LYC3Z(={raU^$5rfehAB$5f zZ_t01@Lp1vT7MD*MlY_aQyY8Vc=VJ7i4QbJ4!5ZAcmzF&wKxTKuWn@DCE*PC*GpTrwoHV)Q$q}#W*|hTU9pDY72_8 z1H=l+J1(z^7quJI0~s*`U&n%TDQw1d|oT2qEw?cD%B(*g)l#0Cs+#_vR+ba?sa}Q zn`TsbdPp<@Fu_OyfQQ5Nx|>OAI0YsHJ6ByV*~0`mIoXlnoMjZmuis|b;B~Oidte7n z7Q?ojNNNCaM$CRWiL4^-*Nx-Db(YBG9#)>@Fk4VnVtsN>Phw`VgW-yA|F zL^qkF`1};8`v%;<3!73SiTE8Sh1qj~%m#*8~09t~`e>k|bm}~m;Qp|pj+TB`#JXe(aw-xW&Sun<2 zacqd}OI}oCtVo=eh@F?|PoNK6vfLR^+mYL=`||g+aXkaEB`<0)8;>jYVNm$0;=N3P z18f7+s*WC#zwsT~H`6X>O(BqCoasLu-~q#NDLPgJu$hZkzu>a|;&_W?u&uCaFg$DR zh4RM`3(OT3ba>eFYD?2y)kS_O97If>XPxafBp0?fL|1E%HP5`9rr5e3;ba2qVXF7m zFN(KmncfdNf()iX%7tii`!n*Uac-hvt=uI5wm`Kgxj$1msKn0JIYZV1y1+q9$@_~D z%H^fX3nxWEMtfMF?+OLLbXtqgneS4P+RuaFFV!dF} z%NpeT>zLZOG)?(@M^W3>S7T90^zv6G-!y5epqYGoHusHt7&Q~v=sqh!l)l_sW2Q)4 z`j?m0PzxHQP#6R3+Gnr&7FSoMffMkfZ(QD@2Z-Xbl|)9D|66$JaOt(}(nmaDkd_x3 zN9Xj!>iX0M`BRn~V_v_nltJh80p-pw@d%AEOyR*lnI*6(xS)O*6l%k;-cv|3mKl)k z_AdRnH6j6LWR7N6h-fZ`TOPmg%ooydER)19z=UxqMN9E3E;Ny5Z?A8v5=r9)q?>Iv zI?Q8PV``M|(G7%wg!u~Dvc*YS1VAr^_#E@hh(GXFzT$dHCeqAW9Vx?8hE;j1_#}b-&i$X?4X^0@iXq ztzSS_otOV(Z;7LL5o9vyJdoqVGBBKdX6$q__VV0!tIMkC?QaKMlJIIR2X5%%qoG=< zlZwb5!n&!*!mlCgcO&L)-x=rs0MY_;=d&-p;zpXVfGk~e^Vr72BsG_?4$akZ5Po|y zx9$6`VWt8_g}&Hv`G(`67uI=l#0;47bRg%aJbG)|-6jT4wnsL*db221bim`(XWqEN zM0;Z|j-1n=!y1XNomOXgR%k$I!!pRPmTr+#y!i5^!L!^|`zQE?zOjhTJ`m|#zsNMZ zUDo4;-M`zNiN>qym-+0&st@TL=0R1H#8fxQ9UteIqN<;fqZckVD9a4PZObOa^L(!5 zdV8D*2}gi7)`o6yIUUm;^%nehC1^8}OK-@oS3D&J%(#e|w;Y8M&963^AFLA--7s&n-#G+^f!3$xO8c;`U376G-sHtCSm%zF#^4c+70gNHJS|hxAnKJdOD)d~Axqzxf3K^PYm?o9TFR-1^Ajb1 z(hl5-NzCUFpNH+F3X^)ozgTLSJnWcCL63s@K*VXjB$H2Esh@7~E1vJp+TvOazyfkB;0Ta!DXn>-Z)$DdidDpT5G{(n zP!g9f{98n`kT0iWk$cCsf=QU->alcu2j_)R`{4}JYOM058VmyJc9%U?&Q~ERu zMAwR3<-Aa?)NGm?(To5@X-`lm6SEMJD(679$jnMkCsQerwPrR_6OK>^m~&~?)WA;mZv97@iz~pEZJ<)|<7`fuuMz-_ z38N1B^fBKGka0&(B|;+w4vqvIU`tu@g9btyAznW3L}EoN3F;JqlR??dKC>aik&pv! zGK$o>Kfpxel4do3$)bL&R2Qz>;oy;~wup&YKNE-AV^Knb&^ptok;h*5imkE@VJ6Ux zZ`eJu&c+Y+Jyew7p(mqB=7wgDht&JcX7JK2G5Hg`k?|HQEY8{x1Niiz|Bq5KP&CE) zr)w3%eTUD`a5vS?>aFiR#{l!jYb=doYSgw><8vB4`T;(Cvxi%)+K9ONBaq4O z=TBg;goIQLiqZ;J3@Mt5m_A6-WW*f)-1NmntD^~4;)ATc&i97%gl_grdGVZa9C>@i zh$mH9Fk0LnW-j(6epp)DFJQSTiBZFuX389p8g|H97w1Lh%|A@|G~xt>DXuq2v% zH|v$i%b#9XVp#YKk;2Di3H!W*|1Mh`mbc1O-Hx@u&5@8uq`{r!}%YiH0 z;n%CkYk@jDS6-5o1Uzkhnc}KdEC{)u#RSg|yz|=qesm2kI0Ng(!Xy6?l=%f(`EK@x z?-nt8_i(8}#9z>mU~|)0$~~)sF=33X90$~YSCb~YK_Q;e%vF zPM!sMB^j(2YBUdoD)2F#Egsfn9gwD*f6Ji}AQtdk_6-sdRvw5`lev}|{)s&x5g}W= zh1UryspqdS?eLWE=nY!j;m_AgdsvzR$xf-sa9QKPcL01W^M^_62!s%Sx#%7o5(zUv z>yh(|^Lc9+%w3K>0V-ZOyh#*=!fOfOIK7AC)G`K^IEpz+SI2qzTdjV%iD(aSPA#5^=BLcU%$n$y@D*O;j;-ZE z!Mo>+{{FsM;DH`bwT#~79+pN#g+-gsyW{@izkLuf_8l9#Rm~|4E2C-4$)!&mqth!W zB!(as)89AJP_r^k&RUbPzSLX48yE7dsNv?r=7 zRO9U>4A+#gaCYXh)=>buWgu~L;F0-seTl^xB<~7$G2((^HEvxIerH!8f_AW9*|#fv z`1g=d^JV@vb_@Ojcv@f+X|Zb21Adx&80&P2;eC;6Hk>ou8y)Vh8q(P?T|}gwpl? z5MmZ8U_=)vvDsZ%N2*e5=i*F-bC26Dvq2dE!*uKnU}znacH8hApaPy^nR#7%B=IqO z^DoF>tskUdp?2|`ebZ*X7S4xxId&VOwu^jGxs2|QmS8{gihS{-y>x;XfP zC1<5xfJg;1daXf_H|g}5f3*8~NuO(dPa{g@0X_aBpLQDP8PJ#o{Dg4+5#OY$t*M^~Z5fpxL592$4uOyA|SC7mD`Ox%%ig3$1eAz9~hT`9hF zB|4h@S38zPrRCZ};)IbE^zld&?T1}UwCFr7nYV>4k2PtE-V{&XX@+M_T9#MOuY(gadgX4a~(BD5UXyqFC& zG2i+ObnOpa&#f{GtDspRJoCgyA|OHS%lRL%r4|40r_CBabiirYqa3G#G{MR4EVEylI z@JTyRg0Bqhu$v3~ghF5b)N#q*z3?UVG->%WbB~US3c5-ty3sg(a-vFBI?|U%i}|d7 z3oQ^ENOmCnL7;xkeED19$0e2Sj5Eg7BOF@wHt7JE#AJgg{CW zsQ_d9tUKR^(!Ylloj34$Mw!|40&A7YrLEK3KQ&LPhzfLiNEp^G2G|pLx>-0lDa7Zb=yz}<)_&sT#!_uUESu*#5 z85Eg)i9Qx#_UkiVx&Z-|T3{HGB^quw--G3Tzi!JD3lkVxm$0G#!)Zhd?DXK`yM}T! zeP2zFRjh~*sNb6~tR|Qx*G+hJMbz;eHw?Rc2LNt6W!F=~JdM{T<(y|z7J=WL48rJQ zv?$v!>x~UJEoICSQU(}e%GU5xAW!_$8O+Ps+0zXA7S3fxNgl<0O-<6~Fz+`&d!G>q zMPCPBc!8rT8YHeW^rr+x7QU~kf$>oK-Sl`!JHHIs_^DD+cyI4NY`n1T;<8Bk7(K)!eB~^?Q~ZpxXscH_3hq9Z-b!>Yg7W7& z;?JhKOuxoG*P=CmY2AdJ9xPXx$o3)|^CF;|9*wW0aYdLInEwV@1Zs2E);r=CTA4&}Kx_s`~ z*sdXSE(*Z2Chw%QAW`YE6<>hx(J4?htdL|5LxhvtD2oATRV%hBWTVYx9xO1M(7MJtQ?c`>u)(^4qnoRz= zVlAUXmmbYF*~lRnJ+_V)=(giEl7>pqqy|6Jm_;!=@G540Y?W|OyDK&^_ZMso^+mn(}y46q4QyOm*cY6`>xD%$={tF@!Mo+)+X2WYR!c)?!*`tJ7ftTKV4 zd;D^v@$HNghbLhdEPq}QNwOx~ODMm_1h+&$y8}?ZE?2DUZ?r}T0(_S?6iS2*SS01@ zh?ZLakH-4x``KG&BjATCqB5eA+Y@l540iq2#W z4TtqcQ-y*EhZLSd%Atup)~iE;^fzs!>%9s+q6u+pmA7ei%VGkr6EB3I-ZLJ80IRnE z5Wn9|w9VFuxl;4vet;?~qt`kC03?hL((2Enn|TJ9chWT-(zVTg`;>)}fiQ(0AWr$X>j}FE=pkF6AraVEKWi-NRJt=B<1283Cc^T{= zNUiGrZHsi4GigZ16mp5}oqcV^6E{1)rB1m3$p+<}bBIwV9V+A`Y>Za3SQ_wP3Dq8* zlBKEMEyT1rL80#qhOiwdqaCCF+xkoJp^3V-M9QF``ZiJ__kwP|bTs$1hvhi|E5h)o zQ8U5DjMpUs9IpPdr`66)*Zq4lWnSsCup+IJzK`e5v&WJ-er#F=6V3XbZX1(u><@^* zFdm-&%uvPzFlDGftCbWx##bob|&P3~y>Kr3U1Wt6l-`Y4Tf?2qV*NRep8nEw4!9mw4kY@O5t!lXX= zR)8%n0ScT@W4q3pTcf1`-jAHONKZG$oC&h`x7Xx=>Z?RYs5f;O3YL4B&f){(&AAqdIdVZsoCzbz=MRX1TC7oF4qm$H9}U?13Lu=tbaAlTdMR*fCBhsCqB z&K~Irky>YB*gW<}HF+P{eMAXESInVN!n;+3g7ONIoiuU>X=U3~`Lh;dR*aHE@YKrY zEVahAa!7+%M^3%8Ka+6}tDEI%JO{WIO22zd{|blMt;*Hpcw1dvC^ZTI95J{`H}DTL z?L%E_Ye#sa^bUpSe(d>k4*5e~>VG6Cu{+=Zo;EnvHeqGi>+H7(S?Ipv{8z7h^aNEI z9zWX!zL!0*uxYXqcHU@$arl&$%EjIU5fz)}eAgzNCSU;s2 z3a59NUoH>LWQ!U8x#qi-l1Hm^B_c;7=a;PG?Nfx6O1x71f4NQ`I!*A#_JiNW)7BqI zGx=Z|cvGm*V+atcg@~v!{mfeY@>!}k2KcgDZ4sKQtOuQP1?0xV?Xd=u^{ zpQt%<{ScR$x&*pqPm~O20PG^I?r%a0L13gVH2N8+m{8swIm4PJgO)`c7ul*cU(Zwl~RON5__c zgz)4)3VXb3kRkg_NOJFTxr3cUi>0r5I3tQvEt-dMVDa}_pe9g5d60W9)T}Fc6h-zM z(E&19;=1)T6bIDGWkF5Fajz5k6rjzVI^A!J{L@9G0JaDsH1%0+5GbXlqP~~UOF|w|6-`? z%{NMeSua?10v{oqb)`UxcZ^1XD8$Ei^@hdgR}EMet9ZpwKJkFu`omxR@u^ur+*hvu zeg$FBx(;%+F&D!WP%ppg=h)20Nxmg>PPJU3{^eNhv#w-_r+1*cbr#>XR`z*ZFAFO% z?9XC)Hd0%lMd?$j=f4Z~ym%i@>B6|5vi>|Yk*hFpOU2mWh=MuP%CVVVlPg~RaTpd> zM(@uRS*X(J9ROiE?MooTM}YHei|&|$xYT7#S#RBuW5(}qjzq4;vLE(NYv`1JN^M76po=EkGbAFG8&a=i9~OXeG*0f zXfZA)dmPhHn1LEA#`H~_mfRF=Bw&h+tGS7S9_FnVX*l6MMbxX^d*v*LTx4;(QF3mD zAlyjc!`L||<{_!GZkT&cFK&(OQ{E?WyvVAbKt}NNJaNx`JsB%2w)AB(KkMKz`L^fu{RI))5ID;U`k$f+f(~0#sCb7CvHusVIp? zY^EC@jo5R6dLu&XUn7S~YuAS~n?iiF$?50@)r1Oy?>$nf#ijrEwZ2ahA3d-Cwks#} zj#;#@CxcF7^wYm?enad2wtxuSyvJXt`=$b{;Iimv#AeUy;Q@kI(-X=p>r)UI9w-}W zT!#G}uPk5<%reK=HsCn$TnX3{H{{Iu8<17c_9RhdOI4<()gpc9ylis>=IKqE`K>dH zq=seRkr&T?^va5VYZc)W?&&A0YM=pLgaqIQ)iL-@8DD|Y?*f^MON5BvWaT$2f;uop z97RJtekpQLqbXMG03M~hcUA>Sh*%IF^OvaE6pjw^H||qQV2QG!!13tSyL-W$gQyOq z%EG;&Cr#at*i}hb(x2|7!bkLIeZ}CQW2*ZPeORFcLd<80|163eLSSFt(4MhMeZ_e` zW+VylXgnb(zm2mh0t%#pWLH6UudL~K2HTCtBRV-#^~6C8xwH=E5FT?Bo9=(CNUMp$ zEiuXaod!w$^PBi?8x=5HiPV@*ki2`NTAQ zwznQAppTzDbpgh6Ap_9PsDxXcdU`#s0XzRU%*%!Ku+XfkgPeJes8A#9G;WEyhn&wpiR_3QRCmK2PxE zo*h*QC+;RK7zr^b3b)%(-mN%eYEA8mXm# z#H9RW#VuKi7Ag)N0GHiNjZIvy5$1F$m@MVw)SuqPJgD4HvV5qyB^p;vBvm#KGz4PZ zl{zZin`$ABFogJJezK&Dx#!8_^qOc?J_g)|x8*Ur`fh-qF04u=;)^D!D)4jwML@d0 z(-hkJ2#9Xbi0zqJ1_^PIyqnHS`hkgFO`yJCki0Mv3<`7ZvArP-L1@6vL0&^LdKHeq zoKx_D33UKuzt8Ui)@plXIKT$(@yZcT@`q$yvUCbjRom~NiVrF~YQWn8#M?O3vl=mm ziH7-J(@s1Oh_+Kwdm~h=H!AE!(F+>u%~F*ZT2hgxJ2pIMe5og(BPkHYXslS}E!K-KTMv!r zk$UTfhB3S3;Ab@=5Yl0_&Y|g)AN{p!VUIPGmr&d}jK)(?V;r7b8RR^JeF==A9=+OLI z9a&F|sl{oR*lym~?VKvm?n}xhC#qnp=2q~K$Ms8^qKVI@59J!xMy~$k%(QRv;-!$g zO2pp0XTT(F*lRtO7AAavzXisOp}eDeHGxcFf;we*O!l#^cp(B;MZb?wat~HM$@Q*7 zU(E=)9llkoG`z7i=y=?kM*SBf4B=;&Bgkv!PHn#Dv(&v0vj_fA=CUo77`u@4+ERh# zDVvK;u5G@8<-T38X#jZ zLtbQmvOGOd4pBuyodd2veiVMOSW))-U;3ogllzDI`+zOI-AeS}LlqdLO}F_?yZ1=| zs;`IgclBfaQP5w;%5#agw%P$^u-&c!I^Kv%;%)^k%eB&tqm!270VJOy;mW$f`rxT6 zLR{d-^w{IgPaVL0OS)h#Sj?@P@F^t*=(4Og$wN{3h0E3ij-~-^!w5~mJrxU$HZ}yto zA+(kZETU&VG1xF68yLoB&FWxa6tyXfqc{>K4AGb>r5Yx#QtCFw4=Ms8y+J$B7Xy1* zJiVjT@i;hy=%%jO8+R2p7*oqS+ac*<175;{Nz|{ahdG)4!l^clnLA@xbg63B^y1W| zo#Ie^_m$cvpJhiVXDUVl3!^0Xx_gu0wJhFj`te`l>?5YaEgs#a_jBLVao#bTf}Ay_ z*0**p*+6B)Ec>P36!(R3-W281d1G$b;E|&uk7xw>k;`%OL|C#siIRP3)8^7r>U7dK z)H~`NPbF()M>$Jes+sDeF~n-Z8(i1FOIVwtD-_$vjH3$NS9yHe7{1!6`a+*!htm+! zA0FDc{98HUOOds_aOT5hs2<#X(CQ_mcQ$Zn!_HXVtIGQxs4>e!xwa@Bb%Zk8Y`*>o zrc_xtHl@nRi+$k{BHosed+^p=7 zl_yBBzpG8x(9&YBriknJSeP&FPOmiOHd_qxSdGg>W`1s+@SAx-MZuv0OK+@QU__*1 z7;Lv>(b}{!1c8+%*$f=nQu;&vk2a_vY53PBZ5ercdx9gHkMn`I3h69X^s3ZZxB&Rn20dWpcQv=$&-N#OLX2e0MWw2c|j zh)>(!sTjV*o5Tm3l|jcRbN^qiFv}OvS)At^y+%fC5H~-mKZe?wI&jq9vu<}QHRKR7 z*wFa9>|VgGb`^yir|ptdR|g?Z;!ZetG4Y~AU%n!3OvQHMO?VOms)KRe)26?k=dxU3 zJaRmoC+vLWY7Ync%<@hzvVw$pFFtUp$|HHEAnzcmYL_etsp<`PG^kD3}cHy|EWgFVKyh^8(){3s;|J{hfo%B8kgl9(e>WH{I;TS-P@frGN1_Pp-_ zy^pWBr&Cy;I^JW*bc~X z3{hREoQrAbt2U`$5fK%cPl1WfhV3B3xtVaKTjSU|JCZ?8#=sL=@@Sds5qb2irT};A z_ojuw*uPUUJdzd^#c zowk%eo9qFG3(v}iYq%piS;4Y*dY1uxBM1h%do7jNL7VJAeOQ458#tC*p<8N|MF}^< z(4l|${yYXm2^;ow)w_=jX=GVU{WzuQHk0)AIgZFg=-NcVR&B7wy?vY0HH`GX_eOv6 zU5np_?7YwViOAM`mmz4t}GcWx%Db(Lkt4#0zh zXNh#bv-I4|$3{(;iD{Kz^kvIw>EBc72xfFEnvp^zli=wEbaex!Z8WmDXr4p!1p>&) zHRxYMBXx?sAQ8_MwTxw zU(f;Bj$1x z`uR}r^s;sUE*JXcy8VDj;#z!Lvrq3dx^U^%mxf{)aQR)6Q$FPp+KxG$=D@9ImPL;g~cL z)|s*;bsyPLSTe7GmvXJWT^wDNYxipTd2>!?bLS#0cuZ9p8y_kdHZaH)Sq~;Xd|nAc z{=8?YL)BPsH%0x%RT>=ja{X=C-DJ0~rDzpzYmeWgP}Sw9MM$B2Xucu9 zRky7t?6#@&s039kSgN98qh6bu$Y7L@PrzJWeQOJx)<4d_MrdAd{C02v1hfzEmlGU0 zB2fOxCR4IDS2*o4-X3C!WRy5oP~BaqNy@PJfLhJV6Ft1aj+FxAISmCuvK*E z=Z@;4fv>Dm6DYI&MYR;y`ktq!9IOOB?ZyK06Ygny<7ig*KntB7VnCagwOLW}p4D<7Yu5)0HgI@> zo3UG`R_9&pRzsP|7dM=YlD`c3dfg(Z`#JmOfmbKHlZ{P~d*GJ}$V+wm<7QL|O}l04 zx03O8T#P(aM_u*OrCj2h+aI5pg;}z_?ASZtu9$)0`2J5 z^*0s5{WZFBAy9Vwlhs=rT+bE195mUUxlMk2sIWK-2=3n!kdxIDNu|0y)9z_2f%xh! zBeBWc*>)EW6V50osSW~6CT>5bw?f3xssfd@?C_mIzM5VlK7j9L*dIFjTS^&{-9MC} zu_oi$@aehT`N{BZSoh$#@{2qHBmF1j^clL>=Dv2Do+~_xQq3+>O*8b_tg!~X^)N%C zlg!wWP*Qy#NzsSJDQ2DPd`%i~`kd)@D9DybaxwH<<8v7N#A+Fuk~CSGgvs&`u@u|n z%0wT!e^)`>D>ey?o^TYTdlC-}-MlXz!mG;)F&CcDeM;epqZPxS#CHjo&Dp zmziX)ne9%G=>ihx1q-(e2$sO_cKNBtr1So2tQTQU_o_Y24;e(U1V$q-H%^qw9abOb zV0+lW)x$4XXz-Wh9-GJU1XOf#yd3P#LQmw)sj_$-^;o9Zxn+l#5Ocq$TH?_`bN#3D zHx4~N@&e<%7w3{U-LHH%N|z5dT8;``3LsE2K8?uUboy4xuEX`5f0qNVuq;8Q4ZTY; z_FA{fY`}SKR6M8>6v$3^YpaMrAFn9e1WDz5D8pNvokoLUs-Gm$kcpxX4>H?-bH7%QS%9h1J{r1)x2?)3aoh4|ow#8iC(52? zh?djBFDva&p|#m)O@fFD*C6yFyI*Fbhn|&W312Yht)TNeFoWEMLc5dBAjS9BqG%Az z&ailS!T;0}SMb=~&artj7#>(X_L4k(Mq(gew6LC&NQW+tLf!8w4RQ4{1z|tK^~T1F zRFH7H1Q-YOUW{P`U%SV0GE&dGJwxRq-F;SQHsxaq#gO*jT){kl`el<#gv0yHIndCUHyi8 z2C1UczgK5#l^Rz^rWp$kw5b`5U)kZJH#^3+Z%^-hbOT8d6vr{-L8FdD*zeYr9k)R4 z6#lk39Vw9r=;;A8wMKtR}}JL-ID;&yjE6Pm#Y@+dOE_l1Z`E5*fvuZptCyFEjk}{`o_VrtJ$4%W9E90T z&U?W#*2cO(q#~$OyA0gr`G*NsrQ?i^m~X02RKjS)0^7Y<$G!2F>Z55{_qM8Hx2bE& znuU|>`eHauUAgGC(52st>b@~vrC)Zg7V2-?#^Q`VUSXWgr=~F^GPT_=1dT8wE`_{- zJ_*!OwAUFQkR1TM0*wXT`A;qfVQ6g``u^9o-F527M@!R0r%Fc^>@ph%orH3$7&tic z@g?frB}mlRmsl%kjG$t(iFItb1jy~(6ZCk^B|1rNl&<9DxBo9)S!MO9G{}$8+hotK z%ZE@z>0YRlHD{&228UrEdxEZv8V^#(PIbcgqXEuY-nvS66)|3Vgc>0oG3;EfG^cx} zQoH$Aq}_>9l181c0Xk~hPwY!wX?hh`v9XjB1Ck4OnW@uC-xPow1`my$>H1YpP5)3S zHrq!O)cqvVCcEf^a9w-9FIm*-cCA+ZV zdZlRQ;7NpqVFR)+eA^T}Yn6kJH!+#^GzLCExj!dl6)Z#)GYx7qcc6#pK72x3B`l4f zopj3q5o?r4{tA$YkafLfUeg6ua&vwx1DeP1i!2&BNJ^eyA$H^bo_unVORMwOP|)=* zTG#_)_eK|(L=+@?Qv;@WHq=TlB_sMtigAPEN_#GIknub+BMYO(;tQ4xc9<#7;t?54 zW9Bm9*rvXW#Au~+nh%&kb{OYFZ?{#I=MbGCy@(NLwWI*HKqF0mk0nCG4Dm;kaPV|8 zgRQ66B{d_++qZ_GZSLM@leH#CkKev{gstemrlZ6UzVN4;0SL;5-%E zA9Oq~#go}U8RP^f+Yh1JCbfM)0F)X#{?sxw!gVZ+cL?n9qXg{6L&`Xr4Qy`~Ow#A~ z-ZbiKgTU=rrwavCL0pMqPx3GV36-0m2WsCBi$fZcD>G=}2NFJOq5e{lmH?#00RK)_&u`WgulK zZ+*(-2+&jg1rmwTXQ*ABNiCIfZEmw`Qnqq3%f7lfr5>MVsP)E%7?%gm+F3|#B-@}U z#oJ7=aIx3zI1t>1f!@J{%?nAyo@F!KI52(a@lE#gBjB$>_@RK>y=~U&i)Mk7vs6;* ztUuW>vNJnAn3x4IbsoZ4t@sE^c`xv&&^}VnP?nCg^OdGm zNrx`=iX1HG<+|j?2xaTMx~`W62DT|0W+uO%S8*c0ww#g4Cv!Athb^U&&lHz|5JQYm zApIvx*FQ-#zN&vI?*L!&WR=q_RzswOj;olRpTU&MbudEF&VB>M$$7M1uZo8xx-89W%>ddGeBIz?BCBwq+~aBei!X7wvtO7_fjnGtT!yorh#^rM*!uY z04d;_q)FKI6DZ6T+4m*h4Ant!tS?V(Kz$^LprQzQ5s#rHam)ivqBTjjpS^Z#B6NMY z1xnkhG!npFqM6w#`rP<+js=jTmTjMwqjl_e$|1^MA`s%+WV|5_ zKHn{b+M)I0H63Xv&Ci{iHYispM(3fH#frC@OUzHe7WQdt=Wwq>L+umShU{juPf1A- zYu`kMks5Fv&gb{~cAgqY&T3vReOnv@f(7!B|L>bCqbp=GH+>?azDub$3@}O?s0&fD zi@mgnJeF6UuK4hYB>55^eKTQsN!dO29e@5ElZt_dM$egcG@p+f_3|sa5j$kT@A1H; z@!;_YN$Rb$;|dePt!1)n;8sQDJl0_kczj3zy4D$>{{#*>qAKPuM0%)#Ly;&xOmQpC9)viUVRw#E{4#AeD26 zj=uyl6(6tzl^7->Ow)i`{lLMsJ_(Biu}Xm=36CO*!cIVVHNvNs*D7`;Q4HXNw>k)U zVav|J$^Srtrx%Tm{gPZb=_-$8I4-`RxjJVINq_AmMtuv>bA2iHLwwtqdvg=W0$^C=Z6z>urm&;?VV_z;{n_*Vm4?!L6asUv8D$D(Tn`#DsIjT-@0%vv}dMdPiQ&v*o>E&CMgdz{~crKm3FlzzH1vXg5^+WOvB4Z#U zce*Y3lqo@14uCEbKd7driCwv^lsz60PH&ruhT*i?!Kp3<=0>=7)EXS_F7yRF?Fhz0)C+f+!_br9w z4&`P_N0L?wYiB}P$NTgz5^A}{Y7?p>2(n901FOxAY`-49RfUzR#k4N7dTF|$=UqXH z;PE(t{s_B4`WITA{6iA_dJ1?L?m6@S)^#Y52P{zQP6v0aJLMW3XY95|Kh)zr3Kv88 zbE2fjmB@0Nj~J4pxB#MKA&v5csKVZRl}bVz;`9N0MsD1=m55c3uLznT*6u`J@i#&9 zg)@*lpdhQ*&-e3dtou{G$X%YiEXe}~YTp^={DPzPeis+A{JxvG9y0T?^-*h>@^Oiv zJC{8gJY2uZO)RsZ;gN*$HU-qpjRnBoO|BH-0Ep*driZYh<8q=J5s_x*U~ z#B;>uU>vS~zFSkbmw>n}6VOOjk8xT*(J=3`Jo0dlw*@C$ipSc*Q3Cl*usN9cvXmyJ z>*ZM>X}d!BSya74YK#8(?Z|8gj%qV&x}i)_&p}Om>!QM1T@giZ8$8(9J4qFbIqsn? zcr&RoAO6_PfwWv;Rm8IjgrJ;a>Oq$|B-8!c!I6VkrQQr@I+SRX3kWIbPg|%N&t?x< z7?vN{`c$i!s~N0_h5;yE`6|UJjfnyNt=PJ9?LEEAcr&)Jrw$gTZS$nEW#2}a7m_T?l!UZ0`7{r|$482`=VkLW3DH1Z*) zn&6qN{9zLJfBs%ox1PE_6|H$xaR2g#X=Q~eRk zbYxE)NDvTZYY&m<$Zs&nxQl=D6C#GC0j0{(c(LgTxnZM4;~F5oJips6wf(9Emk|=^ zu&mpbz%zjm)mjOMaf+;2p#j&$4%*6wh-?JU%n)R4U@-`A@uEE9q}~BQ=Ob>EOX!0n zl@#yIGe7N)j+rO(%(+J{vr&|`A1EeKhR<(8lkOHB)Kn|)D}s`VzKWSrAWhLqEodrx zAbO9D>?kO4vmBWD-0N=Q1L)JV@AU&ZCWD^I;$DvQ%6O}3BpAlcITo|fcnU3mfa?L>Nh}J<`sbcRj{l~C3gJU)k2t|lNS=P;Jb<+>BnX>csC6Rt&lML@_b~rz+!Sg^Ir7`%O{re%K6MTZ1N& zxn+9ww9u@5pQ+tbyJ&fS(~dk$=3iX`T=UIi*?0z*xIYGcM(1u8OB7B1<=0?xrt6-x zDfDLOnY);*2qAkND1YsKQ_%L=>oKVIF+L^~PC@>-hpyG<#s}-f2&54%{-;*nzS?ps zX6if?G$2=F2#N0-G4*J}+y z1Kh$$lpCrAQ+(r)Y&U=Pi-aGD8k3KEoLvY?*yx3tvL+bhD9(C_Ae}gAcw|Mza3Wn5 z`ee0sdL-f5Obj(kh9axM^AdV?^3uJ}KMp3Z1@9TS4^CA`r&ZP`tAC_SO}&h+N9nLr zM>QIq?V+aCAJlRr&?ge(Xwqbsra=$IFN5HS!+A)+BGQsOtFC53afUQ+b(NWAAz`MN zZSevwnb#Iytj);Dw=y=s?H4t5xjAV~yxm@oZ?^vi#TB{nl)V8t8r9DauI;kvb4baG zL{P@6w+D2;920;1oE-~iLhS+r4`VYfO7=jzYAY!XTT?_&>S6Mw_L66Lb}*dLQ{Gak ze&GeP9v~AdJWX+>6eO#-*W4KoJ1uJPdZF7?>vqhdx~$40kmmE?Z=#)Kkd=ETG$6|g zj2;gq=iSaw1*|nT-#5ajpLLi7Us>lIG?k1yDl0Iq6CTc~mzUcZQ@6(8e2Kx@z#f>u zraZ0G@Tu~L`IYB%=+r#wV-~<(*Jl9yBX}?v&Rs*F(x|*~LJMJkErVpNw~U>r5KhLV z#Jltu`39x5;U=)pu<{H)Gdq}B*zbSMIjTT#N4*4p80z#*WK2+*d11&WQ&K3^16)L$V8)#Av z*f#@2ihXINTOBPH zKvlv$XK<|j8bXScx{-uv|`5R`@Qa1BD$HH>Ghen<=vtdvcVX>*;V^IX!xz-wlN$X*ov zkqTCF;YmSG{kj(yVdc9Ma>UX$!#qulvg<4N(_lX64}W`K!X=V|*fl~S5JJ^8Pt|4H zs^b&b1Uc|-KWLq!nJ;q3b4A)eiY+6hys)vDE=PKVin2&RGu1M`hg)qj-Qmz-o+g&9 z0?TIRk-q7Kt7EoL$7xtqKr_aYjs=nge6h%Rd!WcQNXi|SZ-EMh+=N_5C2|6l zWBh|=SY-wjsalK%qO81N%i@rOzC&e`{>Wui;>u}3p?ZXgO1jsT<8&e{{S%ja%4SIn zrA%SuYx%qeGO`lu6e1A*^QDf2m;jwQ7fg5fVVtSar@JRL$pYTcz5qww>A)nNDR~k3 zj;m^OJx{}6YM!K*`Zy+FyqIx^x3KleoCEUC%Xvl~Z_`wh>Y7>>lxv?}`;72cSa1An zD)x#dnBULBkY=RujIvV5njo?#O+o!8idW7Dy?~Dl54AT(r}G$lYLcu!Fq>y_S<9d!k94Y8dfhbVz zuu8j|^ls_>ItSmAVQlKqa0T~^G7UVOHEULdBxkPh3zw+>SKQioyq8=~b zwQf0X13MAHeYLwXKbh9(CYZ=losH}`JJOwJUm78C^81^!^=IpLc(Px-kxei;)O(b^ zA=vbG-XiVX_1a; zv~{COBf~HmwT7n)^giY^98SOH=di0F7O%>$oOBVH-fOafkJpcW({`tmV-PvjC53K^ z`>LMYAxC|+=J36ag1MRXK1xC~@yxgE77O6GyNEl>3~>psaQvF?sYfKyZ%}9T!sZh7 z!#e*;Qq$ac9Rd#9glworc_JAr)AfRWo_56^f&%JYv-Gv0%T=^X=D-S7osPv)2md_= zk&7%_qXlA+syHg%6|9z?NV^YVV}!qz@_!FgvXuhk`k`p@HY#C&?RaQ^MmgS1rq+n1 zf)uM$XJ#4L!mR5?tT7rZwh%)%s+b>hx8oDtk7f}ec_&(TNuloXC|02L9;oY z(gA`&k$eg~n_4D$jHCY=`e^Y^+JMwFP-YR(N@`B~9K^nH>?wjL)=xdC`3BhRA)J^h z$E)Zj!W&A>RTY`_S>g`}w;Noov${E!VqeE!s6{C2O?zHz)|DVIu%F4`{WC^Mi>U1I+cHWvTeR!nc|;X}+fM`+fIYIW0^)r9wjtc7L5; zKdY;-y)7!GtWqmmPpK{>!uIqEGlfuR6&*@Py@=JG$4y>rHLXXY0XKEr&09nMxO64PMOPPB<-^C`D*21d-Nja;>d_g2F z5N#^Af2@&3|9oqojOaJP1`o{XPkQKosj*-s%ruPnX1)?C|j-OU_6B^7vywR zxVy_bV*A!w^_o%{==>l))N(0O;V6sZREz2)$cSrY!$UNwiJ;Xy9_`oRTS=J8u<3Wa zee~NX|;0`QmPp4E(qnKzMi*r~Ll63-J%LWebz@G?fHXLCLM5|hl057}n z-Hze)pRUrPJ%?gUBkbxX=>bwV znz}xY^vE=6|loV+}PupGtjPWN(U~o5_5&6>55D z!>E~4YVd#P=DDs_nd*OpS}r}112#>z^F-Uh3qo?^O#`~3xm8S+a3KUf&}$at1yeMl zL_&QzO4JpwH9`~;g+_*N=t8`?vKM_;8ax4H#%|>g8T}mRoLD{(9Vo8ul4F;~#0kLq zB*zq1YluOCIdh6EG`GM@psPmV9t%Xr1$esJB_`)xqxb0*Si~y`YJg_q-_2_cqau}~ zOjPfscaijdm=* z{s4KhaOoiu+NuMU%NQiiuzUuNKe6ML6VnH^2mzK32{<~M4DW6~5<}IN&JBUSUzXRn z0Weyj+v1nu7(`2b2`D-2*ZQo&kv^MEv)TqiIrSB)_E38xkNGDJMv!cPd^J@KZ6XbD zEK_BwfZbDyw6#pT{=n!fU8?V;G>w>7fcaz^@n;NQrimf6$J_EzB@vB4#qd5_Hf z@j$N;u7NHY%+Z??u&$We-+zBcuUgK-XBsX@6^ZRs=XaxMKOor9xNVmkjp&sb9Ptm! zbk7s#%Dg8;uo!TXuC%{|xP`j?^`oa*O}X7Svu1Q;DMLMiuBFp{4wro$vYME}CglRi zIc%ewL~KwWZ$0J`^@wwor;u%3$zgU+qEYG;t4i(Ou^P^x<7+7G;JRv!Uz_6G=w*P$acr`Kpa= zp87Q0=&(92>nzD@JH|yUq)qP%5! zz2k54iQ%7hun5?x01MZ1rR=dbM8lpY<|Jb_zozUgHKWf1a(unFi^ti`@EYOJuQiuk zBFwU1s#%Y4%6^wCLfeT$hkr0e0QI-+1n%WD#0mpX zpi@8e$}gJ{62+s0TK>ST=tMb@YRLi_0=E%00i@6Q5l=;0dcEAma6WYuuNN2fw@$3_ zMmgjvZCU9fabNpa#IA??v;EjUFZdby1_+Zmq>67G{Gyt=aKpJh{ zQwz$-S;=20X15TdC%nC)krzrdE@>oy_%5In-5%YLYaUP?GnF4lXpZ=mu|e+d#RQ@= zZ}-6}*p|F5kY1{Io-!VWp+Bl$zviXPbQ&C*W*b=eoba92E^dJ6kYi-!6-y_h{(SBWxpuM-whRQDGHX>+_LLu_B7Cpt`R(QD?5cXc$G7ojjh9sj%Z*MTa4sfZ1hH{oleoZ_S!H%U5(0PQ@#Rv>5l2 zGvvPeWrzk;-k~!&o>bRTdrC{z1`jLiWMGGV@2|kZU?*D(;>q^DW*iDE8s&k7m!L8i zq0Z-$%^<8Pd^9)C*+g!nhjZYK&gM05)T}#I5Q&Uyd&N<1lsY-XoCExHas|U-)k-p7Q<$|D=vVH3NTmif zoXQj)^Iw6_x+{x!C7}Jsug4v1$lcGP&*lg_nnHEEsSyu*rGDux)_x)!jh$fPo$Tx^ z>XlK515r|LZFvwo8g|>?;8P2I`%Og-nm2cPZwhlDwR|?Hf;>{H&`v-bHnF8t5SGup z@H{Ag&)CWR`FS)DQJ9Yb_$95ETtPFNFo)}^bH0X|Bk6VRL&Q-Y-L3o;SWD${a3K3m zaOM>JpP|fQ8P)rX0S6h8a_5`;M+U^5gdjRGecZ&^@)|dw1-_`AhFS#DOoxOqsi7ov z`2z)g9FMRS-dh!l?_Vwrm*Au*>gr|*|^#!CP2 zD`x~&G5fg)&0ft(?T2Rsyi^E3$z9bS$W{*6^yB>2+{2KYTsP=xMCl!tTd+I294FM> z(w@vqWrqS$P#pzr<`6p)*U8#0h&soVnEV+9N;Ec(XXqFd(;-E1`TsG-jGNKfIg3Fp z?`(Ts1nQUt+8@AZsN>qTEpOKv7gps(la)S#9WG;j!>tJRlo%^mfG&{Vf-=cYRFZsR z>`@!32>v_JEjfco?l<~#>UCF|w5?d)VaBK{Fyu#s4X>-!#{*^;c24A|25ZObe`hu> zUJMs~jQ}d8Y$!pAp@QB7T3ad~C3rsiVmE%vX`$0$=_zJDoO*HB8C9NT`DI<%@pVf_ z1s{E{p5ts$Q!Pan0CdXuPgF$KFXu1ltzwRzx^7%Z67YppcBK}{UAnc1%lrKy z(r#A*Ftc3+z>v|2uoMQ_%oWu74Bhz73^06~kJ@ z6yY~S^OVzL%mmEQUgr1{*b22)j_b3Y(%lBnwswqv>LPv5&rBZ~cHv>%23psd9yM)d z=xa$zAAr6?-7HNiqA%3FYMM<%0>KNS@%nJWbz|A6eiZ!FNEt*Xzqi3MstqVzL^)dd78C7??q(6abBA}5lcqDH35;I z>G7|=j4N@*uI*dJrw-YG=_5nG%4mmumV1;%UeXZ6o2ok8P~=bzbKQ-<>#q zL5mdYXGpWCgFPq6r}TS7lj0mU<3i}e=&Rs!zj0!q`)6igRkn~?f@PL&8QC5|%l9r( zBER!7MD}Hyxo5N?AjFgIeXaNP|7&?4e=}891Zk+obn8zDm!9@VyhTFZFcBBN*-kw< zM&Gq!8!)6kzL5lSQ$DBe>;@vTQhCtVzJ@Iru#vTystIM3wL|>4hHCIZ0RGzYJOL_$ z^5_7d6kvJ$Z%I<_LqA(1_(EWl+iVx3{wchz8V_P*>>ah0(91;aZdGGH zK)u*QJuefZ9N&*I>w;)ioKw-NkfEd|ra_L8@CLI=2djAba>Fn-hmYus5#*QRoW9x% z5X*CecrFq3)~pcDp9UZv%bR}!;7Sa%aBJ)cC{z&3+Flh|@_~e9tycGq;b2Z#Jh_5j|XlC6$zjuTf>^>L@Z-b1b4TqGUKo3^Br= z3Vv;exaEU#LH3(Cy0y`v%f22gi)}I#uESk3`|kjj(t8{rkb*Ck>}=B=#xYv<)aI{7Nekh-@u$Cz(?4@R!@R16=>jcW^TjCq5#SWzqCkuqVMu<-_ zL*Mbf1<2qhC@Gan&+8E{WO0#Iz(!Cz>#|`RgXV0!O!fSj)Bh-FM z8X;-kd-Tmu%<%*pqjGQ)MG`zRhHkeP}Az0r}xt6fAa4HQGWBr-7Ib??*V zXj+`_&#%&~t_f-F?TbOYS%+L~L$n2ci?1Ii4G%KXmReH_n3E%@KRHAAh(cbx?DKe; zs6Z$YIdd&S_WP#i07tUXU;X@Uw3@jPHp@=`9YF3lRW=+kd9(nD5?u?zh25c(@9IuW ztB%=3Y=Y9S|G?$2DDG+0A!1;N(2P7BglJ}a2dso%5SE#>zVK%#SekH>b&@M|(LzwW zn;}#Z*(_Ez;JPWO-V5~bbL1FYJ6A$6bZ=&z^~mS_4dvAV)hLHUdr{%*PAAIL|E_Km z6kRD6>pyq*7?HoE2mnR#o{q?Zu}*e&@xx|ngOdMtT4*2g=?G;MF9~({FQlz%b}FW z!CrwRWD(11Jm&pMTivFkP5+rvy>`HEOwX~o-(3LoVUGfYJ^m$pxdh-3u2U8xU$wj~ z^Rc2)Su0_I8HG$EOhjN5te{r=u==J@Y!Y0N|07`(y9H$%-MtgMGDC*-)YOBB+_0e& z>>kxy#W{yk=USX6iS_#Z(%$~5bOK6%A~b-%uM=NSdz8>(9g6*sJFI~9WMV)WtPqngfsI)40ba|%l~)tLfc`!_{o1Vf{I1hk%ytwJCS7(XMH7Cr(;zR zO=zI!fxZY!!AlSc_5?>S;QM=y;fTk8b}QXUJ7g(%;WD+_Q#n5sxT>MOX;Kme_=MS| zv~Y$kBvZmiWGH^Ue1HhDKac-;Hik4L;^TUREALN>L+)b7DZM){y~TKt#Y_VQx90z3@S`fOqp?YZ=|HT7E=r{% zg+;n>QW6P7BFuNUXH2hyvnoxt!c(_~-48O|!A6FP{7e0KaTde@$zv^EUShLy^RtAe zuVf|QQ!F3(dt`bu6ERH1VlMI79NgQyFGre(VONrBM(Bf<^mk2s_gZ75U66HE5r6=yIzO%~Fns+O9#zS&x$L0% z=2&$RzE_7W6QID;ibT;rT$hC)?UDyjgiGCmNc`OgYzPmTpcRj|j}6yWdD>MdIr{me z%Fjynok+@rigKm8yg_?lyFr0sFC<41DK!N|?HjDNie|N3 zmrr>2=F2r?%-MN&D^?Sl(c~8nWD{t-l&&`tg3D33zmb9I!_5N}rWMusF^ItgcHW>) zf7@qgGP9gC8fwWIA-j`}L|sDjgk!-~_B==9>pf9yr@=>CR$5_$KkJr)p!+XMFs4Qz zB*m{_^7zrVR?;|OMKoM3Rc@aczKpHa2$j5w``Mt-*w_{H0?W{z>#kR{LF8XaKSUOD zu-;Z8;DQDbok9<)U>a?pE8WI800BW4zUaorMrxLUE4%cPx@!WDxD_B_W@$}c9UZa$ z6WKA^ZYm_#(RA6pr)~Cl&wVfI9c4JfQVFAO*3I#idr>2;SDQvUp8%=B2QYSUtHE0a zhNJr=sFvNA@NHqaWW>Z6?+(Udj69`A0L~d<%=V(WsB<7;C>IPYv#jdvujn=Mn0?UHQ8808cZHZvQ z@2hvHLE`cI)*eX1JMyhdP*qM03Q_COHlM!CC$n$YlH+CcPQCjG*E?OS2(k;AKJuZ$ z*ub1hz(-!DMpJ*~-6fGF#kM1d+{NEG!!+CBOSqZoUsPz+1fpsv7DsFUZ%6_^@1$;O zE*e+ZG`kTr_7JghS)?DX1HjJ7)TWjqbaH2;R+&$ALEHMzmhToZAO8uW`e@%i-RK7$ zOQ6TLnP7nUaZ-c4j1yT8^h9 zbdaoFj=N6mj9JD{Je?Q2)Mj+sOB45RzueLne7U~WAP>QDIT*JuW6g+qx()Y?2cLRx z77d^{9-SR?gVhs^*orjRYH>%(JBYRx7z2|$ue8$ow2A3XQBXq<*FByOS449xo=#25 zTg5n;2={&@*xry3UvLiPQr`GVwuyJ8y8et&>~^JSeT(?Q3A^!5na_O*tSMu# z>$UQB|At;(b%aqMYU4l!bo9-a=Kz~v4wyN}(Gg<~^yT_h?cdeYTsiGw{f#X{v?JvW z3Gxz)iKsmLdbaa<^KcFv&NX^9Kf6*#z>PbwO2|0#Pwg>@{)o&>b zpZC2plqqg0Lg@q!O=VYkq9NSgyoRkhETPyNstFUp5FKoITd7EA6yiC}gYtq0Jq^3S7_1xNHMj?O7RLnVKH2kQV5DF|e{ZQ!{R zv@pf`hg6*q)_DOk)OW53dm3W{Ponl@YF)v!^47UyS`L;QamQ)Wo^4e5*HP=@3_*y5 z^#Z-3oZivm`m2SN`MMNvQVmHvCGGJ=iSjHaziEF&)}2v}i6R9H@8(#xCb*=M@I*Z! z%p>7khkqt6X9W*U$2Gy%o#VO&n~o{!4S$U%Y!g=&lVN4QrNgX&fyn2Vd=3jghBozqkd! zr?5##Ri#=W#vpX-2UbJtoCu>?jM!pU)E3a9Rp@*bVPW!8&`3WBX1022Z2U}XF*=ZL zWQqd_3!W5lNu9MUOImd(tys-6?;4M6bwY8&8>N7vJ4uxjM{3av{g%gdJ>$!gWURS%+W=%R{g;?bR3y9>35B9XEXv5Qrw*URo&FHn#A<)qCxSMf9^P&)gxL7W;>%00P?U{ zLxlrx6z=3<6`P3gh^EbB(nLG`x>*M@rDYZB@8c{IB9kifTAU9mj)|Zwiwy0)xD%yl z4@Hbi8SXO@;tEj`%HYf$0;GiCUuLC?GqUNQsqtu44W%CIT{9D#DWx%7O|@ECqowq$ zB>kSPL3DQc)Tudg zcWWN7yN4;@;!AKYy7FlI>_6zt>GfF{U|c?FvjCFeX0IRHjfQ9unFmy8o*|@oWx*N9 z^JA3Ve%}=5mCZ?8m}Gf=;{G&SX=mw0`hVN^}<3d#4PeZ^>cK zJrH(!$(bq4wV}WC&JK)~9T1fA$4zI}(eO;HrVRGfJREch#c%M>xYzGbGGsLK?9@NB z|F7%$umorX4G6&QU*MF}6(UiKx#Sg5D{$AcyHCg&H``kUNCM@K|6}I= z<_-@K33J(q;=RE>v>)(G!2g~)O$4TOl5NhO6T|Y)=d_*r)1KsU!9&y6p@VKqrjyuO zElUlIvQ=BoX96B9UHF>yNJ+b>5(^(K9QsmQZtnz?eO+6VEmD|JATCq{(>VSQPLR=_ z2%9g`E)>OpE?m>F7>$WYVIw%MvrPO+#Zyi~`K<}=_j&#KbM&%m4;J3R(u8o^Olbab zqAR91Jxcx|_v1hn3bp|vZcgPp1R5e0 zquyIYKN0`OqaAshpb)*x-&Q++D}8!QWyH`5q0aV;ohG5u7tzaYXM=jZbH-P@fMw7N z)6#UeAnB4RwZ9i;Wxyb6kC2R|ocdViZ+QFt1J3uBxlVyPyn9EptX_i6?7;!++y6L` zS2Tp0EJTQNFVe24*Tn}!gKibs#4QxybU7yg9dUeG>L$lGm^o5~xJ%4c$f%%(* z$iXUUWJh{JYjtB*p;3EYI;-!3 zp@n?PX!_g(Ymf84fvJgmSzRp|oDi$M0}g zoOr}X*e?=Yk+f7f%16Bu!#?NW+!5XE&*>plboAW)N1l%=u2%RfeV~^b+ncrp2^J$t zXvXl)-<3bly2S=xWR%AwewW540vXQ&$k~`z3=Bh>)6c%7B|K9Yj+Fmn*{?eH%B-jX~%}C8?FdKvIBF_OpG&~CGmjFDAR+yq$YcL z9t1cKtG#^kyRdGn5#HfwpMU@$T{UWdt1Xx9N(yOa2FMn66t70ExQnJ;q*5KqGuGo_WdzO0j*TovJQN=Kth+8v6)-es7WlfVj*Cn- z(QOo_pyqCYk>`jjQYta5uN>Se!{`Ds?iCiLE}*DfA#A-eVHE1wrE0I$I_#7TQ>BIK zSq^-YPVs2YG&4AhEeH0L<23zDy}+3Q_qC}^&jUy@=+1KaR#`X4`2)JN2ZgnQ99iyS zu*SyJ=pvo23rWw);(Wd*9ALY>JV^NDjR;F~HKKCNUTD4Z?Ce3*yP8`rGQy-TVt9qDY5`c}4iiiOB zbgvD@D)u#N4?%r4ZF9Ah9HybE`qpF#N3d(^G1^k*d=F1T_1+iNT*Zx?L{3l(P*SL- zgCJ!TAV>9=sH-=fo^1yDS8vjlxa<8Y~7oD$aawvV$g z=%g)>mHlLek^7mbYB}@42VMN(m|7R{3AO1Wq&^OiN)zJtSN3#`4$yzEMOVLP*Z@Rp zD=gg7QK`TAdPcouwD6%bICuNDBFPLCAnLQPon5U*XIo|ECx8ZPszrq0|5rS_{G_T} z$96u$;kqRj`Y4`4aVLLchoZuZtG+Dq@U$V-%l#%Js=zGqJRw-@xy%;KFqaA{@D;t{ zXIo_Y)^OvD>R4WFiH_~En`^~o3ibpu4NAr}Lzkt9?^(60buX&la17$JVtsIp@+`Mg zg1T^4Vcr>dK^ITbWf+MAni(TBRO>-7Fgf+_laGy0`Ga!U5cUyTPo_^=PEtp`CmWP5 z*Szq6hSe-o8}cZEKY8V7A7Sg+DpYDhQ6x)$Xp{btx-W0ns0Qf^CwL*q<_XJ1N6$2W zcLORMnepiZWs8~}VpDgA=q)1;oQ#H-vn?(XSME7p%&hfk;m#VurkInnaMlEce`3S9 z8x9}vBoJ3YhVA^l20^_SGbCga`$7ehf!j zL}yP`_hJyyByVEQYXfN?0y8poSF{0-YQAXThhQ*3q6PUuMAcQXv=!q*84G73s}_&F zuTuJX@Kqm(=+ztZ3}gL-AJBeBjHphZydX=6YR}Xvd+BuAM;mv5vm9wu{dw#wxZ(2} zB^y=-hALfP%ZWHeMEB2}-X970)PE zmQ+g{BvpQDGy~@5z zB7@S_wqqY6Zh9Sxd%!B%YmApC`C<9C@X%Yb|wu-9e=vq zr1Cv{Js9N)Hq~gWu9|-yUBZ3KhWe5B5d`$mB4)AtOfT|u(+$#^IyelzlY3@`0hm9G zi_GfF1#i#(jYB5CLzt;V%F6!~_WHC1pOA{h%Q;0!vQSB?b~Nl%QiaVxfH;HQG$fV{LPCY1}>aE_?cs-9ovZ%DNPXSIFe6>5Zs5>y zhjbbNr6yQ%!(s@%Jj=Uh*xI45 zv=*%fC1p>O`RhvTQif6IN_;JKrY;Vnk zH6h(uboirFAkKs@I^S1d2>RNJj)Ni7rFpH^$xvJT3Ao{!iev6^p{*`w76$7nmyW7E zdGq;}8VAnf8|%mmHBhY~e~rnyqdPT3jFwGP%A_iGj0G>-v-m>+@LF!e<2U{F$TO)w zK&QQ~5J!g(ak;T6TUF99vJGvEK0P1@QDf&LJxvA@XBuhop;?Av<~kUtXHTC1;?NAC zzzdc8$N=r}Y=}*Ovk;2<%mq@Yd#9*-pM7ru)t9vTS6KgId{tydJKGu;Qg0cF`Mmym zfWuh~&P3N#_NQ%6S!D@BcdWd#1uy1*_E$bjU0k_yvM1q`p`(Y3SK@CvELCBGC!=ur zPo3)S09~knJiS`~*6-&v+;yA(SmFlOi0C*(++rPYB!{~aQY<&Vc=*$7Q-1;-2UX?vHn4Fk9yDQ@)0bP z)}_lMmz|b$@iKI>16QrJ>;^>r^jFW=dsb|z+@JtBt%P88I9PBhiMX79NIV~DD{d8a0ri9 z`vf{qO69jX#oSmreh_M&kYdeT${I*GZ?2?z{VcPeYN?ir32~9fNrhf$v6ZlDVOIo$ z&%Ho;f<)ZweCn>ioD2920(Yio?Qlfn`>E3YHwqiFVB9?AY3=v3n@2SY=!OZzZTSqE zDIn+RZPvr9P~`HT_f^-HH-UDbQ5CDSTMeFDP1r_TArjPd6{9M)V>73(e)$9$6fq^U zSa@B)fxkONt%8uBNSuf<9-9^tJ>n&Kn5{P--?c(gC+ME4LbAis%!X||!rURb9IAqt z|B~a{m2r812c}ye<7z6nDz1Y0FtSIm1+SvIiTj%Ea zhiAi^g4yyxX{YZ;2G103-UOW2NVG^IlA7?l<$3!SwKch;EgG&~K-sKLr!0c$kRt89 ziu<|iMKrX0RyoJI%;QU+4v5tzv9aLLK~B1U1j^lK z8VqrjRkcwL7##^F=s#9unSLr@E$E(+v-w)5sGeBB!h>61o5SX|ttuCkh?0mj`VXJO z)43pV2%QrfP}Zzq>{6J=nM3*5Ny1e-=cL+aSy9==;tiyWhQIfDAo-_@HM$&w3Ddp+ z5%%oEqDoJmj3#XZ=0E-ltD$kL^<1hJWyJ!lm11qD*#U0g@P+j8g{4Ll={^uw`(zrn z&vs5Og=h<3Q+Fq%Nzqb1Udvt)Mpd(VJMYNJ^8}V~cJ~$BTR64ks^`_pMQ`b}9T`{X zUhB#8w)*INxla~=Yd!1Dk`-s{w@F(y<7pVbpjB+o*v41@1;UeO1c1l5G{gXcO+_d` z{cuZ4L$tfC^{x5_A<3uS9oq^_zx%*yqqToE&yOy&h1as}_#qHdA-t=z==X9h_iPn* zW?8<|>a(bg+%IWrFbuf?swh=EQvhpUvy&RIRSTza@6$@oc-#VnhT1zGqd5wg64pQg zCGn2-@@PsLf;gjmz>vUc-Dtwk=KZ|YtVc?wB1mr#U X25519 6qyRroFA/Sem9DW44cCAPPARsg33Z6qh+unNk8dpokc +k/KmaY0p4Wu9/Ar7SwMejAxxfC+Miwcya7fZ5ZC/DmE +--- Of6T8BTyPjhupUy1dLeeZTUnJANQ077umLCtyNXDb7c +Ò°nö»!“k³¸mQ‹Ù¨9 +\žc²[jrý/>R)çœFØÖ2ô«>ïê \ No newline at end of file diff --git a/cmd/teploy-dash/frontend/js/age.vendor.LICENSE b/cmd/teploy-dash/frontend/js/age.vendor.LICENSE new file mode 100644 index 0000000..7c15e0c --- /dev/null +++ b/cmd/teploy-dash/frontend/js/age.vendor.LICENSE @@ -0,0 +1,27 @@ +Copyright 2023 The age Authors + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of the age project nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/cmd/teploy-dash/frontend/js/age.vendor.README.txt b/cmd/teploy-dash/frontend/js/age.vendor.README.txt new file mode 100644 index 0000000..1be4570 --- /dev/null +++ b/cmd/teploy-dash/frontend/js/age.vendor.README.txt @@ -0,0 +1,7 @@ +age.vendor.js is a bundle of age-encryption@0.3.1 (MIT, see age.vendor.LICENSE), built +with esbuild@0.25.12 from scripts/vendor-age/entry.mjs: + + sh scripts/vendor-age/build.sh + +It exposes window.TeployAge: generate, derivePublic, decryptStart, parseIdentityFile. +Dash loads it only on the setup flow. Do not edit the bundle by hand. diff --git a/cmd/teploy-dash/frontend/js/age.vendor.js b/cmd/teploy-dash/frontend/js/age.vendor.js new file mode 100644 index 0000000..32bf7f6 --- /dev/null +++ b/cmd/teploy-dash/frontend/js/age.vendor.js @@ -0,0 +1 @@ +(()=>{var mc=Object.defineProperty;var Ec=(e,t,n)=>t in e?mc(e,t,{enumerable:!0,configurable:!0,writable:!0,value:n}):e[t]=n;var g=(e,t,n)=>Ec(e,typeof t!="symbol"?t+"":t,n);function ln(e){return e instanceof Uint8Array||ArrayBuffer.isView(e)&&e.constructor.name==="Uint8Array"&&"BYTES_PER_ELEMENT"in e&&e.BYTES_PER_ELEMENT===1}var ar=e=>e?`"${e}" `:"";function et(e,t=""){if(typeof e!="number")throw new TypeError(ar(t)+"expected number, got "+typeof e);if(!Number.isSafeInteger(e)||e<0)throw new RangeError(ar(t)+"expected integer >= 0, got "+e);return e}function pt(e,t,n=""){if(ln(e)&&(t===void 0||e.length===t))return e;t!==void 0&&et(t,"length");let r=ln(e),o=t!==void 0?` of length ${t}`:"",s=r?`length=${e.length}`:`type=${typeof e}`,i=ar(n)+"expected Uint8Array"+o+", got "+s;throw r?new RangeError(i):new TypeError(i)}function Ht(e){if(typeof e!="function"||typeof e.create!="function")throw new TypeError("expected hash wrapped by utils.createHasher");if(et(e.outputLen),et(e.blockLen),e.outputLen<1||e.blockLen<1)throw new Error("hash blockLen / outputLen must be >= 1")}var Bc=(e,t)=>{if(e===null||typeof e!="object"||Array.isArray(e))throw new TypeError((t==="object"?"":`"${t}" `)+"expected object, got type="+typeof e)},Oo=(e,t)=>{Bc(e,t);let n=Object.getPrototypeOf(e);if(n!==Object.prototype&&n!==null)throw new TypeError(`"${t}" expected plain object`);if(Object.hasOwn(e,"__proto__"))throw new TypeError(`"${t}.__proto__" is not allowed`)};function be(e,t=!0){if(e.destroyed)throw new Error("hash was destroyed");if(t&&e.finished)throw new Error("digest() was already called")}function dn(e,t){pt(e,void 0,"output");let n=t.outputLen;if(!(e.length>=n))throw new RangeError('"output" expected length >= '+n)}function hn(e){return new Uint32Array(e.buffer,e.byteOffset,Math.floor(e.byteLength/4))}function nt(...e){for(let t=0;t>>t}function z(e,t){return e<>>32-t>>>0}var Ac=new Uint8Array(new Uint32Array([287454020]).buffer)[0]===68;function Sc(e){return e<<24&4278190080|e<<8&16711680|e>>>8&65280|e>>>24&255}function _c(e){for(let t=0;te:_c,ko=typeof Uint8Array.from([]).toHex=="function"&&typeof Uint8Array.fromHex=="function",vc=Array.from({length:256},(e,t)=>t.toString(16).padStart(2,"0"));function pn(e){if(pt(e),ko)return e.toHex();let t="";for(let n=0;n=48&&e<=57?e-48:e>=65&&e<=70?e-55:e>=97&&e<=102?e-87:void 0}function lr(e){if(typeof e!="string")throw new TypeError("hex string expected, got "+typeof e);if(ko)try{return Uint8Array.fromHex(e)}catch(o){throw o instanceof SyntaxError?new RangeError(o.message):o}let t=e.length,n=t/2;if(t%2)throw new RangeError("hex string expected, got unpadded hex of length "+t);let r=new Uint8Array(n);for(let o=0,s=0;o255*s)throw new Error("Length must be <= 255*HashLen");let i=Math.ceil(r/s);if(n===void 0?n=Lc:pt(n,void 0,"info"),!i)return o&&nt(t),new Uint8Array;let c=o&&i===1?t:new Uint8Array(i*s),{iHash:f,oHash:a}=Nt.create(e,t),u=o?t:new Uint8Array(s),l=i>1?o?.iHash||e.create():void 0;for(let h=0;h1&&f.update(u),f.update(n).update(Ne).digestInto(u),a.update(u).digestInto(u),c.set(u,s*(i-1)),f.destroy(),a.destroy(),l?.destroy(),u!==c&&nt(u),nt(Ne),r===c.length)return c;let d=c.slice(0,r);return nt(c),d}var Me=(e,t,n,r,o)=>{Ht(e),n===void 0&&(n=new Uint8Array(e.outputLen));let s=Nt.create(e,n).update(t);return hr(e,s.digest(),r,o,s)};var Ic=e=>e/2**32|0,Oc=e=>e>>>0;function No(e,t,n,r){let o=Ic(n),s=Oc(n);e.setUint32(t,r?s:o,r),e.setUint32(t+4,r?o:s,r)}function Mo(e,t,n){return e&t^~e&n}function Ko(e,t,n){return e&t^e&n^t&n}var gn=class{constructor(t,n,r,o){g(this,"blockLen");g(this,"outputLen");g(this,"canXOF",!1);g(this,"padOffset");g(this,"isLE");g(this,"buffer");g(this,"view");g(this,"finished",!1);g(this,"length",0);g(this,"pos",0);g(this,"destroyed",!1);this.blockLen=t,this.outputLen=n,this.padOffset=r,this.isLE=o,this.buffer=new Uint8Array(t),this.view=we(this.buffer)}update(t){be(this),pt(t);let{view:n,buffer:r,blockLen:o}=this,s=t.length,i=!1;for(let c=0;co-i&&(this.process(r,0),n.fill(0)),No(r,o-8,this.length*8,s),this.process(r,0),this.roundClean();let c=t===n?r:we(t),f=this.outputLen,a=f/4,u=this.get();if(f%4||a>u.length)throw new Error("invalid outputLen");for(let l=0;l>>3,b=wt(y,17)^wt(y,19)^y>>>10;Mt[d]=b+Mt[d-7]+w+Mt[d-16]|0}let{A:o,B:s,C:i,D:c,E:f,F:a,G:u,H:l}=this;for(let d=0;d<64;d++){let h=wt(f,6)^wt(f,11)^wt(f,25),y=l+h+Mo(f,a,u)+Rc[d]+Mt[d]|0,b=(wt(o,2)^wt(o,13)^wt(o,22))+Ko(o,s,i)|0;l=u,u=a,a=f,f=c+y|0,c=i,i=s,s=o,o=y+b|0}o=o+this.A|0,s=s+this.B|0,i=i+this.C|0,c=c+this.D|0,f=f+this.E|0,a=a+this.F|0,u=u+this.G|0,l=l+this.H|0,this.set(o,s,i,c,f,a,u,l)}roundClean(){nt(Mt)}destroy(){this.destroyed=!0,this.set(0,0,0,0,0,0,0,0),nt(this.buffer)}},yr=class extends pr{constructor(){super(32,Do)}};var mt=Uo(()=>new yr,Co(1));var Yo=e=>Object.freeze(e());function Xo(e){return e instanceof Uint8Array||ArrayBuffer.isView(e)&&e.constructor.name==="Uint8Array"&&"BYTES_PER_ELEMENT"in e&&e.BYTES_PER_ELEMENT===1}function $o(e){if(!Xo(e))throw new TypeError("Uint8Array expected")}function kc(e,t){return Array.isArray(t)?t.length===0?!0:e?t.every(n=>typeof n=="string"):t.every(n=>Number.isSafeInteger(n)):!1}function Uc(e){if(typeof e!="function")throw new TypeError("function expected");return!0}function gr(e,t){if(typeof t!="string")throw new TypeError(`${e}: string expected`);return!0}function br(e,t="number"){if(typeof e!="number")throw new TypeError(`${t}: expected number, got ${typeof e}`);if(!Number.isSafeInteger(e))throw new RangeError(`${t}: expected safe integer, got ${e}`)}function qo(e,t){if(!kc(!1,t))throw new TypeError(`${e}: array of numbers expected`)}function Cc(...e){let t=s=>s,n=(s,i)=>c=>s(i(c)),r=e.map(s=>s.encode).reduceRight(n,t),o=e.map(s=>s.decode).reduce(n,t);return{encode:r,decode:o}}var Hc=(()=>{let e=[];for(let t=0;t<40;t++)e.push(2**t);return e})();function Fo(e,t=e.length){let n=new Array(t);for(let r=0;r{try{let e=new TextDecoder;return e.decode(Uint8Array.of(65,48,43,127))==="A0+\x7F"?e:void 0}catch{return}})(),xr=8192;function Nc(e){let t=e.length;if(Po!==void 0&&t>=12)return Po.decode(e);if(t<=xr)return String.fromCharCode.apply(null,e);let n="";for(let r=0;r8)throw new RangeError("radix2: bits should be in (0..8]");let t=Hc[e]-1;return{encode:n=>{$o(n);let r=n.length,o=new Uint8Array(Math.ceil(r*8/e)),s=0,i=0,c=0;for(let f=0;f>i&t,!(i0&&(o[c]=s<{let r=n.length,o=new Uint8Array(Math.floor(r*e/8)),s=0,i=0,c=0;for(let f=0;f=8;i-=8)o[c++]=s>>i-8&255;if(s=s<<8-i&255,i>=e)throw new Error("Excess padding");if(s>0)throw new Error(`Non-zero padding: ${s}`);return o}}}function Wo(e,t){let n=e.length;if(n>128)throw new Error("alphabet: max 128 letters");let r=new Uint8Array(n),o=new Int8Array(128).fill(-1);for(let s=0;s127)throw new Error("alphabet: single-char ASCII letters only");r[s]=i,o[i]=s}if(t!==void 0)for(let s of Object.keys(t)){let i=s.charCodeAt(0),c=o[t[s].charCodeAt(0)];if(s.length!==1||i>127||c===void 0||c===-1)throw new Error(`alphabet: invalid alias ${s}`);o[i]=c}return{encode:s=>{let i=new Uint8Array(s.length);for(let c=0;c{gr("decode",s);let i=s.length,c=new Uint8Array(i);for(let f=0;fCc(Vo(6),Wo("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")));var wr=Wo("qpzry9x8gf2tvdw0s3jn54khce6mua7l"),Mc=/^[\x21-\x60\x7b-\x7e]+$/;function Zo(e,t){for(let n=0;n126)throw new Error(`${e}: printable ASCII expected`)}}function Kc(e){let t=e.length,n=new Uint8Array(t);for(let r=0;r=32)throw new Error(`alphabet.encode: invalid digit ${o}`);n[r]=o}return n}var zo=[996825010,642813549,513874426,1027748829,705979059];function Ke(e){let t=e>>25,n=(e&33554431)<<5;for(let r=0;r>r&1)===1&&(n^=zo[r]);return n}function Go(e,t,n=1){let r=e.length,o=1;for(let i=0;i126)throw new Error(`Invalid prefix (${e})`);o=Ke(o)^c>>5}o=Ke(o);for(let i=0;i>>5*(5-i)&31;return wr.encode(s)}function Dc(e){let t=e==="bech32"?1:734539939,n=Vo(5),r=l=>{$o(l);let d=l.length,h=new Array(Math.ceil(d*8/5)),y=0,w=0,b=0;for(let _=0;_=5;w-=5)h[b++]=y>>w-5&31;return w>0&&(h[b]=y<<5-w&31),h},o=l=>{qo("radix2.decode",l);let d=l.length,h=new Uint8Array(d);for(let y=0;y=32)throw new Error(`convertRadix2: invalid word=${w}`);h[y]=w}return n.decode(h)},s=jo(o);function i(l,d,h=90){gr("bech32.encode prefix",l),h!==!1&&br(h,"limit"),Xo(d)&&(d=Fo(d)),qo("bech32.encode",d);let y=l.length;if(y===0)throw new TypeError(`Invalid prefix length ${y}`);let w=y+7+d.length;if(h!==!1&&w>h)throw new TypeError(`Length ${w} exceeds limit ${h}`);Zo("bech32.encode prefix",l);let b=l.toLowerCase(),_=Go(b,d,t);return`${b}1${wr.encode(Kc(d))}${_}`}function c(l,d=90){gr("bech32.decode input",l),d!==!1&&br(d,"limit");let h=l.length;if(h<8||d!==!1&&h>d)throw new TypeError(`invalid string length ${h}, expected (8..${d})`);let y=l.toLowerCase();if(l!==y&&!Mc.test(l))throw Zo("bech32.decode input",l),new Error("mixed-case string not allowed");let w=y.lastIndexOf("1");if(w===0||w===-1)throw new Error('invalid separator "1"');let b=y.slice(0,w),_=y.slice(w+1);if(_.length<6)throw new Error("invalid data length");let B=wr.decode(_),v=Fo(B,B.length-6),A=Go(b,v,t);if(!_.endsWith(A))throw new Error(`Invalid checksum in ${l}`);return{prefix:b,words:v}}let f=jo(c);function a(l,d=90){let{prefix:h,words:y}=c(l,d);return{prefix:h,words:y,bytes:o(y)}}function u(l,d){return i(l,r(d))}return{encode:i,decode:c,encodeFromBytes:u,decodeToBytes:a,decodeUnsafe:f,fromWords:o,fromWordsUnsafe:s,toWords:r}}var _t=Yo(()=>Dc("bech32"));function Fc(e,t,n,r){Ht(e);let o=He({dkLen:32,asyncTick:10},r),{c:s,dkLen:i,asyncTick:c}=o;if(et(s,"c"),et(i,"dkLen"),et(c,"asyncTick"),s<1)throw new Error('"c" (iterations) must be >= 1');if(i<1)throw new Error('"dkLen" must be >= 1');if(i>(2**32-1)*e.outputLen)throw new Error("derived key too long");let f=dr(t,"password");try{let a=dr(n,"salt");try{let u=new Uint8Array(i),{iHash:l,oHash:d,outputLen:h}=Nt.create(e,f),y=new Uint8Array(h),w=Pc(l,d,a,y);return{c:s,dkLen:i,asyncTick:c,DK:u,outputLen:h,eng:w}}finally{typeof n=="string"&&nt(a)}}finally{typeof t=="string"&&nt(f)}}function Pc(e,t,n,r){let o=new Uint8Array(4),s=we(o),i=e._cloneInto().update(n),c=t._cloneInto(),f=e._cloneInto,a=t._cloneInto;return{u1:(u,l)=>{s.setInt32(0,u,!1),i._cloneInto(c).update(o).digestInto(r),t._cloneInto(c).update(r).digestInto(r),l.set(r.subarray(0,l.length))},rounds:(u,l)=>{for(let d=1;d(e.destroy(),t.destroy(),i.destroy(),c.destroy(),nt(r),u)}}function mr(e,t,n,r){let{c:o,dkLen:s,DK:i,outputLen:c,eng:f}=Fc(e,t,n,r);for(let a=1,u=0;u0&&(i+=16),Qo(n,s,e,t+=16,n,i)}var jc=1024*(2**20+1+1);function Zc(e,t,n){let r=He({dkLen:32,asyncTick:10,maxmem:jc},n),{N:o,r:s,p:i,dkLen:c,asyncTick:f,maxmem:a,onProgress:u}=r;if(et(o,"N"),et(s,"r"),et(i,"p"),et(c,"dkLen"),et(f,"asyncTick"),et(a,"maxmem"),u!==void 0&&typeof u!="function")throw new Error('"onProgress" must be a function');if(s<1)throw new Error('"r" expected integer >= 1');let l=128*s,d=l/4,h=Math.pow(2,32);if(o<=1||(o&o-1)!==0||o>h)throw new Error('"N" expected a power of 2, and 2^1 <= N <= 2^32');if(i<1||i>(h-1)*32/l)throw new Error('"p" expected integer 1..((2^32 - 1) * 32) / (128 * r)');if(c<1||c>(h-1)*32)throw new Error('"dkLen" expected integer 1..(2^32 - 1) * 32');let y=l*(o+i+1);if(y>a)throw new Error('"maxmem" limit was hit: memUsed(128*r*(N+p+1))='+y+", maxmem="+a);let w=mr(mt,e,t,{c:1,dkLen:l*i}),b=hn(w),_=hn(new Uint8Array(l*o)),B=hn(new Uint8Array(l)),v=()=>{};if(u){let A=2*o*i,R=Math.max(Math.floor(A/1e4),1),T=0;v=()=>{if(T++,u&&(!(T%R)||T===A))try{u(T/A)}catch(D){throw nt(w,_,B),D}}}return{N:o,r:s,p:i,dkLen:c,blockSize32:d,V:_,B32:b,B:w,tmp:B,blockMixCb:v,asyncTick:f}}function zc(e,t,n,r,o){let s=mr(mt,e,n,{c:1,dkLen:t});return nt(n,r,o),s}function Jo(e,t,n){let{N:r,r:o,p:s,dkLen:i,blockSize32:c,V:f,B32:a,B:u,tmp:l,blockMixCb:d}=Zc(e,t,n);ur(a);for(let h=0;h>>0;for(let _=0;_e?`"${e}" `:"";function wn(e,t=""){if(typeof e!="boolean")throw new TypeError(bn(t)+"expected boolean, got type="+typeof e);return e}function Qt(e,t=""){if(typeof e!="number")throw new TypeError(bn(t)+"expected number, got "+typeof e);if(!Number.isSafeInteger(e)||e<0)throw new RangeError(bn(t)+"expected integer >= 0, got "+e);return e}function tt(e,t,n=""){if(Br(e)&&(t===void 0||e.length===t))return e;t!==void 0&&Qt(t,"length");let r=Br(e),o=t!==void 0?` of length ${t}`:"",s=r?`length=${e.length}`:`type=${typeof e}`,i=bn(n)+"expected Uint8Array"+o+", got "+s;throw r?new RangeError(i):new TypeError(i)}var ts=(e,t)=>{if(e===null||typeof e!="object"||Array.isArray(e))throw new TypeError(t==="object"?"expected valid options object":`"${t}" expected object, got type=${typeof e}`)};function Ar(e,t=!0){if(e.destroyed)throw new Error("hash was destroyed");if(t&&e.finished)throw new Error("digest() was already called")}function es(e,t){tt(e,void 0,"output");let n=t.outputLen;if(!(e.length>=n))throw new RangeError('"output" expected length >= '+n)}function At(e){return new Uint32Array(e.buffer,e.byteOffset,Math.floor(e.byteLength/4))}function Et(...e){for(let t=0;t>>8&65280|e>>>24&255}function Xc(e){for(let t=0;te:Xc;function $c(e,t){return!e.byteLength||!t.byteLength?!1:e.buffer===t.buffer&&e.byteOffset[]),s=(c,f)=>r(f,...o(c)).update(c).digest(),i=r(new Uint8Array(e),...o(new Uint8Array(0)));return s.outputLen=i.outputLen,s.blockLen=i.blockLen,s.create=(c,...f)=>r(c,...f),s}var is=(e,t)=>{function n(r,...o){if(tt(r,void 0,"key"),e.nonceLength!==void 0){let l=o[0];tt(l,e.varSizeNonce?void 0:e.nonceLength,"nonce")}let s=e.tagLength,i=e.nonceLength!==void 0?1:0;if(!e.withAAD){for(let l=i;l{if(d!==void 0){if(l!==2)throw new Error("cipher output not supported");tt(d,void 0,"output")}},a=!1;return{encrypt(l,d){if(a)throw new Error("cannot encrypt() twice with same key + nonce");return a=!0,tt(l,void 0,"data"),f(c.encrypt.length,d),c.encrypt(l,d)},decrypt(l,d){if(tt(l,void 0,"data"),s&&l.length= tagLength='+s);return f(c.decrypt.length,d),c.decrypt(l,d)}}}return Object.assign(n,e),n};function De(e,t,n=!0){if(t===void 0)return new Uint8Array(e);if(tt(t,e,"output"),n&&!qe(t))throw new Error("invalid output, must be aligned");return t}function cs(e,t,n){Qt(e),Qt(t),wn(n);let r=new Uint8Array(16),o=Gc(r);return o.setBigUint64(0,BigInt(t),n),o.setBigUint64(8,BigInt(e),n),r}function qe(e){return e.byteOffset%4===0}function me(e){return Uint8Array.from(tt(e))}var as=e=>Uint8Array.from(e.split(""),t=>t.charCodeAt(0)),Vc=vt(At(as("expand 16-byte k"))),Wc=vt(At(as("expand 32-byte k")));function G(e,t){return e<>>32-t}var Fe=64,Qc=16,Sr=2**32-1,fs=Uint32Array.of();function Jc(e,t,n,r,o,s,i,c){let f=o.length,a=new Uint8Array(Fe),u=At(a),l=Kt&&qe(o)&&qe(s),d=l?At(o):fs,h=l?At(s):fs;if(!Kt){for(let y=0;y=Sr)throw new Error("arx: counter overflow");let w=Math.min(Fe,f-y);for(let b=0,_;b=Sr)throw new Error("arx: counter overflow");let w=Math.min(Fe,f-y);if(l&&w===Fe){let b=y/4;if(y%4!==0)throw new Error("arx: invalid block position");for(let _=0,B;_{tt(c,void 0,"key"),tt(f,void 0,"nonce"),tt(a,void 0,"data");let d=a.length,h=u!==void 0;if(u=De(d,u,!1),h&&ns(a,u),Qt(l),l<0||l>=Sr)throw new Error("arx: counter overflow");let y=[],w=c.length,b,_;if(w===32)y.push(b=me(c)),_=Wc;else if(w===16&&n)b=new Uint8Array(32),b.set(c),b.set(c,16),_=Vc,y.push(b);else throw tt(c,32,"arx key"),new Error("invalid key size");(!Kt||!qe(f))&&y.push(f=me(f));let B=At(b);if(r){if(f.length!==24)throw new Error("arx: extended nonce must be 24 bytes");let R=f.subarray(0,16);if(Kt)r(_,B,At(R),B);else{let T=vt(Uint32Array.from(_));r(T,B,At(R),B),Et(T),vt(B)}f=f.subarray(16)}else Kt||vt(B);let v=16-o;if(v!==f.length)throw new Error(`arx: nonce must be ${v} or 16 bytes`);if(v!==12){let R=new Uint8Array(12);R.set(f,s?0:12-f.length),f=R,y.push(f)}let A=vt(At(f));try{return Jc(e,_,B,A,a,u,l,i),u}finally{Et(...y)}}}function ct(e,t){return e[t++]&255|(e[t++]&255)<<8}var _r=class{constructor(t){g(this,"blockLen",16);g(this,"outputLen",16);g(this,"buffer",new Uint8Array(16));g(this,"r",new Uint16Array(10));g(this,"h",new Uint16Array(10));g(this,"pad",new Uint16Array(8));g(this,"pos",0);g(this,"finished",!1);g(this,"destroyed",!1);t=me(tt(t,32,"key"));let n=ct(t,0),r=ct(t,2),o=ct(t,4),s=ct(t,6),i=ct(t,8),c=ct(t,10),f=ct(t,12),a=ct(t,14);this.r[0]=n&8191,this.r[1]=(n>>>13|r<<3)&8191,this.r[2]=(r>>>10|o<<6)&7939,this.r[3]=(o>>>7|s<<9)&8191,this.r[4]=(s>>>4|i<<12)&255,this.r[5]=i>>>1&8190,this.r[6]=(i>>>14|c<<2)&8191,this.r[7]=(c>>>11|f<<5)&8065,this.r[8]=(f>>>8|a<<8)&8191,this.r[9]=a>>>5&127;for(let u=0;u<8;u++)this.pad[u]=ct(t,16+2*u)}process(t,n,r=!1){let o=r?0:2048,{h:s,r:i}=this,c=i[0],f=i[1],a=i[2],u=i[3],l=i[4],d=i[5],h=i[6],y=i[7],w=i[8],b=i[9],_=ct(t,n+0),B=ct(t,n+2),v=ct(t,n+4),A=ct(t,n+6),R=ct(t,n+8),T=ct(t,n+10),D=ct(t,n+12),K=ct(t,n+14),I=s[0]+(_&8191),M=s[1]+((_>>>13|B<<3)&8191),H=s[2]+((B>>>10|v<<6)&8191),C=s[3]+((v>>>7|A<<9)&8191),U=s[4]+((A>>>4|R<<12)&8191),p=s[5]+(R>>>1&8191),x=s[6]+((R>>>14|T<<2)&8191),m=s[7]+((T>>>11|D<<5)&8191),S=s[8]+((D>>>8|K<<8)&8191),O=s[9]+(K>>>5|o),E=0,k=E+I*c+M*(5*b)+H*(5*w)+C*(5*y)+U*(5*h);E=k>>>13,k&=8191,k+=p*(5*d)+x*(5*l)+m*(5*u)+S*(5*a)+O*(5*f),E+=k>>>13,k&=8191;let L=E+I*f+M*c+H*(5*b)+C*(5*w)+U*(5*y);E=L>>>13,L&=8191,L+=p*(5*h)+x*(5*d)+m*(5*l)+S*(5*u)+O*(5*a),E+=L>>>13,L&=8191;let N=E+I*a+M*f+H*c+C*(5*b)+U*(5*w);E=N>>>13,N&=8191,N+=p*(5*y)+x*(5*h)+m*(5*d)+S*(5*l)+O*(5*u),E+=N>>>13,N&=8191;let q=E+I*u+M*a+H*f+C*c+U*(5*b);E=q>>>13,q&=8191,q+=p*(5*w)+x*(5*y)+m*(5*h)+S*(5*d)+O*(5*l),E+=q>>>13,q&=8191;let Z=E+I*l+M*u+H*a+C*f+U*c;E=Z>>>13,Z&=8191,Z+=p*(5*b)+x*(5*w)+m*(5*y)+S*(5*h)+O*(5*d),E+=Z>>>13,Z&=8191;let j=E+I*d+M*l+H*u+C*a+U*f;E=j>>>13,j&=8191,j+=p*c+x*(5*b)+m*(5*w)+S*(5*y)+O*(5*h),E+=j>>>13,j&=8191;let P=E+I*h+M*d+H*l+C*u+U*a;E=P>>>13,P&=8191,P+=p*f+x*c+m*(5*b)+S*(5*w)+O*(5*y),E+=P>>>13,P&=8191;let Y=E+I*y+M*h+H*d+C*l+U*u;E=Y>>>13,Y&=8191,Y+=p*a+x*f+m*c+S*(5*b)+O*(5*w),E+=Y>>>13,Y&=8191;let X=E+I*w+M*y+H*h+C*d+U*l;E=X>>>13,X&=8191,X+=p*u+x*a+m*f+S*c+O*(5*b),E+=X>>>13,X&=8191;let V=E+I*b+M*w+H*y+C*h+U*d;E=V>>>13,V&=8191,V+=p*l+x*u+m*a+S*f+O*c,E+=V>>>13,V&=8191,E=(E<<2)+E|0,E=E+k|0,k=E&8191,E=E>>>13,L+=E,s[0]=k,s[1]=L,s[2]=N,s[3]=q,s[4]=Z,s[5]=j,s[6]=P,s[7]=Y,s[8]=X,s[9]=V}finalize(){let{h:t,pad:n}=this,r=new Uint16Array(10),o=t[1]>>>13;t[1]&=8191;for(let c=2;c<10;c++)t[c]+=o,o=t[c]>>>13,t[c]&=8191;t[0]+=o*5,o=t[0]>>>13,t[0]&=8191,t[1]+=o,o=t[1]>>>13,t[1]&=8191,t[2]+=o,r[0]=t[0]+5,o=r[0]>>>13,r[0]&=8191;for(let c=1;c<10;c++)r[c]=t[c]+o,o=r[c]>>>13,r[c]&=8191;r[9]-=8192;let s=(o^1)-1;for(let c=0;c<10;c++)r[c]&=s;s=~s;for(let c=0;c<10;c++)t[c]=t[c]&s|r[c];t[0]=(t[0]|t[1]<<13)&65535,t[1]=(t[1]>>>3|t[2]<<10)&65535,t[2]=(t[2]>>>6|t[3]<<7)&65535,t[3]=(t[3]>>>9|t[4]<<4)&65535,t[4]=(t[4]>>>12|t[5]<<1|t[6]<<14)&65535,t[5]=(t[6]>>>2|t[7]<<11)&65535,t[6]=(t[7]>>>5|t[8]<<8)&65535,t[7]=(t[8]>>>8|t[9]<<5)&65535;let i=t[0]+n[0];t[0]=i&65535;for(let c=1;c<8;c++)i=(t[c]+n[c]|0)+(i>>>16)|0,t[c]=i&65535;Et(r)}update(t){Ar(this),tt(t),t=me(t);let{buffer:n,blockLen:r}=this,o=t.length;for(let s=0;s>>0,t[s++]=r[i]>>>8}digest(){let{buffer:t,outputLen:n}=this;this.digestInto(t);let r=t.slice(0,n);return this.destroy(),r}},ls=ss(32,e=>new _r(e));function tf(e,t,n,r,o,s=20){let i=e[0],c=e[1],f=e[2],a=e[3],u=t[0],l=t[1],d=t[2],h=t[3],y=t[4],w=t[5],b=t[6],_=t[7],B=o,v=n[0],A=n[1],R=n[2],T=i,D=c,K=f,I=a,M=u,H=l,C=d,U=h,p=y,x=w,m=b,S=_,O=B,E=v,k=A,L=R;for(let q=0;q{e.update(t);let n=t.length%16;n&&e.update(nf.subarray(n))},rf=new Uint8Array(32);function hs(e,t,n,r,o){o!==void 0&&tt(o,void 0,"AAD");let s=e(t,n,rf),i=cs(r.length,o?o.length:0,!0),c=ls.create(s);o&&ds(c,o),ds(c,r),c.update(i);let f=c.digest();return Et(s,i),f}var of=e=>(t,n,r)=>({encrypt(s,i){let c=s.length;i=De(c+16,i,!1),i.set(s);let f=i.subarray(0,-16);e(t,n,f,f,1);let a=hs(e,t,n,f,r);return i.set(a,c),Et(a),i},decrypt(s,i){i=De(s.length-16,i,!1);let c=s.subarray(0,-16),f=s.subarray(-16),a=hs(e,t,n,c,r);if(!os(f,a))throw Et(a),new Error("invalid tag");return i.set(s.subarray(0,-16)),e(t,n,i,i,1),Et(a),i}}),Ee=is({blockSize:64,nonceLength:12,tagLength:16,withAAD:!0},of(ef));function Be(e){return e instanceof Uint8Array||ArrayBuffer.isView(e)&&e.constructor.name==="Uint8Array"}function rt(e,t=""){if(!Number.isSafeInteger(e)||e<0){let n=t&&`"${t}" `;throw new Error(`${n}expected integer >= 0, got ${e}`)}}function F(e,t,n=""){let r=Be(e),o=e?.length,s=t!==void 0;if(!r||s&&o!==t){let i=n&&`"${n}" `,c=s?` of length ${t}`:"",f=r?`length=${o}`:`type=${typeof e}`;throw new Error(i+"expected Uint8Array"+c+", got "+f)}return e}function yt(e){if(typeof e!="function"||typeof e.create!="function")throw new Error("Hash must wrapped by utils.createHasher");rt(e.outputLen),rt(e.blockLen)}function Lt(e,t=!0){if(e.destroyed)throw new Error("Hash instance has been destroyed");if(t&&e.finished)throw new Error("Hash#digest() has already been called")}function mn(e,t){F(e,void 0,"digestInto() output");let n=t.outputLen;if(e.length='+n)}function En(e){return new Uint32Array(e.buffer,e.byteOffset,Math.floor(e.byteLength/4))}function ht(...e){for(let t=0;t>>t}var sf=new Uint8Array(new Uint32Array([287454020]).buffer)[0]===68;function cf(e){return e<<24&4278190080|e<<8&16711680|e>>>8&65280|e>>>24&255}function ff(e){for(let t=0;te:ff,ys=typeof Uint8Array.from([]).toHex=="function"&&typeof Uint8Array.fromHex=="function",af=Array.from({length:256},(e,t)=>t.toString(16).padStart(2,"0"));function Jt(e){if(F(e),ys)return e.toHex();let t="";for(let n=0;n=Tt._0&&e<=Tt._9)return e-Tt._0;if(e>=Tt.A&&e<=Tt.F)return e-(Tt.A-10);if(e>=Tt.a&&e<=Tt.f)return e-(Tt.a-10)}function te(e){if(typeof e!="string")throw new Error("hex string expected, got "+typeof e);if(ys)return Uint8Array.fromHex(e);let t=e.length,n=t/2;if(t%2)throw new Error("hex string expected, got unpadded hex of length "+t);let r=new Uint8Array(n);for(let o=0,s=0;oe(s).update(o).digest(),r=e(void 0);return n.outputLen=r.outputLen,n.blockLen=r.blockLen,n.create=o=>e(o),Object.assign(n,t),Object.freeze(n)}function Dt(e=32){let t=typeof globalThis=="object"?globalThis.crypto:null;if(typeof t?.getRandomValues!="function")throw new Error("crypto.getRandomValues must be defined");return t.getRandomValues(new Uint8Array(e))}var qt=e=>({oid:Uint8Array.from([6,9,96,134,72,1,101,3,4,2,e])});var Lr=BigInt(0),Tr=BigInt(1);function Pe(e,t=""){if(typeof e!="boolean"){let n=t&&`"${t}" `;throw new Error(n+"expected boolean, got type="+typeof e)}return e}function xs(e){if(typeof e=="bigint"){if(!An(e))throw new Error("positive bigint expected, got "+e)}else rt(e);return e}function je(e){let t=xs(e).toString(16);return t.length&1?"0"+t:t}function gs(e){if(typeof e!="string")throw new Error("hex string expected, got "+typeof e);return e===""?Lr:BigInt("0x"+e)}function Ft(e){return gs(Jt(e))}function Pt(e){return gs(Jt(Sn(F(e)).reverse()))}function Se(e,t){rt(t),e=xs(e);let n=te(e.toString(16).padStart(t*2,"0"));if(n.length!==t)throw new Error("number too large");return n}function Ze(e,t){return Se(e,t).reverse()}function Sn(e){return Uint8Array.from(e)}function ee(e){return Uint8Array.from(e,(t,n)=>{let r=t.charCodeAt(0);if(t.length!==1||r>127)throw new Error(`string contains non-ASCII character "${e[n]}" with code ${r} at position ${n}`);return r})}var An=e=>typeof e=="bigint"&&Lr<=e;function uf(e,t,n){return An(e)&&An(t)&&An(n)&&t<=e&&eLr;e>>=Tr,t+=1);return t}var Ge=e=>(Tr<new Uint8Array(b),o=Uint8Array.of(),s=Uint8Array.of(0),i=Uint8Array.of(1),c=1e3,f=r(e),a=r(e),u=0,l=()=>{f.fill(1),a.fill(0),u=0},d=(...b)=>n(a,ot(f,...b)),h=(b=o)=>{a=d(s,b),f=d(),b.length!==0&&(a=d(i,b),f=d())},y=()=>{if(u++>=c)throw new Error("drbg: tried max amount of iterations");let b=0,_=[];for(;b{l(),h(b);let B;for(;!(B=_(y()));)h();return l(),B}}function ne(e,t={},n={}){if(!e||typeof e!="object")throw new Error("expected valid options object");function r(s,i,c){let f=e[s];if(c&&f===void 0)return;let a=typeof f;if(a!==i||f===null)throw new Error(`param "${s}" is invalid: expected ${i}, got ${a}`)}let o=(s,i)=>Object.entries(s).forEach(([c,f])=>r(c,f,i));o(t,!1),o(n,!0)}function Or(e){let t=new WeakMap;return(n,...r)=>{let o=t.get(n);if(o!==void 0)return o;let s=e(n,...r);return t.set(n,s),s}}var dt=BigInt(0),at=BigInt(1),re=BigInt(2),Es=BigInt(3),Bs=BigInt(4),As=BigInt(5),lf=BigInt(7),Ss=BigInt(8),df=BigInt(9),_s=BigInt(16);function lt(e,t){let n=e%t;return n>=dt?n:t+n}function xt(e,t,n){let r=e;for(;t-- >dt;)r*=r,r%=n;return r}function ws(e,t){if(e===dt)throw new Error("invert: expected non-zero number");if(t<=dt)throw new Error("invert: expected positive modulus, got "+t);let n=lt(e,t),r=t,o=dt,s=at,i=at,c=dt;for(;n!==dt;){let a=r/n,u=r%n,l=o-i*a,d=s-c*a;r=n,n=u,o=i,s=c,i=l,c=d}if(r!==at)throw new Error("invert: does not exist");return lt(o,t)}function kr(e,t,n){if(!e.eql(e.sqr(t),n))throw new Error("Cannot find square root")}function vs(e,t){let n=(e.ORDER+at)/Bs,r=e.pow(t,n);return kr(e,r,t),r}function hf(e,t){let n=(e.ORDER-As)/Ss,r=e.mul(t,re),o=e.pow(r,n),s=e.mul(t,o),i=e.mul(e.mul(s,re),o),c=e.mul(s,e.sub(i,e.ONE));return kr(e,c,t),c}function pf(e){let t=_n(e),n=Ts(e),r=n(t,t.neg(t.ONE)),o=n(t,r),s=n(t,t.neg(r)),i=(e+lf)/_s;return(c,f)=>{let a=c.pow(f,i),u=c.mul(a,r),l=c.mul(a,o),d=c.mul(a,s),h=c.eql(c.sqr(u),f),y=c.eql(c.sqr(l),f);a=c.cmov(a,u,h),u=c.cmov(d,l,y);let w=c.eql(c.sqr(u),f),b=c.cmov(a,u,w);return kr(c,b,f),b}}function Ts(e){if(e1e3)throw new Error("Cannot find square root: probably non-prime P");if(n===1)return vs;let s=o.pow(r,t),i=(t+at)/re;return function(f,a){if(f.is0(a))return a;if(ms(f,a)!==1)throw new Error("Cannot find square root");let u=n,l=f.mul(f.ONE,s),d=f.pow(a,t),h=f.pow(a,i);for(;!f.eql(d,f.ONE);){if(f.is0(d))return f.ZERO;let y=1,w=f.sqr(d);for(;!f.eql(w,f.ONE);)if(y++,w=f.sqr(w),y===u)throw new Error("Cannot find square root");let b=at<(r[o]="function",r),t);return ne(e,n),e}function gf(e,t,n){if(ndt;)n&at&&(r=e.mul(r,o)),o=e.sqr(o),n>>=at;return r}function Ye(e,t,n=!1){let r=new Array(t.length).fill(n?e.ZERO:void 0),o=t.reduce((i,c,f)=>e.is0(c)?i:(r[f]=i,e.mul(i,c)),e.ONE),s=e.inv(o);return t.reduceRight((i,c,f)=>e.is0(c)?i:(r[f]=e.mul(i,r[f]),e.mul(i,c)),s),r}function ms(e,t){let n=(e.ORDER-at)/re,r=e.pow(t,n),o=e.eql(r,e.ONE),s=e.eql(r,e.ZERO),i=e.eql(r,e.neg(e.ONE));if(!o&&!s&&!i)throw new Error("invalid Legendre symbol result");return o?1:s?0:-1}function bf(e,t){t!==void 0&&rt(t);let n=t!==void 0?t:e.toString(2).length,r=Math.ceil(n/8);return{nBitLength:n,nByteLength:r}}var Rr=class{constructor(t,n={}){g(this,"ORDER");g(this,"BITS");g(this,"BYTES");g(this,"isLE");g(this,"ZERO",dt);g(this,"ONE",at);g(this,"_lengths");g(this,"_sqrt");g(this,"_mod");if(t<=dt)throw new Error("invalid field: expected ORDER > 0, got "+t);let r;this.isLE=!1,n!=null&&typeof n=="object"&&(typeof n.BITS=="number"&&(r=n.BITS),typeof n.sqrt=="function"&&(this.sqrt=n.sqrt),typeof n.isLE=="boolean"&&(this.isLE=n.isLE),n.allowedLengths&&(this._lengths=n.allowedLengths?.slice()),typeof n.modFromBytes=="boolean"&&(this._mod=n.modFromBytes));let{nBitLength:o,nByteLength:s}=bf(t,r);if(s>2048)throw new Error("invalid field: expected ORDER of <= 2048 bytes");this.ORDER=t,this.BITS=o,this.BYTES=s,this._sqrt=void 0,Object.preventExtensions(this)}create(t){return lt(t,this.ORDER)}isValid(t){if(typeof t!="bigint")throw new Error("invalid field element: expected bigint, got "+typeof t);return dt<=t&&to)throw new Error("Field.fromBytes: expected "+r+" bytes, got "+t.length);let a=new Uint8Array(o);a.set(t,s?0:a.length-t.length),t=a}if(t.length!==o)throw new Error("Field.fromBytes: expected "+o+" bytes, got "+t.length);let f=s?Pt(t):Ft(t);if(c&&(f=lt(f,i)),!n&&!this.isValid(f))throw new Error("invalid field element: outside of range 0..ORDER");return f}invertBatch(t){return Ye(this,t)}cmov(t,n,r){return r?n:t}};function _n(e,t={}){return new Rr(e,t)}function Ls(e){if(typeof e!="bigint")throw new Error("field order must be bigint");let t=e.toString(2).length;return Math.ceil(t/8)}function Cr(e){let t=Ls(e);return t+Math.ceil(t/2)}function Is(e,t,n=!1){F(e);let r=e.length,o=Ls(t),s=Cr(t);if(r<16||r1024)throw new Error("expected "+s+"-1024 bytes of input, got "+r);let i=n?Pt(e):Ft(e),c=lt(i,t-at)+at;return n?Ze(c,o):Se(c,o)}var _e=BigInt(0),oe=BigInt(1);function Xe(e,t){let n=t.negate();return e?n:t}function Kr(e,t){let n=Ye(e.Fp,t.map(r=>r.Z));return t.map((r,o)=>e.fromAffine(r.toAffine(n[o])))}function Us(e,t){if(!Number.isSafeInteger(e)||e<=0||e>t)throw new Error("invalid window size, expected [1.."+t+"], got W="+e)}function Hr(e,t){Us(e,t);let n=Math.ceil(t/e)+1,r=2**(e-1),o=2**e,s=Ge(e),i=BigInt(e);return{windows:n,windowSize:r,mask:s,maxNumber:o,shiftBy:i}}function Os(e,t,n){let{windowSize:r,mask:o,maxNumber:s,shiftBy:i}=n,c=Number(e&o),f=e>>i;c>r&&(c-=s,f+=oe);let a=t*r,u=a+Math.abs(c)-1,l=c===0,d=c<0,h=t%2!==0;return{nextN:f,offset:u,isZero:l,isNeg:d,isNegF:h,offsetF:a}}var Nr=new WeakMap,Cs=new WeakMap;function Mr(e){return Cs.get(e)||1}function Rs(e){if(e!==_e)throw new Error("invalid wNAF")}var vn=class{constructor(t,n){g(this,"BASE");g(this,"ZERO");g(this,"Fn");g(this,"bits");this.BASE=t.BASE,this.ZERO=t.ZERO,this.Fn=t.Fn,this.bits=n}_unsafeLadder(t,n,r=this.ZERO){let o=t;for(;n>_e;)n&oe&&(r=r.add(o)),o=o.double(),n>>=oe;return r}precomputeWindow(t,n){let{windows:r,windowSize:o}=Hr(n,this.bits),s=[],i=t,c=i;for(let f=0;f_e||r>_e;)n&oe&&(s=s.add(o)),r&oe&&(i=i.add(o)),o=o.double(),n>>=oe,r>>=oe;return{p1:s,p2:i}}function ks(e,t,n){if(t){if(t.ORDER!==e)throw new Error("Field.ORDER must match order: Fp == p, Fn == n");return Ur(t),t}else return _n(e,{isLE:n})}function Ns(e,t,n={},r){if(r===void 0&&(r=e==="edwards"),!t||typeof t!="object")throw new Error(`expected valid ${e} CURVE object`);for(let f of["p","n","h"]){let a=t[f];if(!(typeof a=="bigint"&&a>_e))throw new Error(`CURVE.${f} must be positive bigint`)}let o=ks(t.p,n.Fp,r),s=ks(t.n,n.Fn,r),c=["Gx","Gy","a",e==="weierstrass"?"b":"d"];for(let f of c)if(!o.isValid(t[f]))throw new Error(`CURVE.${f} must be valid field element of CURVE.Fp`);return t=Object.freeze(Object.assign({},t)),{CURVE:t,Fp:o,Fn:s}}function Tn(e,t){return function(r){let o=e(r);return{secretKey:o,publicKey:t(o)}}}var $e=BigInt(0),ve=BigInt(1),Ln=BigInt(2);function wf(e){return ne(e,{adjustScalarBytes:"function",powPminus2:"function"}),Object.freeze({...e})}function Ms(e){let t=wf(e),{P:n,type:r,adjustScalarBytes:o,powPminus2:s,randomBytes:i}=t,c=r==="x25519";if(!c&&r!=="x448")throw new Error("invalid type");let f=i||Dt,a=c?255:448,u=c?32:56,l=BigInt(c?9:5),d=BigInt(c?121665:39081),h=c?Ln**BigInt(254):Ln**BigInt(447),y=c?BigInt(8)*Ln**BigInt(251)-ve:BigInt(4)*Ln**BigInt(445)-ve,w=h+y+ve,b=p=>lt(p,n),_=B(l);function B(p){return Ze(b(p),u)}function v(p){let x=Sn(F(p,u,"uCoordinate"));return c&&(x[31]&=127),b(Pt(x))}function A(p){return Pt(o(Sn(F(p,u,"scalar"))))}function R(p,x){let m=M(v(x),A(p));if(m===$e)throw new Error("invalid private or public key received");return B(m)}function T(p){return R(p,_)}let D=T,K=R;function I(p,x,m){let S=b(p*(x-m));return x=b(x-S),m=b(m+S),{x_2:x,x_3:m}}function M(p,x){ze("u",p,$e,n),ze("scalar",x,h,w);let m=x,S=p,O=ve,E=$e,k=p,L=ve,N=$e;for(let Z=BigInt(a-1);Z>=$e;Z--){let j=m>>Z&ve;N^=j,{x_2:O,x_3:k}=I(N,O,k),{x_2:E,x_3:L}=I(N,E,L),N=j;let P=O+E,Y=b(P*P),X=O-E,V=b(X*X),$=Y-V,J=k+L,Vt=k-L,Ct=b(Vt*P),an=b(J*X),un=Ct+an,Ce=Ct-an;k=b(un*un),L=b(S*b(Ce*Ce)),O=b(Y*V),E=b($*(Y+b(d*$)))}({x_2:O,x_3:k}=I(N,O,k)),{x_2:E,x_3:L}=I(N,E,L);let q=s(E);return b(O*q)}let H={secretKey:u,publicKey:u,seed:u},C=(p=f(u))=>(F(p,H.seed,"seed"),p),U={randomSecretKey:C};return Object.freeze({keygen:Tn(C,D),getSharedSecret:K,getPublicKey:D,scalarMult:R,scalarMultBase:T,utils:U,GuBytes:_.slice(),lengths:H})}var In=class{constructor(t,n){g(this,"oHash");g(this,"iHash");g(this,"blockLen");g(this,"outputLen");g(this,"finished",!1);g(this,"destroyed",!1);if(yt(t),F(n,void 0,"key"),this.iHash=t.create(),typeof this.iHash.update!="function")throw new Error("Expected instance of class which extends utils.Hash");this.blockLen=this.iHash.blockLen,this.outputLen=this.iHash.outputLen;let r=this.blockLen,o=new Uint8Array(r);o.set(n.length>r?t.create().update(n).digest():n);for(let s=0;snew In(e,t).update(n).digest();Te.create=(e,t)=>new In(e,t);var Ks=(e,t)=>(e+(e>=0?t:-t)/Ds)/t;function mf(e,t,n){let[[r,o],[s,i]]=t,c=Ks(i*e,n),f=Ks(-o*e,n),a=e-c*r-f*s,u=-c*o-f*i,l=a=h||u=h)throw new Error("splitScalar (endomorphism): failed, k="+e);return{k1neg:l,k1:a,k2neg:d,k2:u}}function qr(e){if(!["compact","recovered","der"].includes(e))throw new Error('Signature format must be "compact", "recovered", or "der"');return e}function Dr(e,t){let n={};for(let r of Object.keys(t))n[r]=e[r]===void 0?t[r]:e[r];return Pe(n.lowS,"lowS"),Pe(n.prehash,"prehash"),n.format!==void 0&&qr(n.format),n}var Fr=class extends Error{constructor(t=""){super(t)}},jt={Err:Fr,_tlv:{encode:(e,t)=>{let{Err:n}=jt;if(e<0||e>256)throw new n("tlv.encode: wrong tag");if(t.length&1)throw new n("tlv.encode: unpadded data");let r=t.length/2,o=je(r);if(o.length/2&128)throw new n("tlv.encode: long form length too big");let s=r>127?je(o.length/2|128):"";return je(e)+s+o+t},decode(e,t){let{Err:n}=jt,r=0;if(e<0||e>256)throw new n("tlv.encode: wrong tag");if(t.length<2||t[r++]!==e)throw new n("tlv.decode: wrong tlv");let o=t[r++],s=!!(o&128),i=0;if(!s)i=o;else{let f=o&127;if(!f)throw new n("tlv.decode(long): indefinite length not supported");if(f>4)throw new n("tlv.decode(long): byte length is too big");let a=t.subarray(r,r+f);if(a.length!==f)throw new n("tlv.decode: length bytes not complete");if(a[0]===0)throw new n("tlv.decode(long): zero leftmost byte");for(let u of a)i=i<<8|u;if(r+=f,i<128)throw new n("tlv.decode(long): not minimal encoding")}let c=t.subarray(r,r+i);if(c.length!==i)throw new n("tlv.decode: wrong value length");return{v:c,l:t.subarray(r+i)}}},_int:{encode(e){let{Err:t}=jt;if(e{let{X:x,Y:m,Z:S}=U;if(r.eql(S,r.ONE))return{x,y:m};let O=U.is0();p==null&&(p=O?r.ONE:r.inv(S));let E=r.mul(x,p),k=r.mul(m,p),L=r.mul(S,p);if(O)return{x:r.ZERO,y:r.ZERO};if(!r.eql(L,r.ONE))throw new Error("invZ was invalid");return{x:E,y:k}}),D=Or(U=>{if(U.is0()){if(t.allowInfinityPoint&&!r.is0(U.Y))return;throw new Error("bad point: ZERO")}let{x:p,y:x}=U.toAffine();if(!r.isValid(p)||!r.isValid(x))throw new Error("bad point: x or y not field elements");if(!b(p,x))throw new Error("bad point: equation left != right");if(!U.isTorsionFree())throw new Error("bad point: not in prime-order subgroup");return!0});function K(U,p,x,m,S){return x=new I(r.mul(x.X,U),x.Y,x.Z),p=Xe(m,p),x=Xe(S,x),p.add(x)}let C=class C{constructor(p,x,m){g(this,"X");g(this,"Y");g(this,"Z");this.X=v("x",p),this.Y=v("y",x,!0),this.Z=v("z",m),Object.freeze(this)}static CURVE(){return s}static fromAffine(p){let{x,y:m}=p||{};if(!p||!r.isValid(x)||!r.isValid(m))throw new Error("invalid affine point");if(p instanceof C)throw new Error("projective point not allowed");return r.is0(x)&&r.is0(m)?C.ZERO:new C(x,m,r.ONE)}static fromBytes(p){let x=C.fromAffine(y(F(p,void 0,"point")));return x.assertValidity(),x}static fromHex(p){return C.fromBytes(te(p))}get x(){return this.toAffine().x}get y(){return this.toAffine().y}precompute(p=8,x=!0){return H.createCache(this,p),x||this.multiply(On),this}assertValidity(){D(this)}hasEvenY(){let{y:p}=this.toAffine();if(!r.isOdd)throw new Error("Field doesn't support isOdd");return!r.isOdd(p)}equals(p){A(p);let{X:x,Y:m,Z:S}=this,{X:O,Y:E,Z:k}=p,L=r.eql(r.mul(x,k),r.mul(O,S)),N=r.eql(r.mul(m,k),r.mul(E,S));return L&&N}negate(){return new C(this.X,r.neg(this.Y),this.Z)}double(){let{a:p,b:x}=s,m=r.mul(x,On),{X:S,Y:O,Z:E}=this,k=r.ZERO,L=r.ZERO,N=r.ZERO,q=r.mul(S,S),Z=r.mul(O,O),j=r.mul(E,E),P=r.mul(S,O);return P=r.add(P,P),N=r.mul(S,E),N=r.add(N,N),k=r.mul(p,N),L=r.mul(m,j),L=r.add(k,L),k=r.sub(Z,L),L=r.add(Z,L),L=r.mul(k,L),k=r.mul(P,k),N=r.mul(m,N),j=r.mul(p,j),P=r.sub(q,j),P=r.mul(p,P),P=r.add(P,N),N=r.add(q,q),q=r.add(N,q),q=r.add(q,j),q=r.mul(q,P),L=r.add(L,q),j=r.mul(O,E),j=r.add(j,j),q=r.mul(j,P),k=r.sub(k,q),N=r.mul(j,Z),N=r.add(N,N),N=r.add(N,N),new C(k,L,N)}add(p){A(p);let{X:x,Y:m,Z:S}=this,{X:O,Y:E,Z:k}=p,L=r.ZERO,N=r.ZERO,q=r.ZERO,Z=s.a,j=r.mul(s.b,On),P=r.mul(x,O),Y=r.mul(m,E),X=r.mul(S,k),V=r.add(x,m),$=r.add(O,E);V=r.mul(V,$),$=r.add(P,Y),V=r.sub(V,$),$=r.add(x,S);let J=r.add(O,k);return $=r.mul($,J),J=r.add(P,X),$=r.sub($,J),J=r.add(m,S),L=r.add(E,k),J=r.mul(J,L),L=r.add(Y,X),J=r.sub(J,L),q=r.mul(Z,$),L=r.mul(j,X),q=r.add(L,q),L=r.sub(Y,q),q=r.add(Y,q),N=r.mul(L,q),Y=r.add(P,P),Y=r.add(Y,P),X=r.mul(Z,X),$=r.mul(j,$),Y=r.add(Y,X),X=r.sub(P,X),X=r.mul(Z,X),$=r.add($,X),P=r.mul(Y,$),N=r.add(N,P),P=r.mul(J,$),L=r.mul(V,L),L=r.sub(L,P),P=r.mul(V,Y),q=r.mul(J,q),q=r.add(q,P),new C(L,N,q)}subtract(p){return this.add(p.negate())}is0(){return this.equals(C.ZERO)}multiply(p){let{endo:x}=t;if(!o.isValidNot0(p))throw new Error("invalid scalar: out of range");let m,S,O=E=>H.cached(this,E,k=>Kr(C,k));if(x){let{k1neg:E,k1:k,k2neg:L,k2:N}=R(p),{p:q,f:Z}=O(k),{p:j,f:P}=O(N);S=Z.add(P),m=K(x.beta,q,j,E,L)}else{let{p:E,f:k}=O(p);m=E,S=k}return Kr(C,[m,S])[0]}multiplyUnsafe(p){let{endo:x}=t,m=this;if(!o.isValid(p))throw new Error("invalid scalar: out of range");if(p===It||m.is0())return C.ZERO;if(p===Le)return m;if(H.hasCache(this))return this.multiply(p);if(x){let{k1neg:S,k1:O,k2neg:E,k2:k}=R(p),{p1:L,p2:N}=Hs(C,m,O,k);return K(x.beta,L,N,S,E)}else return H.unsafe(m,p)}toAffine(p){return T(this,p)}isTorsionFree(){let{isTorsionFree:p}=t;return i===Le?!0:p?p(C,this):H.unsafe(this,c).is0()}clearCofactor(){let{clearCofactor:p}=t;return i===Le?this:p?p(C,this):this.multiplyUnsafe(i)}isSmallOrder(){return this.multiplyUnsafe(i).is0()}toBytes(p=!0){return Pe(p,"isCompressed"),this.assertValidity(),h(C,this,p)}toHex(p=!0){return Jt(this.toBytes(p))}toString(){return``}};g(C,"BASE",new C(s.Gx,s.Gy,r.ONE)),g(C,"ZERO",new C(r.ZERO,r.ONE,r.ZERO)),g(C,"Fp",r),g(C,"Fn",o);let I=C,M=o.BITS,H=new vn(I,t.endo?Math.ceil(M/2):M);return I.BASE.precompute(8),I}function qs(e){return Uint8Array.of(e?2:3)}function Fs(e,t){return{secretKey:t.BYTES,publicKey:1+e.BYTES,publicKeyUncompressed:1+2*e.BYTES,publicKeyHasPrefix:!0,signature:2*t.BYTES}}function Bf(e,t={}){let{Fn:n}=e,r=t.randomBytes||Dt,o=Object.assign(Fs(e.Fp,n),{seed:Cr(n.ORDER)});function s(h){try{let y=n.fromBytes(h);return n.isValidNot0(y)}catch{return!1}}function i(h,y){let{publicKey:w,publicKeyUncompressed:b}=o;try{let _=h.length;return y===!0&&_!==w||y===!1&&_!==b?!1:!!e.fromBytes(h)}catch{return!1}}function c(h=r(o.seed)){return Is(F(h,o.seed,"seed"),n.ORDER)}function f(h,y=!0){return e.BASE.multiply(n.fromBytes(h)).toBytes(y)}function a(h){let{secretKey:y,publicKey:w,publicKeyUncompressed:b}=o;if(!Be(h)||"_lengths"in n&&n._lengths||y===w)return;let _=F(h,void 0,"key").length;return _===w||_===b}function u(h,y,w=!0){if(a(h)===!0)throw new Error("first arg must be private key");if(a(y)===!1)throw new Error("second arg must be public key");let b=n.fromBytes(h);return e.fromBytes(y).multiply(b).toBytes(w)}let l={isValidSecretKey:s,isValidPublicKey:i,randomSecretKey:c},d=Tn(c,f);return Object.freeze({getPublicKey:f,getSharedSecret:u,keygen:d,Point:e,utils:l,lengths:o})}function jr(e,t,n={}){yt(t),ne(n,{},{hmac:"function",lowS:"boolean",randomBytes:"function",bits2int:"function",bits2int_modN:"function"}),n=Object.assign({},n);let r=n.randomBytes||Dt,o=n.hmac||((p,x)=>Te(t,p,x)),{Fp:s,Fn:i}=e,{ORDER:c,BITS:f}=i,{keygen:a,getPublicKey:u,getSharedSecret:l,utils:d,lengths:h}=Bf(e,n),y={prehash:!0,lowS:typeof n.lowS=="boolean"?n.lowS:!0,format:"compact",extraEntropy:!1},w=c*Ds>Le;return p>x}function _(p,x){if(!i.isValidNot0(x))throw new Error(`invalid signature ${p}: out of range 1..Point.Fn.ORDER`);return x}function B(){if(w)throw new Error('"recovered" sig type is not supported for cofactor >2 curves')}function v(p,x){qr(x);let m=h.signature,S=x==="compact"?m:x==="recovered"?m+1:void 0;return F(p,S)}class A{constructor(x,m,S){g(this,"r");g(this,"s");g(this,"recovery");if(this.r=_("r",x),this.s=_("s",m),S!=null){if(B(),![0,1,2,3].includes(S))throw new Error("invalid recovery id");this.recovery=S}Object.freeze(this)}static fromBytes(x,m=y.format){v(x,m);let S;if(m==="der"){let{r:L,s:N}=jt.toSig(F(x));return new A(L,N)}m==="recovered"&&(S=x[0],m="compact",x=x.subarray(1));let O=h.signature/2,E=x.subarray(0,O),k=x.subarray(O,O*2);return new A(i.fromBytes(E),i.fromBytes(k),S)}static fromHex(x,m){return this.fromBytes(te(x),m)}assertRecovery(){let{recovery:x}=this;if(x==null)throw new Error("invalid recovery id: must be present");return x}addRecoveryBit(x){return new A(this.r,this.s,x)}recoverPublicKey(x){let{r:m,s:S}=this,O=this.assertRecovery(),E=O===2||O===3?m+c:m;if(!s.isValid(E))throw new Error("invalid recovery id: sig.r+curve.n != R.x");let k=s.toBytes(E),L=e.fromBytes(ot(qs((O&1)===0),k)),N=i.inv(E),q=T(F(x,void 0,"msgHash")),Z=i.create(-q*N),j=i.create(S*N),P=e.BASE.multiplyUnsafe(Z).add(L.multiplyUnsafe(j));if(P.is0())throw new Error("invalid recovery: point at infinify");return P.assertValidity(),P}hasHighS(){return b(this.s)}toBytes(x=y.format){if(qr(x),x==="der")return te(jt.hexFromSig(this));let{r:m,s:S}=this,O=i.toBytes(m),E=i.toBytes(S);return x==="recovered"?(B(),ot(Uint8Array.of(this.assertRecovery()),O,E)):ot(O,E)}toHex(x){return Jt(this.toBytes(x))}}let R=n.bits2int||function(x){if(x.length>8192)throw new Error("input is too large");let m=Ft(x),S=x.length*8-f;return S>0?m>>BigInt(S):m},T=n.bits2int_modN||function(x){return i.create(R(x))},D=Ge(f);function K(p){return ze("num < 2^"+f,p,It,D),i.toBytes(p)}function I(p,x){return F(p,void 0,"message"),x?F(t(p),void 0,"prehashed message"):p}function M(p,x,m){let{lowS:S,prehash:O,extraEntropy:E}=Dr(m,y);p=I(p,O);let k=T(p),L=i.fromBytes(x);if(!i.isValidNot0(L))throw new Error("invalid private key");let N=[K(L),K(k)];if(E!=null&&E!==!1){let P=E===!0?r(h.secretKey):E;N.push(F(P,void 0,"extraEntropy"))}let q=ot(...N),Z=k;function j(P){let Y=R(P);if(!i.isValidNot0(Y))return;let X=i.inv(Y),V=e.BASE.multiply(Y).toAffine(),$=i.create(V.x);if($===It)return;let J=i.create(X*i.create(Z+$*L));if(J===It)return;let Vt=(V.x===$?0:2)|Number(V.y&Le),Ct=J;return S&&b(J)&&(Ct=i.neg(J),Vt^=1),new A($,Ct,w?void 0:Vt)}return{seed:q,k2sig:j}}function H(p,x,m={}){let{seed:S,k2sig:O}=M(p,x,m);return bs(t.outputLen,i.BYTES,o)(S,O).toBytes(m.format)}function C(p,x,m,S={}){let{lowS:O,prehash:E,format:k}=Dr(S,y);if(m=F(m,void 0,"publicKey"),x=I(x,E),!Be(p)){let L=p instanceof A?", use sig.toBytes()":"";throw new Error("verify expects Uint8Array signature"+L)}v(p,k);try{let L=A.fromBytes(p,k),N=e.fromBytes(m);if(O&&L.hasHighS())return!1;let{r:q,s:Z}=L,j=T(x),P=i.inv(Z),Y=i.create(j*P),X=i.create(q*P),V=e.BASE.multiplyUnsafe(Y).add(N.multiplyUnsafe(X));return V.is0()?!1:i.create(V.x)===q}catch{return!1}}function U(p,x,m={}){let{prehash:S}=Dr(m,y);return x=I(x,S),A.fromBytes(p,"recovered").recoverPublicKey(x).toBytes()}return Object.freeze({keygen:a,getPublicKey:u,getSharedSecret:l,utils:d,lengths:h,Point:e,sign:H,verify:C,recoverPublicKey:U,Signature:A,hash:t})}function Ps(e,t,n){return e&t^~e&n}function js(e,t,n){return e&t^e&n^t&n}var Ve=class{constructor(t,n,r,o){g(this,"blockLen");g(this,"outputLen");g(this,"padOffset");g(this,"isLE");g(this,"buffer");g(this,"view");g(this,"finished",!1);g(this,"length",0);g(this,"pos",0);g(this,"destroyed",!1);this.blockLen=t,this.outputLen=n,this.padOffset=r,this.isLE=o,this.buffer=new Uint8Array(t),this.view=Bn(this.buffer)}update(t){Lt(this),F(t);let{view:n,buffer:r,blockLen:o}=this,s=t.length;for(let i=0;io-i&&(this.process(r,0),i=0);for(let l=i;lu.length)throw new Error("_sha2: outputLen bigger than state");for(let l=0;l>Zs&Rn)}:{h:Number(e>>Zs&Rn)|0,l:Number(e&Rn)|0}}function kn(e,t=!1){let n=e.length,r=new Uint32Array(n),o=new Uint32Array(n);for(let s=0;se>>>n,zr=(e,t,n)=>e<<32-n|t>>>n,se=(e,t,n)=>e>>>n|t<<32-n,ie=(e,t,n)=>e<<32-n|t>>>n,We=(e,t,n)=>e<<64-n|t>>>n-32,Qe=(e,t,n)=>e>>>n-32|t<<64-n;var zs=(e,t,n)=>e<>>32-n,Gs=(e,t,n)=>t<>>32-n,Ys=(e,t,n)=>t<>>64-n,Xs=(e,t,n)=>e<>>64-n;function St(e,t,n,r){let o=(t>>>0)+(r>>>0);return{h:e+n+(o/2**32|0)|0,l:o|0}}var $s=(e,t,n)=>(e>>>0)+(t>>>0)+(n>>>0),Vs=(e,t,n,r)=>t+n+r+(e/2**32|0)|0,Ws=(e,t,n,r)=>(e>>>0)+(t>>>0)+(n>>>0)+(r>>>0),Qs=(e,t,n,r,o)=>t+n+r+o+(e/2**32|0)|0,Js=(e,t,n,r,o)=>(e>>>0)+(t>>>0)+(n>>>0)+(r>>>0)+(o>>>0),ti=(e,t,n,r,o,s)=>t+n+r+o+s+(e/2**32|0)|0;var _f=Uint32Array.from([1116352408,1899447441,3049323471,3921009573,961987163,1508970993,2453635748,2870763221,3624381080,310598401,607225278,1426881987,1925078388,2162078206,2614888103,3248222580,3835390401,4022224774,264347078,604807628,770255983,1249150122,1555081692,1996064986,2554220882,2821834349,2952996808,3210313671,3336571891,3584528711,113926993,338241895,666307205,773529912,1294757372,1396182291,1695183700,1986661051,2177026350,2456956037,2730485921,2820302411,3259730800,3345764771,3516065817,3600352804,4094571909,275423344,430227734,506948616,659060556,883997877,958139571,1322822218,1537002063,1747873779,1955562222,2024104815,2227730452,2361852424,2428436474,2756734187,3204031479,3329325298]),Zt=new Uint32Array(64),Gr=class extends Ve{constructor(t){super(64,t,8,!1)}get(){let{A:t,B:n,C:r,D:o,E:s,F:i,G:c,H:f}=this;return[t,n,r,o,s,i,c,f]}set(t,n,r,o,s,i,c,f){this.A=t|0,this.B=n|0,this.C=r|0,this.D=o|0,this.E=s|0,this.F=i|0,this.G=c|0,this.H=f|0}process(t,n){for(let l=0;l<16;l++,n+=4)Zt[l]=t.getUint32(n,!1);for(let l=16;l<64;l++){let d=Zt[l-15],h=Zt[l-2],y=Bt(d,7)^Bt(d,18)^d>>>3,w=Bt(h,17)^Bt(h,19)^h>>>10;Zt[l]=w+Zt[l-7]+y+Zt[l-16]|0}let{A:r,B:o,C:s,D:i,E:c,F:f,G:a,H:u}=this;for(let l=0;l<64;l++){let d=Bt(c,6)^Bt(c,11)^Bt(c,25),h=u+d+Ps(c,f,a)+_f[l]+Zt[l]|0,w=(Bt(r,2)^Bt(r,13)^Bt(r,22))+js(r,o,s)|0;u=a,a=f,f=c,c=i+h|0,i=s,s=o,o=r,r=h+w|0}r=r+this.A|0,o=o+this.B|0,s=s+this.C|0,i=i+this.D|0,c=c+this.E|0,f=f+this.F|0,a=a+this.G|0,u=u+this.H|0,this.set(r,o,s,i,c,f,a,u)}roundClean(){ht(Zt)}destroy(){this.set(0,0,0,0,0,0,0,0),ht(this.buffer)}},Yr=class extends Gr{constructor(){super(32);g(this,"A",Ot[0]|0);g(this,"B",Ot[1]|0);g(this,"C",Ot[2]|0);g(this,"D",Ot[3]|0);g(this,"E",Ot[4]|0);g(this,"F",Ot[5]|0);g(this,"G",Ot[6]|0);g(this,"H",Ot[7]|0)}};var ei=kn(["0x428a2f98d728ae22","0x7137449123ef65cd","0xb5c0fbcfec4d3b2f","0xe9b5dba58189dbbc","0x3956c25bf348b538","0x59f111f1b605d019","0x923f82a4af194f9b","0xab1c5ed5da6d8118","0xd807aa98a3030242","0x12835b0145706fbe","0x243185be4ee4b28c","0x550c7dc3d5ffb4e2","0x72be5d74f27b896f","0x80deb1fe3b1696b1","0x9bdc06a725c71235","0xc19bf174cf692694","0xe49b69c19ef14ad2","0xefbe4786384f25e3","0x0fc19dc68b8cd5b5","0x240ca1cc77ac9c65","0x2de92c6f592b0275","0x4a7484aa6ea6e483","0x5cb0a9dcbd41fbd4","0x76f988da831153b5","0x983e5152ee66dfab","0xa831c66d2db43210","0xb00327c898fb213f","0xbf597fc7beef0ee4","0xc6e00bf33da88fc2","0xd5a79147930aa725","0x06ca6351e003826f","0x142929670a0e6e70","0x27b70a8546d22ffc","0x2e1b21385c26c926","0x4d2c6dfc5ac42aed","0x53380d139d95b3df","0x650a73548baf63de","0x766a0abb3c77b2a8","0x81c2c92e47edaee6","0x92722c851482353b","0xa2bfe8a14cf10364","0xa81a664bbc423001","0xc24b8b70d0f89791","0xc76c51a30654be30","0xd192e819d6ef5218","0xd69906245565a910","0xf40e35855771202a","0x106aa07032bbd1b8","0x19a4c116b8d2d0c8","0x1e376c085141ab53","0x2748774cdf8eeb99","0x34b0bcb5e19b48a8","0x391c0cb3c5c95a63","0x4ed8aa4ae3418acb","0x5b9cca4f7763e373","0x682e6ff3d6b2b8a3","0x748f82ee5defb2fc","0x78a5636f43172f60","0x84c87814a1f0ab72","0x8cc702081a6439ec","0x90befffa23631e28","0xa4506cebde82bde9","0xbef9a3f7b2c67915","0xc67178f2e372532b","0xca273eceea26619c","0xd186b8c721c0c207","0xeada7dd6cde0eb1e","0xf57d4f7fee6ed178","0x06f067aa72176fba","0x0a637dc5a2c898a6","0x113f9804bef90dae","0x1b710b35131c471b","0x28db77f523047d84","0x32caab7b40c72493","0x3c9ebe0a15c9bebc","0x431d67c49c100d4c","0x4cc5d4becb3e42b6","0x597f299cfc657e2a","0x5fcb6fab3ad6faec","0x6c44198c4a475817"].map(e=>BigInt(e))),vf=ei[0],Tf=ei[1],zt=new Uint32Array(80),Gt=new Uint32Array(80),Xr=class extends Ve{constructor(t){super(128,t,16,!1)}get(){let{Ah:t,Al:n,Bh:r,Bl:o,Ch:s,Cl:i,Dh:c,Dl:f,Eh:a,El:u,Fh:l,Fl:d,Gh:h,Gl:y,Hh:w,Hl:b}=this;return[t,n,r,o,s,i,c,f,a,u,l,d,h,y,w,b]}set(t,n,r,o,s,i,c,f,a,u,l,d,h,y,w,b){this.Ah=t|0,this.Al=n|0,this.Bh=r|0,this.Bl=o|0,this.Ch=s|0,this.Cl=i|0,this.Dh=c|0,this.Dl=f|0,this.Eh=a|0,this.El=u|0,this.Fh=l|0,this.Fl=d|0,this.Gh=h|0,this.Gl=y|0,this.Hh=w|0,this.Hl=b|0}process(t,n){for(let v=0;v<16;v++,n+=4)zt[v]=t.getUint32(n),Gt[v]=t.getUint32(n+=4);for(let v=16;v<80;v++){let A=zt[v-15]|0,R=Gt[v-15]|0,T=se(A,R,1)^se(A,R,8)^Zr(A,R,7),D=ie(A,R,1)^ie(A,R,8)^zr(A,R,7),K=zt[v-2]|0,I=Gt[v-2]|0,M=se(K,I,19)^We(K,I,61)^Zr(K,I,6),H=ie(K,I,19)^Qe(K,I,61)^zr(K,I,6),C=Ws(D,H,Gt[v-7],Gt[v-16]),U=Qs(C,T,M,zt[v-7],zt[v-16]);zt[v]=U|0,Gt[v]=C|0}let{Ah:r,Al:o,Bh:s,Bl:i,Ch:c,Cl:f,Dh:a,Dl:u,Eh:l,El:d,Fh:h,Fl:y,Gh:w,Gl:b,Hh:_,Hl:B}=this;for(let v=0;v<80;v++){let A=se(l,d,14)^se(l,d,18)^We(l,d,41),R=ie(l,d,14)^ie(l,d,18)^Qe(l,d,41),T=l&h^~l&w,D=d&y^~d&b,K=Js(B,R,D,Tf[v],Gt[v]),I=ti(K,_,A,T,vf[v],zt[v]),M=K|0,H=se(r,o,28)^We(r,o,34)^We(r,o,39),C=ie(r,o,28)^Qe(r,o,34)^Qe(r,o,39),U=r&s^r&c^s&c,p=o&i^o&f^i&f;_=w|0,B=b|0,w=h|0,b=y|0,h=l|0,y=d|0,{h:l,l:d}=St(a|0,u|0,I|0,M|0),a=c|0,u=f|0,c=s|0,f=i|0,s=r|0,i=o|0;let x=$s(M,C,p);r=Vs(x,I,H,U),o=x|0}({h:r,l:o}=St(this.Ah|0,this.Al|0,r|0,o|0)),{h:s,l:i}=St(this.Bh|0,this.Bl|0,s|0,i|0),{h:c,l:f}=St(this.Ch|0,this.Cl|0,c|0,f|0),{h:a,l:u}=St(this.Dh|0,this.Dl|0,a|0,u|0),{h:l,l:d}=St(this.Eh|0,this.El|0,l|0,d|0),{h,l:y}=St(this.Fh|0,this.Fl|0,h|0,y|0),{h:w,l:b}=St(this.Gh|0,this.Gl|0,w|0,b|0),{h:_,l:B}=St(this.Hh|0,this.Hl|0,_|0,B|0),this.set(r,o,s,i,c,f,a,u,l,d,h,y,w,b,_,B)}roundClean(){ht(zt,Gt)}destroy(){ht(this.buffer),this.set(0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)}};var $r=class extends Xr{constructor(){super(48);g(this,"Ah",ft[0]|0);g(this,"Al",ft[1]|0);g(this,"Bh",ft[2]|0);g(this,"Bl",ft[3]|0);g(this,"Ch",ft[4]|0);g(this,"Cl",ft[5]|0);g(this,"Dh",ft[6]|0);g(this,"Dl",ft[7]|0);g(this,"Eh",ft[8]|0);g(this,"El",ft[9]|0);g(this,"Fh",ft[10]|0);g(this,"Fl",ft[11]|0);g(this,"Gh",ft[12]|0);g(this,"Gl",ft[13]|0);g(this,"Hh",ft[14]|0);g(this,"Hl",ft[15]|0)}};var Un=Ae(()=>new Yr,qt(1));var ni=Ae(()=>new $r,qt(2));var Lf=BigInt(1),ri=BigInt(2),If=BigInt(3),Of=BigInt(5),Iu=BigInt(8),oi=BigInt("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed");function Rf(e){let t=BigInt(10),n=BigInt(20),r=BigInt(40),o=BigInt(80),s=oi,c=e*e%s*e%s,f=xt(c,ri,s)*c%s,a=xt(f,Lf,s)*e%s,u=xt(a,Of,s)*a%s,l=xt(u,t,s)*u%s,d=xt(l,n,s)*l%s,h=xt(d,r,s)*d%s,y=xt(h,o,s)*h%s,w=xt(y,o,s)*h%s,b=xt(w,t,s)*u%s;return{pow_p_5_8:xt(b,ri,s)*e%s,b2:c}}function kf(e){return e[0]&=248,e[31]&=127,e[31]|=64,e}var si=(()=>{let e=oi;return Ms({P:e,type:"x25519",powPminus2:t=>{let{pow_p_5_8:n,b2:r}=Rf(t);return lt(xt(n,If,e)*r,e)},adjustScalarBytes:kf})})();var Uf={p:BigInt("0xffffffff00000001000000000000000000000000ffffffffffffffffffffffff"),n:BigInt("0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc632551"),h:BigInt(1),a:BigInt("0xffffffff00000001000000000000000000000000fffffffffffffffffffffffc"),b:BigInt("0x5ac635d8aa3a93e7b3ebbd55769886bc651d06b0cc53b0f63bce3c3e27d2604b"),Gx:BigInt("0x6b17d1f2e12c4247f8bce6e563a440f277037d812deb33a0f4a13945d898c296"),Gy:BigInt("0x4fe342e2fe1a7f9b8ee7eb4a7c0f9e162bce33576b315ececbb6406837bf51f5")},Cf={p:BigInt("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeffffffff0000000000000000ffffffff"),n:BigInt("0xffffffffffffffffffffffffffffffffffffffffffffffffc7634d81f4372ddf581a0db248b0a77aecec196accc52973"),h:BigInt(1),a:BigInt("0xfffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffeffffffff0000000000000000fffffffc"),b:BigInt("0xb3312fa7e23ee7e4988e056be3f82d19181d9c6efe8141120314088f5013875ac656398d8a2ed19d2a85c8edd3ec2aef"),Gx:BigInt("0xaa87ca22be8b05378eb1c71ef320ad746e1d3b628ba79b9859f741e082542a385502f25dbf55296c3a545e3872760ab7"),Gy:BigInt("0x3617de4a96262c6f5d9e98bf9292dc29f8f41dbd289a147ce9da3113b5f0b8c00a60b1ce1d7e819d7a431d7c90ea0e5f")};var Hf=Pr(Uf),ii=jr(Hf,Un);var Nf=Pr(Cf),ci=jr(Nf,ni);function ai(e,t,n){return yt(e),n===void 0&&(n=new Uint8Array(e.outputLen)),Te(e,n,t)}var Vr=Uint8Array.of(0),fi=Uint8Array.of();function ui(e,t,n,r=32){yt(e),rt(r,"length");let o=e.outputLen;if(r>255*o)throw new Error("Length must be <= 255*HashLen");let s=Math.ceil(r/o);n===void 0?n=fi:F(n,void 0,"info");let i=new Uint8Array(s*o),c=Te.create(e,t),f=c._cloneInto(),a=new Uint8Array(c.outputLen);for(let u=0;u>Df)*Ff)%qf,t&Kf&&(o^=Je<<(Je<n>32?Ys(e,t,n):zs(e,t,n),di=(e,t,n)=>n>32?Xs(e,t,n):Gs(e,t,n);function Zf(e,t=24){let n=new Uint32Array(10);for(let r=24-t;r<24;r++){for(let i=0;i<10;i++)n[i]=e[i]^e[i+10]^e[i+20]^e[i+30]^e[i+40];for(let i=0;i<10;i+=2){let c=(i+8)%10,f=(i+2)%10,a=n[f],u=n[f+1],l=li(a,u,1)^n[c],d=di(a,u,1)^n[c+1];for(let h=0;h<50;h+=10)e[i+h]^=l,e[i+h+1]^=d}let o=e[2],s=e[3];for(let i=0;i<24;i++){let c=pi[i],f=li(o,s,c),a=di(o,s,c),u=hi[i];o=e[u],s=e[u+1],e[u]=f,e[u+1]=a}for(let i=0;i<50;i+=10){for(let c=0;c<10;c++)n[c]=e[i+c];for(let c=0;c<10;c++)e[i+c]^=~n[(c+2)%10]&n[(c+4)%10]}e[0]^=Pf[r],e[1]^=jf[r]}ht(n)}var Cn=class e{constructor(t,n,r,o=!1,s=24){g(this,"state");g(this,"pos",0);g(this,"posOut",0);g(this,"finished",!1);g(this,"state32");g(this,"destroyed",!1);g(this,"blockLen");g(this,"suffix");g(this,"outputLen");g(this,"enableXOF",!1);g(this,"rounds");if(this.blockLen=t,this.suffix=n,this.outputLen=r,this.enableXOF=o,this.rounds=s,rt(r,"outputLen"),!(0=r&&this.keccak();let i=Math.min(r-this.posOut,s-o);t.set(n.subarray(this.posOut,this.posOut+i),o),this.posOut+=i,o+=i}return t}xofInto(t){if(!this.enableXOF)throw new Error("XOF is not possible for this instance");return this.writeInto(t)}xof(t){return rt(t),this.xofInto(new Uint8Array(t))}digestInto(t){if(mn(t,this),this.finished)throw new Error("digest() was already called");return this.writeInto(t),this.destroy(),t}digest(){return this.digestInto(new Uint8Array(this.outputLen))}destroy(){this.destroyed=!0,ht(this.state)}_cloneInto(t){let{blockLen:n,suffix:r,outputLen:o,rounds:s,enableXOF:i}=this;return t||(t=new e(n,r,o,i,s)),t.state32.set(this.state32),t.pos=this.pos,t.posOut=this.posOut,t.finished=this.finished,t.rounds=s,t.suffix=r,t.outputLen=o,t.enableXOF=i,t.destroyed=this.destroyed,t}},gi=(e,t,n,r={})=>Ae(()=>new Cn(t,e,n),r);var Ie=gi(6,136,32,qt(8));var bi=gi(6,72,64,qt(10));var wi=(e,t,n,r={})=>Ae((o={})=>new Cn(t,e,o.dkLen===void 0?n:o.dkLen,!0),r),mi=wi(31,168,16,qt(11)),Rt=wi(31,136,32,qt(12));function Wr(e){if(!Number.isSafeInteger(e)||e<0||e>4294967295)throw new Error("wrong u32 integer:"+e);return e}function Bi(e){return Wr(e),(e&e-1)===0&&e!==0}function Qr(e,t){Wr(e);let n=0;for(let r=0;r>>=1)n=n<<1|e&1;return n}function Ai(e){return Wr(e),31-Math.clz32(e)}function Ei(e){let t=e.length;if(t<2||!Bi(t))throw new Error("n must be a power of 2 and greater than 1. Got "+t);let n=Ai(t);for(let r=0;r{let{N:n,roots:r,dit:o,invertButterflies:s=!1,skipStages:i=0,brp:c=!0}=t,f=Ai(n);if(!Bi(n))throw new Error("FFT: Polynomial size should be power of two");let a=o!==s;return u=>{if(u.length!==n)throw new Error("FFT: wrong Polynomial length");o&&c&&Ei(u);for(let l=0,d=1;l>1,b=n>>h;for(let _=0;_typeof o=="number"?o:o.bytesLen,r=t.reduce((o,s)=>o+n(s),0);return{bytesLen:r,encode:o=>{let s=new Uint8Array(r);for(let i=0,c=0;i{F(o,r,e);let s=[];for(let i of t){let c=n(i),f=o.subarray(0,c);s.push(typeof i=="number"?f:i.decode(f)),o=o.subarray(c)}return s}}}function Nn(e,t){let n=t*e.bytesLen;return{bytesLen:n,encode:r=>{if(r.length!==t)throw new Error(`vecCoder.encode: wrong length=${r.length}. Expected: ${t}`);let o=new Uint8Array(n);for(let s=0,i=0;s{F(r,n);let o=[];for(let s=0;s{let{newPoly:t,N:n,Q:r,F:o,ROOT_OF_UNITY:s,brvBits:i,isKyber:c}=e,f=(B,v=r)=>{let A=B%v|0;return(A>=0?A|0:v+A|0)|0},a=(B,v=r)=>{let A=f(B,v)|0;return(A>v>>1?A-v|0:A)|0};function u(){let B=t(n);for(let v=0;vf((B|0)+(v|0))|0,sub:(B,v)=>f((B|0)-(v|0))|0,mul:(B,v)=>f((B|0)*(v|0))|0,inv:B=>{throw new Error("not implemented")}},h={N:n,roots:l,invertButterflies:!0,skipStages:c?1:0,brp:!1},y=Jr(d,{dit:!1,...h}),w=Jr(d,{dit:!0,...h});return{mod:f,smod:a,nttZetas:l,NTT:{encode:B=>y(B),decode:B=>{w(B);for(let v=0;v{let A=to(B),R=B*(n/8);return{bytesLen:R,encode:T=>{let D=new Uint8Array(R);for(let K=0,I=0,M=0,H=0;K=8;M-=8,I>>=8)D[H++]=I&to(M);return D},decode:T=>{let D=t(n);for(let K=0,I=0,M=0,H=0;K=B;M-=B,I>>=B)D[H++]=v.decode(I&A);return D}}}}},zf=e=>(t,n)=>{n||(n=e.blockLen);let r=new Uint8Array(t.length+2);r.set(t);let o=t.length,s=new Uint8Array(n),i=e.create({}),c=0,f=0;return{stats:()=>({calls:c,xofs:f}),get:(a,u)=>(r[o+0]=a,r[o+1]=u,i.destroy(),i=e.create({}).update(r),c++,()=>(f++,i.xofInto(s))),clean:()=>{i.destroy(),st(s,r)}}},vi=zf(mi);var ut=256,kt=3329,Gf=3303,Yf=17,{mod:nn,nttZetas:Xf,NTT:fe,bitsCoder:$f}=_i({N:ut,Q:kt,F:Gf,ROOT_OF_UNITY:Yf,newPoly:e=>new Uint16Array(e),brvBits:7,isKyber:!0}),eo={512:{N:ut,Q:kt,K:2,ETA1:3,ETA2:2,du:10,dv:4,RBGstrength:128},768:{N:ut,Q:kt,K:3,ETA1:2,ETA2:2,du:10,dv:4,RBGstrength:192},1024:{N:ut,Q:kt,K:4,ETA1:2,ETA2:2,du:11,dv:5,RBGstrength:256}},Vf=e=>{if(e>=12)return{encode:n=>n,decode:n=>n};let t=2**(e-1);return{encode:n=>((n<n*kt+t>>>e}},tn=e=>$f(e,Vf(e));function ae(e,t){for(let n=0;n>1)];n&1&&(r=-r);let{c0:o,c1:s}=Qf(e[2*n+0],e[2*n+1],t[2*n+0],t[2*n+1],r);e[2*n+0]=o,e[2*n+1]=s}return e}function Ti(e){let t=new Uint16Array(ut);for(let n=0;n>0|r[o+1]<<8)&4095,i=(r[o+1]>>4|r[o+2]<<4)&4095;s>=1,c+=1,c===r?(l=u,u=0):c===2*r&&(s[a++]=nn(l-u),u=0,c=0)}if(c)throw new Error(`sampleCBD: leftover bits: ${c}`);return s}var Jf=e=>{let{K:t,PRF:n,XOF:r,HASH512:o,ETA1:s,ETA2:i,du:c,dv:f}=e,a=tn(1),u=tn(f),l=tn(c),d=ce("publicKey",Nn(tn(12),t),32),h=Nn(tn(12),t),y=ce("ciphertext",Nn(l,t),u),w=ce("seed",32,32);return{secretCoder:h,lengths:{secretKey:h.bytesLen,publicKey:d.bytesLen,cipherText:y.bytesLen},keygen:b=>{F(b,32,"seed");let _=new Uint8Array(33);_.set(b),_[32]=t;let B=o(_),[v,A]=w.decode(B),R=[],T=[];for(let I=0;I{let[v,A]=d.decode(b),R=[];for(let H=0;H{let[B,v]=y.decode(b),A=h.decode(_),R=new Uint16Array(ut);for(let T=0;T{F(u,a,"seed");let{publicKey:l,secretKey:d}=t.keygen(u.subarray(0,32)),h=n(l),y=c.encode([d,l,h,u.subarray(32)]);return st(d,h),{publicKey:l,secretKey:y}},getPublicKey:u=>{let[l,d,h,y]=c.decode(u);return Uint8Array.from(d)},encapsulate:(u,l=Yt(f))=>{F(u,i.publicKey,"publicKey"),F(l,f,"message");let d=u.subarray(0,384*e.K),h=s.encode(s.decode(Si(d)));if(!Hn(h,d))throw st(h),new Error("ML-KEM.encapsulate: wrong publicKey modulus");st(h);let y=r.create().update(l).update(n(u)).digest(),w=t.encrypt(u,l,y.subarray(32,64));return st(y.subarray(32)),{cipherText:w,sharedSecret:y.subarray(0,32)}},decapsulate:(u,l)=>{F(l,c.bytesLen,"secretKey"),F(u,i.cipherText,"cipherText");let d=c.bytesLen-96,h=d+32,y=n(l.subarray(d/2,h));if(!Hn(y,l.subarray(h,h+32)))throw new Error("invalid secretKey: hash check failed");let[w,b,_,B]=c.decode(l),v=t.decrypt(u,w),A=r.create().update(v).update(_).digest(),R=A.subarray(0,32),T=t.encrypt(b,v,A.subarray(32,64)),D=Hn(u,T),K=o.create({dkLen:32}).update(B).update(u).digest();return st(v,T,D?K:R),D?R:K}}}function ta(e,t,n){return Rt.create({dkLen:e}).update(t).update(new Uint8Array([n])).digest()}var ro={HASH256:Ie,HASH512:bi,KDF:Rt,XOF:vi,PRF:ta},rl=no({...ro,...eo[512]}),Kn=no({...ro,...eo[768]}),Li=no({...ro,...eo[1024]});function ea(e,t=!1){let n=e.lengths,r=e.keygen;if(t){let s=e.Point.Fn;if(!s)throw new Error("No Point.Fn");r=(i=Yt(n.seed))=>{F(i,n.seed,"seed");let c=s.isLE?Pt(i):Ft(i),f=s.toBytes(s.create(c));return{secretKey:f,publicKey:e.getPublicKey(f)}}}return{lengths:{secretKey:n.secretKey,publicKey:n.publicKey,seed:n.seed},keygen:r,getPublicKey:o=>e.getPublicKey(o)}}function oo(e,t=!1){let n=ea(e,t);if(!e.getSharedSecret)throw new Error("wrong curve");return{lengths:{...n.lengths,msg:n.lengths.seed,cipherText:n.lengths.publicKey},keygen:n.keygen,getPublicKey:n.getPublicKey,encapsulate(r,o=Yt(e.lengths.seed)){let s=this.keygen(o).secretKey,i=this.decapsulate(r,s),c=e.getPublicKey(s);return st(s),{sharedSecret:i,cipherText:c}},decapsulate(r,o){let s=e.getSharedSecret(o,r);return e.lengths.publicKeyHasPrefix?s.subarray(1):s}}}function rn(e,t){return ce(t,...e.map(n=>{if(typeof n.lengths[t]!="number")throw new Error("wrong length: "+t);return n.lengths[t]}))}function so(e){return(t,n)=>e(t,{dkLen:n})}function na(e,t,...n){let r=rn(n,"seed"),o=rn(n,"publicKey");e===void 0&&(e=r.bytesLen),rt(e);function s(i){F(i,e);let c=r.decode(t(i,r.bytesLen)),f=n.map((l,d)=>l.keygen(c[d])),a=f.map(l=>l.secretKey),u=f.map(l=>l.publicKey);return{secretKey:a,publicKey:u}}return{info:{lengths:{seed:e,publicKey:o.bytesLen,secretKey:e}},getPublicKey(i){return this.keygen(i).publicKey},keygen(i=Yt(e)){let{publicKey:c,secretKey:f}=s(i),a=o.encode(c);return st(c),st(f),{secretKey:i,publicKey:a}},expandDecapsulationKey:s,realSeedLen:e}}function io(e,t,n,r,...o){let s=na(e,n,...o),i=rn(o,"cipherText"),c=rn(o,"publicKey"),f=rn(o,"msg");return t===void 0&&(t=f.bytesLen),rt(t),{lengths:{...s.info.lengths,msg:t,msgRand:f.bytesLen,cipherText:i.bytesLen},getPublicKey:s.getPublicKey,keygen:s.keygen,encapsulate(a,u=Yt(f.bytesLen)){let l=c.decode(a),d=f.decode(u),h=o.map((_,B)=>_.encapsulate(l[B],d[B])),y=h.map(_=>_.sharedSecret),w=h.map(_=>_.cipherText),b={sharedSecret:r(l,w,y),cipherText:i.encode(w)};return st(y,w),b},decapsulate(a,u){let l=i.decode(a),{publicKey:d,secretKey:h}=s.expandDecapsulationKey(u),y=o.map((w,b)=>w.decapsulate(l[b],h[b]));return r(d,l,y)}}}function Ii(e,t,n,r,o){return yt(r),yt(o),io(32,32,so(r),(s,i,c)=>o(ot(c[0],c[1],i[1],s[1],ee(e))),t,n)}var pl=Ii("QSF-KEM(ML-KEM-768,P-256)-XOF(SHAKE256)-KDF(SHA3-256)",Kn,oo(ii,!0),Rt,Ie),yl=Ii("QSF-KEM(ML-KEM-1024,P-384)-XOF(SHAKE256)-KDF(SHA3-256)",Li,oo(ci,!0),Rt,Ie);function ra(e,t,n,r,o){return yt(r),yt(o),io(32,32,so(r),(s,i,c)=>{let f=ot(c[0],c[1],i[0],s[0],i[1],s[1],ee(e)),a=32,u=ot(ee("hybrid_prk"),f),l=ai(o,u),d=ot(Se(a,2),ee("shared_secret"),ee("")),h=ui(o,l,d,a);return st(l,d,u,f),h},t,n)}var Oi=oo(si),xl=ra("KitchenSink-KEM(ML-KEM-768,X25519)-XOF(SHAKE256)-KDF(HKDF-SHA-256)",Kn,Oi,Rt,Un),oa=io(32,32,so(Rt),(e,t,n)=>Ie(ot(n[0],n[1],t[1],e[1],ee("\\.//^\\"))),Kn,Oi);var co=oa;function qn(e,t,n=()=>{}){if(!Array.isArray(e))throw new TypeError(`"${t}" expected array, got type=${typeof e}`);for(let r=0;rpt(e,t,n),fo=et;function Xt(e,t="object"){if(e===null||typeof e!="object"||Array.isArray(e))throw new TypeError(t==="object"?"expected valid options object":`"${t}" expected object, got type=${typeof e}`);return e}function ue(e,t){if(typeof e!="function")throw new TypeError(`"${t}" is invalid: expected function, got ${typeof e}`);return e}var ki=pn;var Ui=e=>lr(e),Fn=ln,Pn=e=>yn(e),Dn=BigInt(0),Ri=BigInt(1),sa=e=>e?`"${e}" `:"";function le(e,t=""){if(typeof e!="boolean")throw new TypeError(sa(t)+"expected boolean, got type="+typeof e);return e}function ia(e){if(typeof e=="bigint"){if(!$t(e))throw new RangeError("positive bigint expected, got "+e)}else fo(e);return e}function on(e,t=""){if(typeof e!="number"){let n=t&&`"${t}" `;throw new TypeError(n+"expected number, got type="+typeof e)}if(!Number.isSafeInteger(e)){let n=t&&`"${t}" `;throw new RangeError(n+"expected safe integer, got "+e)}}function Ci(e){if(typeof e!="string")throw new TypeError("hex string expected, got "+typeof e);return e===""?Dn:BigInt("0x"+e)}function jn(e){return Ci(pn(e))}function de(e){return Ci(pn(he(pt(e)).reverse()))}function ao(e,t){if(et(t),t===0)throw new Error("zero output length is invalid");e=ia(e);let n=t*2,r=e.toString(16);if(r.length>n)throw new RangeError("number is too large");return lr(r.padStart(n,"0"))}function Zn(e,t){return ao(e,t).reverse()}function he(e){return Uint8Array.from(Ut(e))}function $t(e){return typeof e=="bigint"&&Dn<=e}function uo(e,t,n){return $t(e)&&$t(t)&&$t(n)&&t<=e&&e(on(e,"n"),(Ri<Object.entries(i).forEach(([f,a])=>o(f,a,c));s(t,!1),s(n,!0)}var it=BigInt(0),Q=BigInt(1),xe=BigInt(2),Ki=BigInt(3),ho=BigInt(4),Di=BigInt(5),ca=BigInt(7),qi=BigInt(8),fa=BigInt(9),aa=BigInt(15),Fi=BigInt(16),ua=BigInt("0x10000000000000000");function W(e,t){if(t<=it)throw new Error("mod: expected positive modulus, got "+t);let n=e%t;return n>=it?n:t+n}function la(e,t,n){if(n<=Q)throw new Error("pow: expected modulus > 1, got "+n);if(typeof t!="bigint")throw new TypeError("invalid exponent: expected bigint, got "+typeof t);if(tit;)t&Q&&(c=c*r%n),r=r*r%n,t>>=Q;return c}let o=[];for(;t>it;)o.push(Number(t&aa)),t>>=ho;let s=new Array(16);s[0]=Q,s[1]=r;for(let c=2;c<16;c++)s[c]=s[c-1]*r%n;let i=s[o[o.length-1]];for(let c=o.length-2;c>=0;c--){i=i*i%n,i=i*i%n,i=i*i%n,i=i*i%n;let f=o[c];f!==0&&(i=i*s[f]%n)}return i}function gt(e,t,n){if(n<=Q)throw new Error("pow2: expected modulus > 1, got "+n);if(tit;)r*=r,r%=n;return r}function Ni(e,t){if(e===it)throw new Error("invert: expected non-zero number");if(t<=Q)throw new Error("invert: expected modulus > 1, got "+t);let n=W(e,t),r=t,o=it,s=Q;for(;n!==it;){let c=r/n,f=r-n*c,a=o-s*c;r=n,n=f,o=s,s=a}if(r!==Q)throw new Error("invert: does not exist");return W(o,t)}function po(e,t,n){let r=e;if(!r.eql(r.sqr(t),n))throw new Error("Cannot find square root")}function yo(e,t){if((e&Q)===it)throw new Error(t+": expected odd modulus, got "+e)}function Pi(e,t){let n=e,r=(n.ORDER+Q)/ho,o=n.pow(t,r);return po(n,o,t),o}function da(e,t){let n=e,r=(n.ORDER-Di)/qi,o=n.mul(t,xe),s=n.pow(o,r),i=n.mul(t,s),c=n.mul(n.mul(i,xe),s),f=n.mul(i,n.sub(c,n.ONE));return po(n,f,t),f}function ha(e){let t=Yn(e),n=ji(e),r=n(t,t.neg(t.ONE)),o=n(t,r),s=n(t,t.neg(r)),i=(e+ca)/Fi;return((c,f)=>{let a=c,u=a.pow(f,i),l=a.mul(u,r),d=a.mul(u,o),h=a.mul(u,s),y=a.eql(a.sqr(l),f),w=a.eql(a.sqr(d),f);u=a.cmov(u,l,y),l=a.cmov(h,d,w);let b=a.eql(a.sqr(l),f),_=a.cmov(u,l,b);return po(a,_,f),_})}function ji(e){if(e1e3)throw new Error("Cannot find square root: probably non-prime P");if(n===1)return Pi;let s=o.pow(r,t),i=(t+Q)/xe;return function(f,a){let u=f;if(u.is0(a))return a;if(sn(u,a)!==1)throw new Error("Cannot find square root");let l=n,d=u.mul(u.ONE,s),h=u.pow(a,t),y=u.pow(a,i);for(;!u.eql(h,u.ONE);){if(u.is0(h))throw new Error("Cannot find square root: probably non-prime P");let w=1,b=u.sqr(h);for(;!u.eql(b,u.ONE);)if(w++,b=u.sqr(b),w===l)throw new Error("Cannot find square root");let _=Q<(W(e,t)&Q)===Q,ya=["create","isValid","is0","neg","inv","sqrt","sqr","eql","add","sub","mul","pow","div","addN","subN","mulN","sqrN"];function Oe(e){if(Xt(e,"field"),typeof e.ORDER!="bigint")throw new TypeError('param "ORDER" is invalid: expected bigint, got '+typeof e.ORDER);on(e.BYTES,"BYTES"),on(e.BITS,"BITS");for(let t of ya)ue(e[t],"field."+t);if(e.BYTES<1||e.BITS<1)throw new Error("invalid field: expected BYTES/BITS > 0");if(e.ORDER<=Q)throw new Error("invalid field: expected ORDER > 1, got "+e.ORDER);return e}function Gn(e,t,n=!1){Oe(e),qn(t,"nums"),le(n,"passZero");let r=e,o=new Array(t.length).fill(n?r.ZERO:void 0),s=t.reduce((c,f,a)=>r.is0(f)?c:(o[a]=c,r.mul(c,f)),r.ONE),i=r.inv(s);return t.reduceRight((c,f,a)=>r.is0(f)?c:(o[a]=r.mul(c,o[a]),r.mul(c,f)),i),o}function sn(e,t){Oe(e);let n=e;yo(n.ORDER,"FpLegendre");let r=(n.ORDER-Q)/xe,o=n.pow(t,r),s=n.eql(o,n.ONE),i=n.eql(o,n.ZERO),c=n.eql(o,n.neg(n.ONE));if(!s&&!i&&!c)throw new Error("invalid Legendre symbol result");return s?1:i?0:-1}function xa(e,t){if(t!==void 0&&fo(t),e<=it)throw new Error("invalid n length: expected positive n, got "+e);if(t!==void 0&&t<1)throw new Error("invalid n length: expected positive bit length, got "+t);let n=lo(e);if(t!==void 0&&t= bitLen(n) (${n})`);let r=t!==void 0?t:n,o=Math.ceil(r/8);return{nBitLength:r,nByteLength:o}}var Mi=new WeakMap,zn=class{constructor(t,n={}){g(this,"ORDER");g(this,"BITS");g(this,"BYTES");g(this,"isLE");g(this,"ZERO",it);g(this,"ONE",Q);g(this,"_lengths");g(this,"_mod");if(t<=Q)throw new Error("invalid field: expected ORDER > 1, got "+t);let r;this.isLE=!1,n!=null&&typeof n=="object"&&(typeof n.BITS=="number"&&(r=n.BITS),typeof n.sqrt=="function"&&Object.defineProperty(this,"sqrt",{value:n.sqrt,enumerable:!0}),typeof n.isLE=="boolean"&&(this.isLE=n.isLE),n.allowedLengths&&(this._lengths=Object.freeze(n.allowedLengths.slice())),typeof n.modFromBytes=="boolean"&&(this._mod=n.modFromBytes));let{nBitLength:o,nByteLength:s}=xa(t,r);if(s>2048)throw new Error("invalid field: expected ORDER of <= 2048 bytes");this.ORDER=t,this.BITS=o,this.BYTES=s,Object.freeze(this)}create(t){return W(t,this.ORDER)}isValid(t){if(typeof t!="bigint")throw new TypeError("invalid field element: expected bigint, got "+typeof t);return it<=t&&to)throw new Error("Field.fromBytes: expected "+r+" bytes, got "+t.length);let a=new Uint8Array(o);a.set(t,s?0:a.length-t.length),t=a}if(t.length!==o)throw new Error("Field.fromBytes: expected "+o+" bytes, got "+t.length);let f=s?de(t):jn(t);if(c&&(f=W(f,i)),!n&&!this.isValid(f))throw new Error("invalid field element: outside of range 0..ORDER");return f}invertBatch(t){return Gn(this,t,!0)}cmov(t,n,r){return le(r,"condition"),r?n:t}};function Yn(e,t={}){return Object.freeze(zn.prototype),new zn(e,t)}var bo=BigInt(0),cn=BigInt(1),ga=BigInt(4),xo=16,zi=128,ba=5,Gi=2**31;function $n(e){let t=e;if(typeof t!="function")throw new TypeError('"Point" expected constructor, got type='+typeof e);ue(t.fromAffine,"Point.fromAffine"),ue(t.fromBytes,"Point.fromBytes"),ue(t.fromHex,"Point.fromHex"),Xt(t.BASE,"Point.BASE"),Xt(t.ZERO,"Point.ZERO"),Oe(t.Fp),Oe(t.Fn)}function Xi(e,t){$n(e),$i(t,e);let n=Gn(e.Fp,t.map(r=>r.Z));return t.map((r,o)=>e.fromAffine(r.toAffine(n[o])))}function wa(e,t,n=1){if(!Number.isSafeInteger(e)||et)throw new Error("invalid window size, expected ["+n+".."+t+"], got W="+e)}function ma(e,t){let n=e*(4*t+128);if(n>Gi)throw new Error("invalid window size: table would need ~"+Math.ceil(n/2**20)+" MiB, max "+Gi/2**20+" MiB")}function Ea(e,t){if(e!==void 0){ue(e,"randomBytes");try{let n=e(t);if(!Fn(n)||n.length!==t)return}catch{return}return e}}function $i(e,t){qn(e,"points"),e.forEach((n,r)=>{if(!(n instanceof t))throw new Error("invalid point at index "+r)})}function Ba(e,t,n){if(!Array.isArray(e))throw new Error("array of scalars expected");e.forEach((r,o)=>{if(!(n===void 0?t.isValid(r):$t(r)&&rbo;){let i=0;e&cn&&(i=Number(e&o),i>=r&&(i-=n),e-=BigInt(i)),s.push(i),e>>=cn}return s}function _a(e,t,n){let r=2**t,o=r/2,s=BigInt(r-1),i=BigInt(t),c=[];for(let f=0;f>=i,a>o&&(a-=r,e+=cn),c.push(a)}if(e!==bo)throw new Error("invalid wnaf");return c}function va(e,t,n){let r=0;for(let s of n)r=Math.max(r,s.length);let o=e;for(let s=r-1;s>=0;s--){s!==r-1&&(o=o.double());for(let i=0;i>1];o=o.add(c<0?f.negate():f)}}}return o}var Xn=class{constructor(t,n){g(this,"Point");g(this,"BASE");g(this,"ZERO");g(this,"randomBytes");g(this,"wnafPrecomputes",new WeakMap);g(this,"baseCanBeBlinded");g(this,"bits");$n(t),this.randomBytes=Ea(n,xo),this.Point=t,this.BASE=t.BASE,this.ZERO=t.ZERO,this.bits=t.Fn.BITS}buildWnafTable(t,n,r){let o=Math.ceil(r/n)+1,s=2**(n-1),i=[],c=t;for(let f=0;fc.W===t&&c.bits===r);return i||(i=this.buildWnafTable(n,t,r),typeof o=="function"&&(i={...i,comp:o(i.comp)}),s||(s=[],this.wnafPrecomputes.set(n,s)),s.push(i)),i}assertPoint(t){if(!(t instanceof this.Point))throw new TypeError('"point" expected Point instance, got type='+typeof t)}validateMulInput(t,n){if(this.assertPoint(t),!uo(n,cn,this.Point.Fn.ORDER))throw new Error("invalid scalar")}runCT(t,n,r,o){let s=go(t);return s===1?this.fixedWindowCT(t,n,r):this.wnafCachedCT(this.getWnafPrecomputes(s,t,r,o),n)}mulCT(t,n,r){return this.validateMulInput(t,n),this.runCT(t,n,this.bits,r)}mulCTBlinded(t,n,r){if(this.validateMulInput(t,n),this.randomBytes===void 0)throw new Error("randomBytes is required for scalar blinding");let o=this.Point.Fn.BITS+zi,s=this.randomBytes(xo);if(!Fn(s)||s.length!==xo)throw new Error("randomBytes returned invalid byte array");s[0]=s[0]&63|128;let i=n+jn(s)*this.Point.Fn.ORDER;return this.runCT(t,i,o,r)}fixedWindowCT(t,n,r){let o=ba,s=1<=0;u--){if(u!==f-1)for(let h=0;h>BigInt(u*o)&i),d=c[0];for(let h=1;h=this.Point.Fn.ORDER)return Ta(this.Point,[t],[n],!0);let s=this.getWnafPrecomputes(o,t,this.bits,r);return this.wnafCachedCT(s,n).p}setWindowSize(t,n){this.assertPoint(t),wa(n,this.bits);let r=Math.ceil((this.bits+zi)/n)+1;ma(r*2**(n-1),this.Point.Fp.BYTES),Vi.set(t,n),this.wnafPrecomputes.delete(t)}hasWindowSize(t){return go(t)!==1}};function Ta(e,t,n,r=!1){if($n(e),$i(t,e),le(r,"allowOversized"),Ba(n,e.Fn,r?e.Fn.ORDER**ga:void 0),t.length!==n.length)throw new Error("arrays of points and scalars must have equal length");let o=t.map(i=>Aa(i,4)),s=n.map(i=>Sa(i,4));return va(e.ZERO,o,s)}function Yi(e,t,n){if(t){if(t.ORDER!==e)throw new Error("Field.ORDER must match order: Fp == p, Fn == n");return Oe(t),t}else return Yn(e,{isLE:n})}function Wi(e,t,n={},r){if(e!=="weierstrass"&&e!=="edwards")throw new Error('expected curve type "weierstrass" or "edwards"');if(r===void 0&&(r=e==="edwards"),!t||typeof t!="object")throw new Error(`expected valid ${e} CURVE object`);ye(n);for(let f of["p","n","h"]){let a=t[f];if(!($t(a)&&a!==bo))throw new Error(`CURVE.${f} must be positive bigint`)}let o=Yi(t.p,n.Fp,r),s=Yi(t.n,n.Fn,r),c=["Gx","Gy","a",e==="weierstrass"?"b":"d"];for(let f of c)if(!o.isValid(t[f]))throw new Error(`CURVE.${f} must be valid field element of CURVE.Fp`);return t=Object.freeze(Object.assign({},t)),{CURVE:t,Fp:o,Fn:s}}function wo(e,t){return function(r){let o=e(r);return{secretKey:o,publicKey:t(o)}}}var Vn=BigInt(0),Wn=BigInt(1),Qn=BigInt(2),La=BigInt(4),Qi=BigInt(8);function Ia(e,t,n,r){let o=e.sqr(n),s=e.sqr(r),i=e.add(e.mul(t.a,o),s),c=e.add(e.ONE,e.mul(t.d,e.mul(o,s)));return e.eql(i,c)}function Ji(e,t={}){ye(t,{},{},"extraOpts");let n=t,r=Wi("edwards",e,n,n.FpFnLE),{Fp:o,Fn:s}=r,i=r.CURVE,{h:c}=i;if(sn(o,i.a)!==1)throw new Error("edwards: CURVE.a must be a square in Fp for complete addition formulas");if(sn(o,i.d)!==-1)throw new Error("edwards: CURVE.d must be a non-square in Fp for complete addition formulas");ye(n,{},{uvRatio:"function",randomBytes:"function"});let f=n.randomBytes===void 0?Pn:n.randomBytes,a=Qn<{try{return{isValid:!0,value:o.sqrt(o.div(v,A))}}catch{return{isValid:!1,value:Vn}}}:n.uvRatio;if(!Ia(o,i,i.Gx,i.Gy))throw new Error("bad curve params: generator point");let d=o.eql(i.a,o.neg(o.ONE))?v=>o.neg(v):o.eql(i.a,o.ONE)?v=>v:v=>o.mul(i.a,v);function h(v,A,R=!1){let T=R?Wn:Vn;return pe("coordinate "+v,A,T,a),A}function y(v){if(!(v instanceof w))throw new Error("EdwardsPoint expected")}let B=class B{constructor(A,R,T,D){g(this,"X");g(this,"Y");g(this,"Z");g(this,"T");this.X=h("x",A),this.Y=h("y",R),this.Z=h("z",T,!0),this.T=h("t",D),Object.freeze(this)}static CURVE(){return i}static fromAffine(A){if(A instanceof B)throw new Error("extended point not allowed");let{x:R,y:T}=A||{};return h("x",R),h("y",T),new B(R,T,o.ONE,o.mul(R,T))}static fromBytes(A,R=!1){let T=o.BYTES,{a:D,d:K}=i;A=he(Ut(A,T,"point")),le(R,"zip215");let I=he(A),M=A[T-1];I[T-1]=M&-129;let H=de(I),C=R?a:o.ORDER;pe("point.y",H,Vn,C);let U=o.sqr(H),p=o.sub(U,o.ONE),x=o.sub(o.mulN(K,U),D),{isValid:m,value:S}=l(p,x);if(!m)throw new Error("bad point: invalid y coordinate");let O=u(S),E=(M&128)!==0;if(!R&&o.is0(S)&&E)throw new Error("bad point: x=0 and x_0=1");return E!==O&&(S=o.neg(S)),B.fromAffine({x:S,y:H})}static fromHex(A,R=!1){return B.fromBytes(Ui(A),R)}get x(){return this.toAffine().x}get y(){return this.toAffine().y}precompute(A=6,R=!0){return _.setWindowSize(this,A),R||this.multiply(Qn),this}assertValidity(){let A=this,{a:R,d:T}=i;if(A.is0())throw new Error("bad point: ZERO");let{X:D,Y:K,Z:I,T:M}=A,H=o.sqr(D),C=o.sqr(K),U=o.sqr(I),p=o.sqr(U),x=o.mul(H,R),m=o.mul(o.add(x,C),U),S=o.add(p,o.mul(T,o.mul(H,C)));if(!o.eql(m,S))throw new Error("bad point: equation left != right (1)");let O=o.mul(D,K),E=o.mul(I,M);if(!o.eql(O,E))throw new Error("bad point: equation left != right (2)")}equals(A){y(A);let{X:R,Y:T,Z:D}=this,{X:K,Y:I,Z:M}=A,H=o.mul(R,M),C=o.mul(K,D),U=o.mul(T,M),p=o.mul(I,D);return o.eql(H,C)&&o.eql(U,p)}is0(){return this.equals(B.ZERO)}negate(){return new B(o.neg(this.X),this.Y,this.Z,o.neg(this.T))}double(){let{X:A,Y:R,Z:T}=this,D=o.sqr(A),K=o.sqr(R),I=o.mul(o.sqr(T),Qn),M=d(D),H=o.addN(A,R),C=o.sub(o.subN(o.sqr(H),D),K),U=o.addN(M,K),p=o.subN(U,I),x=o.subN(M,K),m=o.mul(C,p),S=o.mul(U,x),O=o.mul(C,x),E=o.mul(p,U);return new B(m,S,E,O)}add(A){y(A);let{d:R}=i,{X:T,Y:D,Z:K,T:I}=this,{X:M,Y:H,Z:C,T:U}=A,p=o.mul(T,M),x=o.mul(D,H),m=o.mul(o.mulN(I,R),U),S=o.mul(K,C),O=o.sub(o.subN(o.mulN(o.addN(T,D),o.addN(M,H)),p),x),E=o.subN(S,m),k=o.addN(S,m),L=o.sub(x,d(p)),N=o.mul(O,E),q=o.mul(k,L),Z=o.mul(O,L),j=o.mul(E,k);return new B(N,q,j,Z)}subtract(A){return y(A),this.add(A.negate())}multiply(A){if(!s.isValidNot0(A))throw new RangeError("invalid scalar: expected 1 <= sc < curve.n");let{p:R,f:T}=_.mulSecret(this,A,c,b);return b([R,T])[0]}multiplyUnsafe(A){if(!s.isValid(A))throw new RangeError("invalid scalar: expected 0 <= sc < curve.n");return A===Vn?B.ZERO:this.is0()||A===Wn?this:_.mulUnsafe(this,A,b)}isSmallOrder(){return this.clearCofactor().is0()}isTorsionFree(){return _.mulUnsafe(this,i.n).is0()}toAffine(A){let R=this,T=A;if(T!=null&&typeof T!="bigint")throw new TypeError('"invertedZ" expected bigint, got type='+typeof T);let{X:D,Y:K,Z:I}=R,M=R.is0();T==null&&(T=M?o.create(Qi):o.inv(I));let H=o.mul(D,T),C=o.mul(K,T),U=o.mul(I,T);if(M)return{x:o.ZERO,y:o.ONE};if(!o.eql(U,o.ONE))throw new Error("invZ was invalid");return{x:H,y:C}}clearCofactor(){return c===Wn?this:c===Qn?this.double():c===La?this.double().double():c===Qi?this.double().double().double():this.multiplyUnsafe(c)}toBytes(){let{x:A,y:R}=this.toAffine(),T=o.toBytes(R);return T[T.length-1]|=u(A)?128:0,T}toHex(){return ki(this.toBytes())}toString(){return``}};g(B,"BASE",new B(i.Gx,i.Gy,o.ONE,o.mul(i.Gx,i.Gy))),g(B,"ZERO",new B(o.ZERO,o.ONE,o.ONE,o.ZERO)),g(B,"Fp",o),g(B,"Fn",s);let w=B,b=v=>Xi(w,v),_=new Xn(w,f);return _.bits>=6&&w.BASE.precompute(6),Object.freeze(w.prototype),Object.freeze(w),w}var ge=BigInt(0),bt=BigInt(1),Jn=BigInt(2);function tc(e,t){return e+t-(t>>bt<{let s=r+o,c=((t+o-r)*n+r)%e;return{x_2:c,x_3:s-c}}}function Ra(e){return ye(e,{P:"bigint",type:"string",adjustScalarBytes:"function",powPminus2:"function"},{randomBytes:"function",scalarMultBase:"function"}),Object.freeze({...e})}function ec(e){let t=Ra(e),{P:n,type:r,adjustScalarBytes:o,powPminus2:s,randomBytes:i}=t,c=t.scalarMultBase,f=r==="x25519";if(!f&&r!=="x448")throw new Error("invalid type");let a=i===void 0?Pn:i,u=f?255:448,l=Oa(n),d=f?32:56,h=BigInt(f?9:5),y=BigInt(f?121665:39081),w=f?Jn**BigInt(254):Jn**BigInt(447),b=f?BigInt(8)*(Jn**BigInt(251)-bt):BigInt(4)*(Jn**BigInt(445)-bt),_=w+b+bt,B=m=>W(m,n),v=A(h);function A(m){return Zn(B(m),d)}function R(m){let S=he(Ut(m,d,"uCoordinate"));return f&&(S[31]&=127),B(de(S))}function T(m){return de(o(he(Ut(m,d,"scalar"))))}let D=new Set(f?[ge,bt,n-bt,BigInt("325606250916557431795983626356110631294008115727848805560023387167927233504"),BigInt("39382357235489614581723060781553021112529911719440698176882885853963445705823")]:[ge,bt,n-bt]);function K(m,S){let O=R(S);if(D.has(O))throw new Error("invalid private or public key received");let E=C(O,T(m));if(E===ge)throw new Error("invalid private or public key received");return A(E)}function I(m){if(c===void 0)return K(m,v);let S=T(m);pe("scalar",S,w,_);let O=B(c(S));if(O===ge)throw new Error("invalid private or public key received");return A(O)}let M=I,H=K;function C(m,S){pe("u",m,ge,n),pe("scalar",S,w,_);let O=S,E=m,k=bt,L=ge,N=m,q=bt,Z=O^O>>bt;for(let Y=BigInt(u-1);Y>=ge;Y--){let X=tc(n,Z>>Y);({x_2:k,x_3:N}=l(X,k,N)),{x_2:L,x_3:q}=l(X,L,q);let V=k+L,$=B(V*V),J=k-L,Vt=B(J*J),Ct=$-Vt,an=N+q,un=N-q,Ce=B(un*V),To=B(an*J),Lo=Ce+To,Io=Ce-To;N=B(Lo*Lo),q=B(E*B(Io*Io)),k=B($*Vt),L=B(Ct*($+B(y*Ct)))}let j=tc(n,O);({x_2:k,x_3:N}=l(j,k,N)),{x_2:L,x_3:q}=l(j,L,q);let P=s(L);return B(k*P)}let U={secretKey:d,publicKey:d,seed:d},p=m=>(m=m===void 0?a(d):m,Ut(m,U.seed,"seed"),m),x={randomSecretKey:p};return Object.freeze(U),Object.freeze(x),Object.freeze({keygen:wo(p,M),getSharedSecret:H,getPublicKey:M,scalarMult:K,scalarMultBase:I,utils:x,GuBytes:v.slice(),lengths:U})}var nc=BigInt(0),ka=BigInt(1),rc=BigInt(2),Ua=BigInt(3),Ca=BigInt(5),Ha=BigInt(8),tr=BigInt("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffed"),Na={p:tr,n:BigInt("0x1000000000000000000000000000000014def9dea2f79cd65812631a5cf5d3ed"),h:Ha,a:BigInt("0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffec"),d:BigInt("0x52036cee2b6ffe738cc740797779e89800700a4d4141d8ab75eb4dca135978a3"),Gx:BigInt("0x216936d3cd6e53fec0a4e231fdd6dc5c692cc7609525a7b2c9562d608f25d51a"),Gy:BigInt("0x6666666666666666666666666666666666666666666666666666666666666658")};function ic(e){let t=BigInt(10),n=BigInt(20),r=BigInt(40),o=BigInt(80),s=tr,c=e*e%s*e%s,f=gt(c,rc,s)*c%s,a=gt(f,ka,s)*e%s,u=gt(a,Ca,s)*a%s,l=gt(u,t,s)*u%s,d=gt(l,n,s)*l%s,h=gt(d,r,s)*d%s,y=gt(h,o,s)*h%s,w=gt(y,o,s)*h%s,b=gt(w,t,s)*u%s;return{pow_p_5_8:gt(b,rc,s)*e%s,b2:c}}function Ma(e){return e[0]&=248,e[31]&=127,e[31]|=64,e}var oc=BigInt("19681161376707505956807079304988542015446066515923890162744021073123829784752");function Ka(e,t){let n=tr,r=W(t*t*t,n),o=W(r*r*t,n),s=ic(e*o).pow_p_5_8,i=W(e*r*s,n),c=W(t*i*i,n),f=i,a=W(i*oc,n),u=c===e,l=c===W(-e,n),d=c===W(-e*oc,n);return u&&(i=f),(l||d)&&(i=a),Zi(i,n)&&(i=W(-i,n)),{isValid:u||l,value:i}}var sc=Ji(Na,{uvRatio:Ka});var er=(()=>{let e=tr,t=n=>{let{pow_p_5_8:r,b2:o}=ic(n);return W(gt(r,Ua,e)*o,e)};return ec({P:e,type:"x25519",powPminus2:t,adjustScalarBytes:Ma,scalarMultBase:n=>{let r=W(n,sc.Fn.ORDER);if(r===nc)return nc;let o=sc.BASE.multiply(r);return W((o.Z+o.Y)*t(W(o.Z-o.Y,e)),e)}})})();var cc=!1,fc=!1;function ac(e){fc=e}async function uc(e,t){if(fc)return await t();try{return await e()}catch(n){if(n instanceof ReferenceError||n instanceof DOMException&&n.name==="NotSupportedError")return await t();throw n}}async function Eo(e,t){return await uc(async()=>{let n=mo(e)?e:await qa(e),r=await crypto.subtle.importKey("raw",t,{name:"X25519"},cc,[]);return new Uint8Array(await crypto.subtle.deriveBits({name:"X25519",public:r},n,256))},()=>{if(mo(e))throw new Error("CryptoKey provided but X25519 WebCrypto is not supported");return er.scalarMult(e,t)})}async function nr(e){return await uc(async()=>Eo(e,er.GuBytes),()=>{if(mo(e))throw new Error("CryptoKey provided but X25519 WebCrypto is not supported");return er.scalarMultBase(e)})}var Da=new Uint8Array([48,46,2,1,0,48,5,6,3,43,101,110,4,34,4,32]);async function qa(e){if(e.length!==32)throw new Error("X25519 private key must be 32 bytes");let t=new Uint8Array([...Da,...e]);return crypto.subtle.importKey("pkcs8",t,{name:"X25519"},cc,["deriveBits"])}function mo(e){return typeof CryptoKey<"u"&&e instanceof CryptoKey}var rr=class{constructor(t){g(this,"s");g(this,"transcript",[]);g(this,"buf",new Uint8Array(0));this.s=t.getReader()}async readLine(){let t=[];for(;;){let n=this.buf.indexOf(10);if(n>=0)return t.push(this.buf.subarray(0,n)),this.transcript.push(this.buf.subarray(0,n+1)),this.buf=this.buf.subarray(n+1),Pa(Re(t));this.buf.length>0&&(t.push(this.buf),this.transcript.push(this.buf));let r=await this.s.read();if(r.done)return this.buf=Re(t),null;this.buf=r.value}}close(){return this.s.releaseLock(),{rest:this.buf,transcript:Re(this.transcript)}}};function Pa(e){return e.forEach(t=>{if(t<32||t>126)throw Error("invalid non-ASCII byte in header")}),new TextDecoder().decode(e)}function Re(e){let t=e.reduce(((o,s)=>o+s.length),0),n=new Uint8Array(t),r=0;for(let o of e)n.set(o,r),r+=o.length;return n}function or(e,...t){return e.pipeThrough(new TransformStream({start(n){for(let r of t)n.enqueue(r)}}))}function Bo(e){return new ReadableStream({start(t){t.enqueue(e),t.close()}})}async function lc(e){if(!(e instanceof ReadableStream))throw new Error("readAll expects a ReadableStream");return new Uint8Array(await new Response(e).arrayBuffer())}async function dc(e){if(!(e instanceof ReadableStream))throw new Error("readAllString expects a ReadableStream");return await new Response(e).text()}async function hc(e,t){let n=e.getReader(),r=[],o=0;for(;o")return{next:t};for(let s of n)if(s.length===0)throw Error("invalid stanza");let r=[];for(;;){let s=await e.readLine();if(s===null)throw Error("invalid stanza");let i=Wt.decode(s);if(i.length>48)throw Error("invalid stanza");if(r.push(i),i.length<48)break}let o=Re(r);return{s:new ke(n,o)}}async function pc(e){let t=new rr(e),n=await t.readLine();if(n!=="age-encryption.org/v1")throw Error("invalid version "+(n??"line"));let r=[];for(;;){let{s:o,next:s}=await ja(t);if(o!==void 0){r.push(o);continue}if(!s.startsWith("--- "))throw Error("invalid header");let i=Wt.decode(s.slice(4)),{rest:c,transcript:f}=t.close(),a=f.slice(0,f.length-1-s.length+3);return{stanzas:r,headerNoMAC:a,MAC:i,headerSize:f.length,rest:or(e,c)}}}function So(){return xc()}function xc(){let e=yn(32),t=_t.encodeFromBytes("AGE-SECRET-KEY-",e).toUpperCase();return Promise.resolve(t)}async function cr(e){let t;if(gc(e))t=e;else if(e.startsWith("AGE-SECRET-KEY-PQ-1")){let r=_t.decodeToBytes(e);if(r.prefix.toUpperCase()!=="AGE-SECRET-KEY-PQ-"||r.bytes.length!==32)throw Error("invalid identity");let o=co.getPublicKey(r.bytes);return _t.encode("age1pq",_t.toWords(o),!1)}else{let r=_t.decodeToBytes(e);if(!e.startsWith("AGE-SECRET-KEY-1")||r.prefix.toUpperCase()!=="AGE-SECRET-KEY-"||r.bytes.length!==32)throw Error("invalid identity");t=r.bytes}let n=await nr(t);return _t.encodeFromBytes("age",n)}var sr=class{constructor(t){g(this,"identity");let n=_t.decodeToBytes(t);if(!t.startsWith("AGE-SECRET-KEY-PQ-1")||n.prefix.toUpperCase()!=="AGE-SECRET-KEY-PQ-"||n.bytes.length!==32)throw Error("invalid identity");this.identity=n.bytes}unwrapFileKey(t){for(let n of t){if(n.args.length<1||n.args[0]!=="mlkem768x25519")continue;if(n.args.length!==2)throw Error("invalid mlkem768x25519 stanza");let r=Wt.decode(n.args[1]);if(r.length!==1120||n.body.length!==32)throw Error("invalid mlkem768x25519 stanza");let o=co.decapsulate(r,this.identity),s=new TextEncoder().encode("age-encryption.org/mlkem768x25519"),{key:i,nonce:c}=za(Za,o,s);try{return Ee(i,c).decrypt(n.body)}catch{continue}}return null}},Za=25722;function za(e,t,n){let r=Ga(e),o=Ao(r,void 0,"psk_id_hash",new Uint8Array(0)),s=Ao(r,void 0,"info_hash",n),i=new Uint8Array(1+o.length+s.length);i[0]=0,i.set(o,1),i.set(s,1+o.length);let c=Ao(r,t,"secret",new Uint8Array(0)),f=yc(r,c,"key",i,32),a=yc(r,c,"base_nonce",i,12);return{key:f,nonce:a}}function Ga(e){let t=new Uint8Array(10);return t.set(new TextEncoder().encode("HPKE"),0),t[4]=e>>8&255,t[5]=e&255,t[6]=0,t[7]=1,t[8]=0,t[9]=3,t}function Ao(e,t,n,r){let o=new Uint8Array(7+e.length+n.length+r.length),s=0;return o.set(new TextEncoder().encode("HPKE-v1"),s),s+=7,o.set(e,s),s+=e.length,o.set(new TextEncoder().encode(n),s),s+=n.length,o.set(r,s),Ho(mt,o,t)}function yc(e,t,n,r,o){let s=new Uint8Array(9+e.length+n.length+r.length),i=0;return s[i]=o>>8&255,s[i+1]=o&255,i+=2,s.set(new TextEncoder().encode("HPKE-v1"),i),i+=7,s.set(e,i),i+=e.length,s.set(new TextEncoder().encode(n),i),i+=n.length,s.set(r,i),hr(mt,t,s,o)}var fn=class{constructor(t){g(this,"identity");g(this,"recipient");if(gc(t)){this.identity=t,this.recipient=nr(t);return}let n=_t.decodeToBytes(t);if(!t.startsWith("AGE-SECRET-KEY-1")||n.prefix.toUpperCase()!=="AGE-SECRET-KEY-"||n.bytes.length!==32)throw Error("invalid identity");this.identity=n.bytes,this.recipient=nr(n.bytes)}async unwrapFileKey(t){for(let n of t){if(n.args.length<1||n.args[0]!=="X25519")continue;if(n.args.length!==2)throw Error("invalid X25519 stanza");let r=Wt.decode(n.args[1]);if(r.length!==32)throw Error("invalid X25519 stanza");let o=await Eo(this.identity,r),s=await this.recipient,i=new Uint8Array(r.length+s.length);i.set(r),i.set(s,r.length);let c=new TextEncoder().encode("age-encryption.org/v1/X25519"),f=Me(mt,o,i,c,32),a=_o(n.body,f);if(a!==null)return a}return null}};var ir=class{constructor(t){g(this,"passphrase");this.passphrase=t}unwrapFileKey(t){for(let n of t){if(n.args.length<1||n.args[0]!=="scrypt")continue;if(t.length!==1)throw Error("scrypt recipient is not the only one in the header");if(n.args.length!==3||!/^[1-9][0-9]*$/.test(n.args[2]))throw Error("invalid scrypt stanza");let r=Wt.decode(n.args[1]);if(r.length!==16)throw Error("invalid scrypt stanza");let o=Number(n.args[2]);if(o>20)throw Error("scrypt work factor is too high");let s="age-encryption.org/v1/scrypt",i=new Uint8Array(s.length+16);i.set(new TextEncoder().encode(s)),i.set(r,s.length);let c=Jo(this.passphrase,i,{N:2**o,r:8,p:1,dkLen:32}),f=_o(n.body,c);if(f!==null)return f}return null}};function _o(e,t){if(e.length!==32)throw Error("invalid stanza");let n=new Uint8Array(12);try{return Ee(t,n).decrypt(e)}catch{return null}}function gc(e){return typeof CryptoKey<"u"&&e instanceof CryptoKey}var Ue=16,Ya=64*1024,vo=Ya+Ue;function bc(e){let t=new Uint8Array(12),n=()=>{for(let i=t.length-2;i>=0&&(t[i]++,t[i]===0);i--);},r=!0,o=new Uint8Array(vo),s=0;return new TransformStream({transform(i,c){for(;i.length>0;){if(s===o.length){let a=Ee(e,t).decrypt(o);c.enqueue(a),n(),s=0,r=!1}let f=Math.min(o.length-s,i.length);o.set(i.subarray(0,f),s),s+=f,i=i.subarray(f)}},flush(i){t[11]=1;let c=Ee(e,t).decrypt(o.subarray(0,s));if(!r&&c.length===0)throw new Error("final chunk is empty");i.enqueue(c)}})}function wc(e){if(e0?Ue:0,r}var fr=class{constructor(){g(this,"identities",[])}addPassphrase(t){this.identities.push(new ir(t))}addIdentity(t){if(Qa(t))this.identities.push(new fn(t));else if(typeof t=="string")if(t.startsWith("AGE-SECRET-KEY-1"))this.identities.push(new fn(t));else if(t.startsWith("AGE-SECRET-KEY-PQ-1"))this.identities.push(new sr(t));else throw new Error("unrecognized identity type");else this.identities.push(t)}async decrypt(t,n){let r=t instanceof ReadableStream?t:Bo(t),{fileKey:o,headerSize:s,rest:i}=await this.decryptHeaderInternal(r),{data:c,rest:f}=await hc(i,16),a=new TextEncoder().encode("payload"),u=Me(mt,o,c,a,32),l=bc(u),d=f.pipeThrough(l),h=Object.assign(d,{size:y=>wc(y-s-c.length)});return t instanceof ReadableStream?h:n==="text"?await dc(d):await lc(d)}async decryptHeader(t){return(await this.decryptHeaderInternal(Bo(t))).fileKey}async decryptHeaderInternal(t){let n=await pc(t),r=await this.unwrapFileKey(n.stanzas);if(r===null)throw Error("no identity matched any of the file's recipients");let o=new TextEncoder().encode("header"),s=Me(mt,r,void 0,o,32),i=Nt(mt,s,n.headerNoMAC);if(!Wa(n.MAC,i))throw Error("invalid header HMAC");return{fileKey:r,headerSize:n.headerSize,rest:n.rest}}async unwrapFileKey(t){for(let n of this.identities){let r=await n.unwrapFileKey(t);if(r!==null)return r}return null}};function Wa(e,t){if(e.length!==t.length)return!1;let n=0;for(let r=0;r/dev/null +npm install --no-audit --no-fund --save-exact "age-encryption@$AGE_VERSION" "esbuild@$ESBUILD_VERSION" >/dev/null +npx esbuild entry.mjs --bundle --format=iife --platform=browser --target=es2020 --minify --legal-comments=none \ + --outfile="$root/cmd/teploy-dash/frontend/js/age.vendor.js" +cp node_modules/age-encryption/LICENSE "$root/cmd/teploy-dash/frontend/js/age.vendor.LICENSE" diff --git a/scripts/vendor-age/entry.mjs b/scripts/vendor-age/entry.mjs new file mode 100644 index 0000000..971399e --- /dev/null +++ b/scripts/vendor-age/entry.mjs @@ -0,0 +1,75 @@ +// Bundled into cmd/teploy-dash/frontend/js/age.vendor.js by build.sh. Exposes +// window.TeployAge: the only browser crypto Dash has. The private recovery key is +// created and used here and never leaves the browser. +// +// Dash runs on plain HTTP in many installs, where crypto.subtle is undefined and +// age-encryption would otherwise crash with a TypeError. forceWebCryptoOff makes it +// use its bundled pure-JS X25519. The package's exports expose only dist/index.js, +// hence the relative path through node_modules. +import { Decrypter, generateIdentity, identityToRecipient } from 'age-encryption'; +import { forceWebCryptoOff } from './node_modules/age-encryption/dist/x25519.js'; + +forceWebCryptoOff(true); + +const IDENTITY_RE = /^AGE-SECRET-KEY-1[0-9A-Z]+$/; + +// parseIdentityFile reads a saved key file: comment lines (#) and blank lines are +// skipped, and exactly one identity is expected. Comment lines "# name: value" are +// returned as notes (bucket, endpoint and base live there). +function parseIdentityFile(text) { + const notes = {}; + const identities = []; + for (const raw of String(text).split(/\r?\n/)) { + const line = raw.trim(); + if (line === '') continue; + if (line.startsWith('#')) { + const m = /^#\s*([A-Za-z ]+):\s*(.+)$/.exec(line); + if (m) notes[m[1].trim().toLowerCase()] = m[2].trim(); + continue; + } + if (!IDENTITY_RE.test(line)) throw new Error('This file does not look like a recovery key.'); + identities.push(line); + } + if (identities.length !== 1) { + throw new Error(identities.length === 0 ? 'This file holds no recovery key.' : 'This file holds more than one key.'); + } + return { identity: identities[0], notes }; +} + +async function generate() { + const identity = await generateIdentity(); + return { identity, publicKey: await identityToRecipient(identity) }; +} + +async function derivePublic(identity) { + return identityToRecipient(identity); +} + +// decryptStart decrypts the start of an age file, which may be cut off. age works in +// 64 KiB chunks, each with its own tag, so a truncated prefix yields its whole chunks +// and then an "invalid tag" error: the first chunk is the proof. A wrong key fails +// before any byte comes out. +async function decryptStart(identity, bytes) { + const d = new Decrypter(); + d.addIdentity(identity); + const input = new ReadableStream({ + start(c) { c.enqueue(bytes); c.close(); }, + }); + const reader = (await d.decrypt(input)).getReader(); + let length = 0; + let truncated = false; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + length += value.length; + } + } catch (err) { + if (length === 0) throw err; + truncated = true; + } + if (length === 0) throw new Error('Decryption produced nothing.'); + return { length, truncated }; +} + +globalThis.TeployAge = { generate, derivePublic, decryptStart, parseIdentityFile }; From 628e0b5fb8948450c3eb8e3d7c26e034c10cfa7f Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:43:21 -0700 Subject: [PATCH 03/11] feat(b2): native v4 client and lifecycle rule logic Authorize, buckets, keys and file versions through net/http with an injectable base URL. Replies are validated, API hosts are limited to backblazeb2.com, and errors carry a plain message from the status and code, never the store's message. Rules merge by prefix, keep rules Dash does not own byte for byte, and refuse upload-to-hide over the base. Co-Authored-By: Claude Opus 5.5 --- internal/b2/b2.go | 473 +++++++++++++++++++++++++++++++++++++++ internal/b2/b2_test.go | 229 +++++++++++++++++++ internal/b2/fake_test.go | 97 ++++++++ internal/b2/rules.go | 234 +++++++++++++++++++ 4 files changed, 1033 insertions(+) create mode 100644 internal/b2/b2.go create mode 100644 internal/b2/b2_test.go create mode 100644 internal/b2/fake_test.go create mode 100644 internal/b2/rules.go diff --git a/internal/b2/b2.go b/internal/b2/b2.go new file mode 100644 index 0000000..662d1d9 --- /dev/null +++ b/internal/b2/b2.go @@ -0,0 +1,473 @@ +// Package b2 is a small client for Backblaze's native v4 API: what "Protect my data" +// needs and nothing more. Standard library only. The base URL is injectable so tests +// run against a fake server. +// +// Nothing here logs, and an error never carries the store's message text, which can +// echo what was sent: it holds the status and the code, and says what they mean in +// plain words. +package b2 + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" +) + +// DefaultBaseURL is where every account is authorized. +const DefaultBaseURL = "https://api.backblazeb2.com" + +const maxReply = 8 << 20 + +// Capabilities a setup key needs, and those it can make use of. +const ( + CapListBuckets = "listBuckets" + CapWriteBuckets = "writeBuckets" + CapWriteKeys = "writeKeys" + CapDeleteKeys = "deleteKeys" + CapListFiles = "listFiles" + CapDeleteFiles = "deleteFiles" + CapReadFiles = "readFiles" + CapWriteFiles = "writeFiles" + // CapReadLifecycle lets a server read the bucket's rules. Backblaze's capability + // list does not always name it, so a refusal is handled and reported. + CapReadLifecycle = "readBucketLifecycleRules" +) + +// RequiredSetupCapabilities is what a key must hold for Dash to set up a bucket. +var RequiredSetupCapabilities = []string{CapListBuckets, CapWriteBuckets, CapWriteKeys} + +// Client talks to one base URL. +type Client struct { + BaseURL string + HTTP *http.Client + // AnyHost allows API hosts other than *.backblazeb2.com over any scheme (tests). + AnyHost bool +} + +// APIError is a refusal from Backblaze. +type APIError struct { + Status int + Code string +} + +// Error is the plain wording, safe for a response. +func (e *APIError) Error() string { return plainMessage(e.Status, e.Code) } + +func plainMessage(status int, code string) string { + switch code { + case "transaction_cap_exceeded": + return "Backblaze's daily transaction cap on this account has been reached. Your data is fine. Raise the cap under Caps & Alerts in the Backblaze console, or try again tomorrow." + case "email_not_verified": + return "Backblaze needs this account's email address verified first. Verify it in the Backblaze console, then try again." + case "no_payment_history": + return "Backblaze needs a payment method on this account for this. Add one in the Backblaze console, then try again." + case "bad_auth_token", "expired_auth_token": + return "The Backblaze key was rejected or has expired. Paste a new one." + case "unauthorized": + return "This Backblaze key is not allowed to do that. Check its permissions in the Backblaze console." + case "too_many_buckets": + return "This Backblaze account already has the most buckets it can have. Choose an existing bucket, or delete one in the Backblaze console." + case "bad_bucket_id": + return "Backblaze does not know that bucket any more. Check the bucket and try again." + case "file_lock_conflict": + return "This bucket has Object Lock on, and Backblaze refused the change. Choose another bucket." + case "conflict": + return "Someone else changed this bucket at the same moment. Try again." + case "duplicate_bucket_name": + return "That bucket name is taken." + } + if code != "" { + return fmt.Sprintf("Backblaze answered %d (%s).", status, code) + } + return fmt.Sprintf("Backblaze answered %d.", status) +} + +// IsCode reports whether err is a refusal with this code. +func IsCode(err error, code string) bool { + var e *APIError + return errors.As(err, &e) && e.Code == code +} + +// IsCapped reports whether err is the daily transaction cap: not an outage, and never +// to be shown as one. +func IsCapped(err error) bool { return IsCode(err, "transaction_cap_exceeded") } + +// BucketRef is a bucket a key is limited to. +type BucketRef struct { + ID string `json:"id"` + Name string `json:"name"` +} + +// Session is an authorized key. The token is on it and goes nowhere else. +type Session struct { + client *Client + token string + AccountID string + APIURL string + S3URL string + Region string + Capabilities []string + // Buckets is set when the key is limited to these. + Buckets []BucketRef + NamePrefix string + // ExpiresAt is nil for a key that never expires. + ExpiresAt *time.Time +} + +// Has reports whether the key holds a capability. +func (s *Session) Has(capability string) bool { + for _, c := range s.Capabilities { + if c == capability { + return true + } + } + return false +} + +// Missing lists required capabilities the key lacks. +func (s *Session) Missing() []string { + out := []string{} + for _, c := range RequiredSetupCapabilities { + if !s.Has(c) { + out = append(out, c) + } + } + return out +} + +var s3HostRE = regexp.MustCompile(`^s3\.([a-z0-9][a-z0-9-]{0,24})\.backblazeb2\.com$`) + +func (c *Client) checkAPIURL(raw string) error { + u, err := url.Parse(raw) + if err != nil || u.Host == "" || u.User != nil { + return errors.New("Backblaze gave an API address that is not a URL") + } + if c.AnyHost { + return nil + } + if u.Scheme != "https" || !strings.HasSuffix(u.Hostname(), ".backblazeb2.com") { + return errors.New("Backblaze gave an API address outside backblazeb2.com") + } + return nil +} + +// Authorize exchanges a key for a session. +func (c *Client) Authorize(ctx context.Context, keyID, appKey string) (*Session, error) { + base := c.BaseURL + if base == "" { + base = DefaultBaseURL + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(base, "/")+"/b2api/v4/b2_authorize_account", nil) + if err != nil { + return nil, errors.New("could not build the request to Backblaze") + } + req.SetBasicAuth(keyID, appKey) + var reply struct { + AccountID string `json:"accountId"` + Token string `json:"authorizationToken"` + Expiry *int64 `json:"applicationKeyExpirationTimestamp"` + APIInfo struct { + Storage struct { + APIURL string `json:"apiUrl"` + S3URL string `json:"s3ApiUrl"` + Allowed struct { + Buckets []BucketRef `json:"buckets"` + Capabilities []string `json:"capabilities"` + NamePrefix *string `json:"namePrefix"` + } `json:"allowed"` + } `json:"storageApi"` + } `json:"apiInfo"` + } + if err := c.send(req, &reply); err != nil { + return nil, err + } + st := reply.APIInfo.Storage + if reply.AccountID == "" || reply.Token == "" || st.APIURL == "" || st.S3URL == "" || st.Allowed.Capabilities == nil { + return nil, errors.New("Backblaze's reply to the key was incomplete") + } + if err := c.checkAPIURL(st.APIURL); err != nil { + return nil, err + } + u, err := url.Parse(st.S3URL) + if err != nil { + return nil, errors.New("Backblaze gave an S3 address that is not a URL") + } + m := s3HostRE.FindStringSubmatch(u.Hostname()) + if m == nil || u.Scheme != "https" || u.Port() != "" { + return nil, errors.New("Backblaze gave an S3 address that does not look like one of its own") + } + s := &Session{client: c, token: reply.Token, AccountID: reply.AccountID, + APIURL: strings.TrimRight(st.APIURL, "/"), S3URL: strings.TrimRight(st.S3URL, "/"), Region: m[1], + Capabilities: st.Allowed.Capabilities} + for _, b := range st.Allowed.Buckets { + if b.ID != "" { + s.Buckets = append(s.Buckets, b) + } + } + if st.Allowed.NamePrefix != nil { + s.NamePrefix = *st.Allowed.NamePrefix + } + if reply.Expiry != nil && *reply.Expiry > 0 { + t := time.UnixMilli(*reply.Expiry).UTC() + s.ExpiresAt = &t + } + return s, nil +} + +func (c *Client) httpClient() *http.Client { + base := c.HTTP + if base == nil { + base = &http.Client{Timeout: 45 * time.Second} + } + client := *base + // A redirect would carry the token or the key somewhere else. + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + return &client +} + +func (c *Client) send(req *http.Request, out any) error { + resp, err := c.httpClient().Do(req) + if err != nil { + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return fmt.Errorf("could not reach Backblaze: %v", err) + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxReply)) + if err != nil { + return errors.New("Backblaze's reply did not arrive whole") + } + if resp.StatusCode/100 != 2 { + var e struct { + Code string `json:"code"` + } + _ = json.Unmarshal(data, &e) + return &APIError{Status: resp.StatusCode, Code: e.Code} + } + if out == nil { + return nil + } + if err := json.Unmarshal(data, out); err != nil { + return errors.New("Backblaze's reply could not be read") + } + return nil +} + +func (s *Session) call(ctx context.Context, name string, body, out any) error { + raw, err := json.Marshal(body) + if err != nil { + return errors.New("could not build the request to Backblaze") + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.APIURL+"/b2api/v4/"+name, bytes.NewReader(raw)) + if err != nil { + return errors.New("could not build the request to Backblaze") + } + req.Header.Set("Authorization", s.token) + req.Header.Set("Content-Type", "application/json") + return s.client.send(req, out) +} + +// Bucket is one bucket as listed. +type Bucket struct { + ID string `json:"bucketId"` + Name string `json:"bucketName"` + Type string `json:"bucketType"` + Revision int64 `json:"revision"` + Rules Rules `json:"lifecycleRules"` +} + +// ListBuckets lists what the key can see: a key limited to buckets must name them. +func (s *Session) ListBuckets(ctx context.Context) ([]Bucket, error) { + if len(s.Buckets) == 0 { + return s.listBuckets(ctx, map[string]any{"accountId": s.AccountID, "bucketTypes": []string{"allPrivate", "allPublic"}}) + } + out := []Bucket{} + for _, ref := range s.Buckets { + found, err := s.listBuckets(ctx, map[string]any{"accountId": s.AccountID, "bucketId": ref.ID}) + if err != nil { + return nil, err + } + out = append(out, found...) + } + return out, nil +} + +// FindBucket returns the named bucket, or nil when this key cannot see one. +func (s *Session) FindBucket(ctx context.Context, name string) (*Bucket, error) { + if len(s.Buckets) > 0 { + all, err := s.ListBuckets(ctx) + if err != nil { + return nil, err + } + for i := range all { + if all[i].Name == name { + return &all[i], nil + } + } + return nil, nil + } + found, err := s.listBuckets(ctx, map[string]any{"accountId": s.AccountID, "bucketName": name}) + if err != nil { + return nil, err + } + for i := range found { + if found[i].Name == name { + return &found[i], nil + } + } + return nil, nil +} + +func (s *Session) listBuckets(ctx context.Context, body map[string]any) ([]Bucket, error) { + var reply struct { + Buckets *[]Bucket `json:"buckets"` + } + if err := s.call(ctx, "b2_list_buckets", body, &reply); err != nil { + return nil, err + } + if reply.Buckets == nil { + return nil, errors.New("Backblaze's list of buckets could not be read") + } + for _, b := range *reply.Buckets { + if b.ID == "" || b.Name == "" { + return nil, errors.New("Backblaze's list of buckets could not be read") + } + } + return *reply.Buckets, nil +} + +// CreatePrivateBucket creates a private bucket. +func (s *Session) CreatePrivateBucket(ctx context.Context, name string) (*Bucket, error) { + var b Bucket + if err := s.call(ctx, "b2_create_bucket", map[string]any{"accountId": s.AccountID, "bucketName": name, "bucketType": "allPrivate"}, &b); err != nil { + return nil, err + } + if b.ID == "" || b.Name != name { + return nil, errors.New("Backblaze's reply to the new bucket could not be read") + } + return &b, nil +} + +// UpdateRules replaces the bucket's rules if its revision has not moved. +func (s *Session) UpdateRules(ctx context.Context, bucketID string, rules Rules, revision int64) error { + if rules == nil { + rules = Rules{} + } + var b Bucket + err := s.call(ctx, "b2_update_bucket", map[string]any{"accountId": s.AccountID, "bucketId": bucketID, "lifecycleRules": rules, "ifRevisionIs": revision}, &b) + if err != nil { + return err + } + if b.ID != bucketID { + return errors.New("Backblaze's reply to the bucket change could not be read") + } + return nil +} + +// Key is a new application key. Secret exists only here: Backblaze shows it once, and +// the caller hands it to a server and drops it. +type Key struct { + ID string + Name string + Secret string + Capabilities []string +} + +// CreateKey makes a key limited to a bucket and a name prefix. +func (s *Session) CreateKey(ctx context.Context, name string, capabilities []string, bucketID, prefix string) (*Key, error) { + var reply struct { + ID string `json:"applicationKeyId"` + Secret string `json:"applicationKey"` + Capabilities []string `json:"capabilities"` + KeyName string `json:"keyName"` + } + err := s.call(ctx, "b2_create_key", map[string]any{"accountId": s.AccountID, "capabilities": capabilities, "keyName": name, + "bucketIds": []string{bucketID}, "namePrefix": prefix}, &reply) + if err != nil { + return nil, err + } + if reply.ID == "" || reply.Secret == "" { + return nil, errors.New("Backblaze's reply to the new key could not be read") + } + return &Key{ID: reply.ID, Name: name, Secret: reply.Secret, Capabilities: reply.Capabilities}, nil +} + +// DeleteKey deletes a key by its id. +func (s *Session) DeleteKey(ctx context.Context, keyID string) error { + return s.call(ctx, "b2_delete_key", map[string]any{"applicationKeyId": keyID}, nil) +} + +// ListFolders names what sits directly under prefix, up to max entries. +func (s *Session) ListFolders(ctx context.Context, bucketID, prefix string, max int) ([]string, error) { + var reply struct { + Files []struct { + Name string `json:"fileName"` + } `json:"files"` + } + err := s.call(ctx, "b2_list_file_names", map[string]any{"bucketId": bucketID, "prefix": prefix, "delimiter": "/", "maxFileCount": max}, &reply) + if err != nil { + return nil, err + } + out := make([]string, 0, len(reply.Files)) + for _, f := range reply.Files { + out = append(out, f.Name) + } + return out, nil +} + +// DeleteAllVersions removes every version under prefix and returns how many went. It +// stops at the first refusal, and what it already removed stays removed. +func (s *Session) DeleteAllVersions(ctx context.Context, bucketID, prefix string) (int, error) { + if prefix == "" { + return 0, errors.New("refusing to delete a whole bucket") + } + removed := 0 + startName, startID := "", "" + for { + body := map[string]any{"bucketId": bucketID, "prefix": prefix, "maxFileCount": 1000} + if startName != "" { + body["startFileName"] = startName + } + if startID != "" { + body["startFileId"] = startID + } + var reply struct { + Files []struct { + Name string `json:"fileName"` + ID string `json:"fileId"` + } `json:"files"` + NextName *string `json:"nextFileName"` + NextID *string `json:"nextFileId"` + } + if err := s.call(ctx, "b2_list_file_versions", body, &reply); err != nil { + return removed, err + } + for _, f := range reply.Files { + if !strings.HasPrefix(f.Name, prefix) { + return removed, errors.New("Backblaze listed a file outside the folder being removed, so removal stopped") + } + if err := s.call(ctx, "b2_delete_file_version", map[string]any{"fileName": f.Name, "fileId": f.ID}, nil); err != nil { + return removed, err + } + removed++ + } + if reply.NextName == nil || *reply.NextName == "" { + return removed, nil + } + startName = *reply.NextName + startID = "" + if reply.NextID != nil { + startID = *reply.NextID + } + } +} diff --git a/internal/b2/b2_test.go b/internal/b2/b2_test.go new file mode 100644 index 0000000..75aef83 --- /dev/null +++ b/internal/b2/b2_test.go @@ -0,0 +1,229 @@ +package b2 + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +func auth(t *testing.T, c *Client) *Session { + t.Helper() + s, err := c.Authorize(context.Background(), "KEYID", "APPKEY") + if err != nil { + t.Fatal(err) + } + return s +} + +func TestAuthorize_ReadsCapabilitiesEndpointAndExpiry(t *testing.T) { + f, c := newFake(t) + at := int64(1_900_000_000_000) + f.expiry = &at + s := auth(t, c) + if s.Region != "us-west-004" || s.S3URL != "https://s3.us-west-004.backblazeb2.com" || s.AccountID != "acct" { + t.Fatalf("session = %+v", s) + } + if s.ExpiresAt == nil || s.ExpiresAt.UnixMilli() != at { + t.Fatalf("expiry = %v", s.ExpiresAt) + } + if m := s.Missing(); len(m) != 0 { + t.Fatalf("missing = %v", m) + } + f.caps = []string{CapListBuckets} + f.expiry = nil + s = auth(t, c) + if got := strings.Join(s.Missing(), ","); got != "writeBuckets,writeKeys" { + t.Fatalf("missing = %s", got) + } + if s.ExpiresAt != nil { + t.Fatal("no timestamp means it never expires") + } +} + +func TestAuthorize_ValidatesTheReply(t *testing.T) { + f, c := newFake(t) + for _, bad := range []string{"http://s3.us-west-004.backblazeb2.com", "https://s3.example.com", "https://evil.backblazeb2.com.example.net", "not a url"} { + f.s3 = bad + if _, err := c.Authorize(context.Background(), "KEYID", "APPKEY"); err == nil { + t.Errorf("S3 address %q must be refused", bad) + } + } + // The API address is where the token goes next. + f.s3 = "https://s3.us-west-004.backblazeb2.com" + strict := &Client{BaseURL: c.BaseURL, HTTP: c.HTTP} + if _, err := strict.Authorize(context.Background(), "KEYID", "APPKEY"); err == nil || !strings.Contains(err.Error(), "outside backblazeb2.com") { + t.Fatalf("a foreign API address must be refused: %v", err) + } +} + +func TestErrors_PlainAndNeverEchoing(t *testing.T) { + f, c := newFake(t) + f.echo = "APPKEY" + cases := map[string]string{ + "transaction_cap_exceeded": "daily transaction cap", + "email_not_verified": "email address verified", + "no_payment_history": "payment method", + "bad_auth_token": "rejected or has expired", + "unauthorized": "not allowed", + "something_new": "something_new", + } + for code, want := range cases { + f.fail["b2_authorize_account"] = [2]any{403, code} + _, err := c.Authorize(context.Background(), "KEYID", "APPKEY") + if err == nil || !strings.Contains(err.Error(), want) { + t.Errorf("%s: %v", code, err) + } + if strings.Contains(err.Error(), "APPKEY") { + t.Errorf("%s: the message text leaked: %v", code, err) + } + } + f.fail["b2_authorize_account"] = [2]any{403, "transaction_cap_exceeded"} + _, err := c.Authorize(context.Background(), "KEYID", "APPKEY") + if !IsCapped(err) { + t.Fatal("IsCapped") + } +} + +func TestCreateKey_LimitsToBucketAndPrefix(t *testing.T) { + f, c := newFake(t) + s := auth(t, c) + k, err := s.CreateKey(context.Background(), "n", []string{CapListFiles}, "bid", "teploy/prod/") + if err != nil || k.ID != "K1" || k.Secret != "SECRET" { + t.Fatalf("%+v %v", k, err) + } + body := f.bodies["b2_create_key"][0] + ids, _ := body["bucketIds"].([]any) + if len(ids) != 1 || ids[0] != "bid" || body["namePrefix"] != "teploy/prod/" || body["accountId"] != "acct" { + t.Fatalf("body = %+v", body) + } +} + +func TestUpdateRules_SendsRevisionAndKeepsUnknownFields(t *testing.T) { + f, c := newFake(t) + s := auth(t, c) + var rules Rules + if err := json.Unmarshal([]byte(`[{"fileNamePrefix":"x/","daysFromHidingToDeleting":3,"daysFromUploadingToHiding":null,"novel":"kept"}]`), &rules); err != nil { + t.Fatal(err) + } + if err := s.UpdateRules(context.Background(), "bid", rules, 7); err != nil { + t.Fatal(err) + } + body := f.bodies["b2_update_bucket"][0] + if body["ifRevisionIs"] != float64(7) { + t.Fatalf("revision = %v", body["ifRevisionIs"]) + } + sent := body["lifecycleRules"].([]any)[0].(map[string]any) + if sent["novel"] != "kept" { + t.Fatalf("an untouched rule must go back as it came: %+v", sent) + } +} + +func TestDeleteAllVersions_PagesAndRefusesAWholeBucket(t *testing.T) { + f, c := newFake(t) + s := auth(t, c) + if _, err := s.DeleteAllVersions(context.Background(), "bid", ""); err == nil { + t.Fatal("an empty prefix is a whole bucket") + } + f.files = []map[string]string{{"fileName": "teploy/s/1", "fileId": "1"}} + n, err := s.DeleteAllVersions(context.Background(), "bid", "teploy/s/") + if err != nil || n != 2 || strings.Join(f.deleted, ",") != "1,2" { + t.Fatalf("n=%d err=%v deleted=%v", n, err, f.deleted) + } + f.files = []map[string]string{{"fileName": "elsewhere/1", "fileId": "9"}} + if _, err := s.DeleteAllVersions(context.Background(), "bid", "teploy/s/"); err == nil { + t.Fatal("a file outside the folder must stop the removal") + } +} + +func rule(t *testing.T, raw string) Rule { + t.Helper() + var r Rule + if err := json.Unmarshal([]byte(raw), &r); err != nil { + t.Fatal(err) + } + return r +} + +func TestAnalyze(t *testing.T) { + cases := []struct { + name string + rule string + refused, warned bool + s3 bool + }{ + {"upload-to-hide on the base", `{"fileNamePrefix":"teploy/","daysFromUploadingToHiding":30}`, true, false, false}, + {"upload-to-hide on the whole bucket", `{"fileNamePrefix":"","daysFromUploadingToHiding":1}`, true, false, false}, + {"upload-to-hide inside the base", `{"fileNamePrefix":"teploy/prod/","daysFromUploadingToHiding":7}`, true, false, false}, + {"incoming is allowed", `{"fileNamePrefix":"teploy/prod/.incoming/","daysFromUploadingToHiding":1,"daysFromHidingToDeleting":1}`, false, false, false}, + {"elsewhere is not ours", `{"fileNamePrefix":"photos/","daysFromUploadingToHiding":1}`, false, false, false}, + {"shortens the window", `{"fileNamePrefix":"teploy/","daysFromHidingToDeleting":3}`, false, true, false}, + {"keep only last version", `{"fileNamePrefix":"","daysFromHidingToDeleting":1,"daysFromUploadingToHiding":null}`, false, true, false}, + {"longer is fine", `{"fileNamePrefix":"teploy/","daysFromHidingToDeleting":30}`, false, false, false}, + {"unknown field reads as S3-managed", `{"fileNamePrefix":"x/","id":"abc"}`, false, false, true}, + } + for _, c := range cases { + a := Analyze(Rules{rule(t, c.rule)}, "teploy", 14) + if (len(a.Refusals) > 0) != c.refused || (len(a.Warnings) > 0) != c.warned || a.S3Managed != c.s3 { + t.Errorf("%s: %+v", c.name, a) + } + } + last := Analyze(Rules{rule(t, `{"fileNamePrefix":"","daysFromHidingToDeleting":1}`)}, "teploy", 14) + if !strings.Contains(last.Warnings[0], "keeps only the last version") { + t.Errorf("wording: %v", last.Warnings) + } +} + +func TestMerge_MatchesByPrefixAndKeepsOthers(t *testing.T) { + other := rule(t, `{"fileNamePrefix":"photos/","daysFromHidingToDeleting":5,"novel":1}`) + merged, changed, err := Merge(Rules{other}, "teploy", []string{"teploy/b", "teploy/a"}, 14, false) + if err != nil || !changed || len(merged) != 4 { + t.Fatalf("%+v %v %v", merged, changed, err) + } + raw, _ := json.Marshal(merged[0]) + if !strings.Contains(string(raw), `"novel"`) { + t.Fatalf("a rule Dash does not own must survive: %s", raw) + } + base := merged[1] + if base.FileNamePrefix != "teploy/" || *base.HideToDelete != 14 || *base.CancelUnfinishedLarge != 7 || base.UploadToHide != nil { + t.Fatalf("base = %+v", base) + } + raw, _ = json.Marshal(base) + if !strings.Contains(string(raw), `"daysFromUploadingToHiding":null`) { + t.Fatalf("base must not hide on upload: %s", raw) + } + in := merged[2] + if in.FileNamePrefix != "teploy/a/.incoming/" || *in.UploadToHide != 1 || *in.HideToDelete != 1 { + t.Fatalf("incoming = %+v", in) + } + // Running it again changes nothing. + if _, changed, err := Merge(merged, "teploy", []string{"teploy/a", "teploy/b"}, 14, false); err != nil || changed { + t.Fatalf("a second merge must be a no-op: %v %v", changed, err) + } + // A window a person set stays, unless they ask for another. + if again, changed, _ := Merge(merged, "teploy", nil, 30, false); changed || *again[1].HideToDelete != 14 { + t.Fatal("an existing window must not be overwritten silently") + } + if again, changed, _ := Merge(merged, "teploy", nil, 30, true); !changed || *again[1].HideToDelete != 30 { + t.Fatal("an explicit window replaces it") + } +} + +func TestMerge_RespectsTheRuleLimit(t *testing.T) { + var many Rules + for i := 0; i < MaxRules; i++ { + many = append(many, Rule{FileNamePrefix: "x" + strings.Repeat("y", i) + "/"}) + } + if _, _, err := Merge(many, "teploy", nil, 14, false); err == nil { + t.Fatal("over the limit must be refused") + } +} + +func TestRules_ReadArrayObjectAndNull(t *testing.T) { + for raw, want := range map[string]int{`[]`: 0, `null`: 0, `{"fileNamePrefix":"a/"}`: 1, `[{"fileNamePrefix":"a/"},{"fileNamePrefix":"b/"}]`: 2} { + var rs Rules + if err := json.Unmarshal([]byte(raw), &rs); err != nil || len(rs) != want { + t.Errorf("%s: %d %v", raw, len(rs), err) + } + } +} diff --git a/internal/b2/fake_test.go b/internal/b2/fake_test.go new file mode 100644 index 0000000..e745d96 --- /dev/null +++ b/internal/b2/fake_test.go @@ -0,0 +1,97 @@ +package b2 + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// fakeB2 is enough of the native API for the client's own tests. +type fakeB2 struct { + mu sync.Mutex + caps []string + expiry *int64 + buckets []map[string]any + fail map[string][2]any // call name -> {status, code} + auth string + s3 string + files []map[string]string + deleted []string + bodies map[string][]map[string]any + echo string +} + +func newFake(t *testing.T) (*fakeB2, *Client) { + t.Helper() + f := &fakeB2{caps: []string{CapListBuckets, CapWriteBuckets, CapWriteKeys}, fail: map[string][2]any{}, + s3: "https://s3.us-west-004.backblazeb2.com", bodies: map[string][]map[string]any{}} + srv := httptest.NewServer(http.HandlerFunc(f.serve(t))) + t.Cleanup(srv.Close) + f.auth = srv.URL + return f, &Client{BaseURL: srv.URL, HTTP: srv.Client(), AnyHost: true} +} + +func (f *fakeB2) serve(t *testing.T) func(http.ResponseWriter, *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + if !strings.Contains(r.URL.Path, "/b2api/v4/") { + t.Errorf("not the v4 API: %s", r.URL.Path) + } + writeJSON := func(status int, v any) { + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) + } + if failure, ok := f.fail[name]; ok { + writeJSON(failure[0].(int), map[string]any{"status": failure[0], "code": failure[1], "message": "the key " + f.echo + " was refused"}) + return + } + var body map[string]any + if r.Method == http.MethodPost { + json.NewDecoder(r.Body).Decode(&body) + f.bodies[name] = append(f.bodies[name], body) + if r.Header.Get("Authorization") != "TOKEN" { + t.Errorf("%s without the token", name) + } + } + switch name { + case "b2_authorize_account": + got := r.Header.Get("Authorization") + if got != "Basic "+base64.StdEncoding.EncodeToString([]byte("KEYID:APPKEY")) { + t.Errorf("authorization = %q", got) + } + writeJSON(200, map[string]any{"accountId": "acct", "authorizationToken": "TOKEN", "applicationKeyExpirationTimestamp": f.expiry, + "apiInfo": map[string]any{"storageApi": map[string]any{"apiUrl": f.auth, "s3ApiUrl": f.s3, + "allowed": map[string]any{"capabilities": f.caps, "buckets": nil, "namePrefix": nil}}}}) + case "b2_list_buckets": + writeJSON(200, map[string]any{"buckets": f.buckets}) + case "b2_create_bucket": + writeJSON(200, map[string]any{"bucketId": "new-id", "bucketName": body["bucketName"], "bucketType": body["bucketType"], "revision": 1}) + case "b2_update_bucket": + writeJSON(200, map[string]any{"bucketId": body["bucketId"], "bucketName": "b", "bucketType": "allPrivate", "revision": 2}) + case "b2_create_key": + writeJSON(200, map[string]any{"applicationKeyId": "K1", "applicationKey": "SECRET", "capabilities": body["capabilities"], "keyName": body["keyName"]}) + case "b2_delete_key": + writeJSON(200, map[string]any{"applicationKeyId": body["applicationKeyId"]}) + case "b2_list_file_names": + writeJSON(200, map[string]any{"files": []map[string]any{{"fileName": "teploy/"}}}) + case "b2_list_file_versions": + if _, more := body["startFileName"]; more { + writeJSON(200, map[string]any{"files": []map[string]string{{"fileName": "teploy/s/2", "fileId": "2"}}, "nextFileName": nil, "nextFileId": nil}) + } else { + writeJSON(200, map[string]any{"files": f.files, "nextFileName": "teploy/s/2", "nextFileId": "2"}) + } + case "b2_delete_file_version": + f.deleted = append(f.deleted, body["fileId"].(string)) + writeJSON(200, map[string]any{}) + default: + t.Errorf("unexpected call %s", name) + w.WriteHeader(500) + } + } +} diff --git a/internal/b2/rules.go b/internal/b2/rules.go new file mode 100644 index 0000000..9968dd6 --- /dev/null +++ b/internal/b2/rules.go @@ -0,0 +1,234 @@ +package b2 + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" +) + +// MaxRules is Backblaze's limit per bucket. +const MaxRules = 100 + +// DefaultHideDays is how long a hidden backup stays recoverable. +const DefaultHideDays = 14 + +const incomingSuffix = "/.incoming/" + +// Rule is one lifecycle rule. Rules Dash does not own go back byte for byte, so a field +// this code has never heard of survives a merge. +type Rule struct { + FileNamePrefix string `json:"fileNamePrefix"` + HideToDelete *int `json:"daysFromHidingToDeleting"` + UploadToHide *int `json:"daysFromUploadingToHiding"` + CancelUnfinishedLarge *int `json:"daysFromStartingToCancelingUnfinishedLargeFiles,omitempty"` + // Foreign is true when the rule carries a field the native API does not document: + // the sign of a rule managed some other way. + Foreign bool `json:"-"` + raw json.RawMessage +} + +// UnmarshalJSON keeps the original bytes and notes any undocumented field. +func (r *Rule) UnmarshalJSON(data []byte) error { + type plain Rule + var p plain + if err := json.Unmarshal(data, &p); err != nil { + return err + } + *r = Rule(p) + r.raw = append(json.RawMessage(nil), data...) + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + for name := range fields { + switch name { + case "fileNamePrefix", "daysFromHidingToDeleting", "daysFromUploadingToHiding", "daysFromStartingToCancelingUnfinishedLargeFiles": + default: + r.Foreign = true + } + } + return nil +} + +// MarshalJSON sends an untouched rule back as it came. +func (r Rule) MarshalJSON() ([]byte, error) { + if len(r.raw) > 0 { + return r.raw, nil + } + type plain Rule + return json.Marshal(plain(r)) +} + +// Rules is a bucket's list. Backblaze's schema calls the field an object and its +// examples use an array, so both read. +type Rules []Rule + +// UnmarshalJSON accepts an array, null or a single rule. +func (rs *Rules) UnmarshalJSON(data []byte) error { + trimmed := bytes.TrimSpace(data) + if len(trimmed) > 0 && trimmed[0] == '{' { + var one Rule + if err := json.Unmarshal(trimmed, &one); err != nil { + return err + } + *rs = Rules{one} + return nil + } + var many []Rule + if err := json.Unmarshal(trimmed, &many); err != nil { + return err + } + *rs = many + return nil +} + +func intp(v int) *int { return &v } + +func set(v *int) bool { return v != nil && *v > 0 } + +// BasePrefix turns a base into the prefix rules use: "teploy/", or "" for no base. +func BasePrefix(base string) string { + if base == "" { + return "" + } + return base + "/" +} + +// IncomingPrefix is a server's pending-upload tree, from its full prefix. +func IncomingPrefix(serverPrefix string) string { return serverPrefix + incomingSuffix } + +func overlaps(a, b string) bool { return strings.HasPrefix(a, b) || strings.HasPrefix(b, a) } + +func isIncoming(prefix string) bool { + return strings.HasSuffix(prefix, incomingSuffix) || prefix == strings.TrimPrefix(incomingSuffix, "/") +} + +// Analysis is what the rules mean for backups under a base. +type Analysis struct { + // Refusals: a rule would hide live backups, so setup must stop. + Refusals []string `json:"refusals"` + // Warnings: a person should read these and decide. + Warnings []string `json:"warnings"` + // S3Managed: a rule looks like it is managed through the S3 API, so Dash must not + // send lifecycleRules. Backblaze gives no sure sign, so this is a best effort. + S3Managed bool `json:"s3_managed"` +} + +// Analyze checks a bucket's rules against a base and the recovery window wanted. +func Analyze(rules Rules, base string, windowDays int) Analysis { + out := Analysis{Refusals: []string{}, Warnings: []string{}} + basePrefix := BasePrefix(base) + for _, r := range rules { + if r.Foreign { + out.S3Managed = true + } + if !overlaps(r.FileNamePrefix, basePrefix) || isIncoming(r.FileNamePrefix) { + continue + } + name := r.FileNamePrefix + if name == "" { + name = "the whole bucket" + } + if set(r.UploadToHide) { + out.Refusals = append(out.Refusals, fmt.Sprintf("A lifecycle rule on %s hides files %d days after upload, which would make live backups disappear. Remove that rule or choose another bucket.", name, *r.UploadToHide)) + continue + } + if set(r.HideToDelete) && *r.HideToDelete < windowDays { + if *r.HideToDelete == 1 && r.FileNamePrefix == "" { + out.Warnings = append(out.Warnings, fmt.Sprintf("This bucket keeps only the last version of each file, so a backup hidden by retention or by a compromised server is gone after a day instead of %d.", windowDays)) + } else { + out.Warnings = append(out.Warnings, fmt.Sprintf("A lifecycle rule on %s deletes hidden files after %d days, which shortens the recovery window from %d days.", name, *r.HideToDelete, windowDays)) + } + } + } + return out +} + +// Merge returns the bucket's rules with the base rule and each server's .incoming/ rule +// in place, matched by prefix since rule ids change. Rules it does not own are kept +// untouched. An existing base rule's window stays unless windowExplicit says a person +// chose one. changed is false when nothing needs sending. +func Merge(existing Rules, base string, serverPrefixes []string, windowDays int, windowExplicit bool) (Rules, bool, error) { + out := make(Rules, len(existing)) + copy(out, existing) + changed := false + find := func(prefix string) int { + for i := range out { + if out[i].FileNamePrefix == prefix { + return i + } + } + return -1 + } + basePrefix := BasePrefix(base) + if i := find(basePrefix); i < 0 { + out = append(out, Rule{FileNamePrefix: basePrefix, HideToDelete: intp(windowDays), CancelUnfinishedLarge: intp(7)}) + changed = true + } else { + r := out[i] + update := false + if !set(r.HideToDelete) || (windowExplicit && *r.HideToDelete != windowDays) { + r.HideToDelete = intp(windowDays) + update = true + } + if !set(r.CancelUnfinishedLarge) { + r.CancelUnfinishedLarge = intp(7) + update = true + } + if update { + r.raw = nil + out[i] = r + changed = true + } + } + sorted := append([]string(nil), serverPrefixes...) + sort.Strings(sorted) + for _, sp := range sorted { + prefix := IncomingPrefix(sp) + if i := find(prefix); i < 0 { + out = append(out, Rule{FileNamePrefix: prefix, HideToDelete: intp(1), UploadToHide: intp(1)}) + changed = true + } else if !set(out[i].UploadToHide) || !set(out[i].HideToDelete) || *out[i].UploadToHide != 1 || *out[i].HideToDelete != 1 { + r := out[i] + r.UploadToHide, r.HideToDelete, r.raw = intp(1), intp(1), nil + out[i] = r + changed = true + } + } + if changed && len(out) > MaxRules { + return nil, false, fmt.Errorf("this bucket would need %d lifecycle rules and Backblaze allows %d. Choose another bucket", len(out), MaxRules) + } + return out, changed, nil +} + +// WithoutPrefix drops the rule with this exact prefix. +func WithoutPrefix(rules Rules, prefix string) (Rules, bool) { + out := make(Rules, 0, len(rules)) + removed := false + for _, r := range rules { + if r.FileNamePrefix == prefix { + removed = true + continue + } + out = append(out, r) + } + return out, removed +} + +// RulesText is the rules in words, for a bucket whose rules a person keeps themselves. +func RulesText(base string, serverPrefixes []string, windowDays int) []string { + basePrefix := BasePrefix(base) + name := basePrefix + if name == "" { + name = "the whole bucket" + } + out := []string{fmt.Sprintf("On %s: delete hidden files after %d days, cancel unfinished large files after 7 days, and do not hide on upload.", name, windowDays)} + sorted := append([]string(nil), serverPrefixes...) + sort.Strings(sorted) + for _, sp := range sorted { + out = append(out, fmt.Sprintf("On %s: hide files 1 day after upload and delete hidden files 1 day later.", IncomingPrefix(sp))) + } + return out +} From 0354c86d91a498e40bd84032f7af68a89cbce320 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:43:26 -0700 Subject: [PATCH 04/11] feat(backups): Protect my data setup flow on the server Preflight with consented tool and binary changes, Backblaze bucket and rules, one limited key per server, public recovery key, coverage, turn on through the policy with allow_empty for empty servers, detached runs, proof of recovery through a ranged read, rebuilt and retired servers. Gated on backup-setup-v1, every route under administer.credentials and audited. Records hold key ids, never a secret. Fixes the probe, which read a refused delete on B2 as the recommended setup and would have hidden a retention that could not work: it now checks the object is really hidden, and stops otherwise. Live guidance moves from 30 to 14 days. Protected now has a meaning: a verified backup on each covered target and a passing restore test for each covered database. Dash's own volumes are excluded when there is no recovery key, because they hold its SSH key, and the page reports the real dash/volumes entry. The hand-written-jobs list shows only when such jobs are found. Co-Authored-By: Claude Opus 5.5 --- internal/s3probe/s3probe.go | 6 + internal/s3probe/s3probe_test.go | 31 +- internal/server/backupalerts.go | 15 + internal/server/backupfleet.go | 51 +- internal/server/backuppolicy.go | 112 +- internal/server/backuprunbook_test.go | 65 +- internal/server/backups_test.go | 55 +- internal/server/backupsetup.go | 1982 +++++++++++++++++ internal/server/backupsetup_status.go | 415 ++++ internal/server/backupsetup_test.go | 1701 ++++++++++++++ internal/server/backupstorage.go | 41 +- internal/server/backuptests_test.go | 11 + internal/server/capabilities.go | 4 + internal/server/caps.go | 3 + internal/server/contracts_corpus_test.go | 17 + .../detach-already-running.json | 6 + .../backup-contracts/detach-failed.json | 9 + .../backup-contracts/detach-finished.json | 16 + .../backup-contracts/detach-started.json | 8 + .../preflight-after-actions.json | 61 + .../preflight-needs-setup.json | 41 + .../backup-contracts/preflight-ready.json | 49 + .../backup-contracts/preflight-unknown.json | 44 + .../preflight-unreachable.json | 39 + 24 files changed, 4715 insertions(+), 67 deletions(-) create mode 100644 internal/server/backupsetup.go create mode 100644 internal/server/backupsetup_status.go create mode 100644 internal/server/backupsetup_test.go create mode 100644 internal/server/testdata/backup-contracts/detach-already-running.json create mode 100644 internal/server/testdata/backup-contracts/detach-failed.json create mode 100644 internal/server/testdata/backup-contracts/detach-finished.json create mode 100644 internal/server/testdata/backup-contracts/detach-started.json create mode 100644 internal/server/testdata/backup-contracts/preflight-after-actions.json create mode 100644 internal/server/testdata/backup-contracts/preflight-needs-setup.json create mode 100644 internal/server/testdata/backup-contracts/preflight-ready.json create mode 100644 internal/server/testdata/backup-contracts/preflight-unknown.json create mode 100644 internal/server/testdata/backup-contracts/preflight-unreachable.json diff --git a/internal/s3probe/s3probe.go b/internal/s3probe/s3probe.go index c4e7c4a..756c713 100644 --- a/internal/s3probe/s3probe.go +++ b/internal/s3probe/s3probe.go @@ -59,6 +59,9 @@ type Result struct { Write bool `json:"write"` Read bool `json:"read"` Delete string `json:"delete"` + // Hidden: after an accepted delete, reading the object again found nothing. On a + // store that keeps versions, that is what a real hide looks like. + Hidden bool `json:"hidden"` // Error names the first failed step in words, with the store's code. Error string `json:"error,omitempty"` } @@ -114,6 +117,9 @@ func Run(ctx context.Context, cfg Config) Result { } case status/100 == 2: res.Delete = DeleteRemoved + if status, _, _, err := do(ctx, cfg, http.MethodGet, nil); err == nil && status == http.StatusNotFound { + res.Hidden = true + } case status == http.StatusForbidden: res.Delete = DeleteRefused if res.Error == "" { diff --git a/internal/s3probe/s3probe_test.go b/internal/s3probe/s3probe_test.go index 93b1171..2f4d9a3 100644 --- a/internal/s3probe/s3probe_test.go +++ b/internal/s3probe/s3probe_test.go @@ -38,8 +38,10 @@ type fakeStore struct { readStatus int readCorrupt bool deleteStatus int - methods []string - auth []string + // keepAfterDelete: an accepted delete leaves the object readable. + keepAfterDelete bool + methods []string + auth []string } func (f *fakeStore) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -64,7 +66,11 @@ func (f *fakeStore) ServeHTTP(w http.ResponseWriter, r *http.Request) { fail(f.readStatus) return } - body := f.objects[r.URL.Path] + body, ok := f.objects[r.URL.Path] + if !ok && !f.keepAfterDelete { + w.WriteHeader(http.StatusNotFound) + return + } if f.readCorrupt { body = []byte("different") } @@ -74,7 +80,9 @@ func (f *fakeStore) ServeHTTP(w http.ResponseWriter, r *http.Request) { fail(f.deleteStatus) return } - delete(f.objects, r.URL.Path) + if !f.keepAfterDelete { + delete(f.objects, r.URL.Path) + } w.WriteHeader(http.StatusNoContent) } } @@ -93,9 +101,12 @@ func TestRunWritesReadsDeletes(t *testing.T) { if !res.Write || !res.Read || res.Delete != DeleteRemoved || res.Error != "" { t.Fatalf("res = %+v", res) } - if strings.Join(f.methods, ",") != "PUT,GET,DELETE" { + if strings.Join(f.methods, ",") != "PUT,GET,DELETE,GET" { t.Fatalf("methods = %v", f.methods) } + if !res.Hidden { + t.Fatalf("the object is gone, so it reads as hidden: %+v", res) + } if len(f.objects) != 0 { t.Fatalf("object left behind: %v", f.objects) } @@ -108,7 +119,15 @@ func TestRunWritesReadsDeletes(t *testing.T) { func TestRunReportsARefusedDelete(t *testing.T) { res := runAgainst(t, &fakeStore{deleteStatus: http.StatusForbidden}) - if !res.Write || !res.Read || res.Delete != DeleteRefused { + if !res.Write || !res.Read || res.Delete != DeleteRefused || res.Hidden { + t.Fatalf("res = %+v", res) + } +} + +// An accepted delete that leaves the object readable did not hide anything. +func TestRunSeesAnObjectStillReadable(t *testing.T) { + res := runAgainst(t, &fakeStore{keepAfterDelete: true}) + if res.Delete != DeleteRemoved || res.Hidden { t.Fatalf("res = %+v", res) } } diff --git a/internal/server/backupalerts.go b/internal/server/backupalerts.go index af0c950..d4f983f 100644 --- a/internal/server/backupalerts.go +++ b/internal/server/backupalerts.go @@ -67,6 +67,9 @@ func backupObservations(rec *backupServerRecord, now time.Time) (obs []backupObs switch { case !st.Target.HasCredentials: add(backupObservation{rule, false, "the target has no stored credentials — connect storage again"}) + case !st.Target.Reachable && backupTargetCapped(st.Target.Error): + // Not an outage, and the server is fine: say what it is. + add(backupObservation{rule, false, backupCappedMessage}) case !st.Target.Reachable: msg := "the server cannot reach its backup target" if st.Target.Error != "" { @@ -78,6 +81,18 @@ func backupObservations(rec *backupServerRecord, now time.Time) (obs []backupObs } } + // The bucket's rule, rechecked on every read through the server's own key. A key that + // cannot read rules says unknown, which proves nothing either way and raises nothing. + if st.Target.Configured { + rule := backupRule(rec.Server, "lifecycle", "", "backup lifecycle rule on "+rec.Server, alert.SeverityWarning) + switch st.Target.IncomingLifecycle { + case "covered": + add(backupObservation{rule, true, "the lifecycle rule is in place"}) + case "absent": + add(backupObservation{rule, false, "the bucket has no lifecycle rule for this server's pending uploads, so what retention removes stays billed: " + backupLifecycleRule(nil, st.Target)}) + } + } + if problem := st.AWSCLIProblem; problem != "" { rule := backupRule(rec.Server, "tools", "", "backup tooling on "+rec.Server, alert.SeverityWarning) add(backupObservation{rule, false, problem}) diff --git a/internal/server/backupfleet.go b/internal/server/backupfleet.go index b1b55c0..b94da82 100644 --- a/internal/server/backupfleet.go +++ b/internal/server/backupfleet.go @@ -24,6 +24,7 @@ import ( "sync" "time" + "github.com/useteploy/teploy-dash/internal/b2" "github.com/useteploy/teploy-dash/internal/cli" "github.com/useteploy/teploy-dash/internal/durable" "github.com/useteploy/teploy-dash/internal/remote" @@ -177,6 +178,9 @@ type backupServerRecord struct { // HeartbeatMonitorID is the push monitor Dash owns for this server. HeartbeatMonitorID string `json:"heartbeat_monitor_id,omitempty"` HeartbeatNote string `json:"heartbeat_note,omitempty"` + // SelfExcluded is what Dash added to this server's excludes because the policy had + // no recovery key, so a key arriving later can take it back out. + SelfExcluded []string `json:"self_excluded,omitempty"` // Migration is the per-entry evidence for retiring hand-written backup // jobs (plan P2.9): distinct good policy backups seen and their keys. Migration map[string]*backupMigrateEntry `json:"migration,omitempty"` @@ -197,6 +201,7 @@ type backupRecord struct { Storage *backupStorage `json:"storage,omitempty"` Policy *backupPolicySpec `json:"policy,omitempty"` Recovery *backupRecovery `json:"recovery,omitempty"` + Setup *backupSetupRecord `json:"setup,omitempty"` Window backupWindow `json:"window"` ChecklistDoneAt *time.Time `json:"checklist_done_at,omitempty"` Servers map[string]*backupServerRecord `json:"servers"` @@ -224,6 +229,13 @@ type backupState struct { probe func(context.Context, s3probe.Config) s3probe.Result // resources is the pre-launch free RAM/disk read (a seam for tests). resources func(context.Context, remote.ServerConn) (backupResources, error) + // setupMu keeps the steps of "Protect my data" that change servers single-flight. + setupMu sync.Mutex + // b2Client makes the Backblaze client (a seam for tests). + b2Client func() *b2.Client + // dataDir is where this Dash keeps its state; selfApp is its own app name. + dataDir string + selfApp string } // serverName aliases the fleet's server connection type: the backup files @@ -246,6 +258,7 @@ func (s *Server) initBackups(dataDir string) { b.running = map[string]bool{} b.probe = s3probe.Run b.resources = s.readBackupResources + b.dataDir, b.selfApp = dataDir, backupSelfApp() if dataDir == "" { return } @@ -675,6 +688,7 @@ type backupsPageView struct { Rows []backupRowView `json:"rows"` Tests []backupAutoTestView `json:"tests"` ProjectedBytes int64 `json:"projected_bytes"` + Setup *backupSetupView `json:"setup,omitempty"` SelfBackup backupSelfView `json:"self_backup"` Handwritten []backupHandwrittenJob `json:"handwritten,omitempty"` HandwrittenRule string `json:"handwritten_rule,omitempty"` @@ -704,8 +718,8 @@ func (s *Server) handleBackupsRefresh(w http.ResponseWriter, r *http.Request) { writeData(w, map[string]bool{"ok": true}) } -// handleBackupsAction serves POST /api/backups/{refresh,storage,policy,exclude,override} -// and /api/backups/recovery-key/{generate,confirm,drill}. +// handleBackupsAction serves POST /api/backups/{refresh,storage,policy,exclude,override}, +// /api/backups/recovery-key/{generate,confirm,drill} and /api/backups/setup/*. func (s *Server) handleBackupsAction(w http.ResponseWriter, r *http.Request) { rest := strings.Trim(strings.TrimPrefix(r.URL.Path, "/api/backups/"), "/") action, sub, _ := strings.Cut(rest, "/") @@ -720,6 +734,8 @@ func (s *Server) handleBackupsAction(w http.ResponseWriter, r *http.Request) { s.handleBackupExclude(w, r) case "override": s.handleBackupOverride(w, r) + case "setup": + s.handleBackupSetup(w, r, sub) case "recovery-key": switch sub { case "generate": @@ -741,7 +757,8 @@ func (s *Server) requireBackupFleet(w http.ResponseWriter, r *http.Request) bool } func (s *Server) backupsView(ctx context.Context, now time.Time) backupsPageView { - supported := s.cliInstalled() && s.capabilities(ctx).Features.BackupFleet + features := s.capabilities(ctx).Features + supported := s.cliInstalled() && features.BackupFleet channels := s.alertChannelsConfigured() publicURL := s.alertPublicURL() manual := s.manualRestoreTests() @@ -754,7 +771,6 @@ func (s *Server) backupsView(ctx context.Context, now time.Time) backupsPageView Storage: b.rec.storageViewLocked(), Policy: b.rec.Policy, Window: b.rec.Window, Presets: backupPresetNames(), Intervals: backupIntervals, PublicURLSet: publicURL != "", Servers: []backupServerView{}, Rows: []backupRowView{}, Tests: []backupAutoTestView{}, - SelfBackup: backupSelfGap(), Handwritten: backupHandwrittenJobs, HandwrittenRule: HandwrittenRule, } if !supported { view.Note = "Backups needs teploy " + backupFleetMinCLI + " or later on the dashboard host." @@ -780,6 +796,12 @@ func (s *Server) backupsView(ctx context.Context, now time.Time) backupsPageView } view.Tests = b.testViewsLocked(now) view.Recovery = b.rec.recoveryViewLocked(now) + view.SelfBackup = backupSelfStatus(b.selfApp, b.dataDir, view.Rows, b.rec.Recovery != nil) + view.Setup = b.setupViewLocked(s.cliInstalled() && features.BackupSetup, channels, &view) + // The hand-written-jobs list is for those who have some: new users never see it. + if b.backupHandwrittenFound(view.Rows) { + view.Handwritten, view.HandwrittenRule = backupHandwrittenJobs, HandwrittenRule + } view.Checklist = b.checklistLocked(view.Servers, view.Rows, channels) view.ChecklistDone = b.rec.ChecklistDoneAt != nil if !view.ChecklistDone { @@ -997,24 +1019,3 @@ func (b *backupState) checklistLocked(servers []backupServerView, rows []backupR } return []backupChecklistItem{storage, alert, drill, verified} } - -// backupSelfView says where Dash's own backup stands. See docs/rebuild-dash.md. -type backupSelfView struct { - Available bool `json:"available"` - Reason string `json:"reason"` - Runbook string `json:"runbook"` -} - -// backupSelfGap: plan P2.7 needs dash-state, teploy-config, ssh-key and the secrets -// file stored through the policy, encrypted to the owner key ALONE. CLI v0.1.52 -// encrypts every object to the owner key and the server's identity, and backs up -// accessories, app volumes and adopted containers, with no way to name a recipient -// set per target. That is the CLI's to add; Dash does not build a second backup -// path around it (plan principle 1). -func backupSelfGap() backupSelfView { - return backupSelfView{ - Reason: "Dash does not back itself up yet. The teploy CLI cannot yet encrypt a backup to the recovery key alone, " + - "and Dash's SSH key must not be readable by a server's own key. Copy Dash's state by hand until it can.", - Runbook: "docs/rebuild-dash.md", - } -} diff --git a/internal/server/backuppolicy.go b/internal/server/backuppolicy.go index c2cc3c2..a32c8c1 100644 --- a/internal/server/backuppolicy.go +++ b/internal/server/backuppolicy.go @@ -260,6 +260,14 @@ func (s *Server) handleBackupKeyConfirm(w http.ResponseWriter, r *http.Request) ctx := context.WithoutCancel(r.Context()) results := s.forEachBackupServer(ctx, servers, func(ctx context.Context, name string) error { args := []string{"server", "backup-policy", "set", "--host", name, "--recipient", body.PublicKey, "--json"} + // With a key, Dash's own volumes no longer have to stay out. + restored := false + if b.selfExcludedOn(name) { + if current, err := s.readBackupPolicyShow(ctx, name); err == nil { + args = append(args, "--exclude="+strings.Join(b.withoutSelfExcludes(name, current.Excludes), ",")) + restored = true + } + } result, err := s.runCLI(ctx, args...) if err != nil { return err @@ -267,6 +275,9 @@ func (s *Server) handleBackupKeyConfirm(w http.ResponseWriter, r *http.Request) if result.ExitCode != 0 { return commandFailure(args, result) } + if restored { + b.setSelfExcluded(name, nil) + } return nil }) allOK := len(results) > 0 @@ -550,7 +561,7 @@ func (s *Server) handleBackupPolicy(w http.ResponseWriter, r *http.Request) { notes := make(map[string]string, len(servers)) var notesMu sync.Mutex results := s.forEachBackupServer(ctx, servers, func(ctx context.Context, name string) error { - note, err := s.applyBackupPolicy(ctx, name, body.Interval, retention, body.Volumes, body.VolumeMaxGB, recipient) + note, err := s.applyBackupPolicy(ctx, name, backupPolicyApply{Interval: body.Interval, Retention: retention, Volumes: body.Volumes, VolumeMaxGB: body.VolumeMaxGB, Recipient: recipient}) notesMu.Lock() notes[name] = note notesMu.Unlock() @@ -593,9 +604,22 @@ func (s *Server) backupWindowFrom(start, end *int) *backupWindow { return &window } +// backupPolicyApply is one write of the policy. AllowEmpty and Adopt are only for the +// setup flow: the manual form leaves them as they are. +type backupPolicyApply struct { + Interval string + Retention backupRetention + Volumes bool + VolumeMaxGB float64 + Recipient string + AllowEmpty *bool + Adopt []string +} + // applyBackupPolicy writes the policy to one server. The returned note is what the // owner should know about its heartbeat. -func (s *Server) applyBackupPolicy(ctx context.Context, name, interval string, ret backupRetention, volumes bool, volumeMaxGB float64, recipient string) (string, error) { +func (s *Server) applyBackupPolicy(ctx context.Context, name string, p backupPolicyApply) (string, error) { + interval, ret, volumes, volumeMaxGB, recipient := p.Interval, p.Retention, p.Volumes, p.VolumeMaxGB, p.Recipient current, err := s.readBackupPolicyShow(ctx, name) if err != nil { return "", err @@ -608,6 +632,26 @@ func (s *Server) applyBackupPolicy(ctx context.Context, name, interval string, r if recipient != "" { args = append(args, "--recipient", recipient) } + if p.AllowEmpty != nil { + args = append(args, "--allow-empty="+strconv.FormatBool(*p.AllowEmpty)) + } + for _, a := range p.Adopt { + args = append(args, "--adopt", a) + } + // Without any recovery key, a volume backup of Dash would upload its SSH key in + // plaintext, so its own volumes stay out until a key arrives. + var selfExcluded []string + selfChanged := false + if recipient == "" && current.Recipient == "" { + merged, added := s.backups.withSelfExcludes(current.Excludes) + if len(added) > 0 { + args = append(args, "--exclude="+strings.Join(merged, ",")) + selfExcluded, selfChanged = append(s.backups.selfExcludedList(name), added...), true + } + } else if s.backups.selfExcludedOn(name) { + args = append(args, "--exclude="+strings.Join(s.backups.withoutSelfExcludes(name, current.Excludes), ",")) + selfChanged = true + } heartbeat, note := s.backupHeartbeatFor(name, interval, current) var result *cli.Result if heartbeat != "" { @@ -625,17 +669,67 @@ func (s *Server) applyBackupPolicy(ctx context.Context, name, interval string, r return note, err } } + if selfChanged { + s.backups.setSelfExcluded(name, selfExcluded) + } if _, err := cli.ParseJSON(result.Stdout); err != nil { return note, fmt.Errorf("the policy may be stored, but the reply could not be read: %w", err) } return note, nil } +// withSelfExcludes adds Dash's own volumes to an exclude list, returning what it added. +func (b *backupState) withSelfExcludes(excludes []string) (merged, added []string) { + merged = append(merged, excludes...) + for _, self := range backupSelfExcludes(b.selfApp) { + have := false + for _, e := range excludes { + have = have || e == self + } + if !have { + merged = append(merged, self) + added = append(added, self) + } + } + return merged, added +} + +func (b *backupState) selfExcludedList(name string) []string { + b.mu.Lock() + defer b.mu.Unlock() + return append([]string(nil), b.serverLocked(name).SelfExcluded...) +} + +func (b *backupState) selfExcludedOn(name string) bool { return len(b.selfExcludedList(name)) > 0 } + +// withoutSelfExcludes takes back out only what Dash itself added. +func (b *backupState) withoutSelfExcludes(name string, excludes []string) []string { + added := map[string]bool{} + for _, e := range b.selfExcludedList(name) { + added[e] = true + } + out := make([]string, 0, len(excludes)) + for _, e := range excludes { + if !added[e] { + out = append(out, e) + } + } + return out +} + +func (b *backupState) setSelfExcluded(name string, list []string) { + b.mu.Lock() + defer b.mu.Unlock() + b.serverLocked(name).SelfExcluded = list + b.saveLocked() +} + type backupPolicyShow struct { Configured bool `json:"configured"` Excludes []string `json:"excludes"` HeartbeatCount int `json:"heartbeat_count"` HeartbeatHosts []string `json:"heartbeat_hosts"` + Recipient string `json:"recipient"` } func (s *Server) readBackupPolicyShow(ctx context.Context, name string) (*backupPolicyShow, error) { @@ -811,6 +905,20 @@ func (s *Server) handleBackupExclude(w http.ResponseWriter, r *http.Request) { writeError(w, "say why this is excluded: it goes in the audit log") return } + if !body.Excluded { + for _, self := range backupSelfExcludes(s.backups.selfApp) { + if body.Name != self { + continue + } + s.backups.mu.Lock() + hasKey := s.backups.rec.Recovery != nil + s.backups.mu.Unlock() + if !hasKey { + writeError(w, "These volumes hold Dash's SSH key, so they stay out until a recovery key encrypts them.") + return + } + } + } if _, err := s.lookupServerStrict(r.Context(), body.Server); err != nil { writeError(w, sanitizeDomainError(err.Error())) return diff --git a/internal/server/backuprunbook_test.go b/internal/server/backuprunbook_test.go index 39113eb..21c6aa3 100644 --- a/internal/server/backuprunbook_test.go +++ b/internal/server/backuprunbook_test.go @@ -10,9 +10,10 @@ import ( "path/filepath" "strings" "testing" + "time" ) -const runbookPath = "../../docs/rebuild-dash.md" +const runbookPath = "../../docs/recovery.md" // runbookNamedPaths are the file and directory names the runbook's table // names, each paired with the package tree whose non-test source must contain @@ -85,12 +86,17 @@ func TestRebuildRunbook_KeyOnlyRecovery(t *testing.T) { } doc := string(raw) essentials := []string{ - "backup fetch", // the drill command, for real - "--identity", // the recovery key is the way back - "AGE-SECRET-KEY", // what the owner must have kept - "age -r", // the interim job encrypts to the owner key alone - "CLI gap", // why the job is not a policy entry yet - "test-send", // the alert path is checked after a rebuild + "backup fetch", // the proof, for real + "--identity", // the recovery key is the way back + "AGE-SECRET-KEY", // what the owner must have kept + "age -r", // a non-template Dash encrypts to the owner key alone + "test-send", // the alert path is checked after a rebuild + "has not been rehearsed", // says so until someone has + "Read Only", // the read-only console key setup never showed + "Everything is lost", // the four cases the page covers + "server A's data onto server B", + "Dash itself", + "Hidden copies", } for _, want := range essentials { if !strings.Contains(doc, want) { @@ -99,19 +105,46 @@ func TestRebuildRunbook_KeyOnlyRecovery(t *testing.T) { } } -// The page surfaces the gap and points at the runbook instead of pretending. -func TestBackupSelfView_GapAndRunbook(t *testing.T) { - v := backupSelfGap() - if v.Available { - t.Fatal("the self-backup must not claim to exist yet") +// The page reports the real dash/volumes entry and points at the recovery page, and +// the old claim that Dash cannot be backed up is gone. +func TestBackupSelfStatus_ReportsTheRealEntry(t *testing.T) { + good := time.Now() + covered := []backupRowView{{Server: "infra", Name: "dash/volumes", Kind: "volumes", App: "dash", State: "covered", LastGood: &good, Encrypted: true}} + v := backupSelfStatus("dash", "/var/teploy-dash", covered, true) + if !v.Available || v.Entry != "dash/volumes" || !strings.Contains(v.Reason, "recovery key opens them on its own") { + t.Fatalf("covered: %+v", v) } - if !strings.Contains(v.Reason, "cannot yet") || !strings.Contains(v.Reason, "recovery key alone") { - t.Fatalf("reason = %q", v.Reason) + if strings.Contains(v.Reason, "cannot yet") { + t.Fatalf("the old claim survives: %q", v.Reason) } - if v.Runbook != "docs/rebuild-dash.md" { + if v.Runbook != "docs/recovery.md" { t.Fatalf("runbook = %q", v.Runbook) } if _, err := os.Stat(runbookPath); err != nil { - t.Fatalf("the named runbook must exist: %v", err) + t.Fatalf("the named recovery page must exist: %v", err) + } + + plain := []backupRowView{{Server: "infra", Name: "dash/volumes", Kind: "volumes", App: "dash", State: "covered", LastGood: &good}} + if v := backupSelfStatus("dash", "/var/teploy-dash", plain, false); v.Available || !strings.Contains(v.Warning, "SSH key") { + t.Fatalf("unencrypted volumes must warn: %+v", v) + } + excluded := []backupRowView{{Server: "infra", Name: "dash/volumes", Kind: "volumes", App: "dash", State: "excluded"}} + if v := backupSelfStatus("dash", "/var/teploy-dash", excluded, false); v.Available || !strings.Contains(v.Reason, "no recovery key") { + t.Fatalf("excluded for want of a key: %+v", v) + } + if v := backupSelfStatus("", "/srv/dash", nil, false); v.Available || !strings.Contains(v.Reason, "not deployed from the teploy template") || !strings.Contains(strings.Join(v.Uncovered, " "), "/srv/dash") { + t.Fatalf("a non-template Dash reports its paths as uncovered: %+v", v) + } +} + +func TestBackupSelfExcludes(t *testing.T) { + if got := backupSelfExcludes("dash"); len(got) != 1 || got[0] != "dash/volumes" { + t.Fatalf("template name: %v", got) + } + if got := backupSelfExcludes(""); len(got) != 1 || got[0] != "dash/volumes" { + t.Fatalf("an older deploy has no TEPLOY_APP: %v", got) + } + if got := strings.Join(backupSelfExcludes("control"), ","); got != "control/volumes,dash/volumes" { + t.Fatalf("named app: %v", got) } } diff --git a/internal/server/backups_test.go b/internal/server/backups_test.go index 4ad79c6..bdca87c 100644 --- a/internal/server/backups_test.go +++ b/internal/server/backups_test.go @@ -46,6 +46,8 @@ type backupCLI struct { keygenErr error // verify answers `accessory verify-backup ... --json` per accessory name. verify map[string]func(args []string) (*cli.Result, error) + // hook answers a command first when it returns handled (the setup flow's commands). + hook func(args []string, stdin string) (res *cli.Result, err error, handled bool) // every argv seen, with stdin payloads where a command took one. calls []backupCLIcall } @@ -93,6 +95,11 @@ func (c *backupCLI) run(ctx context.Context, args ...string) (*cli.Result, error func (c *backupCLI) runStdin(ctx context.Context, stdin string, args ...string) (*cli.Result, error) { c.ran(args, stdin) + if c.hook != nil { + if res, err, handled := c.hook(args, stdin); handled { + return res, err + } + } switch { case len(args) >= 1 && args[0] == "version": out := c.handshake @@ -451,7 +458,7 @@ func TestBackupStorageConnect_ProbePushForgetVerify(t *testing.T) { if !strings.Contains(cfg.Key, "/.incoming/dash-probe-") { t.Errorf("the probe object must sit under .incoming/: %s", cfg.Key) } - return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved} + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} } rec := postJSON(t, s, "/api/backups/storage", storageBody()) @@ -529,13 +536,13 @@ func TestBackupStorageConnect_ProbeFailurePushesNothing(t *testing.T) { } } -// A B2 key without deleteFiles is the recommended setup (plan D2): the probe -// notes it, connects, and says why the object stays. -func TestBackupStorageConnect_B2DeleteRefusalIsFine(t *testing.T) { +// A B2 key that hides the object is the recommended setup (plan D2): the probe notes +// it, connects, and names the recovery window. +func TestBackupStorageConnect_B2HideIsFine(t *testing.T) { c := &backupCLI{status: statusesFromTestdata(t)} s := newBackupsServer(t, c) s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { - return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRefused} + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} } rec := postJSON(t, s, "/api/backups/storage", storageBody()) if rec.Code != http.StatusOK { @@ -543,16 +550,48 @@ func TestBackupStorageConnect_B2DeleteRefusalIsFine(t *testing.T) { } var resp backupStorageResponse decodeData(t, rec, &resp) - if resp.ProbeNote == "" || !strings.Contains(resp.ProbeNote, "cannot delete files") { + if !strings.Contains(resp.ProbeNote, "hid the test object") || !strings.Contains(resp.ProbeNote, "14 days") { t.Fatalf("probe note = %q", resp.ProbeNote) } } +// A refused hide used to read as the recommended setup, which hid a retention that +// could never work. It now stops, unless the person says the refusal is deliberate. +func TestBackupStorageConnect_B2RefusedHideStops(t *testing.T) { + for name, probe := range map[string]s3probe.Result{ + "refused": {Write: true, Read: true, Delete: s3probe.DeleteRefused}, + "still-visible": {Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: false}, + } { + t.Run(name, func(t *testing.T) { + c := &backupCLI{status: statusesFromTestdata(t)} + s := newBackupsServer(t, c) + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { return probe } + rec := postJSON(t, s, "/api/backups/storage", storageBody()) + if rec.Code != http.StatusUnprocessableEntity { + t.Fatalf("code = %d body = %s", rec.Code, rec.Body.String()) + } + if n := c.countOf("backup-target", "set"); n != 0 { + t.Fatalf("a key that cannot hide must push nothing: %d", n) + } + }) + } + c := &backupCLI{status: statusesFromTestdata(t)} + s := newBackupsServer(t, c) + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRefused} + } + body := storageBody() + body["delete_refusal_ok"] = true + if rec := postJSON(t, s, "/api/backups/storage", body); rec.Code != http.StatusOK { + t.Fatalf("a deliberate refusal is still the person's call: %d %s", rec.Code, rec.Body.String()) + } +} + func TestBackupStorageRotation_OnlyRevokeWhenEveryServerVerified(t *testing.T) { c := &backupCLI{status: statusesFromTestdata(t)} s := newBackupsServer(t, c) s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { - return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved} + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} } if rec := postJSON(t, s, "/api/backups/storage", storageBody()); rec.Code != http.StatusOK { t.Fatalf("connect failed: %s", rec.Body.String()) @@ -945,7 +984,7 @@ func TestB2LifecycleRuleCoversHiddenBackups(t *testing.T) { rule := backupLifecycleRule(&backupStorage{Provider: backupProviderB2}, backupStatTarget{Prefix: "teploy/prod"}) for _, want := range []string{ "fileNamePrefix teploy/prod/.incoming/, daysFromUploadingToHiding 1, daysFromHidingToDeleting 1", - "fileNamePrefix teploy/prod/, daysFromHidingToDeleting 30, daysFromUploadingToHiding empty", + "fileNamePrefix teploy/prod/, daysFromHidingToDeleting 14, daysFromUploadingToHiding empty", "never set daysFromUploadingToHiding on this one", } { if !strings.Contains(rule, want) { diff --git a/internal/server/backupsetup.go b/internal/server/backupsetup.go new file mode 100644 index 0000000..235a3db --- /dev/null +++ b/internal/server/backupsetup.go @@ -0,0 +1,1982 @@ +package server + +// backupsetup.go — "Protect my data": the simple path to verified, encrypted, off-site +// backups. It is a thin flow over the same engine as the manual form: `server +// backup-target set`, `server backup-policy set`, `backup run --all`. Dash holds no +// secret: the Backblaze key lives in one request, each server's minted key goes to that +// server on stdin and is dropped, and the private recovery key never leaves the browser. +// +// A server with a target is never touched unless the person ticked it, because a push +// replaces its target and redirects its legacy cron lines. Every endpoint names its +// servers, so there is no "all" to get wrong. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/useteploy/teploy-dash/internal/b2" + "github.com/useteploy/teploy-dash/internal/cli" + "github.com/useteploy/teploy-dash/internal/monitor" +) + +const ( + // backupSetupMinCLI is the first CLI with backup-setup-v1. + backupSetupMinCLI = "v0.1.55" + // backupSetupBase is the fixed base prefix of the simple flow. + backupSetupBase = "teploy" + // backupProofBytes is how much of an object the browser gets for the proof. + backupProofBytes = 128 * 1024 + // backupMinFreeBytes: below this a first backup is likely to run out of room. + backupMinFreeBytes = int64(2) << 30 + backupRulesRetries = 3 + backupStartNow = "now" + backupStartNext = "next" + backupKeyMinted = "minted" + backupKeyPushing = "pushing" + backupKeyPushed = "pushed" + backupKeyVerified = "verified" + backupDefaultSetupInterval = "24h" + // backupRecoveryDoc is the recovery page the UI links. + backupRecoveryDoc = "https://github.com/useteploy/teploy-dash/blob/main/docs/recovery.md" +) + +var ( + backupAgeRecipientRE = regexp.MustCompile(`^age1[0-9a-z]{58}$`) + backupFingerprintRE = regexp.MustCompile(`^[A-Za-z0-9:._+/=-]{1,128}$`) + backupKeyNameBadRE = regexp.MustCompile(`[^a-zA-Z0-9-]+`) +) + +// backupSetupRecord is what the flow remembers. Key ids identify a key but cannot use +// it; no secret is ever here. +type backupSetupRecord struct { + // Bucket is recorded before Dash creates it, so a retry never creates a second. + Bucket string `json:"bucket,omitempty"` + BucketID string `json:"bucket_id,omitempty"` + HideDays int `json:"hide_days,omitempty"` + S3Managed bool `json:"s3_managed,omitempty"` + // Keys is each server's key, by server name. + Keys map[string]*backupServerKey `json:"keys,omitempty"` + Leftovers []backupLeftoverKey `json:"leftovers,omitempty"` + Preflight map[string]*backupPreflightSummary `json:"preflight,omitempty"` + Runs map[string]*backupSetupRun `json:"runs,omitempty"` + AlertsSkippedAt *time.Time `json:"alerts_skipped_at,omitempty"` + TurnedOnAt *time.Time `json:"turned_on_at,omitempty"` + Proof *backupProofRecord `json:"proof,omitempty"` +} + +type backupServerKey struct { + KeyID string `json:"key_id"` + KeyName string `json:"key_name"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + State string `json:"state"` + Lifecycle string `json:"lifecycle,omitempty"` + CreatedAt time.Time `json:"created_at"` + Verified *time.Time `json:"verified_at,omitempty"` + // Replaces are older keys that stay until this one is verified. + Replaces []backupLeftoverKey `json:"replaces,omitempty"` +} + +// backupLeftoverKey is a key Dash wanted gone and could not delete. +type backupLeftoverKey struct { + KeyID string `json:"key_id"` + KeyName string `json:"key_name"` + Server string `json:"server"` + Reason string `json:"reason"` +} + +type backupSetupRun struct { + RequestedAt time.Time `json:"requested_at"` + Status string `json:"status"` + Error string `json:"error,omitempty"` +} + +type backupProofRecord struct { + At time.Time `json:"at"` + Server string `json:"server"` + Object string `json:"object"` + Actor string `json:"actor"` +} + +func (b *backupState) setupLocked() *backupSetupRecord { + if b.rec.Setup == nil { + b.rec.Setup = &backupSetupRecord{} + } + if b.rec.Setup.Keys == nil { + b.rec.Setup.Keys = map[string]*backupServerKey{} + } + if b.rec.Setup.Preflight == nil { + b.rec.Setup.Preflight = map[string]*backupPreflightSummary{} + } + if b.rec.Setup.Runs == nil { + b.rec.Setup.Runs = map[string]*backupSetupRun{} + } + return b.rec.Setup +} + +// ── the CLI's envelopes ────────────────────────────────────────────────────── + +type backupPreflightTool struct { + State string `json:"state"` + Version string `json:"version,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type backupPreflight struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + ObservedAt time.Time `json:"observed_at"` + Reachable bool `json:"reachable"` + Error string `json:"error,omitempty"` + Binary struct { + Path string `json:"path"` + State string `json:"state"` + Version string `json:"version,omitempty"` + FleetEngine *bool `json:"fleet_engine"` + CLIVersion string `json:"cli_version"` + SameAsCLI *bool `json:"same_as_cli"` + Detail string `json:"detail,omitempty"` + } `json:"binary"` + Tools struct { + AWS *backupPreflightTool `json:"aws"` + Age *backupPreflightTool `json:"age"` + Python3 *backupPreflightTool `json:"python3"` + } `json:"tools"` + Disk struct { + Path string `json:"path"` + FreeBytes *int64 `json:"free_bytes"` + Detail string `json:"detail,omitempty"` + } `json:"disk"` + Target struct { + State string `json:"state"` + Bucket string `json:"bucket,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Prefix string `json:"prefix,omitempty"` + HasCredentials bool `json:"has_credentials"` + Detail string `json:"detail,omitempty"` + } `json:"target"` + Policy struct { + State string `json:"state"` + Interval string `json:"interval,omitempty"` + Encrypted bool `json:"encrypted"` + CronInstalled *bool `json:"cron_installed"` + ManagedElsewhere string `json:"managed_elsewhere"` + ManagedReasons []string `json:"managed_reasons,omitempty"` + Detail string `json:"detail,omitempty"` + } `json:"policy"` + ToolsInstall []struct { + Tool string `json:"tool"` + Action string `json:"action"` + Detail string `json:"detail,omitempty"` + } `json:"tools_install,omitempty"` + BinaryUpgrade *struct { + Action string `json:"action"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` + Detail string `json:"detail,omitempty"` + } `json:"binary_upgrade,omitempty"` +} + +func decodeBackupPreflight(stdout string) (*backupPreflight, error) { + if _, err := cli.ParseJSON(stdout); err != nil { + return nil, err + } + var env backupPreflight + if err := json.Unmarshal([]byte(stdout), &env); err != nil { + return nil, fmt.Errorf("decoding backup preflight: %w", err) + } + if env.Host == "" || env.ObservedAt.IsZero() || env.Binary.State == "" || env.Target.State == "" || env.Policy.State == "" || + env.Tools.AWS == nil || env.Tools.Age == nil || env.Tools.Python3 == nil { + return nil, errors.New("decoding backup preflight: incomplete machine response") + } + return &env, nil +} + +type backupHead struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + Key string `json:"key"` + Encrypted bool `json:"encrypted"` + RequestedBytes int `json:"requested_bytes"` + Length int `json:"length"` + TotalSize *int64 `json:"total_size"` + DataBase64 string `json:"data_base64"` +} + +func decodeBackupHead(stdout string) (*backupHead, error) { + if _, err := cli.ParseJSON(stdout); err != nil { + return nil, err + } + var env backupHead + if err := json.Unmarshal([]byte(stdout), &env); err != nil { + return nil, fmt.Errorf("decoding backup head: %w", err) + } + if env.Key == "" || env.DataBase64 == "" || env.Length <= 0 { + return nil, errors.New("decoding backup head: incomplete machine response") + } + return &env, nil +} + +type backupDetach struct { + MachineInterface int `json:"machine_interface"` + Host string `json:"host"` + RequestedAt time.Time `json:"requested_at"` + Status string `json:"status"` + PID int `json:"pid,omitempty"` + Log string `json:"log,omitempty"` + Result *struct { + Status string `json:"status"` + Error string `json:"error,omitempty"` + } `json:"result,omitempty"` + Error string `json:"error,omitempty"` +} + +func decodeBackupDetach(stdout string) (*backupDetach, error) { + if _, err := cli.ParseJSON(stdout); err != nil { + return nil, err + } + var env backupDetach + if err := json.Unmarshal([]byte(stdout), &env); err != nil { + return nil, fmt.Errorf("decoding detached run: %w", err) + } + switch env.Status { + case "started", "already_running", "finished", "failed": + return &env, nil + } + return nil, errors.New("decoding detached run: incomplete machine response") +} + +type backupOwnerMarker struct { + Namespace string `json:"namespace"` + Fingerprint string `json:"fingerprint"` + Key string `json:"key"` + Self bool `json:"self"` +} + +type backupSkippedLocation struct { + Prefix string `json:"prefix"` + Reason string `json:"reason"` +} + +// ── preflight ──────────────────────────────────────────────────────────────── + +// backupPreflightSummary is one server's preflight in the flow's terms. +type backupPreflightSummary struct { + Server string `json:"server"` + ObservedAt time.Time `json:"observed_at"` + Reachable bool `json:"reachable"` + Error string `json:"error,omitempty"` + BinaryState string `json:"binary_state"` + BinaryVersion string `json:"binary_version,omitempty"` + CLIVersion string `json:"cli_version,omitempty"` + Tools map[string]backupPreflightTool `json:"tools"` + FreeBytes *int64 `json:"free_bytes"` + TargetState string `json:"target_state"` + TargetBucket string `json:"target_bucket,omitempty"` + TargetPrefix string `json:"target_prefix,omitempty"` + ManagedElsewhere string `json:"managed_elsewhere"` + ManagedReasons []string `json:"managed_reasons,omitempty"` + // NeedsTools and NeedsUpgrade are consented actions on offer. + NeedsTools []string `json:"needs_tools,omitempty"` + NeedsUpgrade bool `json:"needs_upgrade,omitempty"` + // Warnings are worth reading but stop nothing. + Warnings []string `json:"warnings,omitempty"` + // Blocked says why this server cannot be set up now. + Blocked string `json:"blocked,omitempty"` + // Selected is the default tick: a server with a target, or one whose target could + // not be read, starts unticked. + Selected bool `json:"selected"` + // Actions reports consented installs and upgrades, in plain words. + Actions []string `json:"actions,omitempty"` +} + +func summarizeBackupPreflight(server string, p *backupPreflight) *backupPreflightSummary { + sum := &backupPreflightSummary{Server: server, ObservedAt: p.ObservedAt, Reachable: p.Reachable, + Error: sanitizeDomainError(p.Error), BinaryState: p.Binary.State, BinaryVersion: p.Binary.Version, + CLIVersion: p.Binary.CLIVersion, + Tools: map[string]backupPreflightTool{"aws": *p.Tools.AWS, "age": *p.Tools.Age, "python3": *p.Tools.Python3}, + FreeBytes: p.Disk.FreeBytes, + TargetState: p.Target.State, + TargetBucket: p.Target.Bucket, TargetPrefix: p.Target.Prefix, + ManagedElsewhere: p.Policy.ManagedElsewhere, ManagedReasons: p.Policy.ManagedReasons} + if !p.Reachable { + sum.Blocked = "Dash could not reach this server" + if sum.Error != "" { + sum.Blocked += ": " + sum.Error + } + return sum + } + names := make([]string, 0, 3) + for name := range sum.Tools { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + switch sum.Tools[name].State { + case "missing", "outdated": + sum.NeedsTools = append(sum.NeedsTools, name) + case "unknown": + sum.Warnings = append(sum.Warnings, "Could not check "+name+" on this server.") + } + } + switch p.Binary.State { + case "missing", "no_fleet_engine": + sum.NeedsUpgrade = true + case "unknown": + sum.Warnings = append(sum.Warnings, "Could not check the teploy version on this server.") + } + if p.Disk.FreeBytes == nil { + sum.Warnings = append(sum.Warnings, "Could not check the free disk space on this server.") + } else if *p.Disk.FreeBytes < backupMinFreeBytes { + sum.Warnings = append(sum.Warnings, "This server is low on disk space, and backups are built on its disk first.") + } + switch p.Policy.ManagedElsewhere { + case "yes": + sum.Warnings = append(sum.Warnings, "Something else seems to manage backups on this server: "+strings.Join(p.Policy.ManagedReasons, "; ")) + case "unknown": + sum.Warnings = append(sum.Warnings, "Could not tell whether something else manages backups on this server.") + } + switch p.Target.State { + case "absent": + sum.Selected = p.Policy.ManagedElsewhere != "yes" + case "configured": + sum.Warnings = append(sum.Warnings, "This server already sends backups to "+p.Target.Bucket+". It is left alone unless you tick it.") + default: + sum.Warnings = append(sum.Warnings, "Could not read this server's backup settings, so it is left alone unless you tick it.") + } + return sum +} + +// readBackupPreflight reads one server. A flag's failure exits nonzero and still prints +// the report, so the report wins whenever there is one. +func (s *Server) readBackupPreflight(ctx context.Context, name string, install, upgrade bool) (*backupPreflight, error) { + args := []string{"server", "backup-preflight", "--host", name} + if install { + args = append(args, "--install-tools") + } + if upgrade { + args = append(args, "--upgrade-binary") + } + args = append(args, "--json") + result, err := s.runCLI(ctx, args...) + if result != nil { + if env, derr := decodeBackupPreflight(result.Stdout); derr == nil { + return env, nil + } + if err == nil && result.ExitCode != 0 { + err = commandFailure(args, result) + } + } + if err == nil { + err = errors.New("the teploy CLI did not return a readable report") + } + return nil, err +} + +func (s *Server) eachBackupName(ctx context.Context, names []string, fn func(context.Context, int, string)) { + sem := make(chan struct{}, backupPushConcurrency) + var wg sync.WaitGroup + for i, name := range names { + wg.Add(1) + go func(i int, name string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + pctx, cancel := context.WithTimeout(ctx, backupPushTimeout) + defer cancel() + fn(pctx, i, name) + }(i, name) + } + wg.Wait() +} + +// resolveSetupServers checks named servers against the fleet. Empty means the whole +// fleet only where the caller allows it. +func (s *Server) resolveSetupServers(ctx context.Context, requested []string, allowAll bool) ([]string, []string, error) { + servers, err := s.resolveServers(ctx) + if err != nil { + return nil, nil, fmt.Errorf("server discovery failed: %w", err) + } + fleet := backupServerNames(servers) + if len(fleet) == 0 { + return nil, nil, errors.New("Add a server first: there is nothing to protect yet.") + } + if len(requested) == 0 { + if allowAll { + return fleet, fleet, nil + } + return nil, fleet, errors.New("Tick at least one server.") + } + known := map[string]bool{} + for _, name := range fleet { + known[name] = true + } + seen := map[string]bool{} + out := make([]string, 0, len(requested)) + for _, name := range requested { + if !known[name] { + return nil, fleet, fmt.Errorf("%q is not one of your servers", name) + } + if !seen[name] { + seen[name] = true + out = append(out, name) + } + } + sort.Strings(out) + return out, fleet, nil +} + +func (s *Server) handleSetupPreflight(w http.ResponseWriter, r *http.Request) { + var body struct { + Servers []string `json:"servers"` + InstallTools []string `json:"install_tools"` + UpgradeBinary []string `json:"upgrade_binary"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + names, _, err := s.resolveSetupServers(r.Context(), body.Servers, true) + if err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + contains := func(list []string, v string) bool { + for _, x := range list { + if x == v { + return true + } + } + return false + } + ctx := context.WithoutCancel(r.Context()) + results := make([]*backupPreflightSummary, len(names)) + s.eachBackupName(ctx, names, func(ctx context.Context, i int, name string) { + install, upgrade := contains(body.InstallTools, name), contains(body.UpgradeBinary, name) + p, err := s.readBackupPreflight(ctx, name, install, upgrade) + if err != nil { + results[i] = &backupPreflightSummary{Server: name, BinaryState: "unknown", TargetState: "unknown", ManagedElsewhere: "unknown", + Tools: map[string]backupPreflightTool{}, Blocked: sanitizeDomainError("the check did not finish: " + err.Error())} + return + } + sum := summarizeBackupPreflight(name, p) + for _, t := range p.ToolsInstall { + line := t.Tool + ": " + strings.ReplaceAll(t.Action, "_", " ") + if t.Detail != "" { + line += " (" + t.Detail + ")" + } + sum.Actions = append(sum.Actions, sanitizeDomainError(line)) + } + if u := p.BinaryUpgrade; u != nil { + line := "teploy: " + strings.ReplaceAll(u.Action, "_", " ") + if u.Detail != "" { + line += " (" + u.Detail + ")" + } + sum.Actions = append(sum.Actions, sanitizeDomainError(line)) + } + if install || upgrade { + s.auditf(r, "backups.setup.prepare", name, "install_tools=%v upgrade_binary=%v", install, upgrade) + } + results[i] = sum + }) + b := &s.backups + b.mu.Lock() + setup := b.setupLocked() + for _, sum := range results { + setup.Preflight[sum.Server] = sum + } + b.saveLocked() + b.mu.Unlock() + s.auditf(r, "backups.setup.preflight", "", "servers=%d", len(names)) + writeData(w, map[string]any{"servers": results}) +} + +// ── storage: check ─────────────────────────────────────────────────────────── + +func (s *Server) b2Client() *b2.Client { + if s.backups.b2Client != nil { + return s.backups.b2Client() + } + return &b2.Client{HTTP: &http.Client{Timeout: 45 * time.Second, Transport: &http.Transport{DialContext: monitor.PolicyDialContext(false)}}} +} + +// backupSetupWindow is the recovery window: hidden copies last this long. +func backupSetupWindow(requested, current int) (days int, explicit bool, err error) { + switch { + case requested < 0 || requested > 365: + return 0, false, errors.New("the recovery window must be 1 to 365 days") + case requested > 0: + return requested, true, nil + case current > 0: + return current, false, nil + } + return b2.DefaultHideDays, false, nil +} + +type backupBucketFinding struct { + Name string `json:"name"` + Type string `json:"type"` + Refusals []string `json:"refusals"` + Warnings []string `json:"warnings"` + S3Managed bool `json:"s3_managed"` +} + +func bucketFinding(b *b2.Bucket, base string, window int) backupBucketFinding { + a := b2.Analyze(b.Rules, base, window) + f := backupBucketFinding{Name: b.Name, Type: b.Type, Refusals: a.Refusals, Warnings: a.Warnings, S3Managed: a.S3Managed} + if b.Type != "allPrivate" { + f.Refusals = append(f.Refusals, "This bucket is public. Backups need a private bucket.") + } + return f +} + +// oldPrefixWarnings looks for data Dash would be moving in beside. A key without +// listFiles cannot look, and the warning says so instead of implying a clean bucket. +func (s *Server) oldPrefixWarnings(ctx context.Context, sess *b2.Session, bucket *b2.Bucket, base string, fleet []string) []string { + out := []string{} + b := &s.backups + b.mu.Lock() + expected := map[string]string{} + if st := b.rec.Storage; st != nil && st.Bucket == bucket.Name { + for k, v := range st.Prefixes { + expected[k] = v + } + } + if b.rec.Setup != nil { + for name, sum := range b.rec.Setup.Preflight { + if sum.TargetBucket == bucket.Name && sum.TargetPrefix != "" && !strings.HasPrefix(sum.TargetPrefix+"/", b2.BasePrefix(base)) { + out = append(out, fmt.Sprintf("%s now stores backups under %s in this bucket. Moving it leaves those copies there, and retention will not touch them.", name, sum.TargetPrefix)) + } + } + } + b.mu.Unlock() + if !sess.Has(b2.CapListFiles) { + return append(out, "This key cannot list files, so Dash could not check what this bucket already holds.") + } + top, err := sess.ListFolders(ctx, bucket.ID, "", 20) + if err != nil { + return append(out, "Dash could not check what this bucket already holds: "+err.Error()) + } + basePrefix := b2.BasePrefix(base) + for _, name := range top { + if basePrefix == "" || name != basePrefix { + out = append(out, "This bucket already holds "+name+", which Dash leaves alone.") + } + } + if basePrefix == "" { + return out + } + known := map[string]bool{} + for _, name := range fleet { + known[basePrefix+name+"/"] = true + } + for _, p := range expected { + known[p+"/"] = true + } + for _, name := range top { + if name != basePrefix { + continue + } + inner, err := sess.ListFolders(ctx, bucket.ID, basePrefix, 50) + if err != nil { + return append(out, "Dash could not check what this bucket already holds: "+err.Error()) + } + for _, f := range inner { + if !known[f] { + out = append(out, f+" already exists from an older setup. It stays, and retention will not manage it.") + } + } + } + return out +} + +func (s *Server) handleSetupStorageCheck(w http.ResponseWriter, r *http.Request) { + var body struct { + KeyID string `json:"key_id"` + ApplicationKey string `json:"application_key"` + Bucket string `json:"bucket"` + EmptyBase bool `json:"empty_base"` + HideDays int `json:"hide_days"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if !validSecretField(body.KeyID) || !validSecretField(body.ApplicationKey) { + writeError(w, "Paste both parts of the key: the key ID and the application key.") + return + } + scrub := func(text string) string { return scrubCredentials(text, body.KeyID, body.ApplicationKey) } + sess, err := s.b2Client().Authorize(r.Context(), body.KeyID, body.ApplicationKey) + if err != nil { + s.auditf(r, "backups.setup.storage_check", "", "the key was not accepted") + writeErrorStatus(w, scrub(err.Error()), http.StatusUnprocessableEntity) + return + } + base := backupSetupBase + if body.EmptyBase { + base = "" + } + b := &s.backups + b.mu.Lock() + current, recorded := 0, "" + if setup := b.rec.Setup; setup != nil { + current, recorded = setup.HideDays, setup.Bucket + } + if st := b.rec.Storage; st != nil && st.Provider == backupProviderB2 { + recorded = st.Bucket + } + b.mu.Unlock() + window, _, err := backupSetupWindow(body.HideDays, current) + if err != nil { + writeError(w, err.Error()) + return + } + resp := map[string]any{ + "capabilities": sess.Capabilities, "missing": sess.Missing(), + "can_delete_keys": sess.Has(b2.CapDeleteKeys), "can_delete_files": sess.Has(b2.CapDeleteFiles) && sess.Has(b2.CapListFiles), + "endpoint": sess.S3URL, "region": sess.Region, "base": base, "hide_days": window, "suggested_bucket": recorded, + "can_create_bucket": len(sess.Buckets) == 0, "never_expires": sess.ExpiresAt == nil, + } + warnings := []string{} + if sess.ExpiresAt != nil { + resp["expires_at"] = *sess.ExpiresAt + } else { + warnings = append(warnings, "This key never expires. Dash will not delete it for you, so delete it in the Backblaze console when you finish. A key with an expiry is safer.") + } + if sess.NamePrefix != "" { + warnings = append(warnings, "This key is limited to files under "+sess.NamePrefix+", which may stop it from making the bucket's server keys.") + } + resp["warnings"] = warnings + findings := []backupBucketFinding{} + if len(sess.Missing()) == 0 { + buckets, err := sess.ListBuckets(r.Context()) + switch { + case b2.IsCapped(err): + // A capped listing is not an outage: say which it is. + resp["capped"] = true + resp["listing_error"] = scrub(err.Error()) + case err != nil: + resp["listing_error"] = scrub(err.Error()) + default: + for i := range buckets { + f := bucketFinding(&buckets[i], base, window) + if buckets[i].Name == body.Bucket { + servers, serr := s.resolveServers(r.Context()) + if serr == nil { + f.Warnings = append(f.Warnings, s.oldPrefixWarnings(r.Context(), sess, &buckets[i], base, backupServerNames(servers))...) + } + } + findings = append(findings, f) + } + } + } + resp["buckets"] = findings + s.auditf(r, "backups.setup.storage_check", body.Bucket, "buckets=%d missing=%s", len(findings), strings.Join(sess.Missing(), ",")) + writeData(w, resp) +} + +// ── storage: apply ─────────────────────────────────────────────────────────── + +type setupStorageBody struct { + Password string `json:"password"` + KeyID string `json:"key_id"` + ApplicationKey string `json:"application_key"` + Bucket string `json:"bucket"` + Servers []string `json:"servers"` + AcceptWarnings bool `json:"accept_warnings"` + EmptyBase bool `json:"empty_base"` + HideDays int `json:"hide_days"` + S3Managed bool `json:"s3_managed"` + Rotate bool `json:"rotate"` +} + +// setupServerResult reports one server, in the terms of the failure rule: a key is +// deleted only when it was never pushed. +type setupServerResult struct { + Server string `json:"server"` + Prefix string `json:"prefix"` + Skipped string `json:"skipped,omitempty"` + KeyName string `json:"key_name,omitempty"` + Pushed bool `json:"pushed"` + Verified bool `json:"verified"` + // KeyKept: pushed but not verified, so the server may be using it. + KeyKept bool `json:"key_kept,omitempty"` + KeyDeleted bool `json:"key_deleted,omitempty"` + Error string `json:"error,omitempty"` + Replaced string `json:"replaced,omitempty"` + Lifecycle string `json:"lifecycle,omitempty"` + Note string `json:"note,omitempty"` + // OldOwners are markers another machine left under this server's prefix: a rebuilt + // server's old self, which stops retention until cleared with consent. + OldOwners []backupOwnerMarker `json:"old_owners,omitempty"` +} + +type setupStorageResponse struct { + Bucket string `json:"bucket"` + BucketCreated bool `json:"bucket_created"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Base string `json:"base"` + HideDays int `json:"hide_days"` + RulesState string `json:"rules_state"` + RulesText []string `json:"rules_text,omitempty"` + Warnings []string `json:"warnings"` + Servers []setupServerResult `json:"servers"` + AllOK bool `json:"all_ok"` + Stopped string `json:"stopped,omitempty"` + KeyNote string `json:"key_note,omitempty"` +} + +func setupKeyName(server string) string { + slug := strings.Trim(backupKeyNameBadRE.ReplaceAllString(server, "-"), "-") + if slug == "" { + slug = "server" + } + if len(slug) > 60 { + slug = slug[:60] + } + return "teploy-" + slug + "-" + randomHex(3) +} + +// mintServerKey asks for read-lifecycle first and retries without it when Backblaze +// refuses: a retry that also fails was not about the capability. +func mintServerKey(ctx context.Context, sess *b2.Session, name, bucketID, prefix string) (*b2.Key, bool, error) { + caps := []string{b2.CapListFiles, b2.CapReadFiles, b2.CapWriteFiles} + key, err := sess.CreateKey(ctx, name, append(append([]string(nil), caps...), b2.CapReadLifecycle), bucketID, prefix) + if err == nil { + return key, true, nil + } + if !b2.IsCode(err, "bad_request") { + return nil, false, err + } + key, err = sess.CreateKey(ctx, name, caps, bucketID, prefix) + return key, false, err +} + +// resolveSetupBucket finds or creates the bucket. An unchosen bucket's name is saved +// before the create, so a retry finds it rather than making a second. +func (s *Server) resolveSetupBucket(ctx context.Context, sess *b2.Session, chosen string) (*b2.Bucket, bool, error) { + if chosen != "" { + bucket, err := sess.FindBucket(ctx, chosen) + if err != nil { + return nil, false, err + } + if bucket == nil { + return nil, false, fmt.Errorf("Backblaze does not show a bucket called %s to this key", chosen) + } + return bucket, false, nil + } + b := &s.backups + nextName := func() string { + b.mu.Lock() + defer b.mu.Unlock() + setup := b.setupLocked() + if setup.Bucket == "" { + setup.Bucket = "teploy-backups-" + randomHex(8) + b.saveLocked() + } + return setup.Bucket + } + name := nextName() + bucket, err := sess.FindBucket(ctx, name) + if err != nil { + return nil, false, err + } + if bucket != nil { + return bucket, false, nil + } + if len(sess.Buckets) > 0 { + return nil, false, errors.New("This key is limited to certain buckets, so it cannot create one. Choose one of its buckets.") + } + bucket, err = sess.CreatePrivateBucket(ctx, name) + if err == nil { + return bucket, true, nil + } + if !b2.IsCode(err, "duplicate_bucket_name") { + return nil, false, err + } + // Bucket names are global. Not ours, since the search above did not find it, so + // choose again and record that before trying. + b.mu.Lock() + b.setupLocked().Bucket = "" + b.mu.Unlock() + name = nextName() + bucket, err = sess.CreatePrivateBucket(ctx, name) + if err != nil { + return nil, false, err + } + return bucket, true, nil +} + +// applySetupRules merges the rules, retrying when someone changed the bucket meanwhile. +// It returns the warnings a person must accept, or stops when a rule refuses. +func applySetupRules(ctx context.Context, sess *b2.Session, bucket *b2.Bucket, base string, prefixes []string, window int, explicit, forceS3Managed, accept bool) (state string, warnings []string, refusals []string, err error) { + current := bucket + for attempt := 0; attempt < backupRulesRetries; attempt++ { + merged, changed, merr := b2.Merge(current.Rules, base, prefixes, window, explicit) + if merr != nil { + return "", nil, nil, merr + } + analysis := b2.Analyze(merged, base, window) + finding := bucketFinding(&b2.Bucket{Name: current.Name, Type: current.Type, Rules: merged}, base, window) + if len(finding.Refusals) > 0 { + return "", nil, finding.Refusals, nil + } + warnings = analysis.Warnings + if forceS3Managed || analysis.S3Managed { + return "s3_managed", warnings, nil, nil + } + if len(warnings) > 0 && !accept { + return "", warnings, nil, nil + } + if !changed { + return "set", warnings, nil, nil + } + err = sess.UpdateRules(ctx, current.ID, merged, current.Revision) + if err == nil { + return "set", warnings, nil, nil + } + if !b2.IsCode(err, "conflict") { + return "", nil, nil, err + } + fresh, ferr := sess.FindBucket(ctx, current.Name) + if ferr != nil { + return "", nil, nil, ferr + } + if fresh == nil { + return "", nil, nil, errors.New("the bucket disappeared while its rules were being set") + } + current = fresh + } + return "", nil, nil, err +} + +func (s *Server) handleSetupStorageApply(w http.ResponseWriter, r *http.Request) { + var body setupStorageBody + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if !validSecretField(body.KeyID) || !validSecretField(body.ApplicationKey) { + writeError(w, "Paste both parts of the key: the key ID and the application key.") + return + } + if !s.requireReauth(w, r, body.Password) { + return + } + if body.Bucket != "" && (!backupNameRE.MatchString(body.Bucket) || len(body.Bucket) > 63) { + writeError(w, "That bucket name does not look right.") + return + } + names, _, err := s.resolveSetupServers(r.Context(), body.Servers, false) + if err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + b := &s.backups + if !b.setupMu.TryLock() { + writeErrorStatus(w, "Another setup step is running. Wait for it to finish.", http.StatusConflict) + return + } + defer b.setupMu.Unlock() + // Started means it finishes: walking away must not leave a fleet half-done with + // nobody told. The temporary key stays in this function's scope. + ctx := context.WithoutCancel(r.Context()) + scrub := func(text string) string { return scrubCredentials(text, body.KeyID, body.ApplicationKey) } + fail := func(err error, status int) { + s.auditf(r, "backups.setup.storage_failed", body.Bucket, "%s", scrub(err.Error())) + writeErrorStatus(w, scrub(err.Error()), status) + } + + sess, err := s.b2Client().Authorize(ctx, body.KeyID, body.ApplicationKey) + if err != nil { + fail(err, http.StatusUnprocessableEntity) + return + } + if missing := sess.Missing(); len(missing) > 0 { + fail(fmt.Errorf("This key is missing these permissions: %s. Make a key that has them, then paste it again.", strings.Join(missing, ", ")), http.StatusUnprocessableEntity) + return + } + base := backupSetupBase + if body.EmptyBase { + base = "" + } + b.mu.Lock() + current := 0 + if b.rec.Setup != nil { + current = b.rec.Setup.HideDays + } + var known *backupStorage + if st := b.rec.Storage; st != nil { + c := *st + known = &c + } + b.mu.Unlock() + window, explicit, err := backupSetupWindow(body.HideDays, current) + if err != nil { + writeError(w, err.Error()) + return + } + chosen := body.Bucket + if chosen == "" && known != nil && known.Provider == backupProviderB2 { + chosen = known.Bucket + } + bucket, created, err := s.resolveSetupBucket(ctx, sess, chosen) + if err != nil { + fail(err, http.StatusUnprocessableEntity) + return + } + if bucket.Type != "allPrivate" { + fail(errors.New("This bucket is public. Backups need a private bucket."), http.StatusConflict) + return + } + + // Prefixes stay stable across runs, unless an old one sits outside the base. + prefixes := map[string]string{} + taken := map[string]bool{} + basePrefix := b2.BasePrefix(base) + if known != nil && known.Bucket == bucket.Name && known.Endpoint == sess.S3URL { + for k, v := range known.Prefixes { + prefixes[k] = v + taken[v] = true + } + } + for _, name := range names { + if prefixes[name] == "" || !strings.HasPrefix(prefixes[name]+"/", basePrefix) { + prefixes[name] = backupServerPrefix(base, name, taken) + taken[prefixes[name]] = true + } + } + selected := make([]string, 0, len(names)) + for _, name := range names { + selected = append(selected, prefixes[name]) + } + + rulesState, warnings, refusals, err := applySetupRules(ctx, sess, bucket, base, selected, window, explicit, body.S3Managed, body.AcceptWarnings) + if err != nil { + fail(err, http.StatusBadGateway) + return + } + if len(refusals) > 0 { + s.auditf(r, "backups.setup.storage_refused", bucket.Name, "%s", strings.Join(refusals, " ")) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + json.NewEncoder(w).Encode(map[string]any{"error": strings.Join(refusals, " "), "data": map[string]any{"refusals": refusals}}) + return + } + if rulesState == "" { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + json.NewEncoder(w).Encode(map[string]any{"error": "Read the warnings about this bucket, then confirm.", "data": map[string]any{"warnings": warnings}}) + return + } + if warnings == nil { + warnings = []string{} + } + resp := setupStorageResponse{Bucket: bucket.Name, BucketCreated: created, Endpoint: sess.S3URL, Region: sess.Region, Base: base, + HideDays: window, RulesState: rulesState, Warnings: warnings, Servers: make([]setupServerResult, len(names))} + if rulesState == "s3_managed" { + resp.RulesText = b2.RulesText(base, selected, window) + resp.Warnings = append(resp.Warnings, "This bucket's lifecycle rules look like they are managed through the S3 API, so Dash did not change them. Backblaze gives no sure sign of this, so check that these rules exist.") + } + b.mu.Lock() + setup := b.setupLocked() + setup.Bucket, setup.BucketID, setup.HideDays, setup.S3Managed = bucket.Name, bucket.ID, window, rulesState == "s3_managed" + b.saveLocked() + b.mu.Unlock() + + request := func(key *b2.Key) *backupStorageRequest { + return &backupStorageRequest{Provider: backupProviderB2, Bucket: bucket.Name, Region: sess.Region, Endpoint: sess.S3URL, + BasePrefix: base, AccessKey: key.ID, SecretKey: key.Secret} + } + dropKey := func(server, keyID, keyName, why string) bool { + if keyID == "" { + return false + } + if sess.Has(b2.CapDeleteKeys) { + if err := sess.DeleteKey(ctx, keyID); err == nil || b2.IsCode(err, "bad_request") { + return true + } + } + b.mu.Lock() + setup := b.setupLocked() + setup.Leftovers = append(setup.Leftovers, backupLeftoverKey{KeyID: keyID, KeyName: keyName, Server: server, Reason: why}) + b.saveLocked() + b.mu.Unlock() + return false + } + + // Phase one, in order: mint and prove each key before anything reaches a server, so + // a key that cannot hide stops the run with nothing pushed. + type pendingPush struct { + i int + key *b2.Key + prev *backupServerKey + } + var pending []pendingPush + stopped := "" + for i, name := range names { + res := &resp.Servers[i] + res.Server, res.Prefix = name, prefixes[name] + b.mu.Lock() + var prev *backupServerKey + if k := b.setupLocked().Keys[name]; k != nil { + c := *k + prev = &c + } + b.mu.Unlock() + if stopped != "" { + res.Skipped = "stopped before this server" + continue + } + if prev != nil && prev.Bucket == bucket.Name && prev.Prefix == prefixes[name] && !body.Rotate { + switch prev.State { + case backupKeyVerified: + res.Skipped, res.Verified, res.Pushed, res.KeyName = "already verified", true, true, prev.KeyName + continue + case backupKeyPushed, backupKeyPushing: + // It may be the server's key, so see whether it answers now. + if env, serr := s.readBackupStatus(ctx, name); serr == nil && env.Target.Configured && env.Target.Bucket == bucket.Name && + env.Target.Prefix == prefixes[name] && env.Target.HasCredentials && env.Target.Reachable { + s.recordBackupStatus(name, env, nil, time.Now().UTC()) + now := time.Now().UTC() + b.mu.Lock() + k := b.setupLocked().Keys[name] + k.State, k.Verified = backupKeyVerified, &now + replaced := k.Replaces + k.Replaces = nil + b.saveLocked() + b.mu.Unlock() + for _, old := range replaced { + dropKey(name, old.KeyID, old.KeyName, "replaced by a new key") + } + res.Skipped, res.Verified, res.Pushed, res.KeyName = "verified on this try", true, true, prev.KeyName + continue + } + } + } + // A key left at minted never reached a server, and its secret is gone. + if prev != nil && prev.State == backupKeyMinted { + dropKey(name, prev.KeyID, prev.KeyName, "made by an earlier attempt and never pushed") + prev = nil + } + keyName := setupKeyName(name) + key, lifecycleRead, err := mintServerKey(ctx, sess, keyName, bucket.ID, prefixes[name]+"/") + if err != nil { + res.Error = scrub(err.Error()) + if b2.IsCapped(err) { + stopped = res.Error + } + continue + } + res.KeyName = keyName + if !lifecycleRead { + res.Lifecycle = "unknown" + res.Note = "This key cannot read the bucket's lifecycle rules, so whether they still hold cannot be rechecked." + } + now := time.Now().UTC() + record := &backupServerKey{KeyID: key.ID, KeyName: keyName, Bucket: bucket.Name, Prefix: prefixes[name], State: backupKeyMinted, CreatedAt: now} + if !lifecycleRead { + record.Lifecycle = "unknown" + } + if prev != nil { + record.Replaces = append(append([]backupLeftoverKey(nil), prev.Replaces...), backupLeftoverKey{KeyID: prev.KeyID, KeyName: prev.KeyName, Server: name}) + } + b.mu.Lock() + b.setupLocked().Keys[name] = record + b.saveLocked() + b.mu.Unlock() + + req := request(key) + probe, _, ok := s.probeBackupStorage(ctx, req, prefixes[name]) + if !ok { + msg := scrubCredentials(scrub(probe.Error), key.ID, key.Secret) + res.Error = msg + res.KeyDeleted = dropKey(name, key.ID, keyName, "failed its test and never pushed") + restoreSetupKey(b, name, prev) + if probe.Write && probe.Read { + // The key works and cannot hide: every other key would be the same. + stopped = msg + } + continue + } + pending = append(pending, pendingPush{i: i, key: key, prev: prev}) + } + if stopped != "" { + // Nothing reaches any server: drop what this run minted. + for _, p := range pending { + name := names[p.i] + res := &resp.Servers[p.i] + res.Skipped = "stopped before this server" + res.KeyDeleted = dropKey(name, p.key.ID, p.key.Name, "stopped before it was pushed") + restoreSetupKey(b, name, p.prev) + } + pending = nil + resp.Stopped = stopped + } + + // Phase two, concurrently: push and verify, then retire any key this one replaces. + var wg sync.WaitGroup + sem := make(chan struct{}, backupPushConcurrency) + for _, p := range pending { + wg.Add(1) + go func(p pendingPush) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + pctx, cancel := context.WithTimeout(ctx, backupPushTimeout) + defer cancel() + name := names[p.i] + res := &resp.Servers[p.i] + b.mu.Lock() + b.setupLocked().Keys[name].State = backupKeyPushing + b.saveLocked() + b.mu.Unlock() + pushed := s.pushBackupTargetOne(pctx, name, prefixes[name], request(p.key)) + res.Pushed, res.Verified, res.Replaced = pushed.Pushed, pushed.Verified, pushed.Replaced + res.Error = scrub(pushed.Error) + if pushed.Lifecycle != "" && res.Lifecycle == "" { + res.Lifecycle = pushed.Lifecycle + } + now := time.Now().UTC() + switch { + case pushed.Verified: + b.mu.Lock() + k := b.setupLocked().Keys[name] + k.State, k.Verified = backupKeyVerified, &now + if pushed.Lifecycle != "" && k.Lifecycle == "" { + k.Lifecycle = pushed.Lifecycle + } + replaced := k.Replaces + k.Replaces = nil + b.saveLocked() + b.mu.Unlock() + for _, old := range replaced { + if !dropKey(name, old.KeyID, old.KeyName, "replaced by a new key") { + res.Note = strings.TrimSpace(res.Note + " The old key " + old.KeyName + " could not be deleted, so delete it in the Backblaze console.") + } + } + res.OldOwners = s.readOldOwners(pctx, name, res) + case pushed.Pushed || !s.pushProvablyMissed(pctx, name, bucket.Name, prefixes[name]): + // The server may hold this key. Keep it and say so. + res.Pushed = true + res.KeyKept = true + b.mu.Lock() + b.setupLocked().Keys[name].State = backupKeyPushed + b.saveLocked() + b.mu.Unlock() + default: + res.KeyDeleted = dropKey(name, p.key.ID, p.key.Name, "the push did not reach the server") + restoreSetupKey(b, name, p.prev) + } + }(p) + } + wg.Wait() + + resp.AllOK = resp.Stopped == "" + pushedAny := false + verified := 0 + for i := range resp.Servers { + res := &resp.Servers[i] + resp.AllOK = resp.AllOK && res.Verified + pushedAny = pushedAny || res.Pushed + if res.Verified { + verified++ + } + } + if pushedAny { + now := time.Now().UTC() + b.mu.Lock() + st := b.rec.Storage + if st == nil || st.Bucket != bucket.Name || st.Endpoint != sess.S3URL { + st = &backupStorage{Provider: backupProviderB2, Bucket: bucket.Name, Region: sess.Region, Endpoint: sess.S3URL, + BasePrefix: base, Prefixes: map[string]string{}, ConnectedAt: now} + } else if body.Rotate { + st.RotatedAt = &now + } + st.Provider = backupProviderB2 + for _, name := range names { + st.Prefixes[name] = prefixes[name] + } + b.rec.Storage = st + b.saveLocked() + b.mu.Unlock() + } + // The temporary key goes last, only once every server is done: a partial run needs + // it again, and a key without an expiry may not be temporary at all. + if resp.AllOK { + switch { + case sess.ExpiresAt == nil: + resp.KeyNote = "Your key never expires, so Dash left it. Delete it in the Backblaze console." + case !sess.Has(b2.CapDeleteKeys): + resp.KeyNote = "Your key cannot delete keys, so it stays until it expires. You can delete it in the Backblaze console." + default: + if err := sess.DeleteKey(ctx, body.KeyID); err != nil { + resp.KeyNote = "Dash could not delete your temporary key, so delete it in the Backblaze console. " + scrub(err.Error()) + } else { + resp.KeyNote = "Dash deleted your temporary key." + } + } + } else { + resp.KeyNote = "Your key is still in place, because not every server finished. Dash did not save it, so paste it again to retry." + } + s.auditf(r, "backups.setup.storage", bucket.Name, "created=%v rules=%s servers=%d verified=%d stopped=%v", created, rulesState, len(names), verified, resp.Stopped != "") + s.kickBackupSweep() + writeData(w, resp) +} + +func restoreSetupKey(b *backupState, name string, prev *backupServerKey) { + b.mu.Lock() + defer b.mu.Unlock() + setup := b.setupLocked() + if prev != nil { + setup.Keys[name] = prev + } else { + delete(setup.Keys, name) + } + b.saveLocked() +} + +// pushProvablyMissed is what lets a failed push count as never pushed: the server +// answered and stores something else. If it does not answer, or stores this very +// bucket and prefix, the key may be in use. +func (s *Server) pushProvablyMissed(ctx context.Context, name, bucket, prefix string) bool { + args := []string{"server", "backup-target", "show", "--host", name, "--json"} + result, err := s.runCLI(ctx, args...) + if err != nil || result == nil || result.ExitCode != 0 { + return false + } + var out struct { + Configured bool `json:"configured"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + } + if json.Unmarshal([]byte(result.Stdout), &out) != nil { + return false + } + return !out.Configured || out.Bucket != bucket || out.Prefix != prefix +} + +func (s *Server) readOldOwners(ctx context.Context, name string, res *setupServerResult) []backupOwnerMarker { + args := []string{"server", "backup-target", "owners", "--host", name, "--json"} + result, err := s.runCLI(ctx, args...) + if err != nil || result == nil || result.ExitCode != 0 { + res.Note = strings.TrimSpace(res.Note + " Could not check whether an older machine left its mark here.") + return nil + } + var out struct { + Markers []backupOwnerMarker `json:"markers"` + } + if json.Unmarshal([]byte(result.Stdout), &out) != nil { + return nil + } + seen := map[string]bool{} + var others []backupOwnerMarker + for _, m := range out.Markers { + if !m.Self && !seen[m.Fingerprint] { + seen[m.Fingerprint] = true + others = append(others, m) + } + } + return others +} + +// handleSetupOwners clears a rebuilt server's old marker, with the person's consent. +func (s *Server) handleSetupOwners(w http.ResponseWriter, r *http.Request) { + var body struct { + Server string `json:"server"` + Fingerprint string `json:"fingerprint"` + Force bool `json:"force"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if !backupFingerprintRE.MatchString(body.Fingerprint) { + writeError(w, "That marker does not look right.") + return + } + if _, err := s.lookupServerStrict(r.Context(), body.Server); err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + args := []string{"server", "backup-target", "owners", "--host", body.Server, "--clear", body.Fingerprint} + if body.Force { + args = append(args, "--force") + } + args = append(args, "--json") + result, err := s.runCLI(context.WithoutCancel(r.Context()), args...) + if result == nil { + writeErrorStatus(w, sanitizeDomainError(fmt.Sprintf("clearing the marker: %v", err)), http.StatusBadGateway) + return + } + var out struct { + Fingerprint string `json:"fingerprint"` + Removed int `json:"removed"` + Skipped []backupSkippedLocation `json:"skipped_locations"` + } + decoded := json.Unmarshal([]byte(result.Stdout), &out) == nil + failed := err != nil || result.ExitCode != 0 + s.auditf(r, "backups.setup.clear_owner", body.Server, "fingerprint=%s removed=%d force=%v failed=%v", body.Fingerprint, out.Removed, body.Force, failed) + if failed { + msg := strings.TrimSpace(result.Stderr) + if msg == "" && err != nil { + msg = err.Error() + } + if strings.Contains(msg, "last week") || strings.Contains(strings.ToLower(msg), "recent") { + writeErrorStatus(w, "This location was written to in the last week. If the old server is really gone, confirm that and try again.", http.StatusConflict) + return + } + writeErrorStatus(w, sanitizeDomainError("Dash could not clear the marker: "+msg), http.StatusBadGateway) + return + } + if !decoded { + writeErrorStatus(w, "The teploy CLI did not return a readable report.", http.StatusBadGateway) + return + } + s.kickBackupSweep() + writeData(w, map[string]any{"removed": out.Removed, "skipped_locations": out.Skipped}) +} + +// ── recovery key ───────────────────────────────────────────────────────────── + +// handleSetupRecovery records a public key the browser made and proved against the +// saved file. It reaches the servers when backups are turned on. +func (s *Server) handleSetupRecovery(w http.ResponseWriter, r *http.Request) { + var body struct { + Password string `json:"password"` + PublicKey string `json:"public_key"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if !backupAgeRecipientRE.MatchString(body.PublicKey) { + writeError(w, "That is not a public recovery key. It starts with age1.") + return + } + if !s.requireReauth(w, r, body.Password) { + return + } + b := &s.backups + b.mu.Lock() + defer b.mu.Unlock() + if rec := b.rec.Recovery; rec != nil { + if rec.PublicKey != body.PublicKey { + writeErrorStatus(w, "A recovery key is already set. Backups encrypted to it still need it.", http.StatusConflict) + return + } + writeData(w, map[string]string{"public_key": rec.PublicKey}) + return + } + b.rec.Recovery = &backupRecovery{PublicKey: body.PublicKey, ConfirmedAt: time.Now().UTC()} + b.saveLocked() + s.auditf(r, "backups.setup.recovery", body.PublicKey, "public key recorded; the private key stayed in the browser") + writeData(w, map[string]string{"public_key": body.PublicKey}) +} + +func (s *Server) handleSetupAlertsSkip(w http.ResponseWriter, r *http.Request) { + var body struct{} + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + now := time.Now().UTC() + b := &s.backups + b.mu.Lock() + b.setupLocked().AlertsSkippedAt = &now + b.saveLocked() + b.mu.Unlock() + s.auditf(r, "backups.setup.alerts_skip", "", "alert channel skipped for now") + writeData(w, map[string]bool{"ok": true}) +} + +// ── coverage and turning on ────────────────────────────────────────────────── + +type setupCoverageServer struct { + Server string `json:"server"` + Error string `json:"error,omitempty"` + Items []setupCoverageItem `json:"items"` + Unmanaged []setupCoverageItem `json:"unmanaged"` + Target *setupCoverageTarget `json:"target,omitempty"` +} + +type setupCoverageItem struct { + Name string `json:"name"` + Kind string `json:"kind"` + Engine string `json:"engine,omitempty"` + App string `json:"app,omitempty"` + SizeBytes int64 `json:"size_bytes"` +} + +type setupCoverageTarget struct { + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` +} + +func backupAdoptable(e *backupEntry) bool { + return e.Unmanaged && e.Coverage == "uncovered" && e.Engine != "" && !strings.Contains(e.Reason, "no backup method") +} + +func setupCoverageOf(env *backupStatusEnvelope) ([]setupCoverageItem, []setupCoverageItem) { + items, unmanaged := []setupCoverageItem{}, []setupCoverageItem{} + for i := range env.Entries { + e := &env.Entries[i] + item := setupCoverageItem{Name: e.Name, Kind: e.Kind, Engine: e.Engine, App: e.App, SizeBytes: entryStoredBytes(e)} + switch { + case backupAdoptable(e): + unmanaged = append(unmanaged, item) + case e.Coverage == "excluded" || e.Coverage == "uncovered": + default: + items = append(items, item) + } + } + return items, unmanaged +} + +func (s *Server) handleSetupCoverage(w http.ResponseWriter, r *http.Request) { + var body struct { + Servers []string `json:"servers"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + names, _, err := s.resolveSetupServers(r.Context(), body.Servers, false) + if err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + out := make([]setupCoverageServer, len(names)) + s.eachBackupName(r.Context(), names, func(ctx context.Context, i int, name string) { + out[i] = setupCoverageServer{Server: name, Items: []setupCoverageItem{}, Unmanaged: []setupCoverageItem{}} + env, err := s.readBackupStatus(ctx, name) + s.recordBackupStatus(name, env, err, time.Now().UTC()) + if err != nil { + out[i].Error = sanitizeDomainError(err.Error()) + return + } + out[i].Items, out[i].Unmanaged = setupCoverageOf(env) + if env.Target.Configured { + out[i].Target = &setupCoverageTarget{Bucket: env.Target.Bucket, Prefix: env.Target.Prefix} + } + }) + preset, interval, days := s.setupPolicyDefaults() + ret, _ := backupPresetRetention(preset) + writeData(w, map[string]any{"servers": out, "interval": interval, "preset": preset, "hide_days": days, + "multiplier": backupStorageMultiplier(backupStatPolicy{Interval: interval, Retention: ret}, days)}) +} + +func (s *Server) setupPolicyDefaults() (preset, interval string, hideDays int) { + preset, interval, hideDays = backupPresetStandard, backupDefaultSetupInterval, b2.DefaultHideDays + b := &s.backups + b.mu.Lock() + defer b.mu.Unlock() + if p := b.rec.Policy; p != nil { + preset, interval = p.Preset, p.Interval + } + if setup := b.rec.Setup; setup != nil && setup.HideDays > 0 { + hideDays = setup.HideDays + } + return preset, interval, hideDays +} + +// backupStorageMultiplier is how many times one backup's size the bucket holds at +// steady state: the copies retention keeps, the copies it has hidden and that wait out +// the recovery window, and two days of pending uploads under .incoming/. +func backupStorageMultiplier(p backupStatPolicy, hideDays int) int64 { + return retainedCopies(p) + backupRunsPerDay(p.Interval)*(int64(hideDays)+2) +} + +type setupTurnOnResult struct { + Server string `json:"server"` + OK bool `json:"ok"` + Error string `json:"error,omitempty"` + Note string `json:"note,omitempty"` + AllowEmpty bool `json:"allow_empty"` + Adopted []string `json:"adopted,omitempty"` + Run *backupSetupRun `json:"run,omitempty"` +} + +func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { + var body struct { + Servers []string `json:"servers"` + Adopt bool `json:"adopt"` + Start string `json:"start"` + Interval string `json:"interval"` + Preset string `json:"preset"` + SkipVolumes bool `json:"skip_volumes"` + VolumeMaxGB float64 `json:"volume_max_gb"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if body.Start != backupStartNow && body.Start != backupStartNext { + writeError(w, `Choose "Back up now" or "Start with the next scheduled run".`) + return + } + preset, interval, _ := s.setupPolicyDefaults() + if body.Preset != "" { + preset = body.Preset + } + if body.Interval != "" { + interval = body.Interval + } + retention, ok := backupPresetRetention(preset) + if !ok { + writeError(w, "retention must be simple or standard") + return + } + valid := false + for _, v := range backupIntervals { + valid = valid || v == interval + } + if !valid { + writeError(w, "frequency must be one of "+strings.Join(backupIntervals, ", ")) + return + } + if body.VolumeMaxGB < 0 { + writeError(w, "the volume size cap must be 0 or more") + return + } + names, _, err := s.resolveSetupServers(r.Context(), body.Servers, false) + if err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + b := &s.backups + b.mu.Lock() + recipient := "" + if b.rec.Recovery != nil { + recipient = b.rec.Recovery.PublicKey + } + b.mu.Unlock() + if recipient == "" { + writeError(w, "Save a recovery key first: backups are encrypted to it.") + return + } + if !b.setupMu.TryLock() { + writeErrorStatus(w, "Another setup step is running. Wait for it to finish.", http.StatusConflict) + return + } + defer b.setupMu.Unlock() + ctx := context.WithoutCancel(r.Context()) + volumes := !body.SkipVolumes + results := make([]setupTurnOnResult, len(names)) + s.eachBackupName(ctx, names, func(ctx context.Context, i int, name string) { + res := setupTurnOnResult{Server: name} + defer func() { results[i] = res }() + env, err := s.readBackupStatus(ctx, name) + s.recordBackupStatus(name, env, err, time.Now().UTC()) + if err != nil { + res.Error = sanitizeDomainError(err.Error()) + return + } + if !env.Target.Configured || !env.Target.HasCredentials { + res.Error = "This server has no storage yet, so it was skipped." + return + } + var adopt []string + targets := 0 + for j := range env.Entries { + e := &env.Entries[j] + switch { + case backupAdoptable(e): + if body.Adopt { + adopt = append(adopt, e.Name+":"+e.Engine+":all") + res.Adopted = append(res.Adopted, e.Name) + targets++ + } + case e.Coverage == "excluded" || e.Coverage == "uncovered": + case e.Kind == "volumes" && !volumes: + default: + targets++ + } + } + // A server with nothing on it yet must not fail, and only that server gets + // allow_empty: on one with targets, losing them all should stay a failure. + res.AllowEmpty = targets == 0 + note, err := s.applyBackupPolicy(ctx, name, backupPolicyApply{Interval: interval, Retention: retention, Volumes: volumes, + VolumeMaxGB: body.VolumeMaxGB, Recipient: recipient, AllowEmpty: &res.AllowEmpty, Adopt: adopt}) + res.Note = note + if err != nil { + res.Error = sanitizeDomainError(err.Error()) + return + } + res.OK = true + now := time.Now().UTC() + b.mu.Lock() + rec := b.serverLocked(name) + rec.HeartbeatNote = note + rec.PolicyAppliedAt = now + b.saveLocked() + b.mu.Unlock() + if body.Start == backupStartNow { + res.Run = s.startDetachedBackup(ctx, name) + } + }) + now := time.Now().UTC() + allOK := true + for _, res := range results { + allOK = allOK && res.OK + } + b.mu.Lock() + b.rec.Policy = &backupPolicySpec{Interval: interval, Preset: preset, Volumes: volumes, VolumeMaxGB: body.VolumeMaxGB, UpdatedAt: now, UpdatedBy: requestActor(r)} + b.setupLocked().TurnedOnAt = &now + b.saveLocked() + b.mu.Unlock() + s.auditf(r, "backups.setup.turn_on", "", "interval=%s preset=%s volumes=%v adopt=%v start=%s servers=%d all_ok=%v", interval, preset, volumes, body.Adopt, body.Start, len(names), allOK) + s.kickBackupSweep() + writeData(w, map[string]any{"servers": results, "all_ok": allOK, "interval": interval}) +} + +// startDetachedBackup starts the same run a cron line starts, and records how it went. +// finished and failed exit nonzero with a report, so the report wins. +func (s *Server) startDetachedBackup(ctx context.Context, name string) *backupSetupRun { + run := &backupSetupRun{RequestedAt: time.Now().UTC()} + args := []string{"backup", "run", "--all", "--detach", "--host", name, "--json"} + result, err := s.runCLI(ctx, args...) + var env *backupDetach + if result != nil { + env, _ = decodeBackupDetach(result.Stdout) + } + switch { + case env != nil: + run.Status, run.Error = env.Status, env.Error + if env.Result != nil { + if env.Result.Status != "ok" { + run.Status = "failed" + } + if run.Error == "" { + run.Error = env.Result.Error + } + } + case err != nil: + run.Status, run.Error = "failed", err.Error() + case result != nil && result.ExitCode != 0: + run.Status, run.Error = "failed", commandFailure(args, result).Error() + default: + run.Status, run.Error = "failed", "the teploy CLI did not return a readable report" + } + run.Error = sanitizeDomainError(run.Error) + b := &s.backups + b.mu.Lock() + b.setupLocked().Runs[name] = run + b.saveLocked() + b.mu.Unlock() + return run +} + +func (s *Server) handleSetupRun(w http.ResponseWriter, r *http.Request) { + var body struct { + Servers []string `json:"servers"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + names, _, err := s.resolveSetupServers(r.Context(), body.Servers, false) + if err != nil { + writeError(w, sanitizeDomainError(err.Error())) + return + } + runs := map[string]*backupSetupRun{} + var mu sync.Mutex + s.eachBackupName(context.WithoutCancel(r.Context()), names, func(ctx context.Context, _ int, name string) { + run := s.startDetachedBackup(ctx, name) + mu.Lock() + runs[name] = run + mu.Unlock() + s.auditf(r, "backups.setup.run", name, "status=%s", run.Status) + }) + s.kickBackupSweep() + writeData(w, map[string]any{"runs": runs}) +} + +// ── proving recovery ───────────────────────────────────────────────────────── + +func (s *Server) handleSetupProofStart(w http.ResponseWriter, r *http.Request) { + var body struct{} + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + b := &s.backups + now := time.Now().UTC() + b.mu.Lock() + rec := b.rec.Recovery + if rec == nil { + b.mu.Unlock() + writeError(w, "No recovery key has been set yet.") + return + } + // The newest object this key can open: retention may have taken an older one. + if next := b.rec.pickDrillObject(now); next != nil && (rec.Drill == nil || rec.Drill.Key != next.Key) { + rec.PreviousDrill, rec.Drill = rec.Drill, next + b.saveLocked() + } + var object backupDrillObject + found := rec.Drill != nil + if found { + object = *rec.Drill + } + publicKey := rec.PublicKey + b.mu.Unlock() + if !found { + writeErrorStatus(w, "No backup encrypted to this key has finished yet. Try again after the first backup.", http.StatusConflict) + return + } + args := []string{"backup", "head", object.Key, "--bytes", strconv.Itoa(backupProofBytes), "--host", object.Server, "--json"} + result, err := s.runCLI(r.Context(), args...) + if err == nil { + err = cli.CheckExit(result) + } + if err != nil { + writeErrorStatus(w, sanitizeDomainError("Dash could not read the backup from "+object.Server+": "+err.Error()), http.StatusBadGateway) + return + } + head, err := decodeBackupHead(result.Stdout) + if err != nil { + writeErrorStatus(w, sanitizeDomainError(err.Error()), http.StatusBadGateway) + return + } + if !head.Encrypted { + writeErrorStatus(w, "That backup is not encrypted, so it proves nothing about this key.", http.StatusConflict) + return + } + s.auditf(r, "backups.setup.proof_read", head.Key, "server=%s bytes=%d", object.Server, head.Length) + writeData(w, map[string]any{"server": object.Server, "key": head.Key, "length": head.Length, "total_size": head.TotalSize, + "data_base64": head.DataBase64, "public_key": publicKey}) +} + +func (s *Server) handleSetupProofResult(w http.ResponseWriter, r *http.Request) { + var body struct { + Key string `json:"key"` + Passed bool `json:"passed"` + Detail string `json:"detail"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + b := &s.backups + b.mu.Lock() + defer b.mu.Unlock() + rec := b.rec.Recovery + if rec == nil { + writeError(w, "No recovery key has been set yet.") + return + } + var object *backupDrillObject + for _, o := range []*backupDrillObject{rec.Drill, rec.PreviousDrill} { + if o != nil && o.Key == body.Key { + object = o + } + } + if object == nil { + writeError(w, "That is not the backup this proof read.") + return + } + if !body.Passed { + s.auditf(r, "backups.setup.proof_failed", object.Key, "server=%s detail=%s", object.Server, sanitizeDomainError(body.Detail)) + writeData(w, map[string]bool{"recorded": true}) + return + } + now := time.Now().UTC() + rec.DrillPassedAt = &now + b.setupLocked().Proof = &backupProofRecord{At: now, Server: object.Server, Object: object.Key, Actor: requestActor(r)} + b.saveLocked() + s.auditf(r, "backups.setup.proof_passed", object.Key, "server=%s", object.Server) + writeData(w, map[string]any{"recorded": true, "drill_passed_at": now}) +} + +// ── retiring a server ──────────────────────────────────────────────────────── + +func (s *Server) handleSetupRetire(w http.ResponseWriter, r *http.Request) { + var body struct { + Password string `json:"password"` + KeyID string `json:"key_id"` + ApplicationKey string `json:"application_key"` + Server string `json:"server"` + Data string `json:"data"` + Confirm string `json:"confirm"` + } + if err := strictDecode(r, &body); err != nil { + writeError(w, "invalid request body") + return + } + if body.Data != "keep" && body.Data != "remove" { + writeError(w, "Choose whether to keep or remove this server's backups.") + return + } + if body.Data == "remove" && body.Confirm != body.Server { + writeError(w, "Type the server's name to remove its backups.") + return + } + if !validSecretField(body.KeyID) || !validSecretField(body.ApplicationKey) { + writeError(w, "Paste both parts of the key: the key ID and the application key.") + return + } + if !s.requireReauth(w, r, body.Password) { + return + } + b := &s.backups + if !b.setupMu.TryLock() { + writeErrorStatus(w, "Another setup step is running. Wait for it to finish.", http.StatusConflict) + return + } + defer b.setupMu.Unlock() + ctx := context.WithoutCancel(r.Context()) + scrub := func(text string) string { return scrubCredentials(text, body.KeyID, body.ApplicationKey) } + b.mu.Lock() + var key *backupServerKey + if b.rec.Setup != nil { + if k := b.rec.Setup.Keys[body.Server]; k != nil { + c := *k + key = &c + } + } + prefix, bucketName := "", "" + if st := b.rec.Storage; st != nil { + prefix, bucketName = st.Prefixes[body.Server], st.Bucket + } + b.mu.Unlock() + if key != nil { + prefix, bucketName = key.Prefix, key.Bucket + } + if key == nil && prefix == "" { + writeError(w, "Dash has no storage on record for this server.") + return + } + sess, err := s.b2Client().Authorize(ctx, body.KeyID, body.ApplicationKey) + if err != nil { + writeErrorStatus(w, scrub(err.Error()), http.StatusUnprocessableEntity) + return + } + need := []string{} + if key != nil && !sess.Has(b2.CapDeleteKeys) { + need = append(need, b2.CapDeleteKeys) + } + if body.Data == "remove" { + for _, c := range []string{b2.CapListBuckets, b2.CapListFiles, b2.CapDeleteFiles} { + if !sess.Has(c) { + need = append(need, c) + } + } + } + if len(need) > 0 { + writeErrorStatus(w, "This key is missing these permissions: "+strings.Join(need, ", ")+".", http.StatusUnprocessableEntity) + return + } + notes := []string{} + // A server still in the fleet stops trying first, so it does not fail every night + // on a deleted key. One that is gone cannot, and that is fine. + if _, err := s.lookupServerStrict(ctx, body.Server); err == nil { + for _, args := range [][]string{ + {"server", "backup-policy", "set", "--host", body.Server, "--interval", "none", "--json"}, + {"server", "backup-target", "unset", "--host", body.Server}, + } { + result, err := s.runCLI(ctx, args...) + if err == nil { + err = cli.CheckExit(result) + } + if err != nil { + notes = append(notes, sanitizeDomainError("Could not stop backups on the server: "+err.Error())) + break + } + } + } + if key != nil { + if err := sess.DeleteKey(ctx, key.KeyID); err != nil && !b2.IsCode(err, "bad_request") { + s.auditf(r, "backups.setup.retire_failed", body.Server, "%s", scrub(err.Error())) + writeErrorStatus(w, "Dash could not delete this server's key: "+scrub(err.Error()), http.StatusBadGateway) + return + } + } + removed := 0 + if body.Data == "remove" && prefix != "" { + bucket, err := sess.FindBucket(ctx, bucketName) + if err == nil && bucket == nil { + err = errors.New("Backblaze does not show this bucket to this key") + } + if err == nil { + removed, err = sess.DeleteAllVersions(ctx, bucket.ID, prefix+"/") + if err == nil && sess.Has(b2.CapWriteBuckets) { + if rules, changed := b2.WithoutPrefix(bucket.Rules, b2.IncomingPrefix(prefix)); changed { + if uerr := sess.UpdateRules(ctx, bucket.ID, rules, bucket.Revision); uerr != nil { + notes = append(notes, "Could not remove this server's lifecycle rule: "+scrub(uerr.Error())) + } + } + } + } + if err != nil { + s.auditf(r, "backups.setup.retire_failed", body.Server, "removed=%d %s", removed, scrub(err.Error())) + notes = append(notes, fmt.Sprintf("Removed %d files, then stopped: %s", removed, scrub(err.Error()))) + } + } + b.mu.Lock() + setup := b.setupLocked() + delete(setup.Keys, body.Server) + delete(setup.Preflight, body.Server) + delete(setup.Runs, body.Server) + if body.Data == "remove" && b.rec.Storage != nil { + delete(b.rec.Storage.Prefixes, body.Server) + } + b.saveLocked() + b.mu.Unlock() + s.auditf(r, "backups.setup.retire", body.Server, "data=%s files_removed=%d", body.Data, removed) + s.kickBackupSweep() + writeData(w, map[string]any{"server": body.Server, "data": body.Data, "files_removed": removed, "notes": notes}) +} + +// ── routing ────────────────────────────────────────────────────────────────── + +func (s *Server) handleBackupSetup(w http.ResponseWriter, r *http.Request, sub string) { + noStore(w) + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !s.requireFeatureMin(w, r, func(f machineFeatures) bool { return f.BackupSetup }, "Protect my data", backupSetupMinCLI) { + return + } + switch sub { + case "preflight": + s.handleSetupPreflight(w, r) + case "storage/check": + s.handleSetupStorageCheck(w, r) + case "storage/apply": + s.handleSetupStorageApply(w, r) + case "owners": + s.handleSetupOwners(w, r) + case "recovery": + s.handleSetupRecovery(w, r) + case "alerts/skip": + s.handleSetupAlertsSkip(w, r) + case "coverage": + s.handleSetupCoverage(w, r) + case "turn-on": + s.handleSetupTurnOn(w, r) + case "run": + s.handleSetupRun(w, r) + case "proof/start": + s.handleSetupProofStart(w, r) + case "proof/result": + s.handleSetupProofResult(w, r) + case "retire": + s.handleSetupRetire(w, r) + default: + jsonError(w, "not found", http.StatusNotFound) + } +} diff --git a/internal/server/backupsetup_status.go b/internal/server/backupsetup_status.go new file mode 100644 index 0000000..e26deac --- /dev/null +++ b/internal/server/backupsetup_status.go @@ -0,0 +1,415 @@ +package server + +// backupsetup_status.go — what "Protected" means, and Dash's own backup. +// +// Protected: a verified backup on each covered target, plus a passing restore test for +// each covered database. Volume-only servers need no restore test. A status that could +// not be read proves nothing, so it never reads as protected. + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strings" +) + +// backupGuardSkipsShown: this many skips in a row and the page says why, rather than +// waiting silently. +const backupGuardSkipsShown = 3 + +// Per-server setup states. +const ( + setupNotProtected = "not_protected" + setupUnknown = "unknown" + setupCapped = "capped" + setupUnreachable = "unreachable" + setupEmpty = "empty" + setupWaitingBackup = "waiting_first_backup" + setupLate = "late" + setupFailing = "failing" + setupWaitingTest = "waiting_restore_test" + setupGuardSkipped = "guard_skipped" + setupTestFailed = "restore_test_failed" + setupProtected = "protected" +) + +type backupSetupServerView struct { + Server string `json:"server"` + State string `json:"state"` + Detail string `json:"detail,omitempty"` + // Uncovered names what is not in any backup here. + Uncovered []string `json:"uncovered,omitempty"` + Run *backupSetupRun `json:"run,omitempty"` + Key string `json:"key_name,omitempty"` + Lifecycle string `json:"lifecycle,omitempty"` +} + +type backupSetupView struct { + Supported bool `json:"supported"` + MinCLI string `json:"min_cli"` + // State: none | partial | attention | protected. + State string `json:"state"` + Headline string `json:"headline"` + // Configured: storage exists, whichever way it was set up. + Configured bool `json:"configured"` + Servers []backupSetupServerView `json:"servers"` + NewServers []string `json:"new_servers,omitempty"` + RetiredKeys []string `json:"retired_keys,omitempty"` + Leftovers []backupLeftoverKey `json:"leftovers,omitempty"` + AlertsConfigured bool `json:"alerts_configured"` + AlertsSkipped bool `json:"alerts_skipped"` + Bucket string `json:"bucket,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Base string `json:"base"` + HideDays int `json:"hide_days,omitempty"` + S3Managed bool `json:"s3_managed,omitempty"` + TurnedOn bool `json:"turned_on"` + Proof *backupProofRecord `json:"proof,omitempty"` + Multiplier int64 `json:"multiplier,omitempty"` + StoredBytes int64 `json:"stored_bytes,omitempty"` + ProjectedBytes int64 `json:"projected_bytes,omitempty"` + RecoveryDoc string `json:"recovery_doc"` +} + +// backupTargetCapped: the store's daily cap, as the CLI or aws report it. It is not an +// outage and is not shown as one. +func backupTargetCapped(text string) bool { + lower := strings.ToLower(text) + return strings.Contains(lower, "transaction_cap") || strings.Contains(lower, "transactioncap") || strings.Contains(lower, "cap exceeded") +} + +const backupCappedMessage = "Backblaze's daily transaction cap has been reached, so Dash cannot look at this server's backups today. They are still there. Raise the cap in the Backblaze console, or wait for tomorrow." + +// lastRealRun is a test's newest run that actually ran, and how many guard skips sit +// after it. +func (t *backupAutoTest) lastRealRun() (*backupTestRun, int, string) { + skips, reason := 0, "" + for i := len(t.History) - 1; i >= 0; i-- { + run := &t.History[i] + if run.Skipped == "" { + return run, skips, reason + } + if skips == 0 { + reason = run.Skipped + } + skips++ + } + return nil, skips, reason +} + +func (b *backupState) setupServerStateLocked(sv backupServerView, rows []backupRowView) backupSetupServerView { + out := backupSetupServerView{Server: sv.Server} + if setup := b.rec.Setup; setup != nil { + out.Run = setup.Runs[sv.Server] + if k := setup.Keys[sv.Server]; k != nil { + out.Key, out.Lifecycle = k.KeyName, k.Lifecycle + } + } + if out.Lifecycle == "" { + out.Lifecycle = sv.Target.IncomingLifecycle + } + if backupTargetCapped(sv.Target.Error) { + out.State, out.Detail = setupCapped, backupCappedMessage + return out + } + if sv.State != backupServerOK { + out.State, out.Detail = setupUnknown, "Dash could not read this server, so it does not know how its backups are." + if sv.Error != "" { + out.Detail += " " + sv.Error + } + return out + } + if !sv.Target.Configured || !sv.Policy.Configured { + out.State, out.Detail = setupNotProtected, sv.Server+" is not backed up." + return out + } + if !sv.Target.Reachable { + out.State, out.Detail = setupUnreachable, sv.Server+" cannot reach its storage." + if sv.Target.Error != "" { + out.Detail += " " + sv.Target.Error + } + return out + } + + covered := 0 + var firstBackup, late, failing []string + var waitingTest, failedTest []string + guardReason := "" + for i := range rows { + row := &rows[i] + switch row.State { + case "excluded": + continue + case "uncovered": + out.Uncovered = append(out.Uncovered, row.Name) + continue + } + covered++ + switch { + case row.State == "failing": + detail := row.Name + if row.LastError != "" { + detail += ": " + row.LastError + } + failing = append(failing, detail) + continue + case row.LastGood == nil: + firstBackup = append(firstBackup, row.Name) + continue + case row.State == "stale": + late = append(late, row.Name) + continue + case sv.Encrypted && !row.Encrypted: + firstBackup = append(firstBackup, row.Name) + continue + } + if row.Kind != "accessory" && row.Kind != "adopted" { + continue // volumes need no restore test + } + seen := 0 + for _, t := range b.rec.Tests { + if t.Server != sv.Server || t.RetiredAt != nil || t.Container != row.Name { + continue + } + seen++ + run, skips, reason := t.lastRealRun() + switch { + case run != nil && !run.OK: + failedTest = append(failedTest, row.Name) + case run != nil: + // passed; a later guard skip does not undo it + case skips >= backupGuardSkipsShown: + guardReason = reason + waitingTest = append(waitingTest, row.Name) + default: + waitingTest = append(waitingTest, row.Name) + } + } + if seen == 0 && (row.LastRestoreTest == nil || !row.LastRestoreTest.OK) { + waitingTest = append(waitingTest, row.Name) + } + } + join := func(list []string) string { return strings.Join(list, "; ") } + switch { + case covered == 0: + out.State, out.Detail = setupEmpty, "There is nothing to back up on this server yet." + case len(failing) > 0: + out.State, out.Detail = setupFailing, "Backups are failing: "+join(failing) + case len(failedTest) > 0: + out.State, out.Detail = setupTestFailed, "A restore test failed: "+join(failedTest) + case len(late) > 0: + out.State, out.Detail = setupLate, "Backups are late: "+join(late) + case len(firstBackup) > 0: + out.State, out.Detail = setupWaitingBackup, "Waiting for the first backup of "+join(firstBackup)+"." + case guardReason != "": + out.State, out.Detail = setupGuardSkipped, "The restore test keeps being put off. "+guardReason + case len(waitingTest) > 0: + out.State, out.Detail = setupWaitingTest, "Waiting for a restore test of "+join(waitingTest)+". These run in the off-hours window." + default: + out.State = setupProtected + } + return out +} + +func (b *backupState) setupViewLocked(supported, channels bool, view *backupsPageView) *backupSetupView { + out := &backupSetupView{Supported: supported, MinCLI: backupSetupMinCLI, Servers: []backupSetupServerView{}, + AlertsConfigured: channels, RecoveryDoc: backupRecoveryDoc, Base: backupSetupBase} + rowsByServer := map[string][]backupRowView{} + for _, row := range view.Rows { + rowsByServer[row.Server] = append(rowsByServer[row.Server], row) + } + inFleet := map[string]bool{} + protected, attention, active := 0, 0, 0 + for _, sv := range view.Servers { + inFleet[sv.Server] = true + state := b.setupServerStateLocked(sv, rowsByServer[sv.Server]) + out.Servers = append(out.Servers, state) + switch state.State { + case setupProtected: + protected++ + active++ + case setupEmpty: + case setupNotProtected: + if view.Storage != nil { + out.NewServers = append(out.NewServers, sv.Server) + } + active++ + case setupFailing, setupTestFailed, setupLate, setupUnreachable, setupGuardSkipped, setupCapped, setupUnknown: + attention++ + active++ + default: + active++ + } + } + if st := view.Storage; st != nil { + out.Configured, out.Bucket, out.Endpoint, out.Base = true, st.Bucket, st.Endpoint, st.BasePrefix + } + if setup := b.rec.Setup; setup != nil { + out.AlertsSkipped = setup.AlertsSkippedAt != nil + out.HideDays = setup.HideDays + out.S3Managed = setup.S3Managed + out.TurnedOn = setup.TurnedOnAt != nil + out.Proof = setup.Proof + out.Leftovers = setup.Leftovers + if !b.rec.LastSweepAt.IsZero() { + for name := range setup.Keys { + if !inFleet[name] { + out.RetiredKeys = append(out.RetiredKeys, name) + } + } + sort.Strings(out.RetiredKeys) + } + } + hide := out.HideDays + if hide == 0 { + hide = 14 + } + interval, retention := backupDefaultSetupInterval, backupRetention{KeepDaily: 7, KeepWeekly: 4, KeepMonthly: 12} + if p := b.rec.Policy; p != nil { + if r, ok := backupPresetRetention(p.Preset); ok { + interval, retention = p.Interval, r + } + } + out.Multiplier = backupStorageMultiplier(backupStatPolicy{Interval: interval, Retention: retention}, hide) + for _, row := range view.Rows { + if row.State != "excluded" && row.State != "uncovered" && row.LastGood != nil { + out.StoredBytes += row.SizeBytes + } + } + out.ProjectedBytes = out.StoredBytes * out.Multiplier + + switch { + case active > 0 && protected == active: + out.State, out.Headline = "protected", "Your data is protected." + case attention > 0: + out.State, out.Headline = "attention", "Your backups need attention." + case out.Configured || out.TurnedOn: + out.State, out.Headline = "partial", "Your data is not protected yet." + default: + out.State, out.Headline = "none", "Your data is not protected yet." + } + return out +} + +// ── Dash itself ────────────────────────────────────────────────────────────── + +// backupSelfView says where Dash's own backup stands. Objects encrypted to the owner +// and the server open with the owner key alone, so its volumes need nothing special, +// except that they must never go up in plaintext: they hold its SSH key. +type backupSelfView struct { + Available bool `json:"available"` + Reason string `json:"reason"` + Runbook string `json:"runbook"` + // Entry is the row that backs Dash up, such as dash/volumes. + Entry string `json:"entry,omitempty"` + // Warning: something here needs doing. + Warning string `json:"warning,omitempty"` + // Uncovered lists paths outside the deployment template's volumes, which no backup + // reaches. + Uncovered []string `json:"uncovered,omitempty"` +} + +const ( + backupRecoveryRunbook = "docs/recovery.md" + backupSelfTemplateApp = "dash" +) + +// backupSelfApp names Dash's own app: TEPLOY_APP where teploy deployed it (v0.1.49 and +// later), and "" otherwise. +func backupSelfApp() string { return strings.TrimSpace(os.Getenv("TEPLOY_APP")) } + +// backupSelfExcludes is what a policy without a recipient must leave out. A Dash +// deployed by an older CLI has no TEPLOY_APP, so the template's name is excluded too: +// a wrong guess skips an app's volumes and says so, a missed one uploads an SSH key. +func backupSelfExcludes(app string) []string { + if app != "" && app != backupSelfTemplateApp { + return []string{app + "/volumes", backupSelfTemplateApp + "/volumes"} + } + return []string{backupSelfTemplateApp + "/volumes"} +} + +// backupSelfUncovered compares where this Dash keeps its state with the template's +// volumes (/var/teploy-dash, /root/.ssh, /root/.teploy). +func backupSelfUncovered(dataDir string) []string { + var out []string + if dataDir != "" && filepath.Clean(dataDir) != "/var/teploy-dash" { + out = append(out, dataDir) + } + if home, err := os.UserHomeDir(); err == nil && home != "/root" { + out = append(out, filepath.Join(home, ".teploy")) + } + if key := os.Getenv("TEPLOY_SSH_KEY"); key != "" && !strings.HasPrefix(filepath.Clean(key), "/root/.ssh/") { + out = append(out, key) + } else if key == "" { + if home, err := os.UserHomeDir(); err == nil && home != "/root" { + out = append(out, filepath.Join(home, ".ssh")) + } + } + return out +} + +func backupSelfStatus(app, dataDir string, rows []backupRowView, recipient bool) backupSelfView { + view := backupSelfView{Runbook: backupRecoveryRunbook, Uncovered: backupSelfUncovered(dataDir)} + name := app + if name == "" { + name = backupSelfTemplateApp + } + var row *backupRowView + for i := range rows { + if rows[i].Kind == "volumes" && rows[i].App == name { + row = &rows[i] + break + } + } + switch { + case row == nil && app == "": + view.Reason = "This Dash was not deployed from the teploy template, so no backup reaches its state. Copy it yourself; the recovery page shows how." + case row == nil: + view.Reason = "No server reports " + name + "/volumes yet, so Dash's own state is not in a backup." + case row.State == "excluded": + view.Entry = row.Name + if recipient { + view.Reason = row.Name + " on " + row.Server + " is excluded, so Dash's own state is not in a backup." + } else { + view.Reason = row.Name + " on " + row.Server + " is left out because there is no recovery key: it holds Dash's SSH key, which must never be uploaded unencrypted." + } + case row.LastGood == nil: + view.Entry = row.Name + view.Reason = "Waiting for the first backup of " + row.Name + " on " + row.Server + "." + case !row.Encrypted: + view.Entry = row.Name + view.Reason = row.Name + " on " + row.Server + " is backed up unencrypted." + view.Warning = "These backups hold Dash's SSH key in the clear. Set a recovery key, or exclude " + row.Name + ", and remove the old copies from your bucket." + case row.State == "covered" || row.State == "adopted": + view.Available = true + view.Entry = row.Name + view.Reason = fmt.Sprintf("Dash's volumes are backed up as %s on %s. Your recovery key opens them on its own.", row.Name, row.Server) + default: + view.Entry = row.Name + view.Reason = row.Name + " on " + row.Server + " is " + row.State + "." + } + if len(view.Uncovered) > 0 { + view.Reason += " No backup covers " + strings.Join(view.Uncovered, ", ") + ": these are outside the template's volumes, so copy them yourself." + } + return view +} + +// backupHandwrittenFound reports evidence of jobs the policy does not own. Dash cannot +// list a server's cron and systemd jobs, so this is what it can see: legacy cron lines, +// and policy files something else manages. +func (b *backupState) backupHandwrittenFound(rows []backupRowView) bool { + for i := range rows { + if rows[i].LegacySchedule { + return true + } + } + if b.rec.Setup != nil { + for _, sum := range b.rec.Setup.Preflight { + if sum.ManagedElsewhere == "yes" { + return true + } + } + } + return false +} diff --git a/internal/server/backupsetup_test.go b/internal/server/backupsetup_test.go new file mode 100644 index 0000000..299aacb --- /dev/null +++ b/internal/server/backupsetup_test.go @@ -0,0 +1,1701 @@ +package server + +// backupsetup_test.go — "Protect my data" against a fake Backblaze and a stubbed CLI. +// Nothing here has met a real account: see the plan's real-account spike. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/useteploy/teploy-dash/internal/b2" + "github.com/useteploy/teploy-dash/internal/caps" + "github.com/useteploy/teploy-dash/internal/cli" + "github.com/useteploy/teploy-dash/internal/s3probe" +) + +const ( + testTempKeyID = "TEMPKEYID0123" + testTempKey = "TEMPAPPKEYSECRET0123456789" + testMintSecret = "MINTEDSECRET" + testRecipient = "age1lggyhqrw2nlhcxprm67z43rta597azn8gknawjehu9d9dl0jq3yqqvfafg" +) + +// fakeBackblaze is the native API with the behaviours the failure rules turn on. +type fakeBackblaze struct { + mu sync.Mutex + caps []string + expiry *int64 + // buckets by name. + buckets map[string]map[string]any + revision int64 + // keys minted and still alive, by id. + keys map[string]map[string]any + nextKey int + deleted []string + // refuseLifecycleCap makes create_key refuse readBucketLifecycleRules. + refuseLifecycleCap bool + // fail answers a call with this code, once per entry when failOnce. + fail map[string][2]any + failOnce map[string]bool + calls []string + bodies map[string][]map[string]any + echo string + url string +} + +func newFakeBackblaze(t *testing.T) (*fakeBackblaze, func() *b2.Client) { + t.Helper() + f := &fakeBackblaze{caps: []string{"listBuckets", "writeBuckets", "writeKeys", "deleteKeys"}, + buckets: map[string]map[string]any{}, keys: map[string]map[string]any{}, + fail: map[string][2]any{}, failOnce: map[string]bool{}, bodies: map[string][]map[string]any{}, revision: 1} + exp := time.Now().Add(time.Hour).UnixMilli() + f.expiry = &exp + srv := httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(srv.Close) + f.url = srv.URL + return f, func() *b2.Client { return &b2.Client{BaseURL: srv.URL, HTTP: srv.Client(), AnyHost: true} } +} + +func (f *fakeBackblaze) addBucket(name, kind string, rules ...map[string]any) { + f.mu.Lock() + defer f.mu.Unlock() + if rules == nil { + rules = []map[string]any{} + } + f.buckets[name] = map[string]any{"bucketId": "id-" + name, "bucketName": name, "bucketType": kind, "revision": f.revision, "lifecycleRules": rules} +} + +func (f *fakeBackblaze) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + name := r.URL.Path[strings.LastIndex(r.URL.Path, "/")+1:] + f.calls = append(f.calls, name) + reply := func(status int, v any) { + w.WriteHeader(status) + json.NewEncoder(w).Encode(v) + } + if failure, ok := f.fail[name]; ok { + if f.failOnce[name] { + delete(f.fail, name) + } + reply(failure[0].(int), map[string]any{"status": failure[0], "code": failure[1], "message": "echo " + f.echo}) + return + } + var body map[string]any + if r.Method == http.MethodPost { + json.NewDecoder(r.Body).Decode(&body) + f.bodies[name] = append(f.bodies[name], body) + } + switch name { + case "b2_authorize_account": + reply(200, map[string]any{"accountId": "acct", "authorizationToken": "TOKEN", "applicationKeyExpirationTimestamp": f.expiry, + "apiInfo": map[string]any{"storageApi": map[string]any{"apiUrl": f.url, "s3ApiUrl": "https://s3.us-west-004.backblazeb2.com", + "allowed": map[string]any{"capabilities": f.caps, "buckets": nil, "namePrefix": nil}}}}) + case "b2_list_buckets": + out := []map[string]any{} + for n, b := range f.buckets { + if want, ok := body["bucketName"]; ok && want != n { + continue + } + out = append(out, b) + } + reply(200, map[string]any{"buckets": out}) + case "b2_create_bucket": + n := body["bucketName"].(string) + if _, exists := f.buckets[n]; exists { + reply(400, map[string]any{"code": "duplicate_bucket_name"}) + return + } + b := map[string]any{"bucketId": "id-" + n, "bucketName": n, "bucketType": body["bucketType"], "revision": f.revision, "lifecycleRules": []any{}} + f.buckets[n] = b + reply(200, b) + case "b2_update_bucket": + for _, b := range f.buckets { + if b["bucketId"] == body["bucketId"] { + if int64(body["ifRevisionIs"].(float64)) != b["revision"].(int64) { + reply(409, map[string]any{"code": "conflict"}) + return + } + b["lifecycleRules"] = body["lifecycleRules"] + b["revision"] = b["revision"].(int64) + 1 + reply(200, b) + return + } + } + reply(400, map[string]any{"code": "bad_bucket_id"}) + case "b2_create_key": + capsList := body["capabilities"].([]any) + if f.refuseLifecycleCap { + for _, c := range capsList { + if c == b2.CapReadLifecycle { + reply(400, map[string]any{"code": "bad_request", "message": "unknown capability"}) + return + } + } + } + f.nextKey++ + id := fmt.Sprintf("MINTED%d", f.nextKey) + f.keys[id] = body + reply(200, map[string]any{"applicationKeyId": id, "applicationKey": testMintSecret + fmt.Sprint(f.nextKey), "capabilities": capsList, "keyName": body["keyName"]}) + case "b2_delete_key": + id := body["applicationKeyId"].(string) + f.deleted = append(f.deleted, id) + delete(f.keys, id) + reply(200, map[string]any{"applicationKeyId": id}) + case "b2_list_file_names": + reply(200, map[string]any{"files": []any{}}) + case "b2_list_file_versions": + reply(200, map[string]any{"files": []map[string]string{{"fileName": body["prefix"].(string) + "x", "fileId": "f1"}}}) + case "b2_delete_file_version": + reply(200, map[string]any{}) + default: + w.WriteHeader(500) + } +} + +func (f *fakeBackblaze) count(name string) int { + f.mu.Lock() + defer f.mu.Unlock() + n := 0 + for _, c := range f.calls { + if c == name { + n++ + } + } + return n +} + +func newSetupServer(t *testing.T) (*Server, *backupCLI, *fakeBackblaze) { + t.Helper() + c := &backupCLI{status: statusesFromTestdata(t), tokens: []string{"backup-setup-v1"}, + policyShow: map[string]string{"prod": `{"machine_interface":2,"configured":false}`, "stage": `{"machine_interface":2,"configured":false}`}} + s := newBackupsServer(t, c) + f, client := newFakeBackblaze(t) + s.backups.b2Client = client + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} + } + return s, c, f +} + +func applyBody(servers ...string) map[string]any { + return map[string]any{"key_id": testTempKeyID, "application_key": testTempKey, "servers": servers} +} + +func applySetup(t *testing.T, s *Server, body map[string]any) setupStorageResponse { + t.Helper() + rec := postJSON(t, s, "/api/backups/setup/storage/apply", body) + if rec.Code != http.StatusOK { + t.Fatalf("apply: %d %s", rec.Code, rec.Body.String()) + } + var resp setupStorageResponse + decodeData(t, rec, &resp) + return resp +} + +func resultFor(t *testing.T, resp setupStorageResponse, server string) setupServerResult { + t.Helper() + for _, r := range resp.Servers { + if r.Server == server { + return r + } + } + t.Fatalf("no result for %s: %+v", server, resp.Servers) + return setupServerResult{} +} + +// ── the feature gate and capability ────────────────────────────────────────── + +func TestSetup_OlderCLIIsRefusedWithTheVersion(t *testing.T) { + c := &backupCLI{status: statusesFromTestdata(t)} // no backup-setup-v1 token + s := newBackupsServer(t, c) + rec := postJSON(t, s, "/api/backups/setup/preflight", map[string]any{}) + if rec.Code != http.StatusNotImplemented || !strings.Contains(rec.Body.String(), backupSetupMinCLI) { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if s.backupsView(context.Background(), backupsNow).Setup.Supported { + t.Fatal("the page must not offer it") + } +} + +func TestSetup_EveryRouteNeedsAdministerCredentials(t *testing.T) { + for _, sub := range []string{"preflight", "storage/check", "storage/apply", "owners", "recovery", "alerts/skip", "coverage", "turn-on", "run", "proof/start", "proof/result", "retire"} { + got := requiredCapabilities(http.MethodPost, "/api/backups/setup/"+sub) + if len(got) != 1 || got[0] != caps.AdministerCredentials { + t.Errorf("%s = %v", sub, got) + } + } + if got := requiredCapabilities(http.MethodGet, "/api/backups"); got[0] != caps.ViewMetadata { + t.Errorf("the page itself stays readable: %v", got) + } +} + +func TestSetup_UnknownStepIsNotFound(t *testing.T) { + s, _, _ := newSetupServer(t) + if rec := postJSON(t, s, "/api/backups/setup/nope", map[string]any{}); rec.Code != http.StatusNotFound { + t.Fatalf("%d", rec.Code) + } +} + +// ── preflight ──────────────────────────────────────────────────────────────── + +func corpusFixtureFile(t *testing.T, name string) string { + t.Helper() + raw, err := os.ReadFile(filepath.Join("testdata", "backup-contracts", name)) + if err != nil { + t.Fatal(err) + } + return string(raw) +} + +func TestSetupPreflight_SummarizesAndNeverTicksAConfiguredServer(t *testing.T) { + s, c, _ := newSetupServer(t) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if !subsequence(args, []string{"server", "backup-preflight"}) { + return nil, nil, false + } + switch args[3] { + case "prod": // has a target + return &cli.Result{Stdout: corpusFixtureFile(t, "preflight-ready.json")}, nil, true + default: // needs everything + return &cli.Result{Stdout: corpusFixtureFile(t, "preflight-needs-setup.json")}, nil, true + } + } + rec := postJSON(t, s, "/api/backups/setup/preflight", map[string]any{}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var out struct { + Servers []backupPreflightSummary `json:"servers"` + } + decodeData(t, rec, &out) + if len(out.Servers) != 2 || out.Servers[0].Server != "prod" { + t.Fatalf("%+v", out.Servers) + } + prod, stage := out.Servers[0], out.Servers[1] + if prod.Selected || prod.NeedsUpgrade || len(prod.NeedsTools) != 0 { + t.Errorf("a server with a target starts unticked: %+v", prod) + } + if !stage.Selected || !stage.NeedsUpgrade || len(stage.NeedsTools) == 0 { + t.Errorf("needs-setup: %+v", stage) + } + if n := c.countOf("--install-tools"); n != 0 { + t.Fatalf("no tool install without consent: %d", n) + } +} + +func TestSetupPreflight_ConsentedActionsAndAFailedActionStillReports(t *testing.T) { + s, c, _ := newSetupServer(t) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if !subsequence(args, []string{"server", "backup-preflight"}) { + return nil, nil, false + } + // A flag's failure exits nonzero and still prints. + return &cli.Result{Stdout: corpusFixtureFile(t, "preflight-after-actions.json"), ExitCode: 1}, fmt.Errorf("exit 1"), true + } + rec := postJSON(t, s, "/api/backups/setup/preflight", map[string]any{"servers": []string{"prod"}, "install_tools": []string{"prod"}, "upgrade_binary": []string{"prod"}}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var out struct { + Servers []backupPreflightSummary `json:"servers"` + } + decodeData(t, rec, &out) + if len(out.Servers[0].Actions) == 0 || out.Servers[0].Blocked != "" { + t.Fatalf("the report wins: %+v", out.Servers[0]) + } + if c.countOf("backup-preflight", "--install-tools", "--upgrade-binary") != 1 { + t.Fatal("the consented flags must reach the CLI") + } +} + +func TestSetupPreflight_UnreachableAndUnreadable(t *testing.T) { + s, c, _ := newSetupServer(t) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if !subsequence(args, []string{"server", "backup-preflight"}) { + return nil, nil, false + } + if args[3] == "prod" { + return &cli.Result{Stdout: corpusFixtureFile(t, "preflight-unreachable.json")}, nil, true + } + return &cli.Result{Stderr: "boom", ExitCode: 1}, nil, true + } + rec := postJSON(t, s, "/api/backups/setup/preflight", map[string]any{}) + var out struct { + Servers []backupPreflightSummary `json:"servers"` + } + decodeData(t, rec, &out) + for _, sum := range out.Servers { + if sum.Blocked == "" || sum.Selected { + t.Errorf("%s must be blocked and unticked: %+v", sum.Server, sum) + } + } +} + +func TestSetupPreflight_UnknownNeverReadsFine(t *testing.T) { + p, err := decodeBackupPreflight(corpusFixtureFile(t, "preflight-unknown.json")) + if err != nil { + t.Fatal(err) + } + sum := summarizeBackupPreflight("x", p) + if sum.Selected || len(sum.Warnings) == 0 { + t.Fatalf("an unread target starts unticked and says so: %+v", sum) + } +} + +// ── storage: check ─────────────────────────────────────────────────────────── + +func TestSetupCheck_ReportsExactMissingCapabilities(t *testing.T) { + s, _, f := newSetupServer(t) + f.caps = []string{"listBuckets"} + rec := postJSON(t, s, "/api/backups/setup/storage/check", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey}) + var out map[string]any + decodeData(t, rec, &out) + if fmt.Sprint(out["missing"]) != "[writeBuckets writeKeys]" { + t.Fatalf("%+v", out) + } +} + +func TestSetupCheck_WarnsWhenTheKeyNeverExpires(t *testing.T) { + s, _, f := newSetupServer(t) + f.expiry = nil + rec := postJSON(t, s, "/api/backups/setup/storage/check", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey}) + var out struct { + NeverExpires bool `json:"never_expires"` + Warnings []string `json:"warnings"` + } + decodeData(t, rec, &out) + if !out.NeverExpires || !strings.Contains(strings.Join(out.Warnings, " "), "never expires") { + t.Fatalf("%+v", out) + } +} + +func TestSetupCheck_ACappedListingIsNotUnreachable(t *testing.T) { + s, _, f := newSetupServer(t) + f.fail["b2_list_buckets"] = [2]any{403, "transaction_cap_exceeded"} + rec := postJSON(t, s, "/api/backups/setup/storage/check", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var out struct { + Capped bool `json:"capped"` + ListingError string `json:"listing_error"` + } + decodeData(t, rec, &out) + if !out.Capped || !strings.Contains(out.ListingError, "daily transaction cap") || strings.Contains(strings.ToLower(out.ListingError), "unreachable") { + t.Fatalf("%+v", out) + } +} + +func TestSetupCheck_RefusesAndWarnsPerBucket(t *testing.T) { + s, _, f := newSetupServer(t) + f.addBucket("bad", "allPrivate", map[string]any{"fileNamePrefix": "", "daysFromUploadingToHiding": 30}) + f.addBucket("short", "allPrivate", map[string]any{"fileNamePrefix": "teploy/", "daysFromHidingToDeleting": 2}) + f.addBucket("pub", "allPublic") + f.addBucket("fine", "allPrivate") + rec := postJSON(t, s, "/api/backups/setup/storage/check", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey}) + var out struct { + Buckets []backupBucketFinding `json:"buckets"` + } + decodeData(t, rec, &out) + got := map[string]backupBucketFinding{} + for _, b := range out.Buckets { + got[b.Name] = b + } + if len(got["bad"].Refusals) == 0 || len(got["pub"].Refusals) == 0 || len(got["short"].Warnings) == 0 || len(got["short"].Refusals) != 0 || len(got["fine"].Refusals)+len(got["fine"].Warnings) != 0 { + t.Fatalf("%+v", got) + } +} + +// ── storage: apply ─────────────────────────────────────────────────────────── + +func TestSetupApply_HappyPath(t *testing.T) { + s, c, f := newSetupServer(t) + resp := applySetup(t, s, applyBody("prod", "stage")) + if !resp.AllOK || !resp.BucketCreated || resp.Region != "us-west-004" || resp.Base != "teploy" || resp.RulesState != "set" { + t.Fatalf("%+v", resp) + } + if !strings.HasPrefix(resp.Bucket, "teploy-backups-") { + t.Fatalf("bucket = %s", resp.Bucket) + } + if f.bodies["b2_create_bucket"][0]["bucketType"] != "allPrivate" { + t.Fatal("a created bucket is private") + } + // One limited key per server, without deleteFiles, with a clean name. + if len(f.keys) != 2 { + t.Fatalf("keys = %+v", f.keys) + } + seen := map[string]bool{} + for _, body := range f.keys { + capsList := fmt.Sprint(body["capabilities"]) + if strings.Contains(capsList, "deleteFiles") { + t.Fatalf("a server key must not have deleteFiles: %s", capsList) + } + seen[body["namePrefix"].(string)] = true + for _, r := range body["keyName"].(string) { + if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-') { + t.Fatalf("key name %q", body["keyName"]) + } + } + if ids := body["bucketIds"].([]any); len(ids) != 1 { + t.Fatalf("limited to the bucket: %+v", body) + } + } + if !seen["teploy/prod/"] || !seen["teploy/stage/"] { + t.Fatalf("prefixes = %+v", seen) + } + // The rules. + rules := f.buckets[resp.Bucket]["lifecycleRules"] + raw, _ := json.Marshal(rules) + for _, want := range []string{`"daysFromHidingToDeleting":14,"daysFromStartingToCancelingUnfinishedLargeFiles":7,"daysFromUploadingToHiding":null,"fileNamePrefix":"teploy/"`, `"fileNamePrefix":"teploy/prod/.incoming/"`, `"fileNamePrefix":"teploy/stage/.incoming/"`} { + if !strings.Contains(string(raw), want) { + t.Errorf("rules %s miss %s", raw, want) + } + } + // Each secret rode stdin. + if n := c.countOf("backup-target", "set"); n != 2 { + t.Fatalf("pushes = %d", n) + } + c.mu.Lock() + for _, call := range c.calls { + if subsequence(call.args, []string{"backup-target", "set"}) { + if !strings.Contains(call.stdin, testMintSecret) { + t.Errorf("secret must ride stdin: %q", call.stdin) + } + if strings.Contains(strings.Join(call.args, " "), testMintSecret) { + t.Errorf("secret in argv") + } + } + } + c.mu.Unlock() + // The temporary key has an expiry and deleteKeys, so it goes. + if fmt.Sprint(f.deleted) != "["+testTempKeyID+"]" || !strings.Contains(resp.KeyNote, "deleted your temporary key") { + t.Fatalf("deleted=%v note=%q", f.deleted, resp.KeyNote) + } + // Dash remembers ids and names only. + s.backups.mu.Lock() + st := s.backups.rec.Storage + k := s.backups.rec.Setup.Keys["prod"] + s.backups.mu.Unlock() + if st.Provider != "b2" || st.Prefixes["prod"] != "teploy/prod" || k.State != backupKeyVerified || k.Prefix != "teploy/prod" { + t.Fatalf("storage=%+v key=%+v", st, k) + } +} + +func TestSetupApply_MissingCapabilityStopsEarly(t *testing.T) { + s, c, f := newSetupServer(t) + f.caps = []string{"listBuckets", "writeBuckets"} + rec := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("prod")) + if rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "writeKeys") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if f.count("b2_create_bucket")+f.count("b2_create_key") != 0 || c.countOf("backup-target", "set") != 0 { + t.Fatal("nothing may happen on a key that cannot finish") + } +} + +func TestSetupApply_NoServersTicked(t *testing.T) { + s, _, _ := newSetupServer(t) + if rec := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody()); rec.Code != http.StatusBadRequest { + t.Fatalf("%d", rec.Code) + } + if rec := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("nosuch")); rec.Code != http.StatusBadRequest { + t.Fatalf("%d", rec.Code) + } +} + +func TestSetupApply_AnUntickedServerIsNeverTouched(t *testing.T) { + s, c, _ := newSetupServer(t) + resp := applySetup(t, s, applyBody("prod")) + if !resp.AllOK { + t.Fatalf("%+v", resp) + } + c.mu.Lock() + defer c.mu.Unlock() + for _, call := range c.calls { + if strings.Contains(strings.Join(call.args, " "), "stage") { + t.Fatalf("an unticked server must not be touched: %v", call.args) + } + } +} + +func TestSetupApply_RetryNeverMakesASecondBucket(t *testing.T) { + s, _, f := newSetupServer(t) + // The create fails after the name was recorded. + f.fail["b2_create_bucket"] = [2]any{500, "internal_error"} + f.failOnce["b2_create_bucket"] = true + if rec := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("prod")); rec.Code == http.StatusOK { + t.Fatalf("%s", rec.Body.String()) + } + s.backups.mu.Lock() + recorded := s.backups.rec.Setup.Bucket + s.backups.mu.Unlock() + if recorded == "" { + t.Fatal("the name must be recorded before the create") + } + resp := applySetup(t, s, applyBody("prod")) + if resp.Bucket != recorded || len(f.buckets) != 1 { + t.Fatalf("retry made another: %s, %+v", resp.Bucket, f.buckets) + } + // And a create that landed but whose reply was lost finds the bucket, not a new one. + s2, _, f2 := newSetupServer(t) + s2.backups.mu.Lock() + s2.backups.setupLocked().Bucket = "teploy-backups-landed" + s2.backups.mu.Unlock() + f2.addBucket("teploy-backups-landed", "allPrivate") + resp = applySetup(t, s2, applyBody("prod")) + if resp.Bucket != "teploy-backups-landed" || resp.BucketCreated || f2.count("b2_create_bucket") != 0 { + t.Fatalf("%+v", resp) + } +} + +func TestSetupApply_ANameTakenByAnotherAccountChoosesAgain(t *testing.T) { + s, _, f := newSetupServer(t) + s.backups.mu.Lock() + s.backups.setupLocked().Bucket = "teploy-backups-taken" + s.backups.mu.Unlock() + f.fail["b2_create_bucket"] = [2]any{400, "duplicate_bucket_name"} + f.failOnce["b2_create_bucket"] = true + resp := applySetup(t, s, applyBody("prod")) + if resp.Bucket == "teploy-backups-taken" || !strings.HasPrefix(resp.Bucket, "teploy-backups-") { + t.Fatalf("%s", resp.Bucket) + } +} + +func TestSetupApply_RerunSkipsVerifiedServers(t *testing.T) { + s, c, f := newSetupServer(t) + f.expiry = nil // keep the temp key usable across runs + first := applySetup(t, s, applyBody("prod", "stage")) + minted := f.nextKey + second := applySetup(t, s, applyBody("prod", "stage")) + if second.Bucket != first.Bucket || f.nextKey != minted { + t.Fatalf("a rerun must mint nothing: %d -> %d", minted, f.nextKey) + } + if !second.AllOK || !strings.Contains(resultFor(t, second, "prod").Skipped, "verified") { + t.Fatalf("%+v", second) + } + if n := c.countOf("backup-target", "set"); n != 2 { + t.Fatalf("a rerun must push nothing: %d", n) + } +} + +func TestSetupApply_PushFailedAndProvablyMissed_DeletesTheKey(t *testing.T) { + s, c, f := newSetupServer(t) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if subsequence(args, []string{"backup-target", "set"}) && subsequence(args, []string{"stage"}) { + return &cli.Result{ExitCode: 1, Stderr: "no route"}, fmt.Errorf("exit 1: no route"), true + } + if subsequence(args, []string{"backup-target", "show"}) && subsequence(args, []string{"stage"}) { + return &cli.Result{Stdout: `{"machine_interface":2,"configured":false}`}, nil, true + } + return nil, nil, false + } + resp := applySetup(t, s, applyBody("prod", "stage")) + stage := resultFor(t, resp, "stage") + if resp.AllOK || stage.Pushed || stage.KeyKept || !stage.KeyDeleted { + t.Fatalf("%+v", stage) + } + if strings.Contains(fmt.Sprint(f.deleted), testTempKeyID) { + t.Fatal("a partial run keeps the temporary key") + } + if !strings.Contains(resp.KeyNote, "paste it again") { + t.Fatalf("note = %q", resp.KeyNote) + } + s.backups.mu.Lock() + _, kept := s.backups.rec.Setup.Keys["stage"] + _, prodKept := s.backups.rec.Setup.Keys["prod"] + s.backups.mu.Unlock() + if kept || !prodKept { + t.Fatalf("records: stage=%v prod=%v", kept, prodKept) + } +} + +func TestSetupApply_PushFailedButMayHaveLanded_KeepsTheKey(t *testing.T) { + for name, show := range map[string]func() (*cli.Result, error){ + "server stopped answering": func() (*cli.Result, error) { return nil, fmt.Errorf("exit 1: timeout") }, + "it stores this target": func() (*cli.Result, error) { + return &cli.Result{Stdout: `{"configured":true,"bucket":"PLACEHOLDER","prefix":"teploy/stage"}`}, nil + }, + } { + t.Run(name, func(t *testing.T) { + s, c, f := newSetupServer(t) + f.addBucket("chosen", "allPrivate") + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if subsequence(args, []string{"backup-target", "set"}) && subsequence(args, []string{"stage"}) { + return &cli.Result{ExitCode: 1, Stderr: "cron rewrite failed"}, fmt.Errorf("exit 1"), true + } + if subsequence(args, []string{"backup-target", "show"}) && subsequence(args, []string{"stage"}) { + res, err := show() + if res != nil { + res.Stdout = strings.ReplaceAll(res.Stdout, "PLACEHOLDER", "chosen") + } + return res, err, true + } + return nil, nil, false + } + body := applyBody("stage") + body["bucket"] = "chosen" + resp := applySetup(t, s, body) + stage := resultFor(t, resp, "stage") + if !stage.KeyKept || stage.KeyDeleted || len(f.deleted) != 0 { + t.Fatalf("%+v deleted=%v", stage, f.deleted) + } + }) + } +} + +func TestSetupApply_PushedButUnverifiedKeepsTheKeyThenRerunRecovers(t *testing.T) { + s, c, f := newSetupServer(t) + f.expiry = nil + bad := true + orig := c.status["stage"] + c.status["stage"] = func() (*cli.Result, error) { + if bad { + return nil, fmt.Errorf("exit 1: timeout") + } + return orig() + } + resp := applySetup(t, s, applyBody("stage")) + stage := resultFor(t, resp, "stage") + if !stage.Pushed || stage.Verified || !stage.KeyKept || len(f.keys) != 1 { + t.Fatalf("%+v keys=%v", stage, f.keys) + } + if strings.Contains(fmt.Sprint(f.deleted), "MINTED") { + t.Fatalf("a pushed key must stay: %v", f.deleted) + } + // Next try the server answers on the key it already has: nothing new is minted. + bad = false + minted := f.nextKey + resp = applySetup(t, s, applyBody("stage")) + stage = resultFor(t, resp, "stage") + if !stage.Verified || f.nextKey != minted || !resp.AllOK { + t.Fatalf("%+v minted %d->%d", stage, minted, f.nextKey) + } +} + +func TestSetupApply_PushedUnverifiedRerunReplacesThenDeletesTheOldKey(t *testing.T) { + s, c, f := newSetupServer(t) + f.expiry = nil + failing := true + orig := c.status["stage"] + c.status["stage"] = func() (*cli.Result, error) { + if failing { + return nil, fmt.Errorf("exit 1: timeout") + } + return orig() + } + applySetup(t, s, applyBody("stage")) + s.backups.mu.Lock() + oldID := s.backups.rec.Setup.Keys["stage"].KeyID + s.backups.mu.Unlock() + // The re-verify fails too, so a new key is made, and verification succeeds on it. + calls := 0 + c.status["stage"] = func() (*cli.Result, error) { + calls++ + if calls == 1 { + return nil, fmt.Errorf("exit 1: timeout") + } + return orig() + } + resp := applySetup(t, s, applyBody("stage")) + if !resultFor(t, resp, "stage").Verified || !strings.Contains(fmt.Sprint(f.deleted), oldID) { + t.Fatalf("%+v deleted=%v", resp, f.deleted) + } +} + +func TestSetupApply_RotationOrderIsMintPushVerifyDeleteOld(t *testing.T) { + s, c, f := newSetupServer(t) + f.expiry = nil + applySetup(t, s, applyBody("prod")) + s.backups.mu.Lock() + oldID := s.backups.rec.Setup.Keys["prod"].KeyID + s.backups.mu.Unlock() + + // Failed verification: the old key stays. + orig := c.status["prod"] + c.status["prod"] = func() (*cli.Result, error) { return nil, fmt.Errorf("exit 1: timeout") } + body := applyBody("prod") + body["rotate"] = true + resp := applySetup(t, s, body) + if resultFor(t, resp, "prod").Verified || strings.Contains(fmt.Sprint(f.deleted), oldID) { + t.Fatalf("an unverified rotation must keep the old key: %v", f.deleted) + } + // Verified: the new one is in place and the replaced key goes. + c.status["prod"] = orig + s.backups.mu.Lock() + newID := s.backups.rec.Setup.Keys["prod"].KeyID + s.backups.mu.Unlock() + resp = applySetup(t, s, body) + if !resultFor(t, resp, "prod").Verified || !strings.Contains(fmt.Sprint(f.deleted), newID) || !strings.Contains(fmt.Sprint(f.deleted), oldID) { + t.Fatalf("%+v deleted=%v", resp, f.deleted) + } +} + +func TestSetupStorage_AManualKeyReportsTheOneItReplaced(t *testing.T) { + s, _, f := newSetupServer(t) + f.expiry = nil + applySetup(t, s, applyBody("prod")) + s.backups.mu.Lock() + bucket := s.backups.rec.Storage.Bucket + s.backups.mu.Unlock() + body := storageBody() + body["bucket"] = bucket + if rec := postJSON(t, s, "/api/backups/storage", body); rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + view := s.backupsView(context.Background(), backupsNow) + if len(view.Setup.Leftovers) != 1 || !strings.Contains(view.Setup.Leftovers[0].Reason, "by hand") { + t.Fatalf("%+v", view.Setup.Leftovers) + } +} + +func TestSetupApply_RefusesAConflictingRuleBeforeAnyKey(t *testing.T) { + s, c, f := newSetupServer(t) + f.addBucket("old", "allPrivate", map[string]any{"fileNamePrefix": "teploy/", "daysFromUploadingToHiding": 30, "daysFromHidingToDeleting": 1}) + body := applyBody("prod") + body["bucket"] = "old" + body["accept_warnings"] = true + rec := postJSON(t, s, "/api/backups/setup/storage/apply", body) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "hides files") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if f.count("b2_create_key")+f.count("b2_update_bucket") != 0 || c.countOf("backup-target", "set") != 0 { + t.Fatal("a refused bucket gets nothing") + } +} + +func TestSetupApply_APublicBucketIsRefused(t *testing.T) { + s, _, f := newSetupServer(t) + f.addBucket("pub", "allPublic") + body := applyBody("prod") + body["bucket"] = "pub" + if rec := postJSON(t, s, "/api/backups/setup/storage/apply", body); rec.Code != http.StatusConflict { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } +} + +func TestSetupApply_WarningsNeedConfirmation(t *testing.T) { + s, _, f := newSetupServer(t) + f.addBucket("keep-last", "allPrivate", map[string]any{"fileNamePrefix": "", "daysFromHidingToDeleting": 1}) + body := applyBody("prod") + body["bucket"] = "keep-last" + rec := postJSON(t, s, "/api/backups/setup/storage/apply", body) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "keeps only the last version") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if f.count("b2_create_key") != 0 { + t.Fatal("no key before the warning is read") + } + body["accept_warnings"] = true + if !applySetup(t, s, body).AllOK { + t.Fatal("confirmed runs") + } +} + +func TestSetupApply_MergesWithRevisionAndKeepsOtherRules(t *testing.T) { + s, _, f := newSetupServer(t) + f.addBucket("shared", "allPrivate", map[string]any{"fileNamePrefix": "photos/", "daysFromHidingToDeleting": 9}) + // Someone edits the bucket at the same moment: one conflict, then a clean retry. + f.fail["b2_update_bucket"] = [2]any{409, "conflict"} + f.failOnce["b2_update_bucket"] = true + body := applyBody("prod") + body["bucket"] = "shared" + resp := applySetup(t, s, body) + if !resp.AllOK || resp.RulesState != "set" { + t.Fatalf("%+v", resp) + } + raw, _ := json.Marshal(f.buckets["shared"]["lifecycleRules"]) + if !strings.Contains(string(raw), `"photos/"`) || !strings.Contains(string(raw), `"teploy/"`) { + t.Fatalf("rules = %s", raw) + } + for _, b := range f.bodies["b2_update_bucket"] { + if _, ok := b["ifRevisionIs"]; !ok { + t.Fatal("every update must carry ifRevisionIs") + } + } +} + +func TestSetupApply_NeverSendsLifecycleRulesToAnS3ManagedBucket(t *testing.T) { + s, _, f := newSetupServer(t) + // Detected: a rule with a field the native API does not document. + f.addBucket("s3ish", "allPrivate", map[string]any{"fileNamePrefix": "x/", "id": "rule-1"}) + body := applyBody("prod") + body["bucket"] = "s3ish" + resp := applySetup(t, s, body) + if resp.RulesState != "s3_managed" || f.count("b2_update_bucket") != 0 || len(resp.RulesText) == 0 { + t.Fatalf("%+v updates=%d", resp, f.count("b2_update_bucket")) + } + if !strings.Contains(strings.Join(resp.Warnings, " "), "no sure sign") { + t.Fatalf("it must say the detection is a best effort: %v", resp.Warnings) + } + // Declared: the person's tick. + s2, _, f2 := newSetupServer(t) + f2.addBucket("mine", "allPrivate") + body = applyBody("prod") + body["bucket"], body["s3_managed"] = "mine", true + if resp := applySetup(t, s2, body); resp.RulesState != "s3_managed" || f2.count("b2_update_bucket") != 0 { + t.Fatalf("%+v", resp) + } +} + +func TestSetupApply_LifecycleCapabilityRefusedRetriesWithout(t *testing.T) { + s, _, f := newSetupServer(t) + f.refuseLifecycleCap = true + resp := applySetup(t, s, applyBody("prod")) + prod := resultFor(t, resp, "prod") + if !prod.Verified || prod.Lifecycle != "unknown" || !strings.Contains(prod.Note, "lifecycle") { + t.Fatalf("%+v", prod) + } + if f.count("b2_create_key") != 2 { + t.Fatalf("create_key = %d", f.count("b2_create_key")) + } + s.backups.mu.Lock() + defer s.backups.mu.Unlock() + if s.backups.rec.Setup.Keys["prod"].Lifecycle != "unknown" { + t.Fatal("the record must say so too") + } +} + +func TestSetupApply_HideRefusalStopsEverythingAndDeletesTheKey(t *testing.T) { + for name, probe := range map[string]s3probe.Result{ + "refused": {Write: true, Read: true, Delete: s3probe.DeleteRefused, Error: "delete: the store answered 403 (AccessDenied)"}, + "still-visible": {Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: false}, + } { + t.Run(name, func(t *testing.T) { + s, c, f := newSetupServer(t) + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { return probe } + resp := applySetup(t, s, applyBody("prod", "stage")) + if resp.AllOK || !strings.Contains(strings.ToLower(resp.Stopped), "retention") { + t.Fatalf("%+v", resp) + } + if n := c.countOf("backup-target", "set"); n != 0 { + t.Fatalf("nothing may be pushed: %d", n) + } + if len(f.keys) != 0 { + t.Fatalf("a key never pushed must be deleted: %+v", f.keys) + } + if strings.Contains(fmt.Sprint(f.deleted), testTempKeyID) { + t.Fatal("the temporary key stays for the retry") + } + s.backups.mu.Lock() + defer s.backups.mu.Unlock() + if s.backups.rec.Storage != nil || len(s.backups.rec.Setup.Keys) != 0 { + t.Fatalf("nothing is recorded: %+v", s.backups.rec.Setup.Keys) + } + }) + } +} + +func TestSetupApply_ABadKeyDeletesOnlyThatKey(t *testing.T) { + s, c, f := newSetupServer(t) + s.backups.probe = func(_ context.Context, cfg s3probe.Config) s3probe.Result { + if strings.Contains(cfg.Key, "stage") { + return s3probe.Result{Delete: s3probe.DeleteNotReached, Error: "write: the store answered 403"} + } + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} + } + resp := applySetup(t, s, applyBody("prod", "stage")) + if resp.Stopped != "" || !resultFor(t, resp, "prod").Verified || !resultFor(t, resp, "stage").KeyDeleted { + t.Fatalf("%+v", resp) + } + if n := c.countOf("backup-target", "set"); n != 1 { + t.Fatalf("pushes = %d", n) + } + if len(f.keys) != 1 { + t.Fatalf("keys = %+v", f.keys) + } +} + +func TestSetupApply_WithoutDeleteKeysAStrandedKeyIsReported(t *testing.T) { + s, _, f := newSetupServer(t) + f.caps = []string{"listBuckets", "writeBuckets", "writeKeys"} + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { + return s3probe.Result{Delete: s3probe.DeleteNotReached, Error: "write: the store answered 403"} + } + resp := applySetup(t, s, applyBody("prod")) + if resultFor(t, resp, "prod").KeyDeleted { + t.Fatal("this key cannot delete keys") + } + view := s.backupsView(context.Background(), backupsNow) + if len(view.Setup.Leftovers) != 1 || view.Setup.Leftovers[0].Server != "prod" { + t.Fatalf("leftovers = %+v", view.Setup.Leftovers) + } +} + +func TestSetupApply_CapErrorsArePlain(t *testing.T) { + cases := map[string]struct { + call, code, want string + status int + }{ + "cap on keys": {"b2_create_key", "transaction_cap_exceeded", "daily transaction cap", 403}, + "email": {"b2_create_bucket", "email_not_verified", "email address verified", 401}, + "payment": {"b2_create_bucket", "no_payment_history", "payment method", 401}, + "too many": {"b2_create_bucket", "too_many_buckets", "most buckets", 400}, + "cap on bucket": {"b2_list_buckets", "transaction_cap_exceeded", "daily transaction cap", 403}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + s, _, f := newSetupServer(t) + f.echo = testTempKey + f.fail[c.call] = [2]any{c.status, c.code} + rec := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("prod", "stage")) + if !strings.Contains(rec.Body.String(), c.want) { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if strings.Contains(rec.Body.String(), testTempKey) { + t.Fatal("the message text echoed the key") + } + }) + } +} + +func TestSetupApply_TemporaryKeyRules(t *testing.T) { + t.Run("never expires: left alone, and it says so", func(t *testing.T) { + s, _, f := newSetupServer(t) + f.expiry = nil + resp := applySetup(t, s, applyBody("prod")) + if len(f.deleted) != 0 || !strings.Contains(resp.KeyNote, "never expires") { + t.Fatalf("%v %q", f.deleted, resp.KeyNote) + } + }) + t.Run("no deleteKeys: left alone, and it says so", func(t *testing.T) { + s, _, f := newSetupServer(t) + f.caps = []string{"listBuckets", "writeBuckets", "writeKeys"} + resp := applySetup(t, s, applyBody("prod")) + if len(f.deleted) != 0 || !strings.Contains(resp.KeyNote, "cannot delete keys") { + t.Fatalf("%v %q", f.deleted, resp.KeyNote) + } + }) + t.Run("delete fails: it says so", func(t *testing.T) { + s, _, f := newSetupServer(t) + f.fail["b2_delete_key"] = [2]any{500, "internal_error"} + resp := applySetup(t, s, applyBody("prod")) + if !resp.AllOK || !strings.Contains(resp.KeyNote, "could not delete your temporary key") { + t.Fatalf("%q", resp.KeyNote) + } + }) +} + +func TestSetupApply_TemporaryKeyGoesNowhere(t *testing.T) { + var logged bytes.Buffer + log.SetOutput(&logged) + t.Cleanup(func() { log.SetOutput(os.Stderr) }) + s, c, f := newSetupServer(t) + f.echo = testTempKey + dir := t.TempDir() + s.backups.path = filepath.Join(dir, "backups.json") + s.backups.dataDir = dir + + var out bytes.Buffer + check := postJSON(t, s, "/api/backups/setup/storage/check", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey}) + out.Write(check.Body.Bytes()) + apply := postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("prod", "stage")) + out.Write(apply.Body.Bytes()) + // The failure paths too. + f.fail["b2_authorize_account"] = [2]any{401, "bad_auth_token"} + out.Write(postJSON(t, s, "/api/backups/setup/storage/apply", applyBody("prod")).Body.Bytes()) + delete(f.fail, "b2_authorize_account") + f.fail["b2_create_key"] = [2]any{403, "transaction_cap_exceeded"} + body := applyBody("prod") + body["rotate"] = true + out.Write(postJSON(t, s, "/api/backups/setup/storage/apply", body).Body.Bytes()) + + if s.alerts.audit == nil { + t.Fatal("this test needs the audit log") + } + audited, _ := json.Marshal(s.alerts.audit.Recent(200)) + if !strings.Contains(string(audited), "backups.setup.storage") { + t.Fatal("the steps must be audited") + } + s.backups.mu.Lock() + record, _ := json.Marshal(s.backups.rec) + s.backups.mu.Unlock() + sinks := map[string]string{"responses": out.String(), "logs": logged.String(), "audit": string(audited), "record": string(record)} + for name, text := range sinks { + for _, secret := range []string{testTempKey, testTempKeyID} { + if strings.Contains(text, secret) { + t.Errorf("the temporary key reached %s: %s", name, text) + } + } + if strings.Contains(text, testMintSecret) { + t.Errorf("a minted secret reached %s", name) + } + } + // Nor disk, anywhere under the data directory. + filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return nil + } + raw, _ := os.ReadFile(path) + for _, secret := range []string{testTempKey, testTempKeyID, testMintSecret} { + if bytes.Contains(raw, []byte(secret)) { + t.Errorf("%s holds %s", path, secret) + } + } + return nil + }) + // Nor argv, nor stdin: the temporary key is never given to the CLI. + c.mu.Lock() + defer c.mu.Unlock() + for _, call := range c.calls { + text := strings.Join(call.args, " ") + call.stdin + if strings.Contains(text, testTempKey) || strings.Contains(text, testTempKeyID) { + t.Errorf("the temporary key reached the CLI: %v", call.args) + } + if strings.Contains(strings.Join(call.args, " "), testMintSecret) { + t.Errorf("a minted secret reached argv: %v", call.args) + } + } +} + +func TestSetupApply_OldOwnersAreReportedAndClearedOnlyWithConsent(t *testing.T) { + s, c, _ := newSetupServer(t) + cleared := []string{} + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if !subsequence(args, []string{"backup-target", "owners"}) { + return nil, nil, false + } + for i, a := range args { + if a == "--clear" { + cleared = append(cleared, args[i+1]) + if subsequence(args, []string{"--force"}) { + return &cli.Result{Stdout: `{"fingerprint":"oldfp","removed":2,"skipped_locations":[{"prefix":"old","reason":"AccessDenied"}]}`}, nil, true + } + return &cli.Result{ExitCode: 1, Stderr: "a namespace written to in the last week needs --force", Stdout: `{"fingerprint":"oldfp","removed":0,"skipped_locations":[]}`}, nil, true + } + } + return &cli.Result{Stdout: `{"markers":[{"namespace":"blog","fingerprint":"me","key":"k","self":true},{"namespace":"blog","fingerprint":"oldfp","key":"k2","self":false}],"skipped_locations":[]}`}, nil, true + } + resp := applySetup(t, s, applyBody("prod")) + prod := resultFor(t, resp, "prod") + if len(prod.OldOwners) != 1 || prod.OldOwners[0].Fingerprint != "oldfp" { + t.Fatalf("%+v", prod) + } + if len(cleared) != 0 { + t.Fatal("no clear without consent") + } + // A recent namespace needs the person to confirm the old server is gone. + rec := postJSON(t, s, "/api/backups/setup/owners", map[string]any{"server": "prod", "fingerprint": "oldfp"}) + if rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "last week") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + rec = postJSON(t, s, "/api/backups/setup/owners", map[string]any{"server": "prod", "fingerprint": "oldfp", "force": true}) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "AccessDenied") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if rec := postJSON(t, s, "/api/backups/setup/owners", map[string]any{"server": "prod", "fingerprint": "-x y"}); rec.Code != http.StatusBadRequest { + t.Fatalf("a bad fingerprint: %d", rec.Code) + } +} + +// ── recovery key ───────────────────────────────────────────────────────────── + +func TestSetupRecovery_OnlyAPublicKey(t *testing.T) { + s, _, _ := newSetupServer(t) + for _, bad := range []string{"", "AGE-SECRET-KEY-1QQQ", "age1short"} { + if rec := postJSON(t, s, "/api/backups/setup/recovery", map[string]any{"public_key": bad}); rec.Code != http.StatusBadRequest { + t.Errorf("%q: %d", bad, rec.Code) + } + } + if rec := postJSON(t, s, "/api/backups/setup/recovery", map[string]any{"public_key": testRecipient}); rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + // The same again is fine; another would strand backups made to the first. + if rec := postJSON(t, s, "/api/backups/setup/recovery", map[string]any{"public_key": testRecipient}); rec.Code != http.StatusOK { + t.Fatalf("%d", rec.Code) + } + other := "age1" + strings.Repeat("q", 58) + if rec := postJSON(t, s, "/api/backups/setup/recovery", map[string]any{"public_key": other}); rec.Code != http.StatusConflict { + t.Fatalf("%d", rec.Code) + } +} + +// ── turning on ─────────────────────────────────────────────────────────────── + +func setRecipient(s *Server) { + s.backups.mu.Lock() + defer s.backups.mu.Unlock() + s.backups.rec.Recovery = &backupRecovery{PublicKey: testRecipient, ConfirmedAt: backupsNow.Add(-time.Hour)} +} + +func policySetArgs(c *backupCLI, server string) []string { + c.mu.Lock() + defer c.mu.Unlock() + for _, call := range c.calls { + if subsequence(call.args, []string{"backup-policy", "set"}) && subsequence(call.args, []string{server}) { + return call.args + } + } + return nil +} + +func TestSetupTurnOn_NeedsARecoveryKey(t *testing.T) { + s, c, _ := newSetupServer(t) + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "now"}) + if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "recovery key") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if c.countOf("backup-policy", "set") != 0 { + t.Fatal("no plaintext policy from the simple flow") + } +} + +func TestSetupTurnOn_BackUpNowWritesThePolicyAndStartsADetachedRun(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if subsequence(args, []string{"backup", "run", "--all", "--detach"}) { + return &cli.Result{Stdout: corpusFixtureFile(t, "detach-started.json")}, nil, true + } + return nil, nil, false + } + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "now", "adopt": true}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var out struct { + Servers []setupTurnOnResult `json:"servers"` + AllOK bool `json:"all_ok"` + } + decodeData(t, rec, &out) + r := out.Servers[0] + if !out.AllOK || r.Run == nil || r.Run.Status != "started" || r.AllowEmpty { + t.Fatalf("%+v", out) + } + args := strings.Join(policySetArgs(c, "prod"), " ") + for _, want := range []string{"--interval 24h", "--recipient " + testRecipient, "--volumes=true", "--keep-daily=7", "--allow-empty=false", "--adopt redis-cache:redis:all"} { + if !strings.Contains(args, want) { + t.Errorf("policy args %q miss %q", args, want) + } + } + if strings.Contains(args, "analytics-ch") { + t.Error("an engine with no backup method cannot be adopted") + } + if c.countOf("stage") != 0 { + t.Error("an unticked server is untouched") + } +} + +func TestSetupTurnOn_NoAdoptWithoutTheTick(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "next"}) + if strings.Contains(strings.Join(policySetArgs(c, "prod"), " "), "--adopt") { + t.Fatal("adopting needs the tick") + } +} + +func TestSetupTurnOn_NextRunStartsNothingAndAnEmptyServerAllowsEmpty(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + empty := `{"machine_interface":2,"host":"h","observed_at":"2026-11-04T03:17:04Z","target":{"configured":true,"bucket":"b","prefix":"p","has_credentials":true,"reachable":true},"policy":{"configured":false,"cron_installed":false,"retention":{},"volumes":false,"heartbeat_count":0},"encrypted":false,"aws_cli":true,"age_cli":true,"entries":[],"uncovered_databases":[]}` + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: empty}, nil } + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "next"}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(strings.Join(policySetArgs(c, "prod"), " "), "--allow-empty=true") { + t.Fatalf("a new empty server must not fail: %v", policySetArgs(c, "prod")) + } + if c.countOf("--detach") != 0 { + t.Fatal("next scheduled run starts nothing") + } +} + +func TestSetupTurnOn_ServerWithoutStorageIsSkipped(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + c.status["prod"] = func() (*cli.Result, error) { + return &cli.Result{Stdout: corpusFixtureFile(t, "status-unconfigured.json")}, nil + } + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "now"}) + var out struct { + Servers []setupTurnOnResult `json:"servers"` + AllOK bool `json:"all_ok"` + } + decodeData(t, rec, &out) + if out.AllOK || !strings.Contains(out.Servers[0].Error, "no storage") || c.countOf("backup-policy", "set") != 0 { + t.Fatalf("%+v", out) + } +} + +func TestSetupRun_ReportsEachDetachedOutcome(t *testing.T) { + for fixture, want := range map[string]string{"detach-started.json": "started", "detach-already-running.json": "already_running", "detach-finished.json": "finished", "detach-failed.json": "failed"} { + s, c, _ := newSetupServer(t) + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if subsequence(args, []string{"--detach"}) { + // finished and failed exit nonzero with a report. + return &cli.Result{Stdout: corpusFixtureFile(t, fixture), ExitCode: 1}, nil, true + } + return nil, nil, false + } + rec := postJSON(t, s, "/api/backups/setup/run", map[string]any{"servers": []string{"prod"}}) + var out struct { + Runs map[string]backupSetupRun `json:"runs"` + } + decodeData(t, rec, &out) + got := out.Runs["prod"].Status + if want == "finished" { // the fixture's run ended ok or not: both are terminal and truthful + if got != "finished" && got != "failed" { + t.Errorf("%s: %s", fixture, got) + } + continue + } + if got != want { + t.Errorf("%s: %s", fixture, got) + } + } +} + +func TestBackupStorageMultiplier(t *testing.T) { + std := backupStatPolicy{Interval: "24h", Retention: backupRetention{KeepDaily: 7, KeepWeekly: 4, KeepMonthly: 12}} + if got := backupStorageMultiplier(std, 14); got != 39 { + t.Fatalf("14 days: %d", got) + } + if got := backupStorageMultiplier(std, 30); got != 55 { + t.Fatalf("30 days: %d", got) + } +} + +// ── proving recovery ───────────────────────────────────────────────────────── + +func seedEncryptedObject(s *Server, server string) string { + key := "s3://b/teploy/" + server + "/blog/accessories/db/1.pgdump.age" + env := backupStatusEnvelope{Host: server, ObservedAt: time.Now(), Entries: []backupEntry{{Name: "blog-db", Kind: "accessory", App: "blog", Accessory: "db", Coverage: "covered", + Newest: &backupObject{Key: key, SizeBytes: 1 << 20, CreatedAt: time.Now().UTC().Format(time.RFC3339), Encrypted: true}, Encrypted: true}}, UncoveredDatabases: []string{}} + s.recordBackupStatus(server, &env, nil, time.Now().UTC()) + return key +} + +func TestSetupProof_ReadsThenRecordsObjectAndActor(t *testing.T) { + s, c, _ := newSetupServer(t) + if rec := postJSON(t, s, "/api/backups/setup/proof/start", map[string]any{}); rec.Code != http.StatusBadRequest { + t.Fatalf("no key yet: %d", rec.Code) + } + s.backups.mu.Lock() + s.backups.rec.Recovery = &backupRecovery{PublicKey: testRecipient, ConfirmedAt: time.Now().Add(-time.Hour)} + s.backups.mu.Unlock() + if rec := postJSON(t, s, "/api/backups/setup/proof/start", map[string]any{}); rec.Code != http.StatusConflict { + t.Fatalf("no object yet: %d", rec.Code) + } + key := seedEncryptedObject(s, "prod") + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if !subsequence(args, []string{"backup", "head"}) { + return nil, nil, false + } + if !subsequence(args, []string{key, "--bytes", "131072", "--host", "prod", "--json"}) { + t.Errorf("head args: %v", args) + } + raw, _ := json.Marshal(map[string]any{"machine_interface": 2, "host": "prod", "key": key, "encrypted": true, "requested_bytes": 131072, "length": 3, "total_size": 1 << 20, "data_base64": "AAEC"}) + return &cli.Result{Stdout: string(raw)}, nil, true + } + rec := postJSON(t, s, "/api/backups/setup/proof/start", map[string]any{}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + var start struct { + Key string `json:"key"` + Data string `json:"data_base64"` + } + decodeData(t, rec, &start) + if start.Key != key || start.Data != "AAEC" { + t.Fatalf("%+v", start) + } + // Failure is recorded and proves nothing. + if rec := postJSON(t, s, "/api/backups/setup/proof/result", map[string]any{"key": key, "passed": false, "detail": "no identity matched"}); rec.Code != http.StatusOK { + t.Fatalf("%d", rec.Code) + } + s.backups.mu.Lock() + passed := s.backups.rec.Recovery.DrillPassedAt + s.backups.mu.Unlock() + if passed != nil { + t.Fatal("a failed proof is not a pass") + } + if rec := postJSON(t, s, "/api/backups/setup/proof/result", map[string]any{"key": "s3://b/other", "passed": true}); rec.Code != http.StatusBadRequest { + t.Fatalf("another object: %d", rec.Code) + } + if rec := postJSON(t, s, "/api/backups/setup/proof/result", map[string]any{"key": key, "passed": true}); rec.Code != http.StatusOK { + t.Fatalf("%d", rec.Code) + } + s.backups.mu.Lock() + defer s.backups.mu.Unlock() + proof := s.backups.rec.Setup.Proof + if s.backups.rec.Recovery.DrillPassedAt == nil || proof == nil || proof.Object != key || proof.Server != "prod" || proof.Actor == "" { + t.Fatalf("%+v", proof) + } +} + +func TestSetupProof_RefusesAnUnencryptedObject(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + s.backups.mu.Lock() + s.backups.rec.Recovery.ConfirmedAt = time.Now().Add(-time.Hour) + s.backups.mu.Unlock() + key := seedEncryptedObject(s, "prod") + c.hook = func(args []string, _ string) (*cli.Result, error, bool) { + if subsequence(args, []string{"backup", "head"}) { + raw, _ := json.Marshal(map[string]any{"machine_interface": 2, "host": "prod", "key": key, "encrypted": false, "length": 3, "data_base64": "AAEC"}) + return &cli.Result{Stdout: string(raw)}, nil, true + } + return nil, nil, false + } + if rec := postJSON(t, s, "/api/backups/setup/proof/start", map[string]any{}); rec.Code != http.StatusConflict { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } +} + +// ── retiring a server ──────────────────────────────────────────────────────── + +func retireBody(server, data string) map[string]any { + return map[string]any{"key_id": testTempKeyID, "application_key": testTempKey, "server": server, "data": data} +} + +func TestSetupRetire_DeletesTheKeyAndKeepsOrRemovesData(t *testing.T) { + s, c, f := newSetupServer(t) + f.expiry = nil + applySetup(t, s, applyBody("prod", "stage")) + s.backups.mu.Lock() + prodKey := s.backups.rec.Setup.Keys["prod"].KeyID + stageKey := s.backups.rec.Setup.Keys["stage"].KeyID + s.backups.mu.Unlock() + + // An explicit choice, every time. + if rec := postJSON(t, s, "/api/backups/setup/retire", map[string]any{"key_id": testTempKeyID, "application_key": testTempKey, "server": "prod"}); rec.Code != http.StatusBadRequest { + t.Fatalf("%d", rec.Code) + } + rec := postJSON(t, s, "/api/backups/setup/retire", retireBody("prod", "keep")) + if rec.Code != http.StatusOK || !strings.Contains(fmt.Sprint(f.deleted), prodKey) { + t.Fatalf("%d %s %v", rec.Code, rec.Body.String(), f.deleted) + } + if f.count("b2_delete_file_version") != 0 { + t.Fatal("keep means keep") + } + if c.countOf("backup-policy", "set", "--interval", "none") != 1 { + t.Fatal("a server still in the fleet stops trying") + } + + // Removal needs the name typed, and permissions. + if rec := postJSON(t, s, "/api/backups/setup/retire", retireBody("stage", "remove")); rec.Code != http.StatusBadRequest { + t.Fatalf("%d", rec.Code) + } + body := retireBody("stage", "remove") + body["confirm"] = "stage" + if rec := postJSON(t, s, "/api/backups/setup/retire", body); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "deleteFiles") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + f.caps = append(f.caps, "listFiles", "deleteFiles") + rec = postJSON(t, s, "/api/backups/setup/retire", body) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if !strings.Contains(fmt.Sprint(f.deleted), stageKey) || f.count("b2_delete_file_version") != 1 { + t.Fatalf("%v %d", f.deleted, f.count("b2_delete_file_version")) + } + if strings.Contains(fmt.Sprint(f.bodies["b2_list_file_versions"][0]["prefix"]), "prod") { + t.Fatal("only this server's data") + } + raw, _ := json.Marshal(f.buckets) + if strings.Contains(string(raw), "teploy/stage/.incoming/") { + t.Fatalf("its rule goes too: %s", raw) + } + s.backups.mu.Lock() + defer s.backups.mu.Unlock() + if _, ok := s.backups.rec.Setup.Keys["stage"]; ok { + t.Fatal("record stays") + } + if _, ok := s.backups.rec.Storage.Prefixes["stage"]; ok { + t.Fatal("prefix stays") + } +} + +func TestSetupRetire_NeedsDeleteKeysAndARecord(t *testing.T) { + s, _, f := newSetupServer(t) + f.expiry = nil + if rec := postJSON(t, s, "/api/backups/setup/retire", retireBody("prod", "keep")); rec.Code != http.StatusBadRequest { + t.Fatalf("no record: %d", rec.Code) + } + applySetup(t, s, applyBody("prod")) + f.caps = []string{"listBuckets"} + if rec := postJSON(t, s, "/api/backups/setup/retire", retireBody("prod", "keep")); rec.Code != http.StatusUnprocessableEntity || !strings.Contains(rec.Body.String(), "deleteKeys") { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } +} + +// ── what "Protected" means ─────────────────────────────────────────────────── + +func statusWith(entries []backupEntry, encrypted bool) *backupStatusEnvelope { + return &backupStatusEnvelope{Host: "h", ObservedAt: backupsNow, + Target: backupStatTarget{Configured: true, Bucket: "b", HasCredentials: true, Reachable: true, IncomingLifecycle: "covered"}, + Policy: backupStatPolicy{Configured: true, Interval: "24h"}, Encrypted: encrypted, Entries: entries, UncoveredDatabases: []string{}} +} + +func goodEntry(name, kind string) backupEntry { + at := backupsNow.Add(-time.Hour) + return backupEntry{Name: name, Kind: kind, App: "blog", Coverage: "covered", LastSuccess: &at, Encrypted: true, + Newest: &backupObject{Key: "k", SizeBytes: 100, CreatedAt: at.Format(time.RFC3339), Encrypted: true}} +} + +func setupStateOf(t *testing.T, s *Server, server string) backupSetupServerView { + t.Helper() + view := s.backupsView(context.Background(), backupsNow) + for _, sv := range view.Setup.Servers { + if sv.Server == server { + return sv + } + } + t.Fatalf("no state for %s", server) + return backupSetupServerView{} +} + +func TestSetupStatus_VolumeOnlyServerNeedsNoRestoreTest(t *testing.T) { + s, _, _ := newSetupServer(t) + s.recordBackupStatus("prod", statusWith([]backupEntry{goodEntry("blog/volumes", "volumes")}, true), nil, backupsNow) + if got := setupStateOf(t, s, "prod"); got.State != setupProtected { + t.Fatalf("%+v", got) + } + view := s.backupsView(context.Background(), backupsNow) + if view.Setup.State != "protected" || view.Setup.ProjectedBytes != 100*view.Setup.Multiplier { + t.Fatalf("%+v", view.Setup) + } +} + +func TestSetupStatus_ADatabaseNeedsAPassingRestoreTest(t *testing.T) { + s, _, _ := newSetupServer(t) + s.recordBackupStatus("prod", statusWith([]backupEntry{goodEntry("blog-db", "accessory")}, true), nil, backupsNow) + if got := setupStateOf(t, s, "prod"); got.State != setupWaitingTest { + t.Fatalf("no test yet: %+v", got) + } + test := &backupAutoTest{ID: "t1", Server: "prod", Container: "blog-db", App: "blog"} + s.backups.mu.Lock() + s.backups.rec.Tests["t1"] = test + s.backups.mu.Unlock() + if got := setupStateOf(t, s, "prod"); got.State != setupWaitingTest { + t.Fatalf("never run: %+v", got) + } + // Repeated guard skips say why, instead of waiting silently. + for i := 0; i < backupGuardSkipsShown; i++ { + test.record(backupTestRun{At: backupsNow, Skipped: "waiting: 300 MB free RAM (< 1024)"}) + } + if got := setupStateOf(t, s, "prod"); got.State != setupGuardSkipped || !strings.Contains(got.Detail, "300 MB") { + t.Fatalf("%+v", got) + } + test.record(backupTestRun{At: backupsNow, OK: false, Detail: "boom"}) + if got := setupStateOf(t, s, "prod"); got.State != setupTestFailed { + t.Fatalf("%+v", got) + } + test.record(backupTestRun{At: backupsNow, OK: true}) + if got := setupStateOf(t, s, "prod"); got.State != setupProtected { + t.Fatalf("%+v", got) + } + // A skip after a pass does not undo the pass. + test.record(backupTestRun{At: backupsNow, Skipped: "waiting"}) + if got := setupStateOf(t, s, "prod"); got.State != setupProtected { + t.Fatalf("%+v", got) + } +} + +func TestSetupStatus_States(t *testing.T) { + entry := func(mut func(*backupEntry)) []backupEntry { + e := goodEntry("blog/volumes", "volumes") + mut(&e) + return []backupEntry{e} + } + cases := []struct { + name string + env *backupStatusEnvelope + err error + want string + }{ + {"no target", func() *backupStatusEnvelope { e := statusWith(nil, false); e.Target = backupStatTarget{}; return e }(), nil, setupNotProtected}, + {"no policy", func() *backupStatusEnvelope { e := statusWith(nil, false); e.Policy = backupStatPolicy{}; return e }(), nil, setupNotProtected}, + {"empty", statusWith(nil, true), nil, setupEmpty}, + {"first backup", statusWith(entry(func(e *backupEntry) { e.Coverage, e.LastSuccess = "stale", nil }), true), nil, setupWaitingBackup}, + {"late", statusWith(entry(func(e *backupEntry) { e.Coverage = "stale" }), true), nil, setupLate}, + {"failing", statusWith(entry(func(e *backupEntry) { e.Coverage, e.LastError = "failing", "disk full" }), true), nil, setupFailing}, + {"not yet encrypted", statusWith(entry(func(e *backupEntry) { e.Encrypted = false }), true), nil, setupWaitingBackup}, + {"unread", nil, fmt.Errorf("ssh: timeout"), setupUnknown}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + s, _, _ := newSetupServer(t) + s.recordBackupStatus("prod", c.env, c.err, backupsNow) + if got := setupStateOf(t, s, "prod"); got.State != c.want { + t.Fatalf("%+v", got) + } + if s.backupsView(context.Background(), backupsNow).Setup.State == "protected" { + t.Fatal("not protected") + } + }) + } +} + +func TestSetupStatus_ACappedTargetIsNotUnreachable(t *testing.T) { + s, _, _ := newSetupServer(t) + env := statusWith(nil, true) + env.Target.Reachable = false + env.Target.Error = "An error occurred (TransactionCapExceeded) when calling ListObjectsV2" + s.recordBackupStatus("prod", env, nil, backupsNow) + got := setupStateOf(t, s, "prod") + if got.State != setupCapped || !strings.Contains(got.Detail, "daily transaction cap") { + t.Fatalf("%+v", got) + } + s.backups.mu.Lock() + obs, _ := backupObservations(s.backups.rec.Servers["prod"], backupsNow) + s.backups.mu.Unlock() + found := false + for _, o := range obs { + if strings.Contains(o.rule.Key, ":target:") { + found = true + if strings.Contains(o.message, "cannot reach") || !strings.Contains(o.message, "daily transaction cap") { + t.Fatalf("alert = %q", o.message) + } + } + } + if !found { + t.Fatal("no target observation") + } +} + +func TestSetupStatus_NewAndRetiredServers(t *testing.T) { + s, _, _ := newSetupServer(t) + s.backups.mu.Lock() + s.backups.rec.Storage = &backupStorage{Provider: "b2", Bucket: "b", Prefixes: map[string]string{}} + s.backups.setupLocked().Keys["gone"] = &backupServerKey{KeyID: "K", KeyName: "teploy-gone-1", Bucket: "b", Prefix: "teploy/gone", State: backupKeyVerified} + s.backups.rec.LastSweepAt = backupsNow + s.backups.mu.Unlock() + env := statusWith(nil, false) + env.Target = backupStatTarget{} + s.recordBackupStatus("compute-3", env, nil, backupsNow) + view := s.backupsView(context.Background(), backupsNow) + if fmt.Sprint(view.Setup.NewServers) != "[compute-3]" || fmt.Sprint(view.Setup.RetiredKeys) != "[gone]" { + t.Fatalf("%+v", view.Setup) + } + if got := setupStateOf(t, s, "compute-3"); !strings.Contains(got.Detail, "compute-3 is not backed up") { + t.Fatalf("%+v", got) + } +} + +func TestSetupStatus_LifecycleRecheckedOnEachRead(t *testing.T) { + s, _, _ := newSetupServer(t) + for lifecycle, opens := range map[string]bool{"absent": true, "covered": false} { + env := statusWith(nil, true) + env.Target.IncomingLifecycle = lifecycle + s.recordBackupStatus("prod", env, nil, backupsNow) + s.backups.mu.Lock() + obs, _ := backupObservations(s.backups.rec.Servers["prod"], backupsNow) + s.backups.mu.Unlock() + failing := false + for _, o := range obs { + if strings.Contains(o.rule.Key, ":lifecycle:") { + failing = !o.ok + } + } + if failing != opens { + t.Errorf("%s: failing = %v", lifecycle, failing) + } + } + // A key that cannot read rules proves nothing either way. + env := statusWith(nil, true) + env.Target.IncomingLifecycle = "unknown" + s.recordBackupStatus("prod", env, nil, backupsNow) + s.backups.mu.Lock() + obs, _ := backupObservations(s.backups.rec.Servers["prod"], backupsNow) + s.backups.mu.Unlock() + for _, o := range obs { + if strings.Contains(o.rule.Key, ":lifecycle:") { + t.Fatalf("unknown raises nothing: %+v", o) + } + } +} + +// ── Dash's own volumes ─────────────────────────────────────────────────────── + +func TestPolicy_NoRecipientExcludesDashsOwnVolumes(t *testing.T) { + s, c, _ := newSetupServer(t) + s.backups.selfApp = "dash" + c.policyShow["prod"] = `{"machine_interface":2,"configured":true,"excludes":["blog/volumes"]}` + c.policyShow["stage"] = `{"machine_interface":2,"configured":true}` + rec := postJSON(t, s, "/api/backups/policy", map[string]any{"interval": "24h", "preset": "simple", "volumes": true}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + args := strings.Join(policySetArgs(c, "prod"), " ") + if !strings.Contains(args, "--exclude=blog/volumes,dash/volumes") { + t.Fatalf("Dash's SSH key must not go up in plaintext: %s", args) + } + + // Including them back is refused until there is a key. + if rec := postJSON(t, s, "/api/backups/exclude", map[string]any{"server": "prod", "name": "dash/volumes", "excluded": false}); rec.Code != http.StatusBadRequest { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + + // With a key, Dash's own entry comes back out, and a person's stays. + setRecipient(s) + c.mu.Lock() + c.calls = nil + c.mu.Unlock() + c.policyShow["prod"] = `{"machine_interface":2,"configured":true,"excludes":["blog/volumes","dash/volumes"]}` + rec = postJSON(t, s, "/api/backups/policy", map[string]any{"interval": "24h", "preset": "simple", "volumes": true}) + if rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + args = strings.Join(policySetArgs(c, "prod"), " ") + if !strings.Contains(args, "--exclude=blog/volumes") || strings.Contains(args, "dash/volumes") { + t.Fatalf("%s", args) + } +} + +func TestPolicy_AnEncryptedServerKeepsItsOwnVolumes(t *testing.T) { + s, c, _ := newSetupServer(t) + c.policyShow["prod"] = `{"machine_interface":2,"configured":true,"recipient":"` + testRecipient + `"}` + c.policyShow["stage"] = `{"machine_interface":2,"configured":true,"recipient":"` + testRecipient + `"}` + postJSON(t, s, "/api/backups/policy", map[string]any{"interval": "24h", "preset": "simple", "volumes": true}) + if strings.Contains(strings.Join(policySetArgs(c, "prod"), " "), "--exclude") { + t.Fatal("a server encrypting already needs no exclude") + } +} + +func TestSetupState_PersistsWithoutSecrets(t *testing.T) { + dir := t.TempDir() + c := &backupCLI{status: statusesFromTestdata(t), tokens: []string{"backup-setup-v1"}} + withTempGroupsFile(t) + writeNotifications(t, `{}`) + s := New(Config{DataDir: dir, NoAuth: true, CLIInstalled: func() bool { return true }, CLIRunner: c.run, CLIStdinRunner: c.runStdin, OperationMaxConcurrent: 4}) + _, client := newFakeBackblaze(t) + s.backups.b2Client = client + s.backups.probe = func(context.Context, s3probe.Config) s3probe.Result { + return s3probe.Result{Write: true, Read: true, Delete: s3probe.DeleteRemoved, Hidden: true} + } + applySetup(t, s, applyBody("prod")) + raw, err := os.ReadFile(filepath.Join(dir, "backups.json")) + if err != nil { + t.Fatal(err) + } + info, _ := os.Stat(filepath.Join(dir, "backups.json")) + if info.Mode().Perm() != 0600 { + t.Fatalf("mode = %v", info.Mode()) + } + for _, secret := range []string{testTempKey, testTempKeyID, testMintSecret} { + if bytes.Contains(raw, []byte(secret)) { + t.Fatalf("disk holds %s: %s", secret, raw) + } + } + // It reloads. + s2 := New(Config{DataDir: dir, NoAuth: true, CLIInstalled: func() bool { return true }, CLIRunner: c.run, CLIStdinRunner: c.runStdin, OperationMaxConcurrent: 4}) + s2.backups.mu.Lock() + defer s2.backups.mu.Unlock() + if s2.backups.rec.Setup == nil || s2.backups.rec.Setup.Keys["prod"] == nil { + t.Fatalf("%+v", s2.backups.rec.Setup) + } +} diff --git a/internal/server/backupstorage.go b/internal/server/backupstorage.go index 3fd405e..64e885a 100644 --- a/internal/server/backupstorage.go +++ b/internal/server/backupstorage.go @@ -15,10 +15,12 @@ import ( "net/url" "regexp" "sort" + "strconv" "strings" "sync" "time" + "github.com/useteploy/teploy-dash/internal/b2" "github.com/useteploy/teploy-dash/internal/monitor" "github.com/useteploy/teploy-dash/internal/s3probe" ) @@ -32,8 +34,9 @@ const ( const ( backupPushConcurrency = 3 - // backupPushTimeout covers the CLI's set (which may install age and the aws - // CLI, then rewrites cron lines) and the status read that verifies it. + // backupPushTimeout covers the CLI's set (which may install age, and refuses + // rather than installs a missing aws CLI, then rewrites cron lines) and the status + // read that verifies it. backupPushTimeout = 10 * time.Minute backupProbeTimeout = 45 * time.Second ) @@ -325,6 +328,15 @@ func (s *Server) handleBackupStorage(w http.ResponseWriter, r *http.Request) { b.rec.Storage = &backupStorage{Provider: body.Provider, Bucket: body.Bucket, Region: body.Region, Endpoint: body.Endpoint, BasePrefix: body.BasePrefix, AllowInternal: body.AllowInternal, Prefixes: prefixes, ConnectedAt: now} } + // A key entered by hand replaces what the setup flow made: say where that one went. + if setup := b.rec.Setup; setup != nil { + for _, res := range resp.Servers { + if k := setup.Keys[res.Server]; k != nil && res.Pushed { + setup.Leftovers = append(setup.Leftovers, backupLeftoverKey{KeyID: k.KeyID, KeyName: k.KeyName, Server: res.Server, Reason: "replaced by a key entered by hand"}) + delete(setup.Keys, res.Server) + } + } + } b.saveLocked() b.mu.Unlock() } @@ -344,9 +356,10 @@ func (s *Server) handleBackupStorage(w http.ResponseWriter, r *http.Request) { } // probeBackupStorage runs the write/read/delete probe. B2 keys deliberately lack -// deleteFiles (plan D2), so there a refused delete is the expected, safer setup. -// The object sits in the first server's .incoming/ tree, where the lifecycle rule -// the CLI's README asks for expires it if it could not be removed. +// deleteFiles (plan D2), and retention's delete must still hide the object through such +// a key, or retention cannot work: so on B2 a refused delete stops here, and an accepted +// one must leave the object unreadable. The object sits in the first server's .incoming/ +// tree, where the lifecycle rule removes it. func (s *Server) probeBackupStorage(ctx context.Context, body *backupStorageRequest, firstPrefix string) (s3probe.Result, string, bool) { pctx, cancel := context.WithTimeout(ctx, backupProbeTimeout) defer cancel() @@ -363,12 +376,20 @@ func (s *Server) probeBackupStorage(ctx context.Context, body *backupStorageRequ } switch res.Delete { case s3probe.DeleteRemoved: + if body.Provider == backupProviderB2 { + if !res.Hidden { + res.Error = "Backblaze accepted the delete, but the test object can still be read. Retention would not work with this key, so nothing was pushed." + return res, "", false + } + res.Error = "" + return res, "This key hid the test object, which is how retention works on B2. A key without deleteFiles cannot destroy history: hidden copies can still be recovered until the bucket's lifecycle rule removes them after " + strconv.Itoa(b2.DefaultHideDays) + " days.", true + } res.Error = "" return res, "", true case s3probe.DeleteRefused: - if body.Provider == backupProviderB2 { - res.Error = "" - return res, "This key cannot delete files, which is the recommended B2 setup: a compromised server cannot destroy history. Retention hides files, and the bucket's lifecycle rules remove them after 30 days. The test object stays under .incoming/ until that rule expires it.", true + if body.Provider == backupProviderB2 && !body.DeleteRefusalOK { + res.Error = "Backblaze refused to hide the test object through this key, so retention cannot work: old backups would never be removed, and growth would be the only symptom. Nothing was pushed. Give the key writeFiles and try again." + return res, "", false } if body.DeleteRefusalOK { res.Error = "" @@ -471,11 +492,11 @@ func backupLifecycleRule(st *backupStorage, target backupStatTarget) string { warn := " Scope it to that prefix: a rule on the whole bucket deletes the backups." if st != nil && st.Provider == backupProviderB2 || strings.Contains(target.Endpoint, "backblazeb2.com") { // B2 keeps hidden versions forever by default, so retention frees no storage - // without the second rule. Its 30 days are also how long hidden backups can + // without the second rule. Its 14 days are also how long hidden backups can // still be recovered after a compromised server hides them. Overlapping rules // are allowed; B2 applies the smallest value of each property. return "Add two B2 lifecycle rules. First: fileNamePrefix " + incoming + ", daysFromUploadingToHiding 1, daysFromHidingToDeleting 1." + warn + - " Second: fileNamePrefix " + root + ", daysFromHidingToDeleting 30, daysFromUploadingToHiding empty. Without it, backups that retention removes stay stored and billed forever; never set daysFromUploadingToHiding on this one." + " Second: fileNamePrefix " + root + ", daysFromHidingToDeleting 14, daysFromUploadingToHiding empty. Without it, backups that retention removes stay stored and billed forever; never set daysFromUploadingToHiding on this one." } return `aws s3api put-bucket-lifecycle-configuration --bucket ` + target.Bucket + ` --lifecycle-configuration '{"Rules":[{"ID":"teploy-incoming","Status":"Enabled","Filter":{"Prefix":"` + incoming + diff --git a/internal/server/backuptests_test.go b/internal/server/backuptests_test.go index fcc4680..36af8d4 100644 --- a/internal/server/backuptests_test.go +++ b/internal/server/backuptests_test.go @@ -473,3 +473,14 @@ func TestBackupsPage_HandwrittenSurface(t *testing.T) { t.Error("the rule must say removal stays a human action") } } + +// A new user with no such jobs never sees the list. +func TestBackupsPage_HandwrittenHiddenWithoutEvidence(t *testing.T) { + c := &backupCLI{} + s := newBackupsServer(t, c) + s.recordBackupStatus("prod", mustDecodeStatus(t, "status-unconfigured.json"), nil, backupsNow) + view := s.backupsView(context.Background(), backupsNow) + if len(view.Handwritten) != 0 || view.HandwrittenRule != "" { + t.Fatalf("no evidence of hand-written jobs, yet listed: %+v", view.Handwritten) + } +} diff --git a/internal/server/capabilities.go b/internal/server/capabilities.go index 1ce80ea..fe42a60 100644 --- a/internal/server/capabilities.go +++ b/internal/server/capabilities.go @@ -94,6 +94,9 @@ type machineFeatures struct { // for Teploy/forgejo discovery endpoints. The whole Programme B surface // (Siblings page, Observe connect, telemetry opt-in) gates on it. Siblings bool `json:"siblings"` // siblings-v1 + // BackupSetup gates "Protect my data" riding `server backup-preflight`, `backup + // head` and `backup run --all --detach` (CLI v0.1.55; see backupSetupMinCLI). + BackupSetup bool `json:"backup_setup"` // backup-setup-v1 } type capabilityError struct { @@ -197,6 +200,7 @@ func (s *Server) capabilities(ctx context.Context) capabilities { value.Features.BackupFleet = hasToken("backup-fleet-v1") value.Features.ServerAudit = hasToken("server-audit-v1") value.Features.Siblings = hasToken("siblings-v1") + value.Features.BackupSetup = hasToken("backup-setup-v1") } } else { // Pre-MI CLI (or version --json refused): the legacy path. diff --git a/internal/server/caps.go b/internal/server/caps.go index 697028d..5605b56 100644 --- a/internal/server/caps.go +++ b/internal/server/caps.go @@ -77,6 +77,9 @@ var administerCredentialsPrefixes = []string{ // server, and the recovery key's generate endpoint shows the private key. "/api/backups/storage", "/api/backups/recovery-key", + // "Protect my data": each step handles a Backblaze key or a server's storage, or + // reads backup objects. The state it reports is on GET /api/backups. + "/api/backups/setup", // The audit trail records credential and alerting changes (who // configured which channel, who silenced what). "/api/audit", diff --git a/internal/server/contracts_corpus_test.go b/internal/server/contracts_corpus_test.go index 74f3558..25108ac 100644 --- a/internal/server/contracts_corpus_test.go +++ b/internal/server/contracts_corpus_test.go @@ -77,6 +77,23 @@ func TestContractsCorpusVersionHandshakeDecodes(t *testing.T) { } } +// Corpus rev 21: the setup flow's three envelopes go through its real decoders. +func TestContractsCorpusBackupSetupDecodes(t *testing.T) { + for _, name := range []string{"ready", "needs-setup", "unknown", "unreachable", "after-actions"} { + if _, err := decodeBackupPreflight(readCorpusFixture(t, "backup-preflight-envelope/valid/"+name+".json")); err != nil { + t.Errorf("preflight %s: %v", name, err) + } + } + if _, err := decodeBackupHead(readCorpusFixture(t, "backup-head-envelope/valid/encrypted.json")); err != nil { + t.Errorf("head: %v", err) + } + for _, name := range []string{"started", "already-running", "finished", "failed"} { + if _, err := decodeBackupDetach(readCorpusFixture(t, "backup-detach-envelope/valid/"+name+".json")); err != nil { + t.Errorf("detach %s: %v", name, err) + } + } +} + func TestContractsCorpusAppListEnvelopeDecodes(t *testing.T) { raw := readCorpusFixture(t, "app-list-envelope/valid/mi2.json") if _, err := cli.ParseJSON(raw); err != nil { diff --git a/internal/server/testdata/backup-contracts/detach-already-running.json b/internal/server/testdata/backup-contracts/detach-already-running.json new file mode 100644 index 0000000..6c64337 --- /dev/null +++ b/internal/server/testdata/backup-contracts/detach-already-running.json @@ -0,0 +1,6 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "requested_at": "2026-11-06T12:00:00Z", + "status": "already_running" +} diff --git a/internal/server/testdata/backup-contracts/detach-failed.json b/internal/server/testdata/backup-contracts/detach-failed.json new file mode 100644 index 0000000..13d57c5 --- /dev/null +++ b/internal/server/testdata/backup-contracts/detach-failed.json @@ -0,0 +1,9 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "requested_at": "2026-11-06T12:00:00Z", + "status": "failed", + "pid": 4245, + "log": "/root/.config/teploy/backup-runs/run.MnOp3456.log", + "error": "the run ended without a result: Error: docker: permission denied" +} diff --git a/internal/server/testdata/backup-contracts/detach-finished.json b/internal/server/testdata/backup-contracts/detach-finished.json new file mode 100644 index 0000000..3ae97a0 --- /dev/null +++ b/internal/server/testdata/backup-contracts/detach-finished.json @@ -0,0 +1,16 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "requested_at": "2026-11-06T12:00:00Z", + "status": "finished", + "pid": 4244, + "log": "/root/.config/teploy/backup-runs/run.IjKl9012.log", + "result": { + "status": "failed", + "started_at": "2026-11-06T12:00:00Z", + "duration_ms": 0, + "encrypted": false, + "targets": [], + "error": "no backup policy on this server: set one with `teploy server backup-policy set`" + } +} diff --git a/internal/server/testdata/backup-contracts/detach-started.json b/internal/server/testdata/backup-contracts/detach-started.json new file mode 100644 index 0000000..86b0709 --- /dev/null +++ b/internal/server/testdata/backup-contracts/detach-started.json @@ -0,0 +1,8 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "requested_at": "2026-11-06T12:00:00Z", + "status": "started", + "pid": 4242, + "log": "/root/.config/teploy/backup-runs/run.AbCd1234.log" +} diff --git a/internal/server/testdata/backup-contracts/preflight-after-actions.json b/internal/server/testdata/backup-contracts/preflight-after-actions.json new file mode 100644 index 0000000..10e77f6 --- /dev/null +++ b/internal/server/testdata/backup-contracts/preflight-after-actions.json @@ -0,0 +1,61 @@ +{ + "machine_interface": 2, + "host": "h", + "observed_at": "2026-11-06T12:00:00Z", + "reachable": true, + "binary": { + "path": "/usr/local/bin/teploy", + "state": "ok", + "version": "0.1.55", + "fleet_engine": true, + "cli_version": "0.1.55", + "same_as_cli": true + }, + "tools": { + "aws": { + "state": "ok", + "version": "2.30.1" + }, + "age": { + "state": "ok", + "version": "v1.2.1" + }, + "python3": { + "state": "ok", + "version": "Python 3.12.3" + } + }, + "disk": { + "path": "/var/tmp", + "free_bytes": 107374182400 + }, + "target": { + "state": "absent", + "has_credentials": false + }, + "policy": { + "state": "absent", + "encrypted": false, + "cron_installed": null, + "managed_elsewhere": "no" + }, + "tools_install": [ + { + "tool": "python3", + "action": "done" + }, + { + "tool": "age", + "action": "done" + }, + { + "tool": "aws", + "action": "done" + } + ], + "binary_upgrade": { + "action": "done", + "from": "0.1.20", + "to": "0.1.55" + } +} diff --git a/internal/server/testdata/backup-contracts/preflight-needs-setup.json b/internal/server/testdata/backup-contracts/preflight-needs-setup.json new file mode 100644 index 0000000..ec13e27 --- /dev/null +++ b/internal/server/testdata/backup-contracts/preflight-needs-setup.json @@ -0,0 +1,41 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-06T12:00:00Z", + "reachable": true, + "binary": { + "path": "/usr/local/bin/teploy", + "state": "no_fleet_engine", + "version": "0.1.20", + "fleet_engine": false, + "cli_version": "0.1.55", + "same_as_cli": false + }, + "tools": { + "aws": { + "state": "outdated", + "version": "2.15.0", + "detail": "aws CLI 2.15.0 is older than 2.23.0 (or 1.37.0), the first to send a checksum on every upload, which a bucket with Object Lock requires" + }, + "age": { + "state": "missing" + }, + "python3": { + "state": "missing" + } + }, + "disk": { + "path": "/var/tmp", + "free_bytes": 2147483648 + }, + "target": { + "state": "absent", + "has_credentials": false + }, + "policy": { + "state": "absent", + "encrypted": false, + "cron_installed": null, + "managed_elsewhere": "no" + } +} diff --git a/internal/server/testdata/backup-contracts/preflight-ready.json b/internal/server/testdata/backup-contracts/preflight-ready.json new file mode 100644 index 0000000..f391dc3 --- /dev/null +++ b/internal/server/testdata/backup-contracts/preflight-ready.json @@ -0,0 +1,49 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-06T12:00:00Z", + "reachable": true, + "binary": { + "path": "/usr/local/bin/teploy", + "state": "ok", + "version": "0.1.55", + "fleet_engine": true, + "cli_version": "0.1.55", + "same_as_cli": true + }, + "tools": { + "aws": { + "state": "ok", + "version": "2.30.1" + }, + "age": { + "state": "ok", + "version": "v1.2.1" + }, + "python3": { + "state": "ok", + "version": "Python 3.12.3" + } + }, + "disk": { + "path": "/var/tmp", + "free_bytes": 107374182400 + }, + "target": { + "state": "configured", + "bucket": "b", + "prefix": "teploy/srv1", + "has_credentials": true + }, + "policy": { + "state": "configured", + "path": "/etc/teploy/backup-policy.json", + "interval": "1h", + "encrypted": true, + "heartbeat_hosts": [ + "https://dash.example.internal" + ], + "cron_installed": true, + "managed_elsewhere": "no" + } +} diff --git a/internal/server/testdata/backup-contracts/preflight-unknown.json b/internal/server/testdata/backup-contracts/preflight-unknown.json new file mode 100644 index 0000000..8bbaeaf --- /dev/null +++ b/internal/server/testdata/backup-contracts/preflight-unknown.json @@ -0,0 +1,44 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-06T12:00:00Z", + "reachable": true, + "error": "the probe did not finish, so what it did not reach is unknown", + "binary": { + "path": "/usr/local/bin/teploy", + "state": "unknown", + "fleet_engine": null, + "cli_version": "0.1.55", + "same_as_cli": null, + "detail": "the binary is there and did not run" + }, + "tools": { + "aws": { + "state": "unknown", + "detail": "could not read its version: some wrapper" + }, + "age": { + "state": "unknown" + }, + "python3": { + "state": "unknown" + } + }, + "disk": { + "path": "/var/tmp", + "free_bytes": null, + "detail": "could not read the free space there" + }, + "target": { + "state": "unknown", + "has_credentials": false, + "detail": "reading the backup target: unrecognized output framing" + }, + "policy": { + "state": "unknown", + "encrypted": false, + "cron_installed": null, + "managed_elsewhere": "unknown", + "detail": "the backup policy cannot be read: exit status 3: cannot tell whether this server's backups must be encrypted" + } +} diff --git a/internal/server/testdata/backup-contracts/preflight-unreachable.json b/internal/server/testdata/backup-contracts/preflight-unreachable.json new file mode 100644 index 0000000..3e74984 --- /dev/null +++ b/internal/server/testdata/backup-contracts/preflight-unreachable.json @@ -0,0 +1,39 @@ +{ + "machine_interface": 2, + "host": "192.0.2.10", + "observed_at": "2026-11-06T12:00:00Z", + "reachable": false, + "error": "dial tcp 192.0.2.10:22: i/o timeout", + "binary": { + "path": "/usr/local/bin/teploy", + "state": "unknown", + "fleet_engine": null, + "cli_version": "0.1.55", + "same_as_cli": null + }, + "tools": { + "aws": { + "state": "unknown" + }, + "age": { + "state": "unknown" + }, + "python3": { + "state": "unknown" + } + }, + "disk": { + "path": "/var/tmp", + "free_bytes": null + }, + "target": { + "state": "unknown", + "has_credentials": false + }, + "policy": { + "state": "unknown", + "encrypted": false, + "cron_installed": null, + "managed_elsewhere": "unknown" + } +} From 30b0d6a597add65b69a56e1112db7bda6aa88e80 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:43:27 -0700 Subject: [PATCH 05/11] docs(backups): one recovery page, and the simple flow in the README docs/recovery.md covers everything lost, A's data onto B, Dash itself and hidden copies, with how to make the read-only key the flow never shows, and says plainly that it has not been rehearsed. docs/rebuild-dash.md now points to it. Co-Authored-By: Claude Opus 5.5 --- README.md | 75 ++++++++++++++----- docs/rebuild-dash.md | 125 +------------------------------- docs/recovery.md | 166 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 139 deletions(-) create mode 100644 docs/recovery.md diff --git a/README.md b/README.md index 603a500..ad6bf5f 100644 --- a/README.md +++ b/README.md @@ -276,6 +276,34 @@ CLI's fleet engine (v0.1.52 on the dashboard host, capability `backup-fleet-v1`; an older CLI is a note and the page stays empty — mixed fleets keep working because the engine runs on each server, not here). +**Protect my data** (teploy v0.1.55, capability `backup-setup-v1`) takes a new user to +verified, encrypted, off-site backups from this page, in six steps. The manual controls +stay under **Advanced**; both drive the same engine. + +1. **Check your servers**, before any key is asked for: reachable, teploy version, aws, + age, python3, free disk. Tools and old server binaries are only changed with consent. +2. **Storage.** Paste an expiring Backblaze key with listBuckets, writeBuckets and + writeKeys. Dash reads the capabilities and S3 endpoint from it, reuses a bucket or + creates a private `teploy-backups-`, merges its lifecycle rules, and mints + one key per server limited to the bucket and `teploy//`, without deleteFiles. + Each key must really hide a test object before it reaches a server, or setup stops: + retention cannot work without that. The pasted key lives only in that request. +3. **Recovery key.** Made in your browser and saved as a file, which you then pick again + so the browser can check it. Only the public key reaches Dash. +4. **Alerts**, ntfy or email with a test send. Skippable, and it keeps reminding you. +5. **Turn on.** Shows what each server will cover, with one tick to adopt unmanaged + databases, and storage as a multiple of one backup. "Back up now" or "Start with the + next scheduled run". +6. **Prove recovery.** Pick the key file again: Dash reads the first 128 KiB of a real + backup from the server, your browser decrypts it, and Dash records the result. + +**Protected** means a verified backup on each covered target, plus a passing restore +test for each covered database. Volume-only servers need no restore test. A server that +has a target is left alone unless you tick it, because a push replaces its target. New, +rebuilt and retired servers each have their own short action on the page. +[docs/recovery.md](docs/recovery.md) covers getting data back; it has not been rehearsed +yet. + - **First-run checklist** — storage → alert channel → recovery-key drill → first verified backup — shown until complete, then never again. - **Connect storage once.** Presets for Backblaze B2 (recommended: application @@ -284,9 +312,10 @@ because the engine runs on each server, not here). probes write/read/delete first, then pushes `server backup-target set` to every server with the credentials on stdin, then forgets them: the record keeps the bucket name and per-server prefixes only. A B2 key that cannot delete is the - documented good setup (a compromised server cannot destroy history; the bucket's - lifecycle rules remove hidden versions after 30 days). Rotation re-prompts, pushes, verifies - each server and tells you when to revoke the old key. + documented good setup, so long as it can still hide: the probe confirms the test object + is hidden, and stops if the store refuses (a compromised server cannot destroy history; + the bucket's lifecycle rules remove hidden versions after 14 days). Rotation re-prompts, + pushes, verifies each server and tells you when to revoke the old key. - **Recovery key.** `teploy backup keygen` output shown exactly once (re-auth required; the private key is stored nowhere); the public half is pushed to every server and every object is age-encrypted to it *and* the server's own @@ -321,18 +350,20 @@ because the engine runs on each server, not here). remove* once Dash has watched two consecutive good policy backups arrive AND a restore test of the same engine kind passed. The known hand-written job names (akiroo-backup, teploy-backup.sh, teploy-wp-dbbackup, omni-backup, - teploy-nucleus-backup, dash-state-backup) are listed as the owner's checklist; - removal itself stays a human action on the server, and old dumps stay until - they age out. Dash cannot list a server's systemd/cron jobs — no CLI read for - that exists yet (recorded as a gap). -- **Dash itself** is not yet covered by the policy engine: the CLI cannot encrypt - to the owner key alone or target arbitrary paths. Until it can, the page says - so and points at [`docs/rebuild-dash.md`](docs/rebuild-dash.md) — the rebuild - runbook and the interim dash-state job (age to the owner key only). - -Every action is audited; connecting storage and the recovery key need -`administer.credentials`, the policy and excludes `execute.mutate`, and nothing -here is exposed over MCP. + teploy-nucleus-backup, dash-state-backup) are listed as the owner's checklist, + shown only when Dash finds signs of such jobs: legacy cron lines, or a policy + something else manages. Removal itself stays a human action on the server, and + old dumps stay until they age out. Dash cannot list a server's systemd/cron jobs + (no CLI read for that exists yet, recorded as a gap). +- **Dash itself** is backed up as the `dash/volumes` entry when it runs from the + deployment template. Objects encrypted to the owner key and the server open with + the owner key alone. Dash's volumes hold its SSH key, so a policy with no recovery + key excludes them, found through `TEPLOY_APP`. Paths outside the template's volumes + are reported as uncovered. See [`docs/recovery.md`](docs/recovery.md). + +Every action is audited; connecting storage, the recovery key and every step of +"Protect my data" need `administer.credentials`, the policy and excludes +`execute.mutate`, and nothing here is exposed over MCP. ### Cleanup **Cleanup** answers "what is left over on each server?" with the CLI's read-only @@ -739,7 +770,19 @@ so the direct role claim is available here and takes precedence over groups. | GET | `/api/domains` | Domain health: one row per domain (DNS, route, certificate, HTTP, first failing layer, evidence age), per-server check state, CLI gate. `view.metadata`. | | GET | `/api/backups` | The fleet backups view: cached per-server status (coverage rows, targets, tooling, last run), storage record (never credentials), recovery-key state + drill, the applied policy, dash-owned restore tests with history, migration readiness, projected storage. `view.metadata`. | | POST | `/api/backups/refresh` | Ask for a new fleet status read now (the page follows the cache). `execute.mutate`. | -| POST | `/api/backups/storage` | Probe write/read/delete, then push `server backup-target set` to every server with the credentials on stdin and forget them. `{provider: b2\|r2\|custom, bucket, region/account_id/endpoint, base_prefix, access_key, secret_key, rotate?}`; B2 delete-refusal is accepted, others need `delete_refusal_ok`. `administer.credentials`. | +| POST | `/api/backups/storage` | Probe write/read/delete, then push `server backup-target set` to every server with the credentials on stdin and forget them. `{provider: b2\|r2\|custom, bucket, region/account_id/endpoint, base_prefix, access_key, secret_key, rotate?}`; a refused delete needs `delete_refusal_ok`, and on B2 an accepted delete must leave the object hidden. `administer.credentials`. | +| POST | `/api/backups/setup/preflight` | Check servers: `{servers?, install_tools?, upgrade_binary?}`; the last two name servers whose tools or binary may be changed. `administer.credentials`, as is everything under `/api/backups/setup/`. | +| POST | `/api/backups/setup/storage/check` | Read a Backblaze key: `{key_id, application_key, bucket?}` returns its missing capabilities, expiry and buckets with their warnings. The key is forgotten with the request. | +| POST | `/api/backups/setup/storage/apply` | Set up a bucket and a key per named server: `{password, key_id, application_key, servers, bucket?, accept_warnings?, rotate?, empty_base?, hide_days?, s3_managed?}`. Rerunning skips verified servers. | +| POST | `/api/backups/setup/owners` | Clear a rebuilt server's old marker: `{server, fingerprint, force?}`. | +| POST | `/api/backups/setup/recovery` | Record the public key the browser made: `{password, public_key}`. | +| POST | `/api/backups/setup/alerts/skip` | Skip alerts for now; the page keeps reminding. | +| POST | `/api/backups/setup/coverage` | What named servers would cover, with the storage multiplier: `{servers}`. | +| POST | `/api/backups/setup/turn-on` | Write the policy to named servers: `{servers, start: now\|next, adopt?, interval?, preset?, skip_volumes?, volume_max_gb?}`. | +| POST | `/api/backups/setup/run` | Back up now: `{servers}`, through a detached run. | +| POST | `/api/backups/setup/proof/start` | Read the first 128 KiB of the newest encrypted backup, as stored, for the browser to decrypt. | +| POST | `/api/backups/setup/proof/result` | Record the outcome: `{key, passed, detail?}`. | +| POST | `/api/backups/setup/retire` | Retire a server: `{password, key_id, application_key, server, data: keep\|remove, confirm?}`. | | POST | `/api/backups/recovery-key/generate` | `teploy backup keygen` — the private key is returned exactly once and stored nowhere. Re-auth (account password) required. `administer.credentials`. | | POST | `/api/backups/recovery-key/confirm` | "I saved it": pushes the public key as `--recipient` to every server. `administer.credentials`. | | POST | `/api/backups/recovery-key/drill` | Record the decrypt drill: `{output}` is the JSON `teploy backup fetch --identity` printed; checked against the assigned object and its plausible plaintext size. `administer.credentials`. | diff --git a/docs/rebuild-dash.md b/docs/rebuild-dash.md index 2d3938d..edc163d 100644 --- a/docs/rebuild-dash.md +++ b/docs/rebuild-dash.md @@ -1,124 +1,3 @@ -# Rebuilding Dash from its backup +# Rebuilding Dash -This is the runbook for losing the machine teploy-dash runs on: with the -recovery key, the bucket credentials and this page, a clean machine runs the -fleet's dashboard again. It is linked from **Backups → Dash itself**. - -Live rehearsal is deliberately out of scope for this document's first -revision: the paths below are unit-checked against the code -(`internal/server/backuprunbook_test.go` proves every file this page names is -a file Dash actually writes), but the steps have not been walked end-to-end on -a scratch machine yet. Do that rehearsal before relying on it (see -AUDIT_OPEN). - -## What a rebuild needs - -- The **owner recovery key** (`teploy backup keygen` output, kept offline) — - the private key `AGE-SECRET-KEY-...`. -- The **bucket credentials** (access key + secret) — the same pair the fleet's - servers back up to, or a read-capable key for the same bucket. -- The **Dash backup object** from the bucket: the `dash-state` archive made by - the job below (or your interim equivalent). - -## What the backup must contain - -Everything below is dash-side state; none of it exists anywhere else. - -| Path | What it is | -|---|---| -| `/auth.json` | accounts, bcrypt hashes, session token hashes (X03 principals) | -| `/sessions.json` | persisted sessions (sign-ins survive a rebuild) | -| `/homepage.json` | the homepage layout | -| `/backups.json` | Dash Backups state: storage record, recovery public key, drill, auto restore tests, migration evidence | -| `/incidents.json` | open alert incidents (so recoveries can close them) | -| `/silences.json` | active alert silences | -| `/audit.jsonl` | the audit trail | -| `/domain-health.json` | the last domain-health sweep | -| `/sources/` | registered git sources and their delivery ledgers | -| `/operations/` | the durable operation journal | -| `~/.teploy/notifications.json` | alert channels, SMTP/webhook secrets, public URL, heartbeat receivers — **the Dash secrets file** | -| `~/.teploy/servers.yml` | the teploy CLI's server list (teploy-config) | -| the SSH key Dash reaches the fleet with (`~/.ssh/id_ed25519`, `id_rsa`, or `TEPLOY_SSH_KEY`) | fleet access | - -The default `` is `/var/teploy-dash` (or the directory your service -unit / `--data` flag names — check the unit's ExecStart). - -## Taking the Dash backup (interim, until the CLI can do it) - -The teploy CLI's policy engine (v0.1.52) cannot yet back up arbitrary -directories, and it always encrypts to the owner key **and** the running -server's own identity — Dash's state must be readable with the owner key -ALONE (a rebuild means the old machine, and its identity, is gone). That is a -CLI gap (see AUDIT_OPEN); until the engine grows those two abilities, the -dash-state backup is a manual job on the Dash host, encrypted with `age` to -the recovery key only: - -```sh -#!/bin/sh -# dash-state-backup: run from cron on the Dash host (root or the dash user). -# Keeps 30 daily copies. Fill in RECIPIENT (the age1... public key from -# `teploy backup keygen`) and the bucket settings. -set -eu -RECIPIENT=age1... -STAMP=$(date -u +%Y%m%d-%H%M%S) -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT -tar -C / -czf "$WORK/dash-state.tgz" \ - var/teploy-dash \ # <- your - root/.teploy/notifications.json \ - root/.teploy/servers.yml \ - root/.ssh/id_ed25519 # <- or your TEPLOY_SSH_KEY path -age -r "$RECIPIENT" "$WORK/dash-state.tgz" > "dash-state-$STAMP.tgz.age" -AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... aws s3 cp \ - "dash-state-$STAMP.tgz.age" "s3://BUCKET/dash-state/dash-state-$STAMP.tgz.age" \ - --endpoint-url https://... -find . -name 'dash-state-*.tgz.age' -mtime +30 -delete -``` - -The object lands beside the fleet's backups in the same bucket, under its own -`dash-state/` prefix, encrypted to the owner key only. It does not go through -the policy engine yet — when the CLI gains path targets and owner-only -encryption, this job is replaced by a policy entry and this section deletes -itself. - -## Rebuilding - -1. **Prepare the machine**: install the teploy CLI (same version or newer) and - teploy-dash (same release or newer). Create the dash user if you run one. -2. **Fetch and decrypt the newest object** (the recovery drill, for real): - - ```sh - TEPLOY_S3_ACCESS_KEY=... TEPLOY_S3_SECRET_KEY=... teploy backup fetch \ - s3://BUCKET/dash-state/dash-state-LATEST.tgz.age \ - --identity recovery.txt --endpoint https://... --output dash-state.tgz - ``` - - `recovery.txt` is the `teploy backup keygen` output you kept. If this step - fails, nothing else matters — that is why the Backups page keeps the - decrypt drill banner up until it has passed once. - -3. **Restore the files** (adjust the data dir to your install): - - ```sh - tar -C / -xzvf dash-state.tgz # puts files back where the job found them - chmod 600 ~/.teploy/notifications.json ~/.teploy/servers.yml ~/.ssh/id_ed25519 - chmod 700 ~/.ssh - ``` - -4. **Start teploy-dash**. Sign-in works immediately (accounts and sessions - came back with `auth.json`/`sessions.json`); monitors, restore tests, the - backups page state, open incidents and silences are all as they were. - Verify on the Backups page that every server still reads fresh — the - restored SSH key is what reaches them. - -5. **Check the alert path end to end**: Settings → test-send on one channel. - A Dash that lost its channel config silently is a fleet nobody is watching. - -## What is explicitly not covered - -- **The teploy CLI and teploy-dash binaries**: reinstall from their releases; - pin the versions you ran if you care about byte-identical behaviour. -- **The fleet's servers**: they keep running without Dash; their backups run - from their own cron. Rebuild the machine, not the fleet. -- **Anything outside the table above** (for example a TLS certificate on a - reverse proxy in front of Dash): reissue it. +This page moved to [recovery.md](recovery.md), under "Dash itself". diff --git a/docs/recovery.md b/docs/recovery.md new file mode 100644 index 0000000..461529f --- /dev/null +++ b/docs/recovery.md @@ -0,0 +1,166 @@ +# Getting your data back + +**This page has not been rehearsed yet.** The file names and commands are checked +against the code, but nobody has walked these steps end to end on a scratch machine. +Do that once before you rely on it. + +## What you need + +- **Your recovery key file**, the one you saved during setup. It holds a line starting + `AGE-SECRET-KEY-`, and comment lines naming your bucket, endpoint and base folder. It + is the only thing Dash never keeps a copy of. Nobody can open your backups without it. +- **A way to read the bucket**: a read-only key, below. +- **The teploy CLI** and the `aws` CLI, on any machine. + +### Making a read-only key + +Setup never shows you one, because servers each have their own limited key and your +temporary key is gone. Make one when you need it: + +1. Sign in at backblaze.com and open **Application Keys**. +2. Choose **Add a New Application Key**. +3. Under **Allow access to Bucket(s)**, choose your bucket. Under **Type of Access**, + choose **Read Only**. Leave the file name prefix empty unless you want one server. +4. Copy the key ID and the application key. The second is shown once. +5. Your endpoint is on the bucket's page and in your recovery key file. It looks like + `https://s3.us-west-004.backblazeb2.com`. + +Delete the key in the same place when you are done. + +Below, `B` is your bucket, `E` your endpoint, and `recovery.txt` your key file. + +## Everything is lost + +No servers and no Dash, only the bucket and your key file. Each server's backups are +under `teploy//`. + +1. See what is there: + + ```sh + export AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= + aws s3 ls --recursive s3://B/teploy/ --endpoint-url E + ``` + +2. Fetch and decrypt any object. This needs only the bucket key and your key file: + + ```sh + TEPLOY_S3_ACCESS_KEY= TEPLOY_S3_SECRET_KEY= \ + teploy backup fetch s3://B/teploy/prod/blog/accessories/db/.pgdump.age \ + --identity recovery.txt --endpoint E --output blog-db.pgdump + ``` + + A database dump comes out as a normal dump file, and volumes as a `.tar.gz`. + +3. To put it back on a new server, set the server up with `teploy setup`, deploy the + app, then restore directly from the bucket with your key (next section). + +## Moving server A's data onto server B + +The server you restore on does not need to be the one that made the backup. Restore +runs on it and places your key there for that one command: + +```sh +teploy backup restore --app blog --accessory db --identity recovery.txt \ + --bucket B --endpoint E --prefix teploy/A --host B-server +``` + +Volumes are the same without `--accessory`. Adopted containers take `--adopted +--accessory `, and `--database ` picks one database. Use +`teploy backup list` the same way to find the ``. + +Server A's own key is limited to `teploy/A/`, so server B's key cannot read those +objects. Give this command the read-only key in `TEPLOY_S3_ACCESS_KEY` and +`TEPLOY_S3_SECRET_KEY`. + +## Dash itself + +Dash's own state is in a backup when Dash runs from the deployment template and a +recovery key is set. It appears on the Backups page as `dash/volumes`. Your recovery key +opens that backup on its own, with no need for the lost machine's identity. Dash's +volumes hold its SSH key, so Dash leaves them out of any backup with no recovery key. + +| Path | What it is | Template volume | +|---|---|---| +| `/auth.json` | accounts and password hashes | `dash-state` | +| `/sessions.json` | sign-ins | `dash-state` | +| `/homepage.json` | the homepage layout | `dash-state` | +| `/backups.json` | this page's state, never a credential | `dash-state` | +| `/incidents.json` | open alert incidents | `dash-state` | +| `/silences.json` | alert silences | `dash-state` | +| `/audit.jsonl` | the audit trail | `dash-state` | +| `/domain-health.json` | the last domain sweep | `dash-state` | +| `/sources/` | git sources | `dash-state` | +| `/operations/` | the operation journal | `dash-state` | +| `~/.teploy/notifications.json` | alert channels and their secrets | `teploy-config` | +| `~/.teploy/servers.yml` | your server list | `teploy-config` | +| `~/.ssh/id_ed25519`, or `TEPLOY_SSH_KEY` | how Dash reaches your servers | `ssh-key` | + +The template's `` is `/var/teploy-dash`. + +To rebuild: + +1. Fetch the newest backup. `dash` runs on one of your servers, called `S` here: + + ```sh + aws s3 ls s3://B/teploy/S/dash/volumes/ --endpoint-url E + TEPLOY_S3_ACCESS_KEY= TEPLOY_S3_SECRET_KEY= \ + teploy backup fetch s3://B/teploy/S/dash/volumes/.tar.gz.age \ + --identity recovery.txt --endpoint E --output dash-volumes.tar.gz + ``` + +2. On the new machine, deploy Dash from its template once, then stop it and unpack over + its volumes: + + ```sh + teploy stop --app dash + tar -C /deployments/dash/volumes -xzf dash-volumes.tar.gz + chmod 600 /deployments/dash/volumes/ssh-key/id_ed25519 + teploy start --app dash + ``` + +3. Sign in as before, and check the Backups page reads every server. +4. In Settings, use test-send on one alert channel. A Dash that lost its channels is a + fleet nobody is watching. + +### When Dash is not from the template + +The Backups page lists any path no backup covers. Copy those yourself, encrypted to your +recovery key alone. Fill in `RECIPIENT` with your `age1...` public key, and run this from +cron on the Dash machine: + +```sh +#!/bin/sh +set -eu +RECIPIENT=age1... +STAMP=$(date -u +%Y%m%d-%H%M%S) +WORK=$(mktemp -d) +trap 'rm -rf "$WORK"' EXIT +tar -C / -czf "$WORK/dash-state.tgz" var/teploy-dash root/.teploy/notifications.json \ + root/.teploy/servers.yml root/.ssh/id_ed25519 +age -r "$RECIPIENT" "$WORK/dash-state.tgz" > "$WORK/dash-state-$STAMP.tgz.age" +AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... aws s3 cp \ + "$WORK/dash-state-$STAMP.tgz.age" "s3://B/dash-state/dash-state-$STAMP.tgz.age" --endpoint-url E +``` + +Adjust the paths to your install. `teploy backup fetch` opens these too. + +## Hidden copies + +When retention removes a backup, or a compromised server tries to destroy one, Backblaze +only hides it: server keys cannot truly delete. Hidden copies stay for 14 days unless you +changed that under Advanced, then the bucket's lifecycle rule removes them. + +If backups disappeared within that window: + +1. **Act first, then investigate.** The clock is running on each copy. +2. Sign in at backblaze.com, open your bucket with **Browse Files**, and turn on + **Show hidden files**. +3. Unhide what you want back. A read-only key cannot do this, so use the console. +4. If you need longer, raise the days on the `teploy/` rule under **Lifecycle Settings**. +5. Then replace the key of any server you suspect, from the Backups page. + +## Other things to know + +- Backups made before a recovery key changed still need the older key. +- The binaries are not in any backup. Reinstall them from their releases. +- Your servers keep running and backing up without Dash. From 586f87f9cfbd7214b5cb456b0e5113fe101d9d00 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:45:40 -0700 Subject: [PATCH 06/11] feat(frontend): Protect my data on the Backups and home pages Six steps in plain wording: servers, storage, recovery key, alerts, turn on, prove. The recovery key is made and checked in the browser; only its public half is sent. Keys live in memory and go when the flow closes. New, rebuilt and retired servers have short actions. The manual forms sit unchanged under a collapsed Advanced, open on a CLI too old for the flow. Holds at 375px with no horizontal scroll. Also fixes two existing errors on the Backups page, where x-text read fields a row may not have, and the subtitle that joined two counts into "10 covered". Co-Authored-By: Claude Opus 5.5 --- .../frontend-test/backup_setup.test.mjs | 358 +++++++++++++ cmd/teploy-dash/frontend/css/style.css | 30 ++ cmd/teploy-dash/frontend/index.html | 387 +++++++++++++- cmd/teploy-dash/frontend/js/app.js | 502 ++++++++++++++++++ 4 files changed, 1262 insertions(+), 15 deletions(-) create mode 100644 cmd/teploy-dash/frontend-test/backup_setup.test.mjs diff --git a/cmd/teploy-dash/frontend-test/backup_setup.test.mjs b/cmd/teploy-dash/frontend-test/backup_setup.test.mjs new file mode 100644 index 0000000..5f85a6e --- /dev/null +++ b/cmd/teploy-dash/frontend-test/backup_setup.test.mjs @@ -0,0 +1,358 @@ +// "Protect my data" in the browser: against a stubbed API and the real vendored crypto. +import { test, before } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { harness, response } from './identity_harness.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); +const keyFile = readFileSync(join(here, 'testdata/age/test-key.txt'), 'utf8'); +const prefix = readFileSync(join(here, 'testdata/age/prefix-70k.age')).toString('base64'); + +before(() => { + Object.defineProperty(globalThis.crypto, 'subtle', { value: undefined, configurable: true }); + new Function(readFileSync(join(here, '../frontend/js/age.vendor.js'), 'utf8'))(); +}); + +const baseView = (setup = {}, extra = {}) => ({ + cli_supported: true, recovery: {}, servers: [], rows: [], tests: [], intervals: ['24h'], presets: ['standard'], + self_backup: { available: false, reason: 'x', runbook: 'docs/recovery.md' }, + setup: { supported: true, state: 'none', headline: 'Your data is not protected yet.', servers: [], recovery_doc: 'https://example/recovery.md', ...setup }, + ...extra, +}); + +function setupHarness(routes, setup = {}, extra = {}) { + const h = harness('/backups'); + h.respond(url => { + if (url === '/api/backups') return response(baseView(setup, extra)); + for (const [path, fn] of Object.entries(routes)) { + if (url === path) return fn(); + } + return response({}, { status: 404 }); + }); + return h; +} + +const preflight = () => response({ servers: [ + { server: 'a', reachable: true, selected: true, binary_version: '0.1.55', free_bytes: 1 << 30, target_state: 'absent', tools: {} }, + { server: 'b', reachable: true, selected: false, target_state: 'configured', target_bucket: 'old', tools: {} }, + { server: 'c', reachable: true, selected: true, needs_tools: ['aws'], needs_upgrade: true, target_state: 'absent', tools: {} }, + { server: 'd', reachable: false, blocked: 'Dash could not reach this server', selected: false, tools: {} }, +] }); + +test('the page offers the flow only on a CLI that supports it, and keeps the manual form', async () => { + const h = setupHarness({}); + const p = h.page('backupsPage'); + await p.init(); + assert.equal(p.setupSupported(), true); + const html = h.readHTML(); + assert.ok(html.includes('

Advanced')); + assert.ok(html.includes(`:open="!setupSupported()"`), 'an older CLI still gets its manual form open'); + assert.ok(html.includes('@submit.prevent="connectStorage()"'), 'the manual storage form is still there'); + assert.ok(html.includes('@submit.prevent="applyPolicy()"'), 'the manual policy form is still there'); + p.destroy(); +}); + +test('preflight ticks what is safe: a server with a target and a blocked one start unticked', async () => { + const h = setupHarness({ '/api/backups/setup/preflight': preflight }); + const p = h.page('backupsPage'); + await p.init(); + await p.openSetup(); + assert.equal(p.setupStep, 'check'); + assert.deepEqual(p.selectedServers(), ['a', 'c']); + assert.equal(p.blockingNeeds().length, 2); + // A person's choice survives the next check, and a blocked server stays unticked. + p.setupTicked.c = false; + p.setupTicked.b = true; + await p.runPreflight(); + assert.deepEqual(p.selectedServers(), ['a', 'b']); + assert.equal(p.blockingNeeds().length, 0); + p.destroy(); +}); + +test('tools install and upgrades go only for what was ticked', async () => { + const h = setupHarness({ '/api/backups/setup/preflight': preflight }); + const p = h.page('backupsPage'); + await p.init(); + await p.openSetup(); + assert.equal(p.anyPrepareTicked(), false); + p.setupInstall.c = true; + assert.equal(p.anyPrepareTicked(), true); + await p.prepareServers(); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/preflight').at(-1).body; + assert.deepEqual(sent.install_tools, ['c']); + assert.deepEqual(sent.upgrade_binary, []); + p.destroy(); +}); + +test('one new server is the same flow for that server', async () => { + const h = setupHarness({ '/api/backups/setup/preflight': () => response({ servers: [{ server: 'compute-3', reachable: true, selected: false, tools: {} }] }) }); + const p = h.page('backupsPage'); + await p.init(); + await p.openSetup('compute-3'); + assert.deepEqual(h.requests.filter(r => r.url === '/api/backups/setup/preflight').at(-1).body.servers, ['compute-3']); + assert.deepEqual(p.selectedServers(), ['compute-3']); + p.destroy(); +}); + +test('the key check decides what can go on', async () => { + let check = { missing: ['writeKeys'], buckets: [], can_create_bucket: true }; + const h = setupHarness({ '/api/backups/setup/storage/check': () => response(check) }); + const p = h.page('backupsPage'); + await p.init(); + p.setupKey = { id: ' KEY ', secret: ' SECRET ' }; + await p.checkKey(); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/storage/check').at(-1).body; + assert.equal(sent.key_id, 'KEY'); + assert.equal(p.canApplyStorage(), false, 'missing capabilities stop it'); + check = { missing: [], buckets: [{ name: 'x', refusals: ['no'], warnings: [] }, { name: 'y', refusals: [], warnings: ['careful'] }], can_create_bucket: true }; + await p.checkKey(); + assert.equal(p.canApplyStorage(), true, 'a new bucket is fine'); + p.setupBucket = 'x'; + assert.equal(p.canApplyStorage(), false, 'a refused bucket is not'); + p.setupBucket = 'y'; + assert.equal(p.canApplyStorage(), false, 'a warning needs reading'); + p.setupAccept = true; + assert.equal(p.canApplyStorage(), true); + check = { missing: [], capped: true, listing_error: 'daily transaction cap', buckets: [], can_create_bucket: true }; + await p.checkKey(); + assert.equal(p.canApplyStorage(), false, 'a capped listing shows why and stops'); + p.destroy(); +}); + +test('a key limited to buckets starts on its first bucket', async () => { + const h = setupHarness({ '/api/backups/setup/storage/check': () => response({ missing: [], buckets: [{ name: 'only', refusals: [], warnings: [] }], can_create_bucket: false }) }); + const p = h.page('backupsPage'); + await p.init(); + p.setupKey = { id: 'K', secret: 'S' }; + await p.checkKey(); + assert.equal(p.setupBucket, 'only'); + p.destroy(); +}); + +test('applying sends the ticked servers, and a finished run forgets the key', async () => { + let reply = { all_ok: false, servers: [{ server: 'a', verified: false, key_kept: true, error: 'timeout' }], key_note: 'n', warnings: [] }; + const h = setupHarness({ + '/api/backups/setup/preflight': preflight, + '/api/backups/setup/storage/check': () => response({ missing: [], buckets: [], can_create_bucket: true }), + '/api/backups/setup/storage/apply': () => response(reply), + }); + const p = h.page('backupsPage'); + await p.init(); + await p.openSetup(); + p.setupTicked = { a: true }; + p.setupKey = { id: 'KEYID', secret: 'APPKEY' }; + p.setupPassword = 'pw'; + await p.checkKey(); + await p.applyStorage(); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/storage/apply').at(-1).body; + assert.deepEqual(sent.servers, ['a']); + assert.equal(sent.application_key, 'APPKEY'); + assert.equal(p.setupKey.secret, 'APPKEY', 'a partial run keeps it in memory to retry'); + assert.match(p.storageResultLine(reply.servers[0]), /kept the key/); + assert.equal(p.setupPassword, ''); + reply = { all_ok: true, servers: [{ server: 'a', verified: true }], key_note: 'n', warnings: [] }; + p.setupPassword = 'pw'; + await p.applyStorage(); + assert.equal(p.setupKey.secret, ''); + p.setupKey = { id: 'A', secret: 'B' }; + p.closeSetup(); + assert.equal(p.setupKey.secret, '', 'closing forgets it'); + p.destroy(); +}); + +test('the recovery key: only its public half reaches Dash, after the saved file is checked', async () => { + const h = setupHarness({ '/api/backups/setup/recovery': () => response({}) }); + const p = h.page('backupsPage'); + await p.init(); + p.setupStorage = { bucket: 'my-bucket', endpoint: 'https://s3.us-west-004.backblazeb2.com', base: 'teploy' }; + await p.generateRecovery(); + assert.ok(p.recoveryGen.fileText.includes('# bucket: my-bucket')); + assert.ok(p.recoveryGen.fileText.includes('# endpoint: https://s3.us-west-004.backblazeb2.com')); + assert.ok(p.recoveryGen.fileText.includes('# base: teploy')); + + await p.saveRecovery(); + assert.equal(h.requests.filter(r => r.url === '/api/backups/setup/recovery').length, 0, 'not before the file is checked'); + + await p.checkRecoveryFile(keyFile); + assert.equal(p.recoveryProven, false); + assert.match(p.recoveryError, /different key/); + + const publicKey = p.recoveryGen.publicKey; + await p.checkRecoveryFile(p.recoveryGen.fileText); + assert.equal(p.recoveryProven, true); + await p.saveRecovery(); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/recovery').at(-1).body; + assert.equal(sent.public_key, publicKey); + assert.equal(p.recoveryGen, null); + for (const r of h.requests) assert.ok(!JSON.stringify(r.body || {}).includes('AGE-SECRET-KEY'), `${r.url} carried the private key`); + p.destroy(); +}); + +test('the clipboard is optional: plain HTTP has none, and download is the main path', async () => { + const h = setupHarness({}); + const p = h.page('backupsPage'); + assert.equal(p.canCopy(), typeof navigator !== 'undefined' && !!navigator.clipboard); + assert.ok(h.readHTML().includes('Download the key file')); +}); + +test('proving recovery decrypts the real prefix and records a pass', async () => { + const h = setupHarness({ + '/api/backups/setup/proof/start': () => response({ server: 'a', key: 's3://b/k.age', length: 71680, data_base64: prefix }), + '/api/backups/setup/proof/result': () => response({ recorded: true }), + }); + const p = h.page('backupsPage'); + await p.init(); + await p.proveWith(keyFile); + assert.deepEqual(p.proofState, { passed: true, key: 's3://b/k.age' }); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/proof/result').at(-1).body; + assert.deepEqual(sent, { key: 's3://b/k.age', passed: true }); + for (const r of h.requests) assert.ok(!JSON.stringify(r.body || {}).includes('AGE-SECRET-KEY')); + p.destroy(); +}); + +test('proving with the wrong key records a failure and says what it means', async () => { + const h = setupHarness({ + '/api/backups/setup/proof/start': () => response({ server: 'a', key: 's3://b/k.age', length: 71680, data_base64: prefix }), + '/api/backups/setup/proof/result': () => response({ recorded: true }), + }); + const p = h.page('backupsPage'); + await p.init(); + const other = await globalThis.TeployAge.generate(); + await p.proveWith(`# a comment\n${other.identity}\n`); + assert.equal(p.proofState, null); + assert.match(p.setupError, /did not open your backup/); + assert.equal(h.requests.filter(r => r.url === '/api/backups/setup/proof/result').at(-1).body.passed, false); + p.destroy(); +}); + +test('alerts use the settings channel API with a test send, and can be skipped', async () => { + const seen = []; + const h = harness('/backups'); + h.respond((url, options) => { + seen.push([options.method, url]); + if (url === '/api/backups') return response(baseView()); + if (url.endsWith('/test')) return response({ ok: true }); + return response({}); + }); + const p = h.page('backupsPage'); + await p.init(); + p.alertForm.url = 'https://ntfy.sh/t'; + await p.saveAlert(); + assert.deepEqual(p.alertTest, { ok: true }); + assert.ok(seen.some(([m, u]) => m === 'PUT' && u.startsWith('/api/notifications/channels/ntfy-'))); + assert.equal(h.requests.find(r => r.method === 'PUT').body.url, 'https://ntfy.sh/t'); + await p.skipAlerts(); + assert.ok(seen.some(([, u]) => u === '/api/backups/setup/alerts/skip')); + p.destroy(); +}); + +test('turning on names the ticked servers, the start choice and the tick', async () => { + const h = setupHarness({ + '/api/backups/setup/coverage': () => response({ servers: [{ server: 'a', items: [{ name: 'blog/volumes', kind: 'volumes', app: 'blog', size_bytes: 2048 }], unmanaged: [{ name: 'pg' }] }], multiplier: 39, interval: '24h', preset: 'standard' }), + '/api/backups/setup/turn-on': () => response({ all_ok: true, servers: [{ server: 'a', ok: true, run: { status: 'started' } }] }), + }); + const p = h.page('backupsPage'); + await p.init(); + p.setupTicked = { a: true }; + await p.loadCoverage(); + assert.equal(p.setupCoverage.multiplier, 39); + assert.equal(p.hasUnmanaged(), true); + assert.match(p.coverageLine(p.setupCoverage.servers[0].items[0]), /blog files, 2\.0 KB/); + p.setupAdopt = true; + await p.turnOn('now'); + const sent = h.requests.filter(r => r.url === '/api/backups/setup/turn-on').at(-1).body; + assert.deepEqual(sent.servers, ['a']); + assert.equal(sent.start, 'now'); + assert.equal(sent.adopt, true); + assert.equal(sent.interval, '24h'); + assert.match(p.turnOnLine(p.turnOnResult.servers[0]), /backing up now/); + p.destroy(); +}); + +test('retiring needs an explicit choice', async () => { + const h = setupHarness({ '/api/backups/setup/retire': () => response({ notes: [] }) }); + const p = h.page('backupsPage'); + await p.init(); + p.beginRetire('old'); + await p.submitRetire(); + assert.match(p.setupError, /keep or remove/); + assert.equal(h.requests.filter(r => r.url === '/api/backups/setup/retire').length, 0); + p.retireData = 'keep'; + p.setupKey = { id: 'K', secret: 'S' }; + await p.submitRetire(); + assert.equal(h.requests.filter(r => r.url === '/api/backups/setup/retire').at(-1).body.data, 'keep'); + assert.equal(p.retireFor, null); + assert.equal(p.setupKey.secret, ''); + p.destroy(); +}); + +test('plain HTTP outside localhost and a tailnet warns before a key is pasted', () => { + const h = setupHarness({}); + const p = h.page('backupsPage'); + const cases = [['http:', 'dash.example.com', true], ['https:', 'dash.example.com', false], ['http:', 'localhost', false], + ['http:', '127.0.0.1', false], ['http:', '100.108.1.2', false], ['http:', 'infra.tail1234.ts.net', false], ['http:', '100.200.1.2', true], ['http:', '192.168.1.5', true]]; + for (const [protocol, hostname, want] of cases) { + globalThis.location = { pathname: '/backups', search: '', protocol, hostname }; + assert.equal(p.insecurePage(), want, `${protocol}//${hostname}`); + } +}); + +test('states read in plain words', () => { + const h = setupHarness({}); + const p = h.page('backupsPage'); + assert.equal(p.setupStateClass('protected'), 'green'); + assert.equal(p.setupStateClass('failing'), 'red'); + assert.equal(p.setupStateLabel('capped'), 'daily cap reached'); + assert.equal(p.canBackUpNow({ state: 'late' }), true); + assert.equal(p.canBackUpNow({ state: 'protected' }), false); +}); + +test('the hand-written-jobs list shows only when some are found', async () => { + const h = setupHarness({}); + const p = h.page('backupsPage'); + await p.init(); + assert.equal(p.hasHandwritten(), false); + p.view.handwritten = [{ name: 'omni-backup', where: 'cron' }]; + assert.equal(p.hasHandwritten(), true); + assert.ok(h.readHTML().includes('x-show="hasHandwritten()"')); + p.destroy(); +}); + +test('the home page says so until protected, and a failed read says nothing', async () => { + const h = harness('/'); + let view = baseView(); + h.respond(() => response(view)); + const banner = h.components.protectBanner(); + await banner.init(); + assert.equal(banner.visible(), true); + view = baseView({ state: 'protected' }); + await banner.init(); + assert.equal(banner.visible(), false); + view = baseView({ supported: false }); + await banner.init(); + assert.equal(banner.visible(), false); + h.respond(() => response({}, { status: 500 })); + await banner.init(); + assert.equal(banner.visible(), false); + assert.ok(h.readHTML().includes('x-data="protectBanner"')); +}); + +test('no emoji in the page, and the layout holds at phone width', () => { + const root = join(here, '../frontend'); + const emoji = /\p{Extended_Pictographic}/u; + const js = readFileSync(join(root, 'js/app.js'), 'utf8'); + const start = js.indexOf('function backupSetupComponent'); + const end = js.indexOf('// ── Backups Page (plan P2)'); + assert.ok(start > 0 && end > start); + assert.equal(emoji.test(js.slice(start, end)), false); + const html = readFileSync(join(root, 'index.html'), 'utf8'); + const section = html.slice(html.indexOf(' + + - @@ -2249,8 +2294,8 @@

Storage

- -

Dash probes write/read/delete first. On B2, a key without deleteFiles is the recommended setup, because it can only hide files: a compromised server cannot destroy history. Dash checks that the hide really works, and stops if it does not, because retention needs it.

+ +

Dash probes write/read/delete first. On B2, a key without deleteFiles is the recommended setup, because it can only hide files: a compromised server cannot destroy history. Dash checks that the hide really works, and warns if it does not, because retention needs it.

Create a B2 application key limited to this bucket with listFiles, readFiles, writeFiles and readBucketLifecycleRules, and without deleteFiles. If you limit it to a file name prefix, use the same value as the prefix here, because every server stores under it.

diff --git a/cmd/teploy-dash/frontend/js/app.js b/cmd/teploy-dash/frontend/js/app.js index a3f3d21..c808d36 100644 --- a/cmd/teploy-dash/frontend/js/app.js +++ b/cmd/teploy-dash/frontend/js/app.js @@ -3688,6 +3688,7 @@ document.addEventListener('alpine:init', () => { setupCheck: null, setupBucket: '', setupAccept: false, + setupReplaceOK: false, setupAdv: { hide_days: '', empty_base: false, s3_managed: false, rotate: false }, setupStorage: null, ownerForce: {}, @@ -3698,10 +3699,11 @@ document.addEventListener('alpine:init', () => { alertChannelId: '', alertTest: null, setupCoverage: null, - setupAdopt: false, + setupSkipped: {}, setupPolicy: { interval: '', preset: '', skip_volumes: false }, turnOnResult: null, proofState: null, + proofUnrecorded: null, retireFor: null, retireData: '', retireConfirm: '', @@ -3715,12 +3717,14 @@ document.addEventListener('alpine:init', () => { unreachable: 'cannot reach storage', empty: 'nothing to back up yet', waiting_first_backup: 'waiting for the first backup', late: 'backups are late', failing: 'backups are failing', waiting_restore_test: 'waiting for a restore test', guard_skipped: 'restore test put off', - restore_test_failed: 'restore test failed', protected: 'protected', + restore_test_failed: 'restore test failed', restore_test_overdue: 'restore test overdue', + not_covered: 'not fully backed up', protected_except: 'protected, with exceptions', all_skipped: 'nothing backed up', + protected: 'protected', })[state] || state; }, setupStateClass(state) { - if (state === 'protected') return 'green'; - if (state === 'failing' || state === 'restore_test_failed' || state === 'unreachable' || state === 'not_protected') return 'red'; + if (state === 'protected' || state === 'protected_except') return 'green'; + if (state === 'failing' || state === 'restore_test_failed' || state === 'unreachable' || state === 'not_protected' || state === 'not_covered') return 'red'; if (state === 'empty' || state === 'unknown') return 'gray'; return 'yellow'; }, @@ -3728,6 +3732,19 @@ document.addEventListener('alpine:init', () => { return ['waiting_first_backup', 'late', 'failing'].includes(sv.state); }, + // How this viewer confirms a sensitive step: password, none, or local_account when a + // single sign-on session has no password here. + reauthMode() { + return (this.view && this.view.setup && this.view.setup.reauth) || 'none'; + }, + + async dismissLeftover(keyId) { + try { + await api.post('/api/backups/setup/leftovers/dismiss', { key_id: keyId }); + await this.load(); + } catch (e) { showToast(e.message, 'error'); } + }, + clearSetupSecrets() { this.setupKey = { id: '', secret: '' }; this.setupPassword = ''; @@ -3741,7 +3758,9 @@ document.addEventListener('alpine:init', () => { this.setupScope = server || null; this.retireFor = null; const setup = this.view && this.view.setup; - if (!server && setup && setup.turned_on && !this.recovery.drill_passed_at) { + // Continue goes to the proof only when no server is still waiting to be set up. + const unfinished = ((setup && setup.servers) || []).some(sv => sv.state === 'not_protected'); + if (!server && setup && setup.turned_on && !unfinished && !this.recovery.drill_passed_at) { this.setupStep = 'prove'; return; } @@ -3754,6 +3773,11 @@ document.addEventListener('alpine:init', () => { this.setupStep = step; }, closeSetup() { + // A key made and not saved opens nothing: its file must not be kept by mistake. + if (this.recoveryGen && typeof confirm === 'function' && + !confirm('This recovery key was not saved to Dash, so no backup will use it. If you downloaded the file, it is no use: delete it. Close anyway?')) { + return; + } this.setupOpen = false; this.setupError = ''; this.setupCheck = null; @@ -3843,9 +3867,16 @@ document.addEventListener('alpine:init', () => { } this.setupBusy = false; }, + // With nothing chosen, the flow uses the bucket it suggested. chosenBucket() { - if (!this.setupCheck || !this.setupBucket) return null; - return (this.setupCheck.buckets || []).find(b => b.name === this.setupBucket) || null; + if (!this.setupCheck) return null; + const name = this.setupBucket || this.setupCheck.suggested_bucket; + if (!name) return null; + return (this.setupCheck.buckets || []).find(b => b.name === name) || null; + }, + // A ticked server that has a target, or might: Dash cannot put its old key back. + replaceConsentNeeded() { + return this.setupServers.filter(sv => this.setupTicked[sv.server] && sv.target_state !== 'absent' && !sv.own_key).map(sv => sv.server); }, canApplyStorage() { if (!this.setupCheck || (this.setupCheck.missing || []).length) return false; @@ -3853,6 +3884,7 @@ document.addEventListener('alpine:init', () => { const b = this.chosenBucket(); if (b && (b.refusals || []).length) return false; if (b && (b.warnings || []).length && !this.setupAccept) return false; + if (this.replaceConsentNeeded().length && !this.setupReplaceOK) return false; return true; }, async applyStorage() { @@ -3862,7 +3894,7 @@ document.addEventListener('alpine:init', () => { try { this.setupStorage = await api.post('/api/backups/setup/storage/apply', { password: this.setupPassword, key_id: this.setupKey.id.trim(), application_key: this.setupKey.secret.trim(), - bucket: this.setupBucket, servers: this.selectedServers(), accept_warnings: this.setupAccept, + bucket: this.setupBucket, servers: this.selectedServers(), accept_warnings: this.setupAccept, replace_configured: this.setupReplaceOK, s3_managed: this.setupAdv.s3_managed, rotate: this.setupAdv.rotate, ...this.storageExtras(), }); this.setupPassword = ''; @@ -3874,13 +3906,15 @@ document.addEventListener('alpine:init', () => { storageResultLine(r) { if (r.verified) return r.skipped ? 'is ready, as before.' : 'is ready.'; if (r.skipped) return `was not set up: ${r.skipped}.`; - if (r.key_kept) return `may have its key but did not answer on it. Dash kept the key. ${r.error || ''}`; + if (r.key_kept && r.pushed) return `has the new key but did not answer on it. Dash kept every key. Try again. ${r.error || ''}`; + if (r.key_kept) return `may or may not have the new key, because the push failed. Dash kept every key, so nothing it was using is lost. Try again. ${r.error || ''}`; return `did not get a key. ${r.error || ''}`; }, async clearOwner(server, fingerprint) { this.setupError = ''; try { - await api.post('/api/backups/setup/owners', { server, fingerprint, force: !!this.ownerForce[fingerprint] }); + await api.post('/api/backups/setup/owners', { server, fingerprint, force: !!this.ownerForce[fingerprint], password: this.setupPassword }); + this.setupPassword = ''; showToast('Cleared the older machine\'s mark', 'success'); for (const r of (this.setupStorage && this.setupStorage.servers) || []) { if (r.server === server) r.old_owners = (r.old_owners || []).filter(o => o.fingerprint !== fingerprint); @@ -3953,17 +3987,8 @@ document.addEventListener('alpine:init', () => { document.body.appendChild(link); link.click(); link.remove(); - URL.revokeObjectURL(url); - }, - // navigator.clipboard does not exist on plain HTTP, so the download is the main path. - canCopy() { - return typeof navigator !== 'undefined' && !!navigator.clipboard; - }, - async copyRecovery() { - try { - await navigator.clipboard.writeText(this.recoveryGen.fileText); - showToast('Copied. Save it somewhere safe.', 'success'); - } catch (e) { showToast('Could not copy. Use the download.', 'error'); } + // Revoking straight away can cancel the download in some browsers. + setTimeout(() => URL.revokeObjectURL(url), 60000); }, async recoveryFilePicked(event) { const file = event.target.files && event.target.files[0]; @@ -4047,10 +4072,23 @@ document.addEventListener('alpine:init', () => { this.setupCoverage = await api.post('/api/backups/setup/coverage', { servers }); this.setupPolicy.interval = this.setupPolicy.interval || this.setupCoverage.interval; this.setupPolicy.preset = this.setupPolicy.preset || this.setupCoverage.preset; + // Start from what is saved, so setting up one server changes nothing else. + this.setupPolicy.skip_volumes = this.setupCoverage.volumes === false; } catch (e) { this.setupError = e.message; } }, - hasUnmanaged() { - return ((this.setupCoverage && this.setupCoverage.servers) || []).some(sv => (sv.unmanaged || []).length > 0); + isSkipped(server, name) { + return !!(this.setupSkipped[server] && this.setupSkipped[server][name]); + }, + setSkipped(server, name, skipped) { + this.setupSkipped = { ...this.setupSkipped, [server]: { ...(this.setupSkipped[server] || {}), [name]: skipped } }; + }, + skipList() { + const out = {}; + for (const [server, names] of Object.entries(this.setupSkipped)) { + const list = Object.keys(names).filter(n => names[n]); + if (list.length) out[server] = list; + } + return out; }, coverageLine(item) { const what = item.kind === 'volumes' ? `${item.app || item.name} files` : `${item.name}${item.engine ? ' (' + item.engine + ')' : ''}`; @@ -4061,9 +4099,10 @@ document.addEventListener('alpine:init', () => { this.setupError = ''; try { this.turnOnResult = await api.post('/api/backups/setup/turn-on', { - servers: this.selectedServers(), start, adopt: this.setupAdopt, interval: this.setupPolicy.interval, + password: this.setupPassword, servers: this.selectedServers(), start, skip: this.skipList(), interval: this.setupPolicy.interval, preset: this.setupPolicy.preset, skip_volumes: this.setupPolicy.skip_volumes, }); + this.setupPassword = ''; await this.load(); } catch (e) { this.setupError = e.message; } this.setupBusy = false; @@ -4097,27 +4136,52 @@ document.addEventListener('alpine:init', () => { this.setupBusy = true; this.setupError = ''; this.proofState = null; + this.proofUnrecorded = null; let key = ''; + let start; + let identity; + // Each stage fails differently: a bad file, a backup Dash could not read, a key + // that does not open it, and a result Dash could not record are not the same news. try { const age = await this.ageLib(); - const { identity } = age.parseIdentityFile(text); - const start = await api.post('/api/backups/setup/proof/start', {}); + ({ identity } = age.parseIdentityFile(text)); + start = await api.post('/api/backups/setup/proof/start', {}); key = start.key; + } catch (e) { + this.setupError = e.message; + this.setupBusy = false; + return; + } + try { const raw = atob(start.data_base64); const bytes = new Uint8Array(raw.length); for (let i = 0; i < raw.length; i++) bytes[i] = raw.charCodeAt(i); - await age.decryptStart(identity, bytes); + await (await this.ageLib()).decryptStart(identity, bytes); + } catch (e) { + await api.post('/api/backups/setup/proof/result', { key, passed: false, detail: e.message }).catch(() => {}); + this.setupError = 'This key did not open your backup. Check you picked the right file. If you have lost the file, you cannot read these backups.'; + this.setupBusy = false; + return; + } + await this.recordProof(key); + this.setupBusy = false; + }, + async recordProof(key) { + try { await api.post('/api/backups/setup/proof/result', { key, passed: true }); + this.proofUnrecorded = null; this.proofState = { passed: true, key }; await this.load(); } catch (e) { - if (key) { - await api.post('/api/backups/setup/proof/result', { key, passed: false, detail: e.message }).catch(() => {}); - this.setupError = 'This key did not open your backup. Check you picked the right file. If you have lost the file, you cannot read these backups.'; - } else { - this.setupError = e.message; - } + this.proofUnrecorded = { key }; + this.setupError = e.message; } + }, + async retryProofRecord() { + if (!this.proofUnrecorded) return; + this.setupBusy = true; + this.setupError = ''; + await this.recordProof(this.proofUnrecorded.key); this.setupBusy = false; }, diff --git a/docs/recovery.md b/docs/recovery.md index 461529f..e8de54a 100644 --- a/docs/recovery.md +++ b/docs/recovery.md @@ -77,7 +77,8 @@ objects. Give this command the read-only key in `TEPLOY_S3_ACCESS_KEY` and Dash's own state is in a backup when Dash runs from the deployment template and a recovery key is set. It appears on the Backups page as `dash/volumes`. Your recovery key opens that backup on its own, with no need for the lost machine's identity. Dash's -volumes hold its SSH key, so Dash leaves them out of any backup with no recovery key. +volumes hold its SSH key, so with no recovery key Dash leaves them out on the server it +can prove hosts it, and warns on any server where it cannot tell. | Path | What it is | Template volume | |---|---|---| From b511e175bcf26ff7352a095c35873954237aa498 Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:24:13 -0700 Subject: [PATCH 10/11] test(backups): make the already-deleted key check assert something Co-Authored-By: Claude Opus 5.5 --- internal/server/backupsetup_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/backupsetup_test.go b/internal/server/backupsetup_test.go index 30ad2ae..e630839 100644 --- a/internal/server/backupsetup_test.go +++ b/internal/server/backupsetup_test.go @@ -2382,7 +2382,7 @@ func TestSetupApply_ABadRequestOnDeleteLeavesAKnownLeftover(t *testing.T) { s2.backups.setupLocked().Keys["prod"] = &backupServerKey{KeyID: "GONE", KeyName: "n", Bucket: "b", Prefix: "teploy/prod", State: backupKeyMinted} s2.backups.mu.Unlock() applySetup(t, s2, applyBody("prod")) - if got := s2.backupsView(context.Background(), backupsNow).Setup.Leftovers; len(got) != 1 && len(got) != 0 { + if got := s2.backupsView(context.Background(), backupsNow).Setup.Leftovers; len(got) != 0 { t.Fatalf("%+v", got) } } From da5c40b5683db5e7aff9357c4d2a64aa3d05cd5a Mon Sep 17 00:00:00 2001 From: Tyler <53561637+im-tyler@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:27:22 -0700 Subject: [PATCH 11/11] fix(backups): a server with everything unticked is skipped, not failing Turn-on writes no policy or cron line there and starts no run, and leaves an existing policy alone and says so. The server reads 'Skipped: you chose not to back up this server', stays out of the failure and alert paths, and is named in the headline: 'Protected, except (you chose to skip)'. A database that appears later still gets named and alerted, because nobody decided about it. A server with a database that was not deliberately unticked still gets no allow_empty. Re-opening setup can tick it. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +- .../frontend-test/backup_setup.test.mjs | 2 + cmd/teploy-dash/frontend/index.html | 4 +- cmd/teploy-dash/frontend/js/app.js | 5 +- internal/server/backupalerts.go | 16 +++ internal/server/backupfleet.go | 4 + internal/server/backupsetup.go | 37 ++++- internal/server/backupsetup_status.go | 26 +++- internal/server/backupsetup_test.go | 134 ++++++++++++++++-- 9 files changed, 210 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a547bb5..03d42d3 100644 --- a/README.md +++ b/README.md @@ -300,7 +300,9 @@ stay under **Advanced**; both drive the same engine. **Protected** means a verified backup on each covered target, a restore test passed within 21 days for each covered database, and no database left out of a backup. Volume-only servers need no restore test. If you chose to skip a database, the page says "Protected, -except" and names it. A server that has a target is left alone unless you tick it, because a +except" and names it. If you untick everything on a server, Dash writes no policy or cron +line there, leaves any existing policy alone, keeps the server out of failures and alerts, +and shows it as skipped until you tick it again. A server that has a target is left alone unless you tick it, because a push replaces its target and Dash cannot put the old key back. New, rebuilt and retired servers each have their own short action on the page. [docs/recovery.md](docs/recovery.md) covers getting data back; it has not been rehearsed diff --git a/cmd/teploy-dash/frontend-test/backup_setup.test.mjs b/cmd/teploy-dash/frontend-test/backup_setup.test.mjs index f71283d..7cf5382 100644 --- a/cmd/teploy-dash/frontend-test/backup_setup.test.mjs +++ b/cmd/teploy-dash/frontend-test/backup_setup.test.mjs @@ -434,6 +434,8 @@ test('states read in plain words', () => { assert.equal(p.setupStateLabel('capped'), 'daily cap reached'); assert.equal(p.setupStateLabel('restore_test_overdue'), 'restore test overdue'); assert.equal(p.setupStateClass('not_covered'), 'red'); + assert.equal(p.setupStateClass('all_skipped'), 'gray'); + assert.match(p.turnOnLine({ server: 'a', skipped: true }), /skipped, as you chose/); assert.equal(p.canBackUpNow({ state: 'late' }), true); assert.equal(p.canBackUpNow({ state: 'protected' }), false); }); diff --git a/cmd/teploy-dash/frontend/index.html b/cmd/teploy-dash/frontend/index.html index 83ea895..62f372c 100644 --- a/cmd/teploy-dash/frontend/index.html +++ b/cmd/teploy-dash/frontend/index.html @@ -1850,7 +1850,7 @@

Dash cannot recheck the bucket's cleanup rules with this server's key. Reading this bucket through S3 shows no cleanup rule for this server. Check it under Lifecycle Settings in the Backblaze console, because Dash set one up and this read may not show it. - + @@ -2172,7 +2172,7 @@

Turn on backups

-

No server was turned on. Fix what is listed, then try again.

+

No server was turned on. Fix what is listed, then try again.

diff --git a/cmd/teploy-dash/frontend/js/app.js b/cmd/teploy-dash/frontend/js/app.js index c808d36..48d57a4 100644 --- a/cmd/teploy-dash/frontend/js/app.js +++ b/cmd/teploy-dash/frontend/js/app.js @@ -3718,14 +3718,14 @@ document.addEventListener('alpine:init', () => { waiting_first_backup: 'waiting for the first backup', late: 'backups are late', failing: 'backups are failing', waiting_restore_test: 'waiting for a restore test', guard_skipped: 'restore test put off', restore_test_failed: 'restore test failed', restore_test_overdue: 'restore test overdue', - not_covered: 'not fully backed up', protected_except: 'protected, with exceptions', all_skipped: 'nothing backed up', + not_covered: 'not fully backed up', protected_except: 'protected, with exceptions', all_skipped: 'skipped', protected: 'protected', })[state] || state; }, setupStateClass(state) { if (state === 'protected' || state === 'protected_except') return 'green'; if (state === 'failing' || state === 'restore_test_failed' || state === 'unreachable' || state === 'not_protected' || state === 'not_covered') return 'red'; - if (state === 'empty' || state === 'unknown') return 'gray'; + if (state === 'empty' || state === 'unknown' || state === 'all_skipped') return 'gray'; return 'yellow'; }, canBackUpNow(sv) { @@ -4108,6 +4108,7 @@ document.addEventListener('alpine:init', () => { this.setupBusy = false; }, turnOnLine(r) { + if (r.skipped) return 'was skipped, as you chose.'; if (!r.ok) return r.error || 'could not be turned on.'; if (!r.run) return 'starts with its next scheduled run.'; return ({ diff --git a/internal/server/backupalerts.go b/internal/server/backupalerts.go index b366378..54ef3d2 100644 --- a/internal/server/backupalerts.go +++ b/internal/server/backupalerts.go @@ -62,6 +62,22 @@ func backupObservations(rec *backupServerRecord, now time.Time) (obs []backupObs obs = append(obs, o) } + // A server the person left out raises nothing, except a database nobody has decided + // about yet. Rules from before the choice retire with the next complete read. + if len(rec.SkippedByChoice) > 0 { + chosen := map[string]bool{} + for _, name := range rec.SkippedByChoice { + chosen[name] = true + } + for i := range st.Entries { + e := &st.Entries[i] + if e.Coverage == "uncovered" && !chosen[e.Name] { + add(backupObservation{backupRule(rec.Server, "uncovered", e.Name, e.Name+" on "+rec.Server+" is not covered", alert.SeverityWarning), false, "a new database appeared on a server you chose not to back up"}) + } + } + return obs, keys + } + if st.Target.Configured { rule := backupRule(rec.Server, "target", "", "backup target on "+rec.Server, alert.SeverityCritical) switch { diff --git a/internal/server/backupfleet.go b/internal/server/backupfleet.go index 239e9e8..4f8673a 100644 --- a/internal/server/backupfleet.go +++ b/internal/server/backupfleet.go @@ -179,6 +179,10 @@ type backupServerRecord struct { // HeartbeatMonitorID is the push monitor Dash owns for this server. HeartbeatMonitorID string `json:"heartbeat_monitor_id,omitempty"` HeartbeatNote string `json:"heartbeat_note,omitempty"` + // SkippedByChoice names what a person unticked when they left this whole server out. + // Non-empty means the server is skipped: no policy was written, and it stays out of + // the failure and alert paths, apart from a database that appears later. + SkippedByChoice []string `json:"skipped_by_choice,omitempty"` // SelfExcluded is what Dash added to this server's excludes because the policy had // no recovery key, so a key arriving later can take it back out. SelfExcluded []string `json:"self_excluded,omitempty"` diff --git a/internal/server/backupsetup.go b/internal/server/backupsetup.go index f175d6a..b1e854b 100644 --- a/internal/server/backupsetup.go +++ b/internal/server/backupsetup.go @@ -1674,6 +1674,8 @@ type setupTurnOnResult struct { AllowEmpty bool `json:"allow_empty"` Adopted []string `json:"adopted,omitempty"` Run *backupSetupRun `json:"run,omitempty"` + // Skipped: everything here was unticked, so no policy or cron line was written. + Skipped bool `json:"skipped,omitempty"` } func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { @@ -1776,7 +1778,7 @@ func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { skip[n] = true } var adopt, skipped []string - targets, databases := 0, 0 + targets, databases, notChosen := 0, 0, 0 for j := range env.Entries { e := &env.Entries[j] if e.Kind != "volumes" { @@ -1791,12 +1793,34 @@ func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { res.Adopted = append(res.Adopted, e.Name) targets++ } - case e.Coverage == "excluded" || e.Coverage == "uncovered": + case e.Coverage == "uncovered": + notChosen++ + case e.Coverage == "excluded": case e.Kind == "volumes" && !volumes: default: targets++ } } + // Everything unticked is a decision, not a failure: nothing is written, so no run + // reports one. An existing policy is not this flow's to remove. + if targets == 0 && len(skipped) > 0 && notChosen == 0 { + res.Skipped = true + res.Note = "Skipped: you chose not to back up this server." + if env.Policy.Configured { + res.Note += " It already has a backup policy, which Dash left as it was." + } + b.mu.Lock() + b.serverLocked(name).SkippedByChoice = skipped + setup := b.setupLocked() + if setup.Skipped == nil { + setup.Skipped = map[string][]string{} + } + setup.Skipped[name] = skipped + delete(setup.Runs, name) + b.saveLocked() + b.mu.Unlock() + return + } // Only a server with really nothing on it gets allow_empty. One holding a database // is not empty, whatever was chosen for that database, and losing every target on // one with targets should stay a failure. @@ -1809,9 +1833,6 @@ func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { note, err := s.applyBackupPolicy(ctx, name, backupPolicyApply{Interval: interval, Retention: retention, Volumes: volumes, VolumeMaxGB: maxGB, Recipient: recipient, AllowEmpty: &res.AllowEmpty, Adopt: adopt}) res.Note = note - if targets == 0 && databases > 0 { - res.Note = strings.TrimSpace(res.Note + " Nothing on this server will be backed up, so each run will report a failure until you include something.") - } if err != nil { res.Error = sanitizeDomainError(err.Error()) return @@ -1822,6 +1843,7 @@ func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { rec := b.serverLocked(name) rec.HeartbeatNote = note rec.PolicyAppliedAt = now + rec.SkippedByChoice = nil setup := b.setupLocked() if setup.Skipped == nil { setup.Skipped = map[string][]string{} @@ -1840,7 +1862,7 @@ func (s *Server) handleSetupTurnOn(w http.ResponseWriter, r *http.Request) { now := time.Now().UTC() allOK, anyOK := true, false for _, res := range results { - allOK = allOK && res.OK + allOK = allOK && (res.OK || res.Skipped) anyOK = anyOK || res.OK } // Turned on means at least one server really is. @@ -2185,6 +2207,9 @@ func (s *Server) handleSetupRetire(w http.ResponseWriter, r *http.Request) { delete(setup.Preflight, body.Server) delete(setup.Runs, body.Server) delete(setup.Skipped, body.Server) + if rec := b.rec.Servers[body.Server]; rec != nil { + rec.SkippedByChoice = nil + } if body.Data == "remove" && b.rec.Storage != nil { delete(b.rec.Storage.Prefixes, body.Server) } diff --git a/internal/server/backupsetup_status.go b/internal/server/backupsetup_status.go index db23d89..1ba5925 100644 --- a/internal/server/backupsetup_status.go +++ b/internal/server/backupsetup_status.go @@ -43,6 +43,8 @@ const ( setupProtected = "protected" ) +const backupSkippedServerMessage = "Skipped: you chose not to back up this server." + // backupRestoreTestMaxAge: an older pass no longer says a backup restores. const backupRestoreTestMaxAge = 21 * 24 * time.Hour @@ -139,6 +141,26 @@ func (b *backupState) setupServerStateLocked(sv backupServerView, rows []backupR if !flowBucket { out.Lifecycle = "" } + // Left out on purpose, so nothing below applies: not failing, not waiting. A database + // nobody has decided about yet still has to be named. + if rec := b.rec.Servers[sv.Server]; rec != nil && len(rec.SkippedByChoice) > 0 { + chosen := map[string]bool{} + for _, name := range rec.SkippedByChoice { + chosen[name] = true + } + out.Skipped = append(out.Skipped, rec.SkippedByChoice...) + for i := range rows { + if rows[i].State == "uncovered" && !chosen[rows[i].Name] { + out.Uncovered = append(out.Uncovered, rows[i].Name) + } + } + if len(out.Uncovered) > 0 { + out.State, out.Detail = setupNotCovered, "Not backed up: "+strings.Join(out.Uncovered, ", ")+"." + return out + } + out.State, out.Detail = setupAllSkipped, backupSkippedServerMessage + return out + } if backupTargetCapped(sv.Target.Error) { out.State, out.Detail = setupCapped, backupCappedMessage return out @@ -279,7 +301,7 @@ func (b *backupState) setupServerStateLocked(sv backupServerView, rows []backupR case len(waitingTest) > 0: out.State, out.Detail = setupWaitingTest, "Waiting for a restore test of "+join(waitingTest)+". These run in the off-hours window." case covered == 0 && len(out.Skipped) > 0: - out.State, out.Detail = setupAllSkipped, "Nothing here is backed up: you chose to skip "+names(out.Skipped)+"." + out.State, out.Detail = setupAllSkipped, backupSkippedServerMessage case covered == 0: out.State, out.Detail = setupEmpty, "There is nothing to back up on this server yet." case len(out.Skipped) > 0: @@ -313,7 +335,7 @@ func (b *backupState) setupViewLocked(supported, channels bool, view *backupsPag active++ except = append(except, state.Skipped...) case setupAllSkipped: - except = append(except, state.Skipped...) + except = append(except, sv.Server) case setupEmpty: case setupNotProtected: if view.Storage != nil { diff --git a/internal/server/backupsetup_test.go b/internal/server/backupsetup_test.go index e630839..12b601d 100644 --- a/internal/server/backupsetup_test.go +++ b/internal/server/backupsetup_test.go @@ -1508,22 +1508,138 @@ func TestSetupTurnOn_NextRunStartsNothingAndAnEmptyServerAllowsEmpty(t *testing. } } -// B2: a server with only a skipped Postgres is not empty, so a run there must not report ok. -func TestSetupTurnOn_ASkippedDatabaseIsNotAnEmptyServer(t *testing.T) { +func statusWithEntries(entries string) string { + return strings.Replace(emptyStatusJSON(), `"entries":[]`, `"entries":[`+entries+`]`, 1) +} + +const pgEntryJSON = `{"name":"customer-pg","kind":"unmanaged","engine":"postgres","unmanaged":true,"coverage":"uncovered","schedule":"none","adopted":false,"encrypted":false}` + +// A server where everything was unticked is a decision: no policy or cron line, no run, no +// failure and no alert. Ticking it again later is an ordinary turn-on. +func TestSetupTurnOn_EverythingUntickedWritesNothingAndRaisesNothing(t *testing.T) { s, c, _ := newSetupServer(t) setRecipient(s) - withDB := strings.Replace(emptyStatusJSON(), `"entries":[]`, `"entries":[{"name":"customer-pg","kind":"unmanaged","engine":"postgres","unmanaged":true,"coverage":"uncovered","schedule":"none","adopted":false,"encrypted":false}]`, 1) - c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: withDB}, nil } + pg := statusWithEntries(pgEntryJSON) + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: pg}, nil } + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "now", "skip": map[string][]string{"prod": {"customer-pg"}}}) + var out struct { + Servers []setupTurnOnResult `json:"servers"` + AllOK bool `json:"all_ok"` + AnyOK bool `json:"any_ok"` + } + decodeData(t, rec, &out) + r := out.Servers[0] + if !r.Skipped || r.OK || !out.AllOK || out.AnyOK || !strings.Contains(r.Note, "you chose not to back up this server") { + t.Fatalf("%+v", out) + } + if strings.Contains(rec.Body.String(), "report a failure") { + t.Fatalf("%s", rec.Body.String()) + } + if c.countOf("backup-policy", "set") != 0 || c.countOf("--detach") != 0 { + t.Fatal("no policy, cron line or run") + } + got := setupStateOf(t, s, "prod") + if got.State != setupAllSkipped || got.Detail != "Skipped: you chose not to back up this server." { + t.Fatalf("%+v", got) + } + view := s.backupsView(context.Background(), backupsNow) + if view.Setup.TurnedOn || len(view.Setup.NewServers) != 0 { + t.Fatalf("%+v", view.Setup) + } + s.backups.mu.Lock() + obs, _ := backupObservations(s.backups.rec.Servers["prod"], backupsNow) + s.backups.mu.Unlock() + if len(obs) != 0 { + t.Fatalf("a skipped server raises nothing: %+v", obs) + } + + // A database that appears later has not been decided about. + both := statusWithEntries(pgEntryJSON + `,` + strings.ReplaceAll(pgEntryJSON, "customer-pg", "new-pg")) + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: both}, nil } + env, err := decodeBackupStatus(both) + if err != nil { + t.Fatal(err) + } + s.recordBackupStatus("prod", env, nil, backupsNow) + if got := setupStateOf(t, s, "prod"); got.State != setupNotCovered || !strings.Contains(got.Detail, "new-pg") { + t.Fatalf("%+v", got) + } + s.backups.mu.Lock() + obs, _ = backupObservations(s.backups.rec.Servers["prod"], backupsNow) + s.backups.mu.Unlock() + if len(obs) != 1 || !strings.Contains(obs[0].rule.Key, "new-pg") { + t.Fatalf("%+v", obs) + } + + // Re-opening setup and leaving it ticked turns the server on as usual. + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: pg}, nil } + if rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "next"}); rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + if args := strings.Join(policySetArgs(c, "prod"), " "); !strings.Contains(args, "--adopt customer-pg:postgres:all") { + t.Fatalf("%s", args) + } + if got := setupStateOf(t, s, "prod"); got.State == setupAllSkipped { + t.Fatalf("%+v", got) + } +} + +func TestSetupTurnOn_AnExistingPolicyIsLeftAloneAndSaysSo(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + existing := strings.Replace(statusWithEntries(pgEntryJSON), `"policy":{"configured":false`, `"policy":{"configured":true`, 1) + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: existing}, nil } rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "next", "skip": map[string][]string{"prod": {"customer-pg"}}}) var out struct { Servers []setupTurnOnResult `json:"servers"` } decodeData(t, rec, &out) - if out.Servers[0].AllowEmpty || !strings.Contains(strings.Join(policySetArgs(c, "prod"), " "), "--allow-empty=false") { - t.Fatalf("%+v %v", out.Servers[0], policySetArgs(c, "prod")) + if !out.Servers[0].Skipped || !strings.Contains(out.Servers[0].Note, "left as it was") || c.countOf("backup-policy") != 0 { + t.Fatalf("%+v calls=%d", out.Servers[0], c.countOf("backup-policy")) } - if !strings.Contains(out.Servers[0].Note, "Nothing on this server will be backed up") { - t.Fatalf("it says so: %+v", out.Servers[0]) +} + +func TestSetupStatus_ASkippedServerIsNamedInTheHeadlineNeverTheBareClaim(t *testing.T) { + s, _, _ := newSetupServer(t) + s.recordBackupStatus("prod", statusWith([]backupEntry{goodEntry("blog/volumes", "volumes")}, true), nil, backupsNow) + s.recordBackupStatus("stage", statusWith([]backupEntry{{Name: "customer-pg", Kind: "unmanaged", Engine: "postgres", Coverage: "uncovered", Unmanaged: true}}, false), nil, backupsNow) + s.backups.mu.Lock() + s.backups.serverLocked("stage").SkippedByChoice = []string{"customer-pg"} + s.backups.mu.Unlock() + view := s.backupsView(context.Background(), backupsNow) + if view.Setup.State != setupExcept || view.Setup.Headline != "Protected, except stage (you chose to skip)." { + t.Fatalf("%+v", view.Setup) + } +} + +// A database nobody unticked keeps the server off allow_empty, as a failure worth seeing. +func TestSetupTurnOn_ADatabaseNotDeliberatelyUntickedIsNoEmptyServer(t *testing.T) { + s, c, _ := newSetupServer(t) + setRecipient(s) + noMethod := `{"name":"analytics-ch","kind":"unmanaged","engine":"clickhouse","unmanaged":true,"coverage":"uncovered","reason":"teploy has no backup method for clickhouse","schedule":"none","adopted":false,"encrypted":false}` + for name, skip := range map[string]map[string][]string{ + "not coverable, nothing unticked": {}, + "one unticked, one not coverable": {"prod": {"customer-pg"}}, + } { + t.Run(name, func(t *testing.T) { + c.mu.Lock() + c.calls = nil + c.mu.Unlock() + entries := noMethod + if len(skip) > 0 { + entries += "," + pgEntryJSON + } + body := statusWithEntries(entries) + c.status["prod"] = func() (*cli.Result, error) { return &cli.Result{Stdout: body}, nil } + rec := postJSON(t, s, "/api/backups/setup/turn-on", map[string]any{"servers": []string{"prod"}, "start": "next", "skip": skip}) + var out struct { + Servers []setupTurnOnResult `json:"servers"` + } + decodeData(t, rec, &out) + if out.Servers[0].Skipped || out.Servers[0].AllowEmpty || !strings.Contains(strings.Join(policySetArgs(c, "prod"), " "), "--allow-empty=false") { + t.Fatalf("%+v %v", out.Servers[0], policySetArgs(c, "prod")) + } + }) } } @@ -2032,7 +2148,7 @@ func TestSetupStatus_ADeliberateSkipIsNeverTheBareClaim(t *testing.T) { // All skipped, nothing else: never protected at all. s.recordBackupStatus("prod", statusWith([]backupEntry{db}, true), nil, backupsNow) view = s.backupsView(context.Background(), backupsNow) - if got := setupStateOf(t, s, "prod"); got.State != setupAllSkipped || !strings.Contains(got.Detail, "Nothing here is backed up") { + if got := setupStateOf(t, s, "prod"); got.State != setupAllSkipped || !strings.Contains(got.Detail, "Skipped: you chose not to back up this server") { t.Fatalf("%+v", got) } if strings.HasPrefix(view.Setup.Headline, "Protected") || view.Setup.State == "protected" {