From dfd4e346bc4fdb11dadb5b29a8bd49f1129bf617 Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Tue, 10 Nov 2026 17:43:01 +0800 Subject: [PATCH 1/2] emrg: the release tag has a maintained push path when git-over-https is down --- scripts/push-branch-from-api.py | 115 ++++++++++++++++++- tests/test_push_branch_from_api.py | 175 +++++++++++++++++++++++++++++ 2 files changed, 288 insertions(+), 2 deletions(-) diff --git a/scripts/push-branch-from-api.py b/scripts/push-branch-from-api.py index 0ff5a7cc..066d6fd4 100644 --- a/scripts/push-branch-from-api.py +++ b/scripts/push-branch-from-api.py @@ -34,9 +34,25 @@ verifies: GET refs == local rev-parse, and `git diff` of the remote sha against the original local tip is empty +The tag direction is the release's stage 2 and has the same problem: a tag push is +the only trigger of build-release.yml, so an outage with no maintained tag path +means re-deriving one from notes every release, untested. `--tag` publishes an +annotated tag through the same two endpoints a release needs: + + * `git cat-file -t refs/tags/` must answer `tag` - a lightweight tag is + refused before a single request, because build-release.yml's verify-tag job + requires an annotated tag and would refuse it only after the build was spent + * the tag object is created with POST /git/tags, carrying the tagger's own + instant **and offset** (a tag stamped "now" is a different object) + * refs/tags/ is created with POST /git/refs - creating that ref is what + fires the tag workflow, so this is the whole of stage 2 + * an existing tag at a different commit is refused rather than moved: a + published version number is not a branch to rewrite + Usage: python scripts/push-branch-from-api.py [--repo owner/name] [--branch feature/x] [--ref HEAD] [--force] + python scripts/push-branch-from-api.py [--repo owner/name] --tag v0.3.48 Requirements: git on PATH; api.github.com reachable; gh CLI or GH_TOKEN/GITHUB_TOKEN auth (private repos need it; public repos work anonymously but rate-limit). @@ -435,15 +451,97 @@ def remote_sha(sha: str, kind: str) -> str: "content_identical": not diff.strip()} +def parse_tag(raw: bytes) -> dict: + """Parse a raw annotated tag object into the payload POST /git/tags takes. + + Layout: object/type/tag/tagger header lines, a blank line, then the message. + The tagger's epoch+offset is preserved as ISO-8601 carrying that offset, for + the same reason `parse_commit` preserves the author's: the API stores the + offset it is given, and a tag stamped "now" is not the object that was cut. + """ + header, sep, message = raw.partition(b"\n\n") + if not sep: + raise ValueError("malformed tag object: no blank line before message") + fields = {} + for line in header.decode("utf-8").splitlines(): + key, _, value = line.partition(" ") + fields[key] = value + for required in ("object", "type", "tag", "tagger"): + if required not in fields: + raise ValueError(f"malformed tag object: missing {required}") + return {"object": fields["object"], "type": fields["type"], + "tag": fields["tag"], "tagger": _parse_person(fields["tagger"]), + "message": message.decode("utf-8").rstrip("\n")} + + +def _remote_tag_target(repo: str, tag: str) -> str | None: + """The commit refs/tags/ names on the remote, or None when it is absent.""" + try: + ref = api("GET", f"/repos/{repo}/git/ref/tags/{tag}") + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + sha = ref["object"]["sha"] + try: + return api("GET", f"/repos/{repo}/git/tags/{sha}")["object"]["sha"] + except urllib.error.HTTPError as e: + if e.code == 404: + return sha # a lightweight remote tag holds the commit itself + raise + + +def push_tag(repo: str, tag: str, cwd: str | None = None) -> dict: + """Publish an annotated tag through the Git Data API. + + Creating refs/tags/ is what fires the tag workflow, so this is the + release's stage 2 when git-over-https is down. Everything that can be refused + is refused before the first request: a lightweight tag (the release pipeline + requires an annotated one), a missing local tag, and an already-published tag + naming a different commit - a version number is not a branch to rewrite. + """ + ref = f"refs/tags/{tag}" + try: + kind = git("cat-file", "-t", ref, cwd=cwd).decode("utf-8").strip() + except RuntimeError as e: + raise PushError(f"{ref} is not present locally: {e}") from None + if kind != "tag": + raise PushError(f"{ref} is a {kind}, not an annotated tag object; the release " + "pipeline requires an annotated tag, so nothing was sent") + payload = parse_tag(git("cat-file", "tag", ref, cwd=cwd)) + commit = payload["object"] + + existing = _remote_tag_target(repo, tag) + if existing == commit: + return {"result": "no-op", "tag": tag, "commit": commit} + if existing is not None: + raise PushError(f"{ref} on the remote already names {existing}; refusing to move " + "a published tag - cut a new version instead") + + created = api("POST", f"/repos/{repo}/git/tags", payload)["sha"] + api("POST", f"/repos/{repo}/git/refs", {"ref": ref, "sha": created}) + + confirmed = api("GET", f"/repos/{repo}/git/ref/tags/{tag}")["object"]["sha"] + if confirmed != created: + raise PushError(f"verification failed: {ref} reads {confirmed}, not {created}") + return {"result": "pushed", "tag": tag, "tag_object": created, "commit": commit} + + def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--repo", default=None, help="owner/name (default: from origin)") - ap.add_argument("--branch", required=True, help="remote branch name, e.g. feature/x") + ap.add_argument("--branch", default=None, help="remote branch name, e.g. feature/x") + ap.add_argument("--tag", default=None, + help="annotated tag to publish, e.g. v0.3.48 (release stage 2)") ap.add_argument("--ref", default="HEAD", help="local ref to push (default: HEAD)") ap.add_argument("--force", action="store_true", help="force-update the remote ref (default: fail on conflict)") args = ap.parse_args() + if bool(args.branch) == bool(args.tag): + print("pass exactly one of --branch or --tag ") + return 2 + repo = args.repo if not repo: out = git("remote", "get-url", "origin").decode("utf-8").strip() @@ -454,7 +552,10 @@ def main() -> int: repo = m.group(1) try: - result = push_branch(repo, args.branch, args.ref, args.force) + if args.tag: + result = push_tag(repo, args.tag) + else: + result = push_branch(repo, args.branch, args.ref, args.force) except PushError as e: print(f"push aborted: {e}") return 1 @@ -465,6 +566,16 @@ def main() -> int: print(f"push aborted (network error after retries): {e}") return 1 + if args.tag: + print(f"repo {repo} tag {args.tag}") + print(f" release commit: {result['commit']}") + if result["result"] == "no-op": + print(" nothing to do - refs/tags/ already names that commit") + else: + print(f" tag object: {result['tag_object']}") + print(f" created refs/tags/{args.tag} - creating it fires the tag workflow") + return 0 + print(f"repo {repo} branch {args.branch}") print(f" remote sha: {result['sha']}") if result["result"] == "no-op": diff --git a/tests/test_push_branch_from_api.py b/tests/test_push_branch_from_api.py index 45fd2202..2e7a148d 100644 --- a/tests/test_push_branch_from_api.py +++ b/tests/test_push_branch_from_api.py @@ -107,6 +107,23 @@ def person(p): return ("\n".join(lines) + "\n\n" + msg).encode("utf-8") +def _raw_tag_from_payload(payload): + """Reconstruct the raw annotated tag object GitHub stores for a POST /git/tags + payload: the tagger's epoch+offset, and a message with no trailing newline - + the same normalization create-commit is observed to apply.""" + def person(p): + import datetime as dt + d = dt.datetime.fromisoformat(p["date"]) + return f"{p['name']} <{p['email']}> {int(d.timestamp())} {d.strftime('%z')}" + + msg = payload["message"] + if msg.endswith("\n"): + msg = msg[:-1] + header = (f"object {payload['object']}\ntype {payload['type']}\n" + f"tag {payload['tag']}\ntagger {person(payload['tagger'])}") + return (header + "\n\n" + msg).encode("utf-8") + + def _hash_object(repo: Path, obj_type: str, raw: bytes, write: bool = False) -> str: args = ["hash-object", "-t", obj_type] if write: @@ -127,6 +144,8 @@ def __init__(self, repo: Path): self.refs = {} # "heads/x" -> sha self.log = [] # (method, path) call log self.fail_commits = False + self.tag_targets = {} # tag object sha -> the commit it names + self.tag_payloads = [] # every POST /git/tags body, for the offset reading def __call__(self, method, path, body=None): self.log.append((method, path)) @@ -197,6 +216,25 @@ def __call__(self, method, path, body=None): if sha in self.objects: return {"sha": sha} raise _http_error(404) + if path.startswith("/repos/x/git/ref/tags/"): + ref_key = "refs/tags/" + path[len("/repos/x/git/ref/tags/"):] + if method != "GET": + raise AssertionError(f"unexpected singular-ref call {method} {path}") + if ref_key in self.refs: + return {"ref": ref_key, "object": {"sha": self.refs[ref_key]}} + raise _http_error(404) + if path == "/repos/x/git/tags": + self.tag_payloads.append(body) + raw = _raw_tag_from_payload(body) + sha = _hash_object(self.repo, "tag", raw, write=True) + self.objects[sha] = ("tag", raw) + self.tag_targets[sha] = body["object"] + return {"sha": sha, "object": {"sha": body["object"], "type": "commit"}} + if path.startswith("/repos/x/git/tags/"): + sha = path.rsplit("/", 1)[1] + if sha in self.tag_targets: + return {"sha": sha, "object": {"sha": self.tag_targets[sha], "type": "commit"}} + raise _http_error(404) raise AssertionError(f"unexpected call {method} {path}") @@ -568,3 +606,140 @@ def test_a_push_onto_an_existing_remote_base_uploads_only_what_is_missing(tmp_pa "the new tree was never created" ) assert result["sha"] in fake.objects, "the new commit was never created" + + +# ---------------------------------------------------------------- the tag path +# +# Stage 2 of a release is a tag push, and a tag push is the only trigger of +# build-release.yml. The branch path exists so an outage does not mean +# re-deriving a script from notes; the tag path had no home in the repository at +# all, so every outage re-derived one and nothing tested it (#2795). These pin +# the four refusals and the one order that make the tag path safe to run during a +# release: refuse before the network, create the object before the ref, and never +# move a published tag. + + +def _tag_object(repo: Path, name: str = "v9.9.9") -> str: + r = _git("tag", "-a", name, "-m", f"emrg {name}", "HEAD", cwd=repo, env=GIT_ENV) + assert r.returncode == 0, r.stderr + return _git("rev-parse", f"refs/tags/{name}", cwd=repo).stdout.decode().strip() + + +def _with_fake(mod, fake): + orig = mod.api + mod.api = fake + return orig + + +def test_push_tag_creates_the_tag_object_before_the_ref(tmp_path): + """The ref must be created last, pointing at the object just created, and the + tagger's own instant and offset must reach the payload - a tag stamped "now" + is a different object from the one that was cut.""" + mod = _load_module() + repo = _init_repo(tmp_path) + _write_file(repo, "a.txt") + commit = _commit_all(repo, "root") + _tag_object(repo) + fake = FakeGitHub(repo) + orig = _with_fake(mod, fake) + try: + result = mod.push_tag("x", "v9.9.9", cwd=repo) + finally: + mod.api = orig + + assert result["result"] == "pushed" + assert result["commit"] == commit + assert fake.refs["refs/tags/v9.9.9"] == result["tag_object"], ( + "the ref does not point at the tag object that was created" + ) + posts = [(m, p) for m, p in fake.log if m == "POST"] + assert posts == [("POST", "/repos/x/git/tags"), ("POST", "/repos/x/git/refs")], ( + f"unexpected request order/set: {posts} - the ref has to be created after " + "the object it names, and nothing else is uploaded for a tag" + ) + assert fake.tag_payloads, "no tag payload was posted" + payload = fake.tag_payloads[0] + assert payload["object"] == commit and payload["type"] == "commit" + assert payload["tag"] == "v9.9.9" + # GIT_ENV stamps 1700000000 +0800 in every zone: the offset is what is asserted, + # not the instant's spelling (an offset-carrying date is the whole point). + assert payload["tagger"]["date"] == "2023-11-15T06:13:20+08:00", ( + f"the tagger offset was not preserved: {payload['tagger']['date']!r}" + ) + + +def test_a_lightweight_tag_is_refused_before_any_request(tmp_path): + """A lightweight tag would pass here and be refused by build-release.yml's + verify-tag job only after a four-platform build was spent, so the refusal + belongs in front of the first request.""" + mod = _load_module() + repo = _init_repo(tmp_path) + _write_file(repo, "a.txt") + _commit_all(repo, "root") + r = _git("tag", "light", "HEAD", cwd=repo) # no -a: a lightweight tag + assert r.returncode == 0, r.stderr + fake = FakeGitHub(repo) + orig = _with_fake(mod, fake) + try: + with pytest.raises(mod.PushError) as exc: + mod.push_tag("x", "light", cwd=repo) + finally: + mod.api = orig + + assert "annotated" in str(exc.value) + assert fake.log == [], f"a lightweight tag still made {fake.log} request(s)" + + +def test_push_tag_refuses_to_move_a_published_tag(tmp_path): + """A version number is not a branch: an existing tag naming another commit is + refused, and nothing is posted.""" + mod = _load_module() + repo = _init_repo(tmp_path) + _write_file(repo, "a.txt") + _commit_all(repo, "root") + _tag_object(repo) + other = _commit_all(repo, "later", new_file="b.txt") + fake = FakeGitHub(repo) + fake.refs["refs/tags/v9.9.9"] = other # the published tag names another commit + orig = _with_fake(mod, fake) + try: + with pytest.raises(mod.PushError) as exc: + mod.push_tag("x", "v9.9.9", cwd=repo) + finally: + mod.api = orig + + assert "refusing to move" in str(exc.value) + assert [p for m, p in fake.log if m == "POST"] == [], "a refused tag still posted" + assert fake.refs["refs/tags/v9.9.9"] == other, "the published tag was moved anyway" + + +def test_push_tag_is_a_no_op_when_the_ref_already_names_that_commit(tmp_path): + """Re-running stage 2 after a dropped connection must not be an error.""" + mod = _load_module() + repo = _init_repo(tmp_path) + _write_file(repo, "a.txt") + commit = _commit_all(repo, "root") + _tag_object(repo) + fake = FakeGitHub(repo) + fake.refs["refs/tags/v9.9.9"] = commit + orig = _with_fake(mod, fake) + try: + result = mod.push_tag("x", "v9.9.9", cwd=repo) + finally: + mod.api = orig + + assert result["result"] == "no-op" + assert result["commit"] == commit + assert [p for m, p in fake.log if m == "POST"] == [], "a no-op still posted an object" + + +def test_the_cli_takes_exactly_one_of_branch_or_tag(tmp_path): + """Neither or both is a usage error, reported before any git or network call - + the check has to sit in front of the origin lookup, which needs a repository.""" + import sys + + for argv in ([], ["--branch", "feature/x", "--tag", "v9.9.9"]): + r = subprocess.run([sys.executable, str(SCRIPT)] + argv, capture_output=True, + text=True, cwd=tmp_path, timeout=120) + assert r.returncode == 2, (argv, r.returncode, r.stdout, r.stderr) + assert "exactly one of --branch" in r.stdout, (argv, r.stdout) From 5df5507b874e0620b45035cd4245121e2a0bf2dc Mon Sep 17 00:00:00 2001 From: EMRG Evolution Date: Tue, 10 Nov 2026 17:48:02 +0800 Subject: [PATCH 2/2] emrg: pin the decoding of the test that runs the push script --- tests/test_push_branch_from_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_push_branch_from_api.py b/tests/test_push_branch_from_api.py index 2e7a148d..3a32ecb4 100644 --- a/tests/test_push_branch_from_api.py +++ b/tests/test_push_branch_from_api.py @@ -740,6 +740,7 @@ def test_the_cli_takes_exactly_one_of_branch_or_tag(tmp_path): for argv in ([], ["--branch", "feature/x", "--tag", "v9.9.9"]): r = subprocess.run([sys.executable, str(SCRIPT)] + argv, capture_output=True, - text=True, cwd=tmp_path, timeout=120) + text=True, encoding="utf-8", errors="replace", + cwd=tmp_path, timeout=120) assert r.returncode == 2, (argv, r.returncode, r.stdout, r.stderr) assert "exactly one of --branch" in r.stdout, (argv, r.stdout)