emrg: the release tag has a maintained push path when git-over-https is down

scripts/push-branch-from-api.py

@@ -34,9 +34,25 @@
     verifies: GET refs == local rev-parse, and `git diff` of the remote sha
     against the original local tip is empty
 
+The tag direction is the release's stage 2 and has the same problem: a tag push is
+the only trigger of build-release.yml, so an outage with no maintained tag path
+means re-deriving one from notes every release, untested. `--tag` publishes an
+annotated tag through the same two endpoints a release needs:
+
+  * `git cat-file -t refs/tags/<name>` must answer `tag` - a lightweight tag is
+    refused before a single request, because build-release.yml's verify-tag job
+    requires an annotated tag and would refuse it only after the build was spent
+  * the tag object is created with POST /git/tags, carrying the tagger's own
+    instant **and offset** (a tag stamped "now" is a different object)
+  * refs/tags/<name> is created with POST /git/refs - creating that ref is what
+    fires the tag workflow, so this is the whole of stage 2
+  * an existing tag at a different commit is refused rather than moved: a
+    published version number is not a branch to rewrite
+
 Usage:
     python scripts/push-branch-from-api.py [--repo owner/name] [--branch feature/x]
                                            [--ref HEAD] [--force]
+    python scripts/push-branch-from-api.py [--repo owner/name] --tag v0.3.48
 
 Requirements: git on PATH; api.github.com reachable; gh CLI or GH_TOKEN/GITHUB_TOKEN
 auth (private repos need it; public repos work anonymously but rate-limit).
@@ -435,15 +451,97 @@ def remote_sha(sha: str, kind: str) -> str:
             "content_identical": not diff.strip()}
 
 
+def parse_tag(raw: bytes) -> dict:
+    """Parse a raw annotated tag object into the payload POST /git/tags takes.
+
+    Layout: object/type/tag/tagger header lines, a blank line, then the message.
+    The tagger's epoch+offset is preserved as ISO-8601 carrying that offset, for
+    the same reason `parse_commit` preserves the author's: the API stores the
+    offset it is given, and a tag stamped "now" is not the object that was cut.
+    """
+    header, sep, message = raw.partition(b"\n\n")
+    if not sep:
+        raise ValueError("malformed tag object: no blank line before message")
+    fields = {}
+    for line in header.decode("utf-8").splitlines():
+        key, _, value = line.partition(" ")
+        fields[key] = value
+    for required in ("object", "type", "tag", "tagger"):
+        if required not in fields:
+            raise ValueError(f"malformed tag object: missing {required}")
+    return {"object": fields["object"], "type": fields["type"],
+            "tag": fields["tag"], "tagger": _parse_person(fields["tagger"]),
+            "message": message.decode("utf-8").rstrip("\n")}
+
+
+def _remote_tag_target(repo: str, tag: str) -> str | None:
+    """The commit refs/tags/<tag> names on the remote, or None when it is absent."""
+    try:
+        ref = api("GET", f"/repos/{repo}/git/ref/tags/{tag}")
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None
+        raise
+    sha = ref["object"]["sha"]
+    try:
+        return api("GET", f"/repos/{repo}/git/tags/{sha}")["object"]["sha"]
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return sha     # a lightweight remote tag holds the commit itself
+        raise
+
+
+def push_tag(repo: str, tag: str, cwd: str | None = None) -> dict:
+    """Publish an annotated tag through the Git Data API.
+
+    Creating refs/tags/<name> is what fires the tag workflow, so this is the
+    release's stage 2 when git-over-https is down. Everything that can be refused
+    is refused before the first request: a lightweight tag (the release pipeline
+    requires an annotated one), a missing local tag, and an already-published tag
+    naming a different commit - a version number is not a branch to rewrite.
+    """
+    ref = f"refs/tags/{tag}"
+    try:
+        kind = git("cat-file", "-t", ref, cwd=cwd).decode("utf-8").strip()
+    except RuntimeError as e:
+        raise PushError(f"{ref} is not present locally: {e}") from None
+    if kind != "tag":
+        raise PushError(f"{ref} is a {kind}, not an annotated tag object; the release "
+                        "pipeline requires an annotated tag, so nothing was sent")
+    payload = parse_tag(git("cat-file", "tag", ref, cwd=cwd))
+    commit = payload["object"]
+
+    existing = _remote_tag_target(repo, tag)
+    if existing == commit:
+        return {"result": "no-op", "tag": tag, "commit": commit}
+    if existing is not None:
+        raise PushError(f"{ref} on the remote already names {existing}; refusing to move "
+                        "a published tag - cut a new version instead")
+
+    created = api("POST", f"/repos/{repo}/git/tags", payload)["sha"]
+    api("POST", f"/repos/{repo}/git/refs", {"ref": ref, "sha": created})
+
+    confirmed = api("GET", f"/repos/{repo}/git/ref/tags/{tag}")["object"]["sha"]
+    if confirmed != created:
+        raise PushError(f"verification failed: {ref} reads {confirmed}, not {created}")
+    return {"result": "pushed", "tag": tag, "tag_object": created, "commit": commit}
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--repo", default=None, help="owner/name (default: from origin)")
-    ap.add_argument("--branch", required=True, help="remote branch name, e.g. feature/x")
+    ap.add_argument("--branch", default=None, help="remote branch name, e.g. feature/x")
+    ap.add_argument("--tag", default=None,
+                    help="annotated tag to publish, e.g. v0.3.48 (release stage 2)")
     ap.add_argument("--ref", default="HEAD", help="local ref to push (default: HEAD)")
     ap.add_argument("--force", action="store_true",
                     help="force-update the remote ref (default: fail on conflict)")
     args = ap.parse_args()
 
+    if bool(args.branch) == bool(args.tag):
+        print("pass exactly one of --branch <name> or --tag <name>")
+        return 2
+
     repo = args.repo
     if not repo:
         out = git("remote", "get-url", "origin").decode("utf-8").strip()
@@ -454,7 +552,10 @@ def main() -> int:
         repo = m.group(1)
 
     try:
-        result = push_branch(repo, args.branch, args.ref, args.force)
+        if args.tag:
+            result = push_tag(repo, args.tag)
+        else:
+            result = push_branch(repo, args.branch, args.ref, args.force)
     except PushError as e:
         print(f"push aborted: {e}")
         return 1
@@ -465,6 +566,16 @@ def main() -> int:
         print(f"push aborted (network error after retries): {e}")
         return 1
 
+    if args.tag:
+        print(f"repo {repo} tag {args.tag}")
+        print(f"  release commit: {result['commit']}")
+        if result["result"] == "no-op":
+            print("  nothing to do - refs/tags/ already names that commit")
+        else:
+            print(f"  tag object: {result['tag_object']}")
+            print(f"  created refs/tags/{args.tag} - creating it fires the tag workflow")
+        return 0
+
     print(f"repo {repo} branch {args.branch}")
     print(f"  remote sha: {result['sha']}")
     if result["result"] == "no-op":

tests/test_push_branch_from_api.py

@@ -107,6 +107,23 @@ def person(p):
     return ("\n".join(lines) + "\n\n" + msg).encode("utf-8")
 
 
+def _raw_tag_from_payload(payload):
+    """Reconstruct the raw annotated tag object GitHub stores for a POST /git/tags
+    payload: the tagger's epoch+offset, and a message with no trailing newline -
+    the same normalization create-commit is observed to apply."""
+    def person(p):
+        import datetime as dt
+        d = dt.datetime.fromisoformat(p["date"])
+        return f"{p['name']} <{p['email']}> {int(d.timestamp())} {d.strftime('%z')}"
+
+    msg = payload["message"]
+    if msg.endswith("\n"):
+        msg = msg[:-1]
+    header = (f"object {payload['object']}\ntype {payload['type']}\n"
+              f"tag {payload['tag']}\ntagger {person(payload['tagger'])}")
+    return (header + "\n\n" + msg).encode("utf-8")
+
+
 def _hash_object(repo: Path, obj_type: str, raw: bytes, write: bool = False) -> str:
     args = ["hash-object", "-t", obj_type]
     if write:
@@ -127,6 +144,8 @@ def __init__(self, repo: Path):
         self.refs = {}         # "heads/x" -> sha
         self.log = []          # (method, path) call log
         self.fail_commits = False
+        self.tag_targets = {}  # tag object sha -> the commit it names
+        self.tag_payloads = [] # every POST /git/tags body, for the offset reading
 
     def __call__(self, method, path, body=None):
         self.log.append((method, path))
@@ -197,6 +216,25 @@ def __call__(self, method, path, body=None):
             if sha in self.objects:
                 return {"sha": sha}
             raise _http_error(404)
+        if path.startswith("/repos/x/git/ref/tags/"):
+            ref_key = "refs/tags/" + path[len("/repos/x/git/ref/tags/"):]
+            if method != "GET":
+                raise AssertionError(f"unexpected singular-ref call {method} {path}")
+            if ref_key in self.refs:
+                return {"ref": ref_key, "object": {"sha": self.refs[ref_key]}}
+            raise _http_error(404)
+        if path == "/repos/x/git/tags":
+            self.tag_payloads.append(body)
+            raw = _raw_tag_from_payload(body)
+            sha = _hash_object(self.repo, "tag", raw, write=True)
+            self.objects[sha] = ("tag", raw)
+            self.tag_targets[sha] = body["object"]
+            return {"sha": sha, "object": {"sha": body["object"], "type": "commit"}}
+        if path.startswith("/repos/x/git/tags/"):
+            sha = path.rsplit("/", 1)[1]
+            if sha in self.tag_targets:
+                return {"sha": sha, "object": {"sha": self.tag_targets[sha], "type": "commit"}}
+            raise _http_error(404)
         raise AssertionError(f"unexpected call {method} {path}")
 
 
@@ -568,3 +606,141 @@ def test_a_push_onto_an_existing_remote_base_uploads_only_what_is_missing(tmp_pa
         "the new tree was never created"
     )
     assert result["sha"] in fake.objects, "the new commit was never created"
+
+
+# ---------------------------------------------------------------- the tag path
+#
+# Stage 2 of a release is a tag push, and a tag push is the only trigger of
+# build-release.yml. The branch path exists so an outage does not mean
+# re-deriving a script from notes; the tag path had no home in the repository at
+# all, so every outage re-derived one and nothing tested it (#2795). These pin
+# the four refusals and the one order that make the tag path safe to run during a
+# release: refuse before the network, create the object before the ref, and never
+# move a published tag.
+
+
+def _tag_object(repo: Path, name: str = "v9.9.9") -> str:
+    r = _git("tag", "-a", name, "-m", f"emrg {name}", "HEAD", cwd=repo, env=GIT_ENV)
+    assert r.returncode == 0, r.stderr
+    return _git("rev-parse", f"refs/tags/{name}", cwd=repo).stdout.decode().strip()
+
+
+def _with_fake(mod, fake):
+    orig = mod.api
+    mod.api = fake
+    return orig
+
+
+def test_push_tag_creates_the_tag_object_before_the_ref(tmp_path):
+    """The ref must be created last, pointing at the object just created, and the
+    tagger's own instant and offset must reach the payload - a tag stamped "now"
+    is a different object from the one that was cut."""
+    mod = _load_module()
+    repo = _init_repo(tmp_path)
+    _write_file(repo, "a.txt")
+    commit = _commit_all(repo, "root")
+    _tag_object(repo)
+    fake = FakeGitHub(repo)
+    orig = _with_fake(mod, fake)
+    try:
+        result = mod.push_tag("x", "v9.9.9", cwd=repo)
+    finally:
+        mod.api = orig
+
+    assert result["result"] == "pushed"
+    assert result["commit"] == commit
+    assert fake.refs["refs/tags/v9.9.9"] == result["tag_object"], (
+        "the ref does not point at the tag object that was created"
+    )
+    posts = [(m, p) for m, p in fake.log if m == "POST"]
+    assert posts == [("POST", "/repos/x/git/tags"), ("POST", "/repos/x/git/refs")], (
+        f"unexpected request order/set: {posts} - the ref has to be created after "
+        "the object it names, and nothing else is uploaded for a tag"
+    )
+    assert fake.tag_payloads, "no tag payload was posted"
+    payload = fake.tag_payloads[0]
+    assert payload["object"] == commit and payload["type"] == "commit"
+    assert payload["tag"] == "v9.9.9"
+    # GIT_ENV stamps 1700000000 +0800 in every zone: the offset is what is asserted,
+    # not the instant's spelling (an offset-carrying date is the whole point).
+    assert payload["tagger"]["date"] == "2023-11-15T06:13:20+08:00", (
+        f"the tagger offset was not preserved: {payload['tagger']['date']!r}"
+    )
+
+
+def test_a_lightweight_tag_is_refused_before_any_request(tmp_path):
+    """A lightweight tag would pass here and be refused by build-release.yml's
+    verify-tag job only after a four-platform build was spent, so the refusal
+    belongs in front of the first request."""
+    mod = _load_module()
+    repo = _init_repo(tmp_path)
+    _write_file(repo, "a.txt")
+    _commit_all(repo, "root")
+    r = _git("tag", "light", "HEAD", cwd=repo)   # no -a: a lightweight tag
+    assert r.returncode == 0, r.stderr
+    fake = FakeGitHub(repo)
+    orig = _with_fake(mod, fake)
+    try:
+        with pytest.raises(mod.PushError) as exc:
+            mod.push_tag("x", "light", cwd=repo)
+    finally:
+        mod.api = orig
+
+    assert "annotated" in str(exc.value)
+    assert fake.log == [], f"a lightweight tag still made {fake.log} request(s)"
+
+
+def test_push_tag_refuses_to_move_a_published_tag(tmp_path):
+    """A version number is not a branch: an existing tag naming another commit is
+    refused, and nothing is posted."""
+    mod = _load_module()
+    repo = _init_repo(tmp_path)
+    _write_file(repo, "a.txt")
+    _commit_all(repo, "root")
+    _tag_object(repo)
+    other = _commit_all(repo, "later", new_file="b.txt")
+    fake = FakeGitHub(repo)
+    fake.refs["refs/tags/v9.9.9"] = other      # the published tag names another commit
+    orig = _with_fake(mod, fake)
+    try:
+        with pytest.raises(mod.PushError) as exc:
+            mod.push_tag("x", "v9.9.9", cwd=repo)
+    finally:
+        mod.api = orig
+
+    assert "refusing to move" in str(exc.value)
+    assert [p for m, p in fake.log if m == "POST"] == [], "a refused tag still posted"
+    assert fake.refs["refs/tags/v9.9.9"] == other, "the published tag was moved anyway"
+
+
+def test_push_tag_is_a_no_op_when_the_ref_already_names_that_commit(tmp_path):
+    """Re-running stage 2 after a dropped connection must not be an error."""
+    mod = _load_module()
+    repo = _init_repo(tmp_path)
+    _write_file(repo, "a.txt")
+    commit = _commit_all(repo, "root")
+    _tag_object(repo)
+    fake = FakeGitHub(repo)
+    fake.refs["refs/tags/v9.9.9"] = commit
+    orig = _with_fake(mod, fake)
+    try:
+        result = mod.push_tag("x", "v9.9.9", cwd=repo)
+    finally:
+        mod.api = orig
+
+    assert result["result"] == "no-op"
+    assert result["commit"] == commit
+    assert [p for m, p in fake.log if m == "POST"] == [], "a no-op still posted an object"
+
+
+def test_the_cli_takes_exactly_one_of_branch_or_tag(tmp_path):
+    """Neither or both is a usage error, reported before any git or network call -
+    the check has to sit in front of the origin lookup, which needs a repository."""
+    import sys
+
+    for argv in ([], ["--branch", "feature/x", "--tag", "v9.9.9"]):
+        r = subprocess.run([sys.executable, str(SCRIPT)] + argv, capture_output=True,
+                           text=True, encoding="utf-8", errors="replace",
+                           cwd=tmp_path, timeout=120)
+        assert r.returncode == 2, (argv, r.returncode, r.stdout, r.stderr)
+        assert "exactly one of --branch" in r.stdout, (argv, r.stdout)