Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions app/verify/gsmarena_backfill.py
Original file line number Diff line number Diff line change
Expand Up @@ -794,12 +794,16 @@ def write_source_url_if_unchanged(path: Path, record: dict[str, Any], url: str)
current = json.loads(raw)
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
return False
if not isinstance(current, dict) or content_hash(current) != content_hash(record):
if not isinstance(current, dict):
return False
source_urls = current.get("source_urls")
if isinstance(source_urls, list) and url in source_urls:
return True
if content_hash(current) != content_hash(record):
return False
updated = add_source_url_text(raw, url)
if updated is None:
source_urls = current.get("source_urls")
return isinstance(source_urls, list) and url in source_urls
return False
tmp = path.with_suffix(path.suffix + ".tmp")
try:
tmp.write_bytes(updated.encode("utf-8"))
Expand Down Expand Up @@ -1165,7 +1169,15 @@ def fetch(url: str) -> tuple[int | None, str, str]:
digest = content_hash(record)
cached = cache.get(rel)
if cached and cached.get("hash") == digest and cached.get("decision") in DECISIONS:
result.rows.append(_row_from_cache(cached))
row = _row_from_cache(cached)
if not dry_run and cached["decision"] == CONFIRM:
url = cached.get("proposed_url")
if not isinstance(url, str) or not write_source_url_if_unchanged(
repo / rel, record, url
):
row["decision"] = "write-failed"
row["reason"] = "source-urls-write-failed-or-record-changed"
result.rows.append(row)
result.cached += 1
continue
outcome = evaluate_record(record, fetcher, fetch)
Expand All @@ -1190,8 +1202,8 @@ def fetch(url: str) -> tuple[int | None, str, str]:
"source-urls-write-failed-or-record-changed",
outcome.suffix_only,
)
# Transient blocks stay uncached so a later run can retry them. A real
# CONFIRM is cached only after the guarded source_urls write succeeded.
# Transient blocks stay uncached so a later run can retry them. Cached
# dry-run CONFIRMs are written through the same guard on an apply run.
if outcome.liveness not in RETRY_LIVENESS and outcome.reason != "network-error":
append_cache(
cache_entry(rel_path=rel, record=record, result=outcome, ts=_now_iso()),
Expand Down
61 changes: 59 additions & 2 deletions tests/verify/test_gsmarena_backfill.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
add_source_url_text,
backfill,
compare_specs,
content_hash,
gate_page,
parse_cat_file_batch,
parse_page,
Expand Down Expand Up @@ -370,7 +369,65 @@ def test_dry_run_does_not_write_the_record_and_caches(tmp_path: Path):
)
assert client.requests == 0
assert again.cached == 1
assert content_hash(DX650_RECORD) == again.rows[0]["decision"] or again.cached == 1
assert again.rows[0]["decision"] == CONFIRM

# The cached confirmation still needs to be applied to the source file.
applied = backfill(
repo=repo,
cache_path=state / "gsmarena_backfill_cache.jsonl",
index_path=state / "phone_index.json",
summary_path=state / "summary.md",
limit=5,
sleep_s=0,
dry_run=False,
refresh_index=False,
client=client,
paths=[rel],
records={rel: DX650_RECORD},
)
url = "https://www.gsmarena.com/acer_dx650-2888.php"
written = target.read_text(encoding="utf-8")
assert json.loads(written)["source_urls"].count(url) == 1
assert applied.cached == 1
assert applied.rows[0]["decision"] == CONFIRM
assert client.requests == 0

# Applying again must leave the file byte-for-byte unchanged.
reapplied = backfill(
repo=repo,
cache_path=state / "gsmarena_backfill_cache.jsonl",
index_path=state / "phone_index.json",
summary_path=state / "summary.md",
limit=5,
sleep_s=0,
dry_run=False,
refresh_index=False,
client=client,
paths=[rel],
records={rel: DX650_RECORD},
)
assert target.read_text(encoding="utf-8") == written
assert reapplied.rows[0]["decision"] == CONFIRM
assert client.requests == 0

changed = {**DX650_RECORD, "battery_mah": 9999}
target.write_text(json.dumps(changed, indent=2) + "\n", encoding="utf-8")
stale = backfill(
repo=repo,
cache_path=state / "gsmarena_backfill_cache.jsonl",
index_path=state / "phone_index.json",
summary_path=state / "summary.md",
limit=5,
sleep_s=0,
dry_run=False,
refresh_index=False,
client=client,
paths=[rel],
records={rel: DX650_RECORD},
)
assert stale.rows[0]["decision"] == "write-failed"
assert url not in target.read_text(encoding="utf-8")
assert client.requests == 0


def test_retry_after_is_parsed_and_honored_without_a_fixed_delay(tmp_path: Path, monkeypatch):
Expand Down
Loading