Add UK high-value housing sources: OBR surcharge costing, ONS house prices by constituency, and Price Paid, UK HPI and NSPL registrations

.gitignore

@@ -3,3 +3,11 @@ __pycache__/
 micro/us/*.parquet
 venv/
 *.parquet
+# Raw releases archived in R2 only (chronicle#343); each manifest pins the bytes.
+# Price Paid Data carries Royal Mail and OS address data licensed for
+# non-commercial use, so none of its bytes enter this public repository; the
+# NSPL zip is above GitHub's 100 MB file limit; and each monthly UK HPI full file
+# (35 MB) revises the last, so committing vintages would only grow history.
+db/data/hmlr/price_paid_data_*/pp-*.csv
+db/data/hmlr/uk_hpi_full_file_*/UK-HPI-full-file-*.csv
+db/data/ons/nspl_*/NSPL_*.zip

README.md

@@ -393,6 +393,9 @@ uv run chronicle inventory-artifacts --root db/data
 # Upload all existing manifest-declared local artifacts to ledger-raw and write
 # storage.r2 metadata back into the manifests:
 uv run chronicle publish-raw --root db/data
+# Entries marked `storage.local: false` keep their bytes in R2 only (never
+# committed). Both commands above accept them without a local copy, check them
+# when one is present, and leave the R2 object to verify-raw.
 
 # Read back declared R2 objects and verify canonical keys, SHA-256 and size:
 uv run chronicle verify-raw --root db/data

changelog.d/343.added.md

@@ -0,0 +1 @@
+Add high-value housing sources for chronicle#343: the OBR costing of the high value council tax surcharge (properties in scope by band, static and post-behaviour, the costing and its stated parameters, as source projections), ONS sales counts and median prices by 2024 constituency and property type for the years ending March 2023 to 2026, and raw-only registrations, archived in R2 and never committed, of HM Land Registry Price Paid Data (August 2026 release: the 32 yearly files and the monthly change file), the UK House Price Index full file (July 2026) and the National Statistics Postcode Lookup (August 2026). Manifest entries marked `storage.local: false` keep their bytes in R2 only, and `inventory-artifacts` and `publish-raw` no longer require them locally.

chronicle/artifacts.py

@@ -41,6 +41,7 @@
             "desnz",
             "dft",
             "dwp",
+            "hmlr",
             "hmrc",
             "isc",
             "mhclg",
@@ -162,10 +163,11 @@ class ArtifactInventoryEntry:
     source_url: str | None
     r2: dict[str, Any] | None
     errors: tuple[str, ...]
+    r2_only: bool = False
 
     @property
     def valid(self) -> bool:
-        """Whether this artifact is locally available and checksum-valid."""
+        """Whether this artifact is checksum-valid, or R2-only and absent."""
         return not self.errors
 
     def to_dict(self) -> dict[str, Any]:
@@ -182,6 +184,7 @@ def to_dict(self) -> dict[str, Any]:
             "size_bytes": self.size_bytes,
             "source_url": self.source_url,
             "r2": self.r2,
+            "r2_only": self.r2_only,
             "errors": list(self.errors),
         }
 
@@ -226,11 +229,14 @@ class RawArtifactPublishEntry:
     r2_location: ArtifactStorageLocation | None
     upload: ArtifactCommandResult | None
     errors: tuple[str, ...] = ()
+    skipped: bool = False
 
     @property
     def valid(self) -> bool:
-        """Whether this raw artifact uploaded and was registered."""
-        return not self.errors and self.upload is not None and self.upload.ok
+        """Whether this raw artifact uploaded, or is R2-only and absent here."""
+        if self.errors:
+            return False
+        return self.skipped or (self.upload is not None and self.upload.ok)
 
     def to_dict(self) -> dict[str, Any]:
         """Return a JSON-serializable entry."""
@@ -246,6 +252,7 @@ def to_dict(self) -> dict[str, Any]:
             "size_bytes": self.size_bytes,
             "r2_location": (self.r2_location.to_dict() if self.r2_location else None),
             "upload": self.upload.to_dict() if self.upload else None,
+            "skipped": self.skipped,
             "errors": list(self.errors),
         }
 
@@ -270,7 +277,10 @@ def counts(self) -> dict[str, int]:
         return {
             "manifest_count": len(manifest_paths),
             "artifact_count": len(self.entries),
-            "uploaded_count": sum(1 for entry in self.entries if entry.valid),
+            "uploaded_count": sum(
+                1 for entry in self.entries if entry.valid and not entry.skipped
+            ),
+            "skipped_count": sum(1 for entry in self.entries if entry.skipped),
             "failed_count": sum(1 for entry in self.entries if not entry.valid),
             "r2_link_count": sum(
                 1 for entry in self.entries if entry.r2_location is not None
@@ -1134,6 +1144,7 @@ def inventory_source_artifacts(
                 "manifest_count": 0,
                 "artifact_count": 0,
                 "missing_count": 0,
+                "r2_only_count": 0,
                 "checksum_mismatch_count": 0,
                 "r2_link_count": 0,
             },
@@ -1158,7 +1169,11 @@ def inventory_source_artifacts(
     counts = {
         "manifest_count": len(manifests),
         "artifact_count": len(entries),
-        "missing_count": sum(1 for entry in entries if not entry.exists),
+        # R2-only entries are absent by design, so they never count as missing.
+        "missing_count": sum(
+            1 for entry in entries if not entry.exists and not entry.r2_only
+        ),
+        "r2_only_count": sum(1 for entry in entries if entry.r2_only),
         "checksum_mismatch_count": sum(
             1 for entry in entries if "checksum_mismatch" in entry.errors
         ),
@@ -1490,6 +1505,27 @@ def _upload_r2_object(
     return _run_command(command)
 
 
+def _r2_only_storage(spec: dict[str, Any]) -> tuple[bool, list[str]]:
+    """Return whether a file entry keeps its bytes in R2 only, and any issues.
+
+    ``storage.local: false`` declares bytes that are never committed: R2 holds
+    them and ``verify-raw`` checks them, so a checkout without them is complete.
+    The entry must name the R2 object it relies on.
+    """
+    storage = spec.get("storage")
+    if not isinstance(storage, dict) or "local" not in storage:
+        return False, []
+    local = storage["local"]
+    if not isinstance(local, bool):
+        return False, ["malformed_storage_local"]
+    if local:
+        return False, []
+    r2 = storage.get("r2")
+    if not isinstance(r2, dict) or not r2.get("key"):
+        return True, ["r2_only_without_r2_location"]
+    return True, []
+
+
 def _publish_raw_manifest_entry(
     manifest_path: Path,
     source_id: str,
@@ -1510,6 +1546,31 @@ def _publish_raw_manifest_entry(
     sha256_expected = spec.get("sha256")
     sha256_actual = None
     size_bytes = None
+    r2_only, r2_only_errors = _r2_only_storage(spec)
+    errors.extend(r2_only_errors)
+    if r2_only and filename and not errors and not artifact_path.exists():
+        # Nothing to upload from this checkout; verify-raw checks the object.
+        recorded_r2 = spec["storage"]["r2"]
+        return (
+            RawArtifactPublishEntry(
+                manifest_path=str(manifest_path),
+                source_id=source_id,
+                package_id=package_id,
+                year=str(year),
+                filename=filename,
+                local_path=str(artifact_path),
+                sha256=sha256_expected,
+                size_bytes=spec.get("size_bytes"),
+                r2_location=ArtifactStorageLocation(
+                    provider=str(recorded_r2.get("provider") or "r2"),
+                    bucket=str(recorded_r2.get("bucket") or r2_bucket),
+                    key=str(recorded_r2["key"]),
+                ),
+                upload=None,
+                skipped=True,
+            ),
+            None,
+        )
     if not filename:
         errors.append("missing_filename")
     elif not artifact_path.exists():
@@ -1630,10 +1691,14 @@ def _inventory_entry(
     sha256_expected = spec.get("sha256")
     sha256_actual = None
     size_bytes = None
+    r2_only, r2_only_errors = _r2_only_storage(spec)
+    errors.extend(r2_only_errors)
     if not filename:
         errors.append("missing_filename")
     elif not exists:
-        errors.append("missing_file")
+        # An R2-only entry is complete without local bytes; verify-raw checks it.
+        if not r2_only:
+            errors.append("missing_file")
     else:
         content = artifact_path.read_bytes()
         sha256_actual = hashlib.sha256(content).hexdigest()
@@ -1654,6 +1719,7 @@ def _inventory_entry(
         source_url=spec.get("source_url"),
         r2=r2,
         errors=tuple(errors),
+        r2_only=r2_only,
     )
 
 

chronicle/bundle.py

@@ -185,6 +185,7 @@
     "obr-efo-expenditure-march-2026",
     "obr-efo-receipts-march-2026",
     "obr-fuel-duty-receipts-by-vehicle-april-2024",
+    "obr-hvcts-costing-april-2026",
     "obr-salary-sacrifice-costing-february-2026",
     "ofgem-energy-price-cap-levels-2024-2026",
     "ofgem-energy-price-cap-q1-2024",
@@ -231,6 +232,7 @@
     "ons-household-projections-2022-based",
     "ons-household-totals-country-region-2015-2025",
     "ons-households-by-type-country-2025",
+    "ons-hpssa-pcon24-year-ending-march-2026",
     "ons-lad-population-by-age-2024",
     "ons-lad-population-by-age-2025",
     "ons-mye-2023-england-regions",

chronicle/source_package.py

@@ -494,6 +494,13 @@
         "ons/employee_workplace_pensions_summary_2024"
     ),
     # end chronicle#302
+    # chronicle#343: high-value housing facts for the UK local calibration (the
+    # Price Paid, UK HPI and NSPL releases are raw-only registrations, not packages).
+    "obr-hvcts-costing-april-2026": Path("obr/hvcts_costing_april_2026"),
+    "ons-hpssa-pcon24-year-ending-march-2026": Path(
+        "ons/hpssa_pcon24_year_ending_march_2026"
+    ),
+    # end chronicle#343
     "hmrc-cgt-age-2026": Path("hmrc/cgt_age_2026"),
     "hmrc-cgt-asset-type-2026": Path("hmrc/cgt_asset_type_2026"),
     "hmrc-cgt-badr-ir-2026": Path("hmrc/cgt_badr_ir_2026"),

db/data/hmlr/price_paid_data_monthly_update_august_2026/manifest.yaml

@@ -0,0 +1,37 @@
+source_id: hmlr
+package_id: hmlr-price-paid-data-monthly-update-august-2026
+source_name: HM Land Registry Price Paid Data, August 2026 release, monthly change
+  file
+publisher: HM Land Registry
+source_page: https://www.gov.uk/government/statistical-data-sets/price-paid-data-august-2026
+files:
+  2026:
+    filename: pp-monthly-update-new-version.csv
+    source_url: https://price-paid-data.publicdata.landregistry.gov.uk/pp-monthly-update-new-version.csv
+    source_table: 'Price Paid Data: August 2026, current month (CSV)'
+    sha256: 5236cb389b294f3352027fb3d01bf6f6f367fb99658cbf9ba9ffeef3c3e07d43
+    size_bytes: 15840574
+    fetched_at: '2026-10-12T12:55:50+00:00'
+    retrieved_at: '2026-10-12'
+    vintage: ppd_2026_08
+    vintages:
+    - ppd_2026_08
+    storage:
+      local: false
+      r2:
+        provider: r2
+        bucket: ledger-raw
+        key: raw/uk/hmlr/hmlr-price-paid-data-monthly-update-august-2026/2026/5236cb389b294f3352027fb3d01bf6f6f367fb99658cbf9ba9ffeef3c3e07d43/pp-monthly-update-new-version.csv
+        uri: r2://ledger-raw/raw/uk/hmlr/hmlr-price-paid-data-monthly-update-august-2026/2026/5236cb389b294f3352027fb3d01bf6f6f367fb99658cbf9ba9ffeef3c3e07d43/pp-monthly-update-new-version.csv
+registration_kind: raw_only
+provenance_notes: 'Published 29 September 2026 (object last modified 28 September
+  2026). The release''s change file: transactions received in August 2026 and the
+  additions, changes and deletions it makes to earlier releases, one row each, record
+  status A, C or D in column 16 against the transaction unique identifier. Chronicle
+  archives the file and does not apply it; hmlr-price-paid-data-yearly-august-2026
+  is the same release as a snapshot, one file per year of transfer. Field contract:
+  docs/data-sources/uk-high-value-housing.md. Open Government Licence v3.0 with HM
+  Land Registry''s attribution statement; the address fields are Royal Mail and Ordnance
+  Survey address data that HM Land Registry licenses for personal or non-commercial
+  use and for residential property price information services, so the bytes are archived
+  in R2 and not committed to this public repository.'