From fd6f5160ca78e48b291bbf80ace262abb3fee2d3 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Tue, 4 Aug 2026 23:00:31 +0200 Subject: [PATCH 01/23] replace os.rename with os.replace --- rapida/components/population/__init__.py | 2 +- rapida/connectivity/io.py | 2 +- uv.lock | 34 ------------------------ 3 files changed, 2 insertions(+), 36 deletions(-) diff --git a/rapida/components/population/__init__.py b/rapida/components/population/__init__.py index 32ba035..4391879 100644 --- a/rapida/components/population/__init__.py +++ b/rapida/components/population/__init__.py @@ -512,7 +512,7 @@ def import_raster(self, source=None, **kwargs): ) if os.path.exists(source):os.remove(source) - os.rename(imported_local_path, source) + os.replace(imported_local_path, source) return source def _compute_affected_(self, progress=None, **kwargs): diff --git a/rapida/connectivity/io.py b/rapida/connectivity/io.py index 31eb6a4..9c23fa6 100644 --- a/rapida/connectivity/io.py +++ b/rapida/connectivity/io.py @@ -261,7 +261,7 @@ async def prepare_osm_pbf(bbox: tuple[float, float, float, float], dst_dir: str elif len(extracted_chunks) == 1: # If there was only one valid chunk, just rename it to the final output target - os.rename(extracted_chunks[0], final_output_pbf) + os.replace(extracted_chunks[0], final_output_pbf) else: raise ValueError(f"No OSM data found in the provided bbox: {bbox}") diff --git a/uv.lock b/uv.lock index f2bf0a2..28321ff 100644 --- a/uv.lock +++ b/uv.lock @@ -3143,7 +3143,6 @@ dependencies = [ { name = "tensorflow", marker = "platform_machine == 'aarch64'" }, { name = "tensorflow-cpu", marker = "platform_machine == 'x86_64'" }, { name = "tqdm" }, - { name = "uvloop" }, ] [package.optional-dependencies] @@ -3203,7 +3202,6 @@ requires-dist = [ { name = "tensorflow", marker = "platform_machine == 'aarch64'", specifier = "==2.16.2" }, { name = "tensorflow-cpu", marker = "platform_machine == 'x86_64'", specifier = "==2.16.2" }, { name = "tqdm" }, - { name = "uvloop" }, ] provides-extras = ["dev"] @@ -4005,38 +4003,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/a4/0698f3e5c397442ec9323a537e48cc63b846288b6878d38efd04e91005e3/utm-0.8.1-py3-none-any.whl", hash = "sha256:e3d5e224082af138e40851dcaad08d7f99da1cc4b5c413a7de34eabee35f434a", size = 8613, upload-time = "2025-03-06T11:40:54.273Z" }, ] -[[package]] -name = "uvloop" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, - { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, - { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, - { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, - { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, - { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, - { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, - { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, - { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, - { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, - { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, - { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, - { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, - { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, - { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, - { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, - { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, - { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, -] - [[package]] name = "werkzeug" version = "3.1.8" From 4b83140f09f5d0a44eb7762e57906b1f4498f406 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Tue, 4 Aug 2026 23:05:06 +0200 Subject: [PATCH 02/23] add encoding to json open --- rapida/connectivity/io.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rapida/connectivity/io.py b/rapida/connectivity/io.py index 9c23fa6..2e550e6 100644 --- a/rapida/connectivity/io.py +++ b/rapida/connectivity/io.py @@ -300,7 +300,7 @@ async def extract_health_sites(pbf_path: str, dst_dir: str, progress=None) -> st # Step 3: Compute centroids and flatten properties in a background thread def process_geometries(): - with open(raw_geojson, "r") as f: + with open(raw_geojson, "r", encoding="utf-8") as f: data = json.load(f) processed_features = [] @@ -329,7 +329,7 @@ def process_geometries(): data["features"] = processed_features - with open(final_geojson, "w") as f: + with open(final_geojson, "w", encoding="utf-8") as f: json.dump(data, f) if progress: @@ -352,7 +352,7 @@ def extract_origins_from_geojson(geojson_path: str) -> list[tuple[float, float]] """ Extracts a list of (longitude, latitude) tuples from a GeoJSON FeatureCollection. """ - with open(geojson_path, "r") as f: + with open(geojson_path, "r", encoding="utf-8") as f: data = json.load(f) origins = [] @@ -613,7 +613,7 @@ async def extract_water_bodies(pbf_path: str, dst_dir: str, progress=None) -> st # Step 3: Process geometries and ensure clean polygon outputs in a background thread def process_water_geometries(): - with open(raw_geojson, "r") as f: + with open(raw_geojson, "r", encoding="utf-8") as f: data = json.load(f) processed_features = [] @@ -645,7 +645,7 @@ def process_water_geometries(): data["features"] = processed_features - with open(final_geojson, "w") as f: + with open(final_geojson, "w", encoding="utf-8") as f: json.dump(data, f) if progress: From eaa081f56ce05c98dd7a33dc97660c7d7e9b176a Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 18:44:35 +0200 Subject: [PATCH 03/23] prohibit mjolnir from using mmap --- rapida/connectivity/graph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 01c6da9..2bf8a6a 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -199,6 +199,7 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> # --------------------------------------------------------- valhalla_conf["mjolnir"]["min_reachability"] = 0 + valhalla_conf["mjolnir"]["use_mmap"] = False with open(config_path, "w") as f: json.dump(valhalla_conf, f, indent=4) From c366f0b67608b785a71e677d85f2c4b1c3ad03be Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 18:52:21 +0200 Subject: [PATCH 04/23] prohibit mjolnir from using mmap on windows only --- rapida/connectivity/graph.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 2bf8a6a..6ba2b4e 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -6,10 +6,8 @@ from valhalla.config import _sanitize_config, default_config from typing import Union from pathlib import Path -import re import logging import sys -import valhalla logger = logging.getLogger(__name__) DEFAULT_SPEEDS = { @@ -199,7 +197,8 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> # --------------------------------------------------------- valhalla_conf["mjolnir"]["min_reachability"] = 0 - valhalla_conf["mjolnir"]["use_mmap"] = False + if sys.platform == "win32": + valhalla_conf["mjolnir"]["use_mmap"] = False with open(config_path, "w") as f: json.dump(valhalla_conf, f, indent=4) From a7741d50d68689cd016b43e4389c8f5766720555 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 20:08:45 +0200 Subject: [PATCH 05/23] disable error on cleanup in temp folders --- rapida/connectivity/__init__.py | 7 ++++--- rapida/project/project.py | 2 -- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index d1bcd5c..1a5e63c 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -131,7 +131,7 @@ async def run_connectivity_analysis( del isochrones_gdf del results - with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=isochrones_path, comment='temp project for conn isochrones') with click.Context(assess) as ctx: ctx.ensure_object(dict) @@ -161,6 +161,7 @@ async def run_connectivity_analysis( promote_to_multi=True, index=False ) + #Project._instance = None if barriers_dataset is not None and pop_vars: @@ -239,7 +240,7 @@ async def run_connectivity_analysis( del results - with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=barrier_isochrones_path, comment='temp project for conn isochrones') with click.Context(assess) as ctx: ctx.ensure_object(dict) @@ -341,7 +342,7 @@ async def run_connectivity_analysis( gc.collect() - with TemporaryDirectory(dir=dest_dir, delete=True) as admin_project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: logger.info(f'Computing zonal stats for total population ') adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") diff --git a/rapida/project/project.py b/rapida/project/project.py index acff8d8..6fabf05 100644 --- a/rapida/project/project.py +++ b/rapida/project/project.py @@ -159,7 +159,6 @@ def __init__(self, path: str,polygons: str = None, # 2. Project centroids to EPSG:4326 for the remote mask query centroids_4326 = centroids.to_crs(epsg=4326) - centroids_4326.to_file('/tmp/isocen.fgb', engine='pyogrio') # 3. Use 'mask' instead of 'bbox'. This asks GDAL to only download # features that intersect your specific points, ignoring empty space. @@ -656,4 +655,3 @@ def publish(self, target_layers=None, no_input=False): project_path = '/data/rap/bgd' p = Project(path=project_path) src_raster = '/data/rap/bgd/data/population/female_active/BGD_female_active_r.tif' - p.align_raster(source_raster=src_raster) \ No newline at end of file From 8819114ef6b4dca0e106a38a2569007f34b18c46 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 20:30:44 +0200 Subject: [PATCH 06/23] set mjolnir config for admin and tzone --- rapida/connectivity/graph.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 6ba2b4e..34fca73 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -190,7 +190,7 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> } ] - with open(speeds_config_path, "w") as f: + with open(speeds_config_path, "w", encoding='utf-8') as f: json.dump(speed_schema, f, indent=4) valhalla_conf["mjolnir"]["default_speeds_config"] = speeds_config_path @@ -200,7 +200,11 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> if sys.platform == "win32": valhalla_conf["mjolnir"]["use_mmap"] = False - with open(config_path, "w") as f: + # ADD THESE TWO LINES: + valhalla_conf["mjolnir"]["admin"] = os.path.join(dst_dir, "admin.sqlite").replace("\\", "/") + valhalla_conf["mjolnir"]["timezone"] = os.path.join(dst_dir, "tz_world.sqlite").replace("\\", "/") + + with open(config_path, "w", encoding='utf-8') as f: json.dump(valhalla_conf, f, indent=4) From 69b9c7d8b60bffc01f02ed87ca1f03916586a5b7 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 20:32:34 +0200 Subject: [PATCH 07/23] set mjolnir config for admin and tzone --- rapida/connectivity/graph.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 34fca73..8c8919c 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -197,12 +197,12 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> # --------------------------------------------------------- valhalla_conf["mjolnir"]["min_reachability"] = 0 + valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False + if sys.platform == "win32": valhalla_conf["mjolnir"]["use_mmap"] = False - # ADD THESE TWO LINES: - valhalla_conf["mjolnir"]["admin"] = os.path.join(dst_dir, "admin.sqlite").replace("\\", "/") - valhalla_conf["mjolnir"]["timezone"] = os.path.join(dst_dir, "tz_world.sqlite").replace("\\", "/") + with open(config_path, "w", encoding='utf-8') as f: json.dump(valhalla_conf, f, indent=4) From 7d54134a81579c66d16b88673e20b3b1f888f05f Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 20:32:50 +0200 Subject: [PATCH 08/23] unset mjolnir config for admin and tzone --- rapida/connectivity/graph.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 8c8919c..c113baf 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -197,6 +197,7 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> # --------------------------------------------------------- valhalla_conf["mjolnir"]["min_reachability"] = 0 + valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False if sys.platform == "win32": From 2db4402e1af2b192fb3c6dfc0abcf0d6b46861f4 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 21:07:12 +0200 Subject: [PATCH 09/23] added valhall log --- rapida/connectivity/graph.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index c113baf..5d7712e 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -199,6 +199,12 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> valhalla_conf["mjolnir"]["min_reachability"] = 0 valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False + # Enable file logging for the Valhalla C++ engine + valhalla_conf["logging"] = { + "type": "file", + "color": False, + "file_name": os.path.join(dst_dir, "valhalla_build.log").replace("\\", "/") + } if sys.platform == "win32": valhalla_conf["mjolnir"]["use_mmap"] = False From c1585f10d91c7de0ab62677aa123c4e20464ca36 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 23:04:58 +0200 Subject: [PATCH 10/23] improve agg stats --- rapida/connectivity/__init__.py | 96 +++++++++++++++++++++++++++++---- 1 file changed, 86 insertions(+), 10 deletions(-) diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index 1a5e63c..19e4478 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -103,10 +103,11 @@ async def run_connectivity_analysis( if stats_admin_level: url = f"/vsicurl/https://undpngddlsgeohubdev01.blob.core.windows.net/admin/cgaz/geoBoundariesCGAZ_ADM{stats_admin_level}.fgb" adm_gdf = gpd.read_file(url, bbox=bbox, engine="pyogrio") - if clip_country: - adm_gdf = adm_gdf[adm_gdf['iso3'] == clip_country] if 'iso3' in isochrones_gdf.columns.tolist(): adm_gdf.drop(columns=['iso3'], inplace=True) + if clip_country: + adm_gdf = adm_gdf[adm_gdf['iso3'] == clip_country] + isochrones_gdf['iso3'] = clip_country if isochrones_gdf.crs != adm_gdf.crs: adm_gdf.to_crs(isochrones_gdf.crs, inplace=True) results = [] @@ -146,12 +147,12 @@ async def run_connectivity_analysis( project=project.path, force=False ) - stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg') - pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population') - if not disjoint: - pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index] - pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326') - + stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg') + pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population') + if not disjoint: + pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index] + pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326') + if not stats_admin_level: pop_stat_gdf.to_file( filename=isochrones_path, driver="GeoJSON", @@ -161,7 +162,82 @@ async def run_connectivity_analysis( promote_to_multi=True, index=False ) - #Project._instance = None + else: + logger.info('Aggregating zonal stats independently and pivoting to wide format...') + + pop_col_names = [f'{popv}_{year}' for popv in pop_vars] + admin_col_name = f'admin{stats_admin_level}_name' + assert admin_col_name in pop_stat_gdf.columns.tolist() + pop_stat_gdf['contour'] = pop_stat_gdf['contour'].astype(float).astype(int) + pop_stat_gdf[admin_col_name] = pop_stat_gdf[admin_col_name].astype(str).str.strip() + + # 1. Group and sum the base stats independently (vectorized aggregation) + base_agg = pop_stat_gdf.groupby([admin_col_name, 'contour'])[pop_col_names].sum().reset_index() + + + + # 2. Pivot to Wide Format (Option 2: contours become columns) + pivot_df = base_agg.pivot(index=admin_col_name, columns='contour', values=pop_col_names) + + # Flatten the MultiIndex columns (e.g., ('male_total_2026', 15.0) -> 'male_total_2026_15min') + pivot_df.columns = [f"{col[0]}_{int(col[1])}min" for col in pivot_df.columns] + wide_df = pivot_df.reset_index() + + # 6. Attach geometries from the original adm_gdf + logger.info('Merging aggregated stats back onto original admin boundaries...') + + # Handle case where CGAZ admin column might natively be 'shapeName' + adm_join_col = admin_col_name if admin_col_name in adm_gdf.columns else 'shapeName' + + final_gdf = adm_gdf.merge( + wide_df, + left_on=adm_join_col, + right_on=admin_col_name, + how='inner' # Use 'inner' to only keep admin units that actually had isochrones + ) + + gc.collect() + + with TemporaryDirectory(dir=dest_dir, delete=True, ) as admin_project_folder: + logger.info(f'Computing zonal stats for total population ') + adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') + adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") + admin_project = Project(path=admin_project_folder, polygons=adm_ds_path, + comment='temp project for admin stats') + with click.Context(assess) as ctx: + ctx.ensure_object(dict) + ctx.obj['progress'] = progress + # 2. Use invoke. Do NOT pass 'ctx' manually here. + # Click intercepts this and injects it as the first argument automatically. + ctx.invoke( + assess, + components=('population',), + variables=['total'], + year=year, + project=admin_project.path, + force=False + ) + + admin_stat_gpkg_path = os.path.join(admin_project_folder, 'data', f'{admin_project.name}.gpkg') + admin_pop_stat_gdf = gpd.read_file(admin_stat_gpkg_path, layer='stats.population') + final_gdf = final_gdf.merge(admin_pop_stat_gdf[[admin_col_name, f'total_{year}']], + on=admin_col_name) + + if os.path.exists(adm_ds_path): os.remove(adm_ds_path) + + admin_iso_stats = os.path.join(dest_dir, f"admin{stats_admin_level}_iso_stats.geojson") + logger.info(f"Writing final aggregated admin boundaries to {admin_iso_stats}") + + final_gdf.to_file( + filename=admin_iso_stats, # Fixed: this was pointing to barrier_isochrones_path + driver="GeoJSON", + engine="pyogrio", + mode="w", + layer=f"admin{stats_admin_level}_iso_stats", + promote_to_multi=True, + index=False + ) + if barriers_dataset is not None and pop_vars: @@ -342,7 +418,7 @@ async def run_connectivity_analysis( gc.collect() - with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True,ignore_cleanup_errors=True ) as admin_project_folder: logger.info(f'Computing zonal stats for total population ') adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") From 71ff0964361756716fb2960f46db1567329b5eb4 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 23:13:21 +0200 Subject: [PATCH 11/23] fix iso3 bug in simple isochrone --- rapida/connectivity/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index 19e4478..5b5ff74 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -103,9 +103,9 @@ async def run_connectivity_analysis( if stats_admin_level: url = f"/vsicurl/https://undpngddlsgeohubdev01.blob.core.windows.net/admin/cgaz/geoBoundariesCGAZ_ADM{stats_admin_level}.fgb" adm_gdf = gpd.read_file(url, bbox=bbox, engine="pyogrio") - if 'iso3' in isochrones_gdf.columns.tolist(): - adm_gdf.drop(columns=['iso3'], inplace=True) if clip_country: + if 'iso3' in isochrones_gdf.columns.tolist(): + isochrones_gdf.drop(columns=['iso3'], inplace=True) adm_gdf = adm_gdf[adm_gdf['iso3'] == clip_country] isochrones_gdf['iso3'] = clip_country if isochrones_gdf.crs != adm_gdf.crs: From 13a8bb68bc143295e99762a549feb90296090bf9 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Wed, 5 Aug 2026 23:22:11 +0200 Subject: [PATCH 12/23] add ignore_cleanup --- rapida/connectivity/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index 5b5ff74..7120c96 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -198,7 +198,7 @@ async def run_connectivity_analysis( gc.collect() - with TemporaryDirectory(dir=dest_dir, delete=True, ) as admin_project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: logger.info(f'Computing zonal stats for total population ') adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") From 139d23efd7a9df06e2a23f27f37f8c6e7e8354d9 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 00:23:59 +0200 Subject: [PATCH 13/23] add gdalskip netdf for windows in noaaa cloud mask --- rapida/ntl/noaa/cmask.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index 48c48ad..1f3497b 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -17,6 +17,7 @@ from shapely import concave_hull from shapely.ops import unary_union from itertools import combinations +import platform gdal.UseExceptions() @@ -251,7 +252,10 @@ def cloud_coverage_fast(hdf_url: str, bbox: Iterable[float], def cloud_coverage(hdf_url: str, bbox: list) -> int: - + if platform.system() == "Windows": + gdal.SetConfigOption('GDAL_SKIP', 'netCDF') + else: + gdal.SetConfigOption('GDAL_SKIP', '') # 1. Initialize GDAL environment INSIDE the worker process gdal.UseExceptions() gdal.PushErrorHandler('CPLQuietErrorHandler') From b157dbfb04ba1b59bda8ca6659467f37422f732f Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 00:26:50 +0200 Subject: [PATCH 14/23] add gdalskip netdf for windows in noaaa cloud mask --- rapida/ntl/noaa/cmask.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index 1f3497b..62021fc 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -272,14 +272,18 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: if cc is not None:return cc lon_min, lat_min, lon_max, lat_max = bbox subdataset_str = f'NETCDF:"/vsicurl/{hdf_url}":CloudMaskBinary' - + # --- THE WARP FIX --- + # Open the dataset explicitly first to bypass the Warp string-parsing bug + src_ds = gdal.Open(subdataset_str) + if src_ds is None: + raise Exception(f'Failed to open subdataset for {hdf_url}') # 2. Warp directly from the subdataset string over the network # GDAL's C++ engine reads the global polygon, clips the exact byte blocks, # and handles the 750m resolution on-the-fly into a memory buffer. ds = gdal.Warp( '', # Output to RAM - subdataset_str, + src_ds, format='MEM', dstSRS='EPSG:4326', # works here because we are counting pixels not planar metrics outputBounds=[lon_min, lat_min, lon_max, lat_max], @@ -287,6 +291,7 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: dstNodata=-128, geoloc=True ) + src_ds = None if ds is None: raise Exception(f'Failed to compute cloud coverage for {hdf_url}') From b9ffea2851ee772b12b7589e19fca918d4a53e0c Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 00:40:43 +0200 Subject: [PATCH 15/23] redo cm fetching using VRT --- rapida/ntl/noaa/cmask.py | 100 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 1 deletion(-) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index 62021fc..d60d8f1 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -251,7 +251,7 @@ def cloud_coverage_fast(hdf_url: str, bbox: Iterable[float], raise -def cloud_coverage(hdf_url: str, bbox: list) -> int: +def cloud_coverage_winbug(hdf_url: str, bbox: list) -> int: if platform.system() == "Windows": gdal.SetConfigOption('GDAL_SKIP', 'netCDF') else: @@ -306,6 +306,104 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: return cc +import os +import platform +import numpy as np +from osgeo import gdal + + +def cloud_coverage(hdf_url: str, bbox: list) -> int: + gdal.UseExceptions() + gdal.PushErrorHandler('CPLQuietErrorHandler') + + # # --- 1. DYNAMIC DRIVER SELECTION --- + # is_windows = platform.system() == "Windows" + # + # if is_windows: + # gdal.SetConfigOption('GDAL_SKIP', 'netCDF') + # # On Windows, we force the HDF5 driver and its specific string syntax + # base_ds = f'HDF5:"/vsicurl/{hdf_url}"' + # mask_str = f'{base_ds}://CloudMaskBinary' + # lon_str = f'{base_ds}://Longitude' + # lat_str = f'{base_ds}://Latitude' + # else: + # gdal.SetConfigOption('GDAL_SKIP', '') + # On Linux, we use the native NetCDF driver + # base_ds = f'HDF:"/vsicurl/{hdf_url}"' + # mask_str = f'{base_ds}:CloudMaskBinary' + # lon_str = f'{base_ds}:Longitude' + # lat_str = f'{base_ds}:Latitude' + base_ds = f'HDF:"/vsicurl/{hdf_url}"' + mask_str = f'{base_ds}:CloudMaskBinary' + lon_str = f'{base_ds}:Longitude' + lat_str = f'{base_ds}:Latitude' + gdal.SetConfigOption('GDAL_HTTP_TIMEOUT', '600') + gdal.SetConfigOption('GDAL_HTTP_MULTIPLEX', 'YES') + gdal.SetConfigOption('GDAL_HTTP_VERSION', '2') + gdal.SetConfigOption('VSI_CACHE', 'TRUE') + gdal.SetConfigOption('VSI_CACHE_SIZE', '50000000') + gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR') + gdal.SetConfigOption('CPL_VSIL_CURL_ALLOWED_EXTENSIONS', '.nc') + + _, file_name = os.path.split(hdf_url) + + # (Assuming your cache object is available in scope) + cc = cache.fetch(key=file_name) + if cc is not None: + return cc + + lon_min, lat_min, lon_max, lat_max = bbox + + # --- 2. OPEN THE RAW DATASET --- + src_ds = gdal.Open(mask_str) + if src_ds is None: + raise Exception(f'Failed to open subdataset {mask_str}') + + # --- 3. THE GEOLOCATION FIX --- + # Wrap the dataset in an in-memory VRT (Virtual Raster) so we can modify its metadata safely. + # We explicitly tell GDAL exactly where the Lat/Lon datasets are located on the network. + vrt_ds = gdal.Translate('', src_ds, format='VRT') + vrt_ds.SetMetadata({ + 'X_DATASET': lon_str, + 'X_BAND': '1', + 'Y_DATASET': lat_str, + 'Y_BAND': '1', + 'PIXEL_OFFSET': '0', + 'LINE_OFFSET': '0', + 'PIXEL_STEP': '1', + 'LINE_STEP': '1' + }, 'GEOLOCATION') + + # --- 4. WARP --- + # Now warp the VRT wrapper. geoloc=True will read our injected metadata and work perfectly. + ds = gdal.Warp( + '', + vrt_ds, + format='MEM', + dstSRS='EPSG:4326', + outputBounds=[lon_min, lat_min, lon_max, lat_max], + xRes=0.00675, yRes=0.00675, + dstNodata=-128, + geoloc=True + ) + + if ds is None: + src_ds = vrt_ds = None # Release locks + raise Exception(f'Failed to compute cloud coverage for {hdf_url}') + + data = ds.GetRasterBand(1).ReadAsArray() + valid_data = data[(data == 0) | (data == 1)] + + # Clean up all C++ pointers for Windows file locking + src_ds = vrt_ds = ds = None + + if valid_data.size == 0: + raise Exception(f'Failed to compute cloud coverage for {hdf_url}. No valid data.') + + cc = int((np.count_nonzero(valid_data == 1) / valid_data.size) * 100) + + cache.store(key=file_name, value=cc) + return cc def cloud_coverage_batch(urls: list[str], bbox: Iterable[float], max_threads: int = 5, progress: Progress = None): results = {} From 67bbf484f61190e13aa8aea66677805a81d7d9b9 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 00:43:49 +0200 Subject: [PATCH 16/23] redo cm fetching using VRT cond on win --- rapida/ntl/noaa/cmask.py | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index d60d8f1..4a38548 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -316,27 +316,24 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: gdal.UseExceptions() gdal.PushErrorHandler('CPLQuietErrorHandler') - # # --- 1. DYNAMIC DRIVER SELECTION --- - # is_windows = platform.system() == "Windows" - # - # if is_windows: - # gdal.SetConfigOption('GDAL_SKIP', 'netCDF') - # # On Windows, we force the HDF5 driver and its specific string syntax - # base_ds = f'HDF5:"/vsicurl/{hdf_url}"' - # mask_str = f'{base_ds}://CloudMaskBinary' - # lon_str = f'{base_ds}://Longitude' - # lat_str = f'{base_ds}://Latitude' - # else: - # gdal.SetConfigOption('GDAL_SKIP', '') + # --- 1. DYNAMIC DRIVER SELECTION --- + is_windows = platform.system() == "Windows" + + if is_windows: + gdal.SetConfigOption('GDAL_SKIP', 'netCDF') + # On Windows, we force the HDF5 driver and its specific string syntax + base_ds = f'HDF5:"/vsicurl/{hdf_url}"' + mask_str = f'{base_ds}://CloudMaskBinary' + lon_str = f'{base_ds}://Longitude' + lat_str = f'{base_ds}://Latitude' + else: + gdal.SetConfigOption('GDAL_SKIP', '') # On Linux, we use the native NetCDF driver - # base_ds = f'HDF:"/vsicurl/{hdf_url}"' - # mask_str = f'{base_ds}:CloudMaskBinary' - # lon_str = f'{base_ds}:Longitude' - # lat_str = f'{base_ds}:Latitude' - base_ds = f'HDF:"/vsicurl/{hdf_url}"' - mask_str = f'{base_ds}:CloudMaskBinary' - lon_str = f'{base_ds}:Longitude' - lat_str = f'{base_ds}:Latitude' + base_ds = f'HDF:"/vsicurl/{hdf_url}"' + mask_str = f'{base_ds}:CloudMaskBinary' + lon_str = f'{base_ds}:Longitude' + lat_str = f'{base_ds}:Latitude' + gdal.SetConfigOption('GDAL_HTTP_TIMEOUT', '600') gdal.SetConfigOption('GDAL_HTTP_MULTIPLEX', 'YES') gdal.SetConfigOption('GDAL_HTTP_VERSION', '2') From 36ced0ad8d8b3fe818201de44239b623ad835874 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 10:38:22 +0200 Subject: [PATCH 17/23] add ignore_cleanup to outage detection zonal stats temp folder --- rapida/ntl/noaa/cmask.py | 5 ----- rapida/ntl/outage.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index 4a38548..9d5f779 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -306,11 +306,6 @@ def cloud_coverage_winbug(hdf_url: str, bbox: list) -> int: return cc -import os -import platform -import numpy as np -from osgeo import gdal - def cloud_coverage(hdf_url: str, bbox: list) -> int: gdal.UseExceptions() diff --git a/rapida/ntl/outage.py b/rapida/ntl/outage.py index 35b54a1..fc28875 100644 --- a/rapida/ntl/outage.py +++ b/rapida/ntl/outage.py @@ -224,7 +224,7 @@ async def detect_outage( geom_type=ogr.wkbPolygon ) - with TemporaryDirectory(dir=dst_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dst_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=outage_gpkg_path, comment='temp project for outage zonal stats') with click.Context(assess) as ctx: From 6c3c9557ddc3d0d305e097c729a8874af0a09ab9 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Thu, 6 Aug 2026 20:04:12 +0200 Subject: [PATCH 18/23] improve conn isochone aggregation an add admin db sqlite cfg --- rapida/connectivity/__init__.py | 129 ++++++++++++++++---------------- rapida/connectivity/graph.py | 7 +- 2 files changed, 71 insertions(+), 65 deletions(-) diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index 7120c96..aa52532 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -152,6 +152,7 @@ async def run_connectivity_analysis( if not disjoint: pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index] pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326') + if not stats_admin_level: pop_stat_gdf.to_file( filename=isochrones_path, @@ -163,80 +164,82 @@ async def run_connectivity_analysis( index=False ) else: - logger.info('Aggregating zonal stats independently and pivoting to wide format...') - - pop_col_names = [f'{popv}_{year}' for popv in pop_vars] - admin_col_name = f'admin{stats_admin_level}_name' - assert admin_col_name in pop_stat_gdf.columns.tolist() - pop_stat_gdf['contour'] = pop_stat_gdf['contour'].astype(float).astype(int) - pop_stat_gdf[admin_col_name] = pop_stat_gdf[admin_col_name].astype(str).str.strip() + if barriers_dataset is None: # do aggregation here only when no barriers were given - # 1. Group and sum the base stats independently (vectorized aggregation) - base_agg = pop_stat_gdf.groupby([admin_col_name, 'contour'])[pop_col_names].sum().reset_index() + logger.info('Aggregating zonal stats independently and pivoting to wide format...') + pop_col_names = [f'{popv}_{year}' for popv in pop_vars] + admin_col_name = f'admin{stats_admin_level}_name' + assert admin_col_name in pop_stat_gdf.columns.tolist() + pop_stat_gdf['contour'] = pop_stat_gdf['contour'].astype(float).astype(int) + pop_stat_gdf[admin_col_name] = pop_stat_gdf[admin_col_name].astype(str).str.strip() + # 1. Group and sum the base stats independently (vectorized aggregation) + base_agg = pop_stat_gdf.groupby([admin_col_name, 'contour'])[pop_col_names].sum().reset_index() - # 2. Pivot to Wide Format (Option 2: contours become columns) - pivot_df = base_agg.pivot(index=admin_col_name, columns='contour', values=pop_col_names) - # Flatten the MultiIndex columns (e.g., ('male_total_2026', 15.0) -> 'male_total_2026_15min') - pivot_df.columns = [f"{col[0]}_{int(col[1])}min" for col in pivot_df.columns] - wide_df = pivot_df.reset_index() - # 6. Attach geometries from the original adm_gdf - logger.info('Merging aggregated stats back onto original admin boundaries...') + # 2. Pivot to Wide Format (Option 2: contours become columns) + pivot_df = base_agg.pivot(index=admin_col_name, columns='contour', values=pop_col_names) - # Handle case where CGAZ admin column might natively be 'shapeName' - adm_join_col = admin_col_name if admin_col_name in adm_gdf.columns else 'shapeName' + # Flatten the MultiIndex columns (e.g., ('male_total_2026', 15.0) -> 'male_total_2026_15min') + pivot_df.columns = [f"{col[0]}_{int(col[1])}min" for col in pivot_df.columns] + wide_df = pivot_df.reset_index() - final_gdf = adm_gdf.merge( - wide_df, - left_on=adm_join_col, - right_on=admin_col_name, - how='inner' # Use 'inner' to only keep admin units that actually had isochrones - ) + # 6. Attach geometries from the original adm_gdf + logger.info('Merging aggregated stats back onto original admin boundaries...') - gc.collect() - - with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: - logger.info(f'Computing zonal stats for total population ') - adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') - adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") - admin_project = Project(path=admin_project_folder, polygons=adm_ds_path, - comment='temp project for admin stats') - with click.Context(assess) as ctx: - ctx.ensure_object(dict) - ctx.obj['progress'] = progress - # 2. Use invoke. Do NOT pass 'ctx' manually here. - # Click intercepts this and injects it as the first argument automatically. - ctx.invoke( - assess, - components=('population',), - variables=['total'], - year=year, - project=admin_project.path, - force=False - ) + # Handle case where CGAZ admin column might natively be 'shapeName' + adm_join_col = admin_col_name if admin_col_name in adm_gdf.columns else 'shapeName' - admin_stat_gpkg_path = os.path.join(admin_project_folder, 'data', f'{admin_project.name}.gpkg') - admin_pop_stat_gdf = gpd.read_file(admin_stat_gpkg_path, layer='stats.population') - final_gdf = final_gdf.merge(admin_pop_stat_gdf[[admin_col_name, f'total_{year}']], - on=admin_col_name) - - if os.path.exists(adm_ds_path): os.remove(adm_ds_path) - - admin_iso_stats = os.path.join(dest_dir, f"admin{stats_admin_level}_iso_stats.geojson") - logger.info(f"Writing final aggregated admin boundaries to {admin_iso_stats}") + final_gdf = adm_gdf.merge( + wide_df, + left_on=adm_join_col, + right_on=admin_col_name, + how='inner' # Use 'inner' to only keep admin units that actually had isochrones + ) - final_gdf.to_file( - filename=admin_iso_stats, # Fixed: this was pointing to barrier_isochrones_path - driver="GeoJSON", - engine="pyogrio", - mode="w", - layer=f"admin{stats_admin_level}_iso_stats", - promote_to_multi=True, - index=False - ) + gc.collect() + + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: + logger.info(f'Computing zonal stats for total population ') + adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') + adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") + admin_project = Project(path=admin_project_folder, polygons=adm_ds_path, + comment='temp project for admin stats') + with click.Context(assess) as ctx: + ctx.ensure_object(dict) + ctx.obj['progress'] = progress + # 2. Use invoke. Do NOT pass 'ctx' manually here. + # Click intercepts this and injects it as the first argument automatically. + ctx.invoke( + assess, + components=('population',), + variables=['total'], + year=year, + project=admin_project.path, + force=False + ) + + admin_stat_gpkg_path = os.path.join(admin_project_folder, 'data', f'{admin_project.name}.gpkg') + admin_pop_stat_gdf = gpd.read_file(admin_stat_gpkg_path, layer='stats.population') + final_gdf = final_gdf.merge(admin_pop_stat_gdf[[admin_col_name, f'total_{year}']], + on=admin_col_name) + + if os.path.exists(adm_ds_path): os.remove(adm_ds_path) + + admin_iso_stats = os.path.join(dest_dir, f"admin{stats_admin_level}_iso_stats.geojson") + logger.info(f"Writing final aggregated admin boundaries to {admin_iso_stats}") + + final_gdf.to_file( + filename=admin_iso_stats, # Fixed: this was pointing to barrier_isochrones_path + driver="GeoJSON", + engine="pyogrio", + mode="w", + layer=f"admin{stats_admin_level}_iso_stats", + promote_to_multi=True, + index=False + ) diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 5d7712e..e27883c 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -198,12 +198,15 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> valhalla_conf["mjolnir"]["min_reachability"] = 0 - valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False + #valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False + admin_db_path = os.path.join(dst_dir, 'admin.sqlite') + valhalla_conf["mjolnir"]["admin"] = str(Path(admin_db_path)) + # Enable file logging for the Valhalla C++ engine valhalla_conf["logging"] = { "type": "file", "color": False, - "file_name": os.path.join(dst_dir, "valhalla_build.log").replace("\\", "/") + "file_name": os.path.join(dst_dir, "valhalla_build.log") } if sys.platform == "win32": From 1217f829d833e741e92b4e21029fd7323235e470 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Fri, 7 Aug 2026 18:37:48 +0200 Subject: [PATCH 19/23] adjusted chromium args for auth nased on platform --- rapida/az/surgeauth.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/rapida/az/surgeauth.py b/rapida/az/surgeauth.py index acd5d6f..dd9a4a9 100644 --- a/rapida/az/surgeauth.py +++ b/rapida/az/surgeauth.py @@ -17,7 +17,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM import json import getpass - +import platform logger = logging.getLogger(__name__) @@ -234,14 +234,20 @@ def fetch_token_sync(self, *scope, username=None, password=None, quick_selectors auth_code = None with (sync_playwright() as p): - with p.chromium.launch(headless=True, - args=[ "--disable-gpu", - "--disable-dev-shm-usage", - "--no-sandbox", - "--single-process", - "--disable-setuid-sandbox", - "--use-gl=egl" - ]) as browser: # Use headless=True for invisible mode + chromium_args = [ + "--disable-gpu", + "--disable-dev-shm-usage", + "--no-sandbox" + ] + if platform.system() != 'Windows': + chromium_args.extend([ + "--single-process", + "--disable-setuid-sandbox", + "--use-gl=egl" + ]) + + + with p.chromium.launch(headless=True,args=chromium_args) as browser: # Use headless=True for invisible mode error_msg = None page = browser.new_page() From c7dc689e71874e939b8feff556975872f865511a Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Fri, 7 Aug 2026 19:04:50 +0200 Subject: [PATCH 20/23] adjust gdal_config context to sue all threads due to gdal cache max --- rapida/components/deprivation/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rapida/components/deprivation/__init__.py b/rapida/components/deprivation/__init__.py index c64d7cd..07b7611 100644 --- a/rapida/components/deprivation/__init__.py +++ b/rapida/components/deprivation/__init__.py @@ -108,7 +108,7 @@ def download(self, force=False, **kwargs): 'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES', 'GDAL_CACHEMAX':'512', 'GDAL_NUMTHREADS':'4' - }): + }, thread_local=False): geo.import_raster( source=local_path, dst=self.local_path, target_srs=project.target_srs, crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name, From f9ab62d9de532d4c2bd47d9b5320e68d3c295bf0 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Fri, 7 Aug 2026 20:37:32 +0200 Subject: [PATCH 21/23] fix buildings --- rapida/components/buildings/__init__.py | 36 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/rapida/components/buildings/__init__.py b/rapida/components/buildings/__init__.py index 777bb98..9595159 100644 --- a/rapida/components/buildings/__init__.py +++ b/rapida/components/buildings/__init__.py @@ -205,7 +205,6 @@ def compute(self, force=True, **kwargs): assert force, f'invalid force={force}' return self.download(force=force, **kwargs) - def evaluate(self, **kwargs): destination_layer = f'stats.{self.component}' affected_var_name = f'{self.name}_affected' @@ -216,20 +215,20 @@ def evaluate(self, **kwargs): dataset_path, layer_name = self.local_path.split('::') - # Read only necessary columns (avoid loading full geometries if not needed) + # Read buildings buildings_gdf = gpd.read_file(dataset_path, layer=layer_name, columns=['polyid', 'geometry']) - # List layers once to avoid repeated I/O operations layers = pyogrio.list_layers(dataset_path) layer_names = layers[:, 0] polygons_layer = destination_layer if destination_layer in layer_names else project.polygons_layer_name - # Read only necessary columns for polygons - polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer, columns=['h3id', 'geometry']) - polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'}) + # FIX 1: Read ALL columns so we don't drop previously computed variables (like nbuildings) + polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer) + if 'h3id' in polygons_gdf.columns: + polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'}) - # **Efficient Building Count Calculation** + # Calculate base stats if self.name == 'nbuildings': var_gdf = buildings_gdf['polyid'].value_counts().reset_index() var_gdf.columns = ['polyid', self.name] @@ -237,7 +236,7 @@ def evaluate(self, **kwargs): buildings_gdf[self.name] = buildings_gdf.geometry.area var_gdf = buildings_gdf.groupby('polyid', as_index=False)[self.name].sum() - # **Handle Affected Buildings** + # Handle Affected Buildings if project.raster_mask is not None: affected_layer_name = f'{self.component}.affected' if affected_layer_name in layer_names: @@ -251,18 +250,29 @@ def evaluate(self, **kwargs): affected_buildings_gdf[affected_var_name] = affected_buildings_gdf.geometry.area affected_var_gdf = affected_buildings_gdf.groupby('polyid', as_index=False)[affected_var_name].sum() - # Merge affected data - var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='inner') + # FIX 2: Change to 'left' merge to keep polygons that have 0 affected buildings + var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='left') + + # Fill NaNs with 0 (for polygons with buildings, but none affected) + var_gdf[affected_var_name] = var_gdf[affected_var_name].fillna(0) + var_gdf[affected_var_percentage_name] = (var_gdf[affected_var_name] / var_gdf[self.name]) * 100 + var_gdf[affected_var_percentage_name] = var_gdf[affected_var_percentage_name].fillna(0) - # **Remove old columns before merging** + # Remove old columns before merging (to prevent _x and _y suffix duplicates) polygons_gdf.drop(columns=[col for col in [self.name, affected_var_name, affected_var_percentage_name] if col in polygons_gdf.columns], inplace=True) - # **Final Merge and Save** + # Final Merge and Save out_gdf = polygons_gdf.merge(var_gdf, on='polyid', how='left') - out_gdf = out_gdf.rename(columns={'polyid': 'h3id'}) + # FIX 3: Fill NaNs with 0 for polygons that had no buildings at all + out_gdf[self.name] = out_gdf[self.name].fillna(0) + if project.raster_mask is not None and affected_var_name in out_gdf.columns: + out_gdf[affected_var_name] = out_gdf[affected_var_name].fillna(0) + out_gdf[affected_var_percentage_name] = out_gdf[affected_var_percentage_name].fillna(0) + + out_gdf = out_gdf.rename(columns={'polyid': 'h3id'}) out_gdf.to_file(dataset_path, layer=destination_layer, driver='GPKG', mode='w') def resolve(self, **kwargs): From a06b5239dd50aaf4524d3cf8f8d56b7c06b9d9ef Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Fri, 7 Aug 2026 20:51:04 +0200 Subject: [PATCH 22/23] fix buildings with h3id int64 --- rapida/components/buildings/__init__.py | 35 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/rapida/components/buildings/__init__.py b/rapida/components/buildings/__init__.py index 9595159..f63b1f0 100644 --- a/rapida/components/buildings/__init__.py +++ b/rapida/components/buildings/__init__.py @@ -223,15 +223,22 @@ def evaluate(self, **kwargs): polygons_layer = destination_layer if destination_layer in layer_names else project.polygons_layer_name - # FIX 1: Read ALL columns so we don't drop previously computed variables (like nbuildings) + # Read polygons (ALL columns so we keep previous variables like nbuildings) polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer) if 'h3id' in polygons_gdf.columns: polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'}) + # ========================================== + # CRITICAL FIX: FORCE MATCHING DATA TYPES (Integer) + # Using 'Int64' ensures 64-bit integer matching and prevents NaN crashes + # ========================================== + polygons_gdf['polyid'] = polygons_gdf['polyid'].astype('Int64') + buildings_gdf['polyid'] = buildings_gdf['polyid'].astype('Int64') + # Calculate base stats if self.name == 'nbuildings': - var_gdf = buildings_gdf['polyid'].value_counts().reset_index() - var_gdf.columns = ['polyid', self.name] + # This is 100% safe across all Pandas versions + var_gdf = buildings_gdf.groupby('polyid').size().reset_index(name=self.name) else: buildings_gdf[self.name] = buildings_gdf.geometry.area var_gdf = buildings_gdf.groupby('polyid', as_index=False)[self.name].sum() @@ -242,37 +249,41 @@ def evaluate(self, **kwargs): if affected_layer_name in layer_names: affected_buildings_gdf = gpd.read_file(dataset_path, layer=affected_layer_name, columns=['polyid', 'geometry']) + # Force integer on affected buildings too + affected_buildings_gdf['polyid'] = affected_buildings_gdf['polyid'].astype('Int64') if self.name == 'nbuildings': - affected_var_gdf = affected_buildings_gdf['polyid'].value_counts().reset_index() - affected_var_gdf.columns = ['polyid', affected_var_name] + affected_var_gdf = affected_buildings_gdf.groupby('polyid').size().reset_index( + name=affected_var_name) else: affected_buildings_gdf[affected_var_name] = affected_buildings_gdf.geometry.area affected_var_gdf = affected_buildings_gdf.groupby('polyid', as_index=False)[affected_var_name].sum() - # FIX 2: Change to 'left' merge to keep polygons that have 0 affected buildings + # Left merge to keep polygons that have 0 affected buildings var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='left') - # Fill NaNs with 0 (for polygons with buildings, but none affected) var_gdf[affected_var_name] = var_gdf[affected_var_name].fillna(0) - var_gdf[affected_var_percentage_name] = (var_gdf[affected_var_name] / var_gdf[self.name]) * 100 var_gdf[affected_var_percentage_name] = var_gdf[affected_var_percentage_name].fillna(0) - # Remove old columns before merging (to prevent _x and _y suffix duplicates) - polygons_gdf.drop(columns=[col for col in [self.name, affected_var_name, affected_var_percentage_name] if - col in polygons_gdf.columns], inplace=True) + # Remove ONLY the current variable's columns before merging + cols_to_drop = [col for col in [self.name, affected_var_name, affected_var_percentage_name] if + col in polygons_gdf.columns] + if cols_to_drop: + polygons_gdf.drop(columns=cols_to_drop, inplace=True) # Final Merge and Save out_gdf = polygons_gdf.merge(var_gdf, on='polyid', how='left') - # FIX 3: Fill NaNs with 0 for polygons that had no buildings at all + # Fill NaNs with 0 for polygons that had NO buildings out_gdf[self.name] = out_gdf[self.name].fillna(0) if project.raster_mask is not None and affected_var_name in out_gdf.columns: out_gdf[affected_var_name] = out_gdf[affected_var_name].fillna(0) out_gdf[affected_var_percentage_name] = out_gdf[affected_var_percentage_name].fillna(0) out_gdf = out_gdf.rename(columns={'polyid': 'h3id'}) + + # Write back out out_gdf.to_file(dataset_path, layer=destination_layer, driver='GPKG', mode='w') def resolve(self, **kwargs): From 2248a5ec5b40a8b500f86dc676792424033d9ff7 Mon Sep 17 00:00:00 2001 From: Ioan Ferencik Date: Fri, 7 Aug 2026 20:58:35 +0200 Subject: [PATCH 23/23] adjust gdal_config context to use all threads due to gdal cache max in rwi --- rapida/components/rwi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rapida/components/rwi/__init__.py b/rapida/components/rwi/__init__.py index 5a4c70c..42e22eb 100644 --- a/rapida/components/rwi/__init__.py +++ b/rapida/components/rwi/__init__.py @@ -168,7 +168,7 @@ def download(self, force=False, **kwargs): 'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES', 'GDAL_CACHEMAX':'512', 'GDAL_NUMTHREADS':'4' - }): + }, thread_local=False): geo.import_raster( source=local_path, dst=self.local_path, target_srs=project.target_srs, crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name,