Fix/win bugs

rapida/az/surgeauth.py

@@ -17,7 +17,7 @@
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
 import json
 import getpass
-
+import platform
 
 logger = logging.getLogger(__name__)
 
@@ -234,14 +234,20 @@ def fetch_token_sync(self, *scope, username=None, password=None, quick_selectors
         auth_code = None
 
         with (sync_playwright() as p):
-            with p.chromium.launch(headless=True,
-                                   args=[   "--disable-gpu",
-                                            "--disable-dev-shm-usage",
-                                            "--no-sandbox",
-                                            "--single-process",
-                                            "--disable-setuid-sandbox",
-                                            "--use-gl=egl"
-                                    ]) as browser: # Use headless=True for invisible mode
+            chromium_args = [
+                "--disable-gpu",
+                "--disable-dev-shm-usage",
+                "--no-sandbox"
+            ]
+            if platform.system() != 'Windows':
+                chromium_args.extend([
+                    "--single-process",
+                    "--disable-setuid-sandbox",
+                    "--use-gl=egl"
+                ])
+
+
+            with p.chromium.launch(headless=True,args=chromium_args) as browser: # Use headless=True for invisible mode
                 error_msg = None
                 page = browser.new_page()
 

rapida/components/buildings/__init__.py

@@ -205,7 +205,6 @@ def compute(self, force=True, **kwargs):
         assert force, f'invalid force={force}'
         return self.download(force=force, **kwargs)
 
-
     def evaluate(self, **kwargs):
         destination_layer = f'stats.{self.component}'
         affected_var_name = f'{self.name}_affected'
@@ -216,53 +215,75 @@ def evaluate(self, **kwargs):
 
         dataset_path, layer_name = self.local_path.split('::')
 
-        # Read only necessary columns (avoid loading full geometries if not needed)
+        # Read buildings
         buildings_gdf = gpd.read_file(dataset_path, layer=layer_name, columns=['polyid', 'geometry'])
 
-        # List layers once to avoid repeated I/O operations
         layers = pyogrio.list_layers(dataset_path)
         layer_names = layers[:, 0]
 
         polygons_layer = destination_layer if destination_layer in layer_names else project.polygons_layer_name
 
-        # Read only necessary columns for polygons
-        polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer, columns=['h3id', 'geometry'])
-        polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'})
+        # Read polygons (ALL columns so we keep previous variables like nbuildings)
+        polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer)
+        if 'h3id' in polygons_gdf.columns:
+            polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'})
+
+        # ==========================================
+        # CRITICAL FIX: FORCE MATCHING DATA TYPES (Integer)
+        # Using 'Int64' ensures 64-bit integer matching and prevents NaN crashes
+        # ==========================================
+        polygons_gdf['polyid'] = polygons_gdf['polyid'].astype('Int64')
+        buildings_gdf['polyid'] = buildings_gdf['polyid'].astype('Int64')
 
-        # **Efficient Building Count Calculation**
+        # Calculate base stats
         if self.name == 'nbuildings':
-            var_gdf = buildings_gdf['polyid'].value_counts().reset_index()
-            var_gdf.columns = ['polyid', self.name]
+            # This is 100% safe across all Pandas versions
+            var_gdf = buildings_gdf.groupby('polyid').size().reset_index(name=self.name)
         else:
             buildings_gdf[self.name] = buildings_gdf.geometry.area
             var_gdf = buildings_gdf.groupby('polyid', as_index=False)[self.name].sum()
 
-        # **Handle Affected Buildings**
+        # Handle Affected Buildings
         if project.raster_mask is not None:
             affected_layer_name = f'{self.component}.affected'
             if affected_layer_name in layer_names:
                 affected_buildings_gdf = gpd.read_file(dataset_path, layer=affected_layer_name,
                                                        columns=['polyid', 'geometry'])
+                # Force integer on affected buildings too
+                affected_buildings_gdf['polyid'] = affected_buildings_gdf['polyid'].astype('Int64')
 
                 if self.name == 'nbuildings':
-                    affected_var_gdf = affected_buildings_gdf['polyid'].value_counts().reset_index()
-                    affected_var_gdf.columns = ['polyid', affected_var_name]
+                    affected_var_gdf = affected_buildings_gdf.groupby('polyid').size().reset_index(
+                        name=affected_var_name)
                 else:
                     affected_buildings_gdf[affected_var_name] = affected_buildings_gdf.geometry.area
                     affected_var_gdf = affected_buildings_gdf.groupby('polyid', as_index=False)[affected_var_name].sum()
 
-                # Merge affected data
-                var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='inner')
+                # Left merge to keep polygons that have 0 affected buildings
+                var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='left')
+
+                var_gdf[affected_var_name] = var_gdf[affected_var_name].fillna(0)
                 var_gdf[affected_var_percentage_name] = (var_gdf[affected_var_name] / var_gdf[self.name]) * 100
+                var_gdf[affected_var_percentage_name] = var_gdf[affected_var_percentage_name].fillna(0)
 
-        # **Remove old columns before merging**
-        polygons_gdf.drop(columns=[col for col in [self.name, affected_var_name, affected_var_percentage_name] if
-                                   col in polygons_gdf.columns], inplace=True)
+        # Remove ONLY the current variable's columns before merging
+        cols_to_drop = [col for col in [self.name, affected_var_name, affected_var_percentage_name] if
+                        col in polygons_gdf.columns]
+        if cols_to_drop:
+            polygons_gdf.drop(columns=cols_to_drop, inplace=True)
 
-        # **Final Merge and Save**
+        # Final Merge and Save
         out_gdf = polygons_gdf.merge(var_gdf, on='polyid', how='left')
+
+        # Fill NaNs with 0 for polygons that had NO buildings
+        out_gdf[self.name] = out_gdf[self.name].fillna(0)
+        if project.raster_mask is not None and affected_var_name in out_gdf.columns:
+            out_gdf[affected_var_name] = out_gdf[affected_var_name].fillna(0)
+            out_gdf[affected_var_percentage_name] = out_gdf[affected_var_percentage_name].fillna(0)
+
         out_gdf = out_gdf.rename(columns={'polyid': 'h3id'})
 
+        # Write back out
         out_gdf.to_file(dataset_path, layer=destination_layer, driver='GPKG', mode='w')
 
     def resolve(self, **kwargs):

rapida/components/deprivation/__init__.py

@@ -108,7 +108,7 @@ def download(self, force=False, **kwargs):
             'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES',
             'GDAL_CACHEMAX':'512',
             'GDAL_NUMTHREADS':'4'
-        }):
+        }, thread_local=False):
             geo.import_raster(
                 source=local_path, dst=self.local_path, target_srs=project.target_srs,
                 crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name,

rapida/components/population/__init__.py

@@ -512,7 +512,7 @@ def import_raster(self, source=None, **kwargs):
         )
 
         if os.path.exists(source):os.remove(source)
-        os.rename(imported_local_path, source)
+        os.replace(imported_local_path, source)
         return source
 
     def _compute_affected_(self, progress=None, **kwargs):

rapida/components/rwi/__init__.py

@@ -168,7 +168,7 @@ def download(self, force=False, **kwargs):
             'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES',
             'GDAL_CACHEMAX':'512',
             'GDAL_NUMTHREADS':'4'
-        }):
+        }, thread_local=False):
             geo.import_raster(
                 source=local_path, dst=self.local_path, target_srs=project.target_srs,
                 crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name,

rapida/connectivity/__init__.py

@@ -104,9 +104,10 @@ async def run_connectivity_analysis(
             url = f"/vsicurl/https://undpngddlsgeohubdev01.blob.core.windows.net/admin/cgaz/geoBoundariesCGAZ_ADM{stats_admin_level}.fgb"
             adm_gdf = gpd.read_file(url, bbox=bbox, engine="pyogrio")
             if clip_country:
+                if 'iso3' in isochrones_gdf.columns.tolist():
+                    isochrones_gdf.drop(columns=['iso3'], inplace=True)
                 adm_gdf = adm_gdf[adm_gdf['iso3'] == clip_country]
-            if 'iso3' in isochrones_gdf.columns.tolist():
-                adm_gdf.drop(columns=['iso3'], inplace=True)
+                isochrones_gdf['iso3'] = clip_country
             if isochrones_gdf.crs != adm_gdf.crs:
                 adm_gdf.to_crs(isochrones_gdf.crs, inplace=True)
             results = []
@@ -131,7 +132,7 @@ async def run_connectivity_analysis(
             del isochrones_gdf
             del results
 
-        with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder:
+        with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder:
             project = Project(path=project_folder, polygons=isochrones_path, comment='temp project for conn isochrones')
             with click.Context(assess) as ctx:
                 ctx.ensure_object(dict)
@@ -146,12 +147,13 @@ async def run_connectivity_analysis(
                     project=project.path,
                     force=False
                 )
-                stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg')
-                pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population')
-                if not disjoint:
-                    pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index]
-                pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326')
+            stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg')
+            pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population')
+            if not disjoint:
+                pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index]
+            pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326')
 
+            if not stats_admin_level:
                 pop_stat_gdf.to_file(
                     filename=isochrones_path,
                     driver="GeoJSON",
@@ -161,6 +163,84 @@ async def run_connectivity_analysis(
                     promote_to_multi=True,
                     index=False
                 )
+            else:
+                if barriers_dataset is None: # do aggregation here only when no barriers were given
+
+                    logger.info('Aggregating zonal stats independently and pivoting to wide format...')
+
+                    pop_col_names = [f'{popv}_{year}' for popv in pop_vars]
+                    admin_col_name = f'admin{stats_admin_level}_name'
+                    assert admin_col_name in pop_stat_gdf.columns.tolist()
+                    pop_stat_gdf['contour'] = pop_stat_gdf['contour'].astype(float).astype(int)
+                    pop_stat_gdf[admin_col_name] = pop_stat_gdf[admin_col_name].astype(str).str.strip()
+
+                    # 1. Group and sum the base stats independently (vectorized aggregation)
+                    base_agg = pop_stat_gdf.groupby([admin_col_name, 'contour'])[pop_col_names].sum().reset_index()
+
+
+
+                    # 2. Pivot to Wide Format (Option 2: contours become columns)
+                    pivot_df = base_agg.pivot(index=admin_col_name, columns='contour', values=pop_col_names)
+
+                    # Flatten the MultiIndex columns (e.g., ('male_total_2026', 15.0) -> 'male_total_2026_15min')
+                    pivot_df.columns = [f"{col[0]}_{int(col[1])}min" for col in pivot_df.columns]
+                    wide_df = pivot_df.reset_index()
+
+                    # 6. Attach geometries from the original adm_gdf
+                    logger.info('Merging aggregated stats back onto original admin boundaries...')
+
+                    # Handle case where CGAZ admin column might natively be 'shapeName'
+                    adm_join_col = admin_col_name if admin_col_name in adm_gdf.columns else 'shapeName'
+
+                    final_gdf = adm_gdf.merge(
+                        wide_df,
+                        left_on=adm_join_col,
+                        right_on=admin_col_name,
+                        how='inner'  # Use 'inner' to only keep admin units that actually had isochrones
+                    )
+
+                    gc.collect()
+
+                    with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder:
+                        logger.info(f'Computing zonal stats for total population ')
+                        adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb')
+                        adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio")
+                        admin_project = Project(path=admin_project_folder, polygons=adm_ds_path,
+                                                comment='temp project for admin stats')
+                        with click.Context(assess) as ctx:
+                            ctx.ensure_object(dict)
+                            ctx.obj['progress'] = progress
+                            # 2. Use invoke. Do NOT pass 'ctx' manually here.
+                            # Click intercepts this and injects it as the first argument automatically.
+                            ctx.invoke(
+                                assess,
+                                components=('population',),
+                                variables=['total'],
+                                year=year,
+                                project=admin_project.path,
+                                force=False
+                            )
+
+                            admin_stat_gpkg_path = os.path.join(admin_project_folder, 'data', f'{admin_project.name}.gpkg')
+                            admin_pop_stat_gdf = gpd.read_file(admin_stat_gpkg_path, layer='stats.population')
+                            final_gdf = final_gdf.merge(admin_pop_stat_gdf[[admin_col_name, f'total_{year}']],
+                                                        on=admin_col_name)
+
+                        if os.path.exists(adm_ds_path): os.remove(adm_ds_path)
+
+                    admin_iso_stats = os.path.join(dest_dir, f"admin{stats_admin_level}_iso_stats.geojson")
+                    logger.info(f"Writing final aggregated admin boundaries to {admin_iso_stats}")
+
+                    final_gdf.to_file(
+                        filename=admin_iso_stats,  # Fixed: this was pointing to barrier_isochrones_path
+                        driver="GeoJSON",
+                        engine="pyogrio",
+                        mode="w",
+                        layer=f"admin{stats_admin_level}_iso_stats",
+                        promote_to_multi=True,
+                        index=False
+                    )
+
 
 
     if barriers_dataset is not None and pop_vars:
@@ -239,7 +319,7 @@ async def run_connectivity_analysis(
                 del results
 
 
-            with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder:
+            with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder:
                 project = Project(path=project_folder, polygons=barrier_isochrones_path, comment='temp project for conn isochrones')
                 with click.Context(assess) as ctx:
                     ctx.ensure_object(dict)
@@ -341,7 +421,7 @@ async def run_connectivity_analysis(
 
                         gc.collect()
 
-                        with TemporaryDirectory(dir=dest_dir, delete=True) as admin_project_folder:
+                        with TemporaryDirectory(dir=dest_dir, delete=True,ignore_cleanup_errors=True ) as admin_project_folder:
                             logger.info(f'Computing zonal stats for total population ')
                             adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb')
                             adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio")

rapida/connectivity/graph.py

@@ -6,10 +6,8 @@
 from valhalla.config import _sanitize_config, default_config
 from typing import Union
 from pathlib import Path
-import re
 import logging
 import sys
-import valhalla
 logger = logging.getLogger(__name__)
 
 DEFAULT_SPEEDS = {
@@ -192,15 +190,31 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) ->
             }
         ]
 
-        with open(speeds_config_path, "w") as f:
+        with open(speeds_config_path, "w", encoding='utf-8') as f:
             json.dump(speed_schema, f, indent=4)
 
         valhalla_conf["mjolnir"]["default_speeds_config"] = speeds_config_path
         # ---------------------------------------------------------
 
     valhalla_conf["mjolnir"]["min_reachability"] = 0
 
-    with open(config_path, "w") as f:
+    #valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False
+    admin_db_path = os.path.join(dst_dir, 'admin.sqlite')
+    valhalla_conf["mjolnir"]["admin"] =  str(Path(admin_db_path))
+
+    # Enable file logging for the Valhalla C++ engine
+    valhalla_conf["logging"] = {
+        "type": "file",
+        "color": False,
+        "file_name": os.path.join(dst_dir, "valhalla_build.log")
+    }
+
+    if sys.platform == "win32":
+        valhalla_conf["mjolnir"]["use_mmap"] = False
+
+
+
+    with open(config_path, "w", encoding='utf-8') as f:
         json.dump(valhalla_conf, f, indent=4)