diff --git a/rapida/az/surgeauth.py b/rapida/az/surgeauth.py index acd5d6fc..dd9a4a9e 100644 --- a/rapida/az/surgeauth.py +++ b/rapida/az/surgeauth.py @@ -17,7 +17,7 @@ from cryptography.hazmat.primitives.ciphers.aead import AESGCM import json import getpass - +import platform logger = logging.getLogger(__name__) @@ -234,14 +234,20 @@ def fetch_token_sync(self, *scope, username=None, password=None, quick_selectors auth_code = None with (sync_playwright() as p): - with p.chromium.launch(headless=True, - args=[ "--disable-gpu", - "--disable-dev-shm-usage", - "--no-sandbox", - "--single-process", - "--disable-setuid-sandbox", - "--use-gl=egl" - ]) as browser: # Use headless=True for invisible mode + chromium_args = [ + "--disable-gpu", + "--disable-dev-shm-usage", + "--no-sandbox" + ] + if platform.system() != 'Windows': + chromium_args.extend([ + "--single-process", + "--disable-setuid-sandbox", + "--use-gl=egl" + ]) + + + with p.chromium.launch(headless=True,args=chromium_args) as browser: # Use headless=True for invisible mode error_msg = None page = browser.new_page() diff --git a/rapida/components/buildings/__init__.py b/rapida/components/buildings/__init__.py index 777bb984..f63b1f08 100644 --- a/rapida/components/buildings/__init__.py +++ b/rapida/components/buildings/__init__.py @@ -205,7 +205,6 @@ def compute(self, force=True, **kwargs): assert force, f'invalid force={force}' return self.download(force=force, **kwargs) - def evaluate(self, **kwargs): destination_layer = f'stats.{self.component}' affected_var_name = f'{self.name}_affected' @@ -216,53 +215,75 @@ def evaluate(self, **kwargs): dataset_path, layer_name = self.local_path.split('::') - # Read only necessary columns (avoid loading full geometries if not needed) + # Read buildings buildings_gdf = gpd.read_file(dataset_path, layer=layer_name, columns=['polyid', 'geometry']) - # List layers once to avoid repeated I/O operations layers = pyogrio.list_layers(dataset_path) layer_names = layers[:, 0] polygons_layer = destination_layer if destination_layer in layer_names else project.polygons_layer_name - # Read only necessary columns for polygons - polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer, columns=['h3id', 'geometry']) - polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'}) + # Read polygons (ALL columns so we keep previous variables like nbuildings) + polygons_gdf = gpd.read_file(project.geopackage_file_path, layer=polygons_layer) + if 'h3id' in polygons_gdf.columns: + polygons_gdf = polygons_gdf.rename(columns={'h3id': 'polyid'}) + + # ========================================== + # CRITICAL FIX: FORCE MATCHING DATA TYPES (Integer) + # Using 'Int64' ensures 64-bit integer matching and prevents NaN crashes + # ========================================== + polygons_gdf['polyid'] = polygons_gdf['polyid'].astype('Int64') + buildings_gdf['polyid'] = buildings_gdf['polyid'].astype('Int64') - # **Efficient Building Count Calculation** + # Calculate base stats if self.name == 'nbuildings': - var_gdf = buildings_gdf['polyid'].value_counts().reset_index() - var_gdf.columns = ['polyid', self.name] + # This is 100% safe across all Pandas versions + var_gdf = buildings_gdf.groupby('polyid').size().reset_index(name=self.name) else: buildings_gdf[self.name] = buildings_gdf.geometry.area var_gdf = buildings_gdf.groupby('polyid', as_index=False)[self.name].sum() - # **Handle Affected Buildings** + # Handle Affected Buildings if project.raster_mask is not None: affected_layer_name = f'{self.component}.affected' if affected_layer_name in layer_names: affected_buildings_gdf = gpd.read_file(dataset_path, layer=affected_layer_name, columns=['polyid', 'geometry']) + # Force integer on affected buildings too + affected_buildings_gdf['polyid'] = affected_buildings_gdf['polyid'].astype('Int64') if self.name == 'nbuildings': - affected_var_gdf = affected_buildings_gdf['polyid'].value_counts().reset_index() - affected_var_gdf.columns = ['polyid', affected_var_name] + affected_var_gdf = affected_buildings_gdf.groupby('polyid').size().reset_index( + name=affected_var_name) else: affected_buildings_gdf[affected_var_name] = affected_buildings_gdf.geometry.area affected_var_gdf = affected_buildings_gdf.groupby('polyid', as_index=False)[affected_var_name].sum() - # Merge affected data - var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='inner') + # Left merge to keep polygons that have 0 affected buildings + var_gdf = var_gdf.merge(affected_var_gdf, on='polyid', how='left') + + var_gdf[affected_var_name] = var_gdf[affected_var_name].fillna(0) var_gdf[affected_var_percentage_name] = (var_gdf[affected_var_name] / var_gdf[self.name]) * 100 + var_gdf[affected_var_percentage_name] = var_gdf[affected_var_percentage_name].fillna(0) - # **Remove old columns before merging** - polygons_gdf.drop(columns=[col for col in [self.name, affected_var_name, affected_var_percentage_name] if - col in polygons_gdf.columns], inplace=True) + # Remove ONLY the current variable's columns before merging + cols_to_drop = [col for col in [self.name, affected_var_name, affected_var_percentage_name] if + col in polygons_gdf.columns] + if cols_to_drop: + polygons_gdf.drop(columns=cols_to_drop, inplace=True) - # **Final Merge and Save** + # Final Merge and Save out_gdf = polygons_gdf.merge(var_gdf, on='polyid', how='left') + + # Fill NaNs with 0 for polygons that had NO buildings + out_gdf[self.name] = out_gdf[self.name].fillna(0) + if project.raster_mask is not None and affected_var_name in out_gdf.columns: + out_gdf[affected_var_name] = out_gdf[affected_var_name].fillna(0) + out_gdf[affected_var_percentage_name] = out_gdf[affected_var_percentage_name].fillna(0) + out_gdf = out_gdf.rename(columns={'polyid': 'h3id'}) + # Write back out out_gdf.to_file(dataset_path, layer=destination_layer, driver='GPKG', mode='w') def resolve(self, **kwargs): diff --git a/rapida/components/deprivation/__init__.py b/rapida/components/deprivation/__init__.py index c64d7cd5..07b7611a 100644 --- a/rapida/components/deprivation/__init__.py +++ b/rapida/components/deprivation/__init__.py @@ -108,7 +108,7 @@ def download(self, force=False, **kwargs): 'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES', 'GDAL_CACHEMAX':'512', 'GDAL_NUMTHREADS':'4' - }): + }, thread_local=False): geo.import_raster( source=local_path, dst=self.local_path, target_srs=project.target_srs, crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name, diff --git a/rapida/components/population/__init__.py b/rapida/components/population/__init__.py index 32ba0355..4391879b 100644 --- a/rapida/components/population/__init__.py +++ b/rapida/components/population/__init__.py @@ -512,7 +512,7 @@ def import_raster(self, source=None, **kwargs): ) if os.path.exists(source):os.remove(source) - os.rename(imported_local_path, source) + os.replace(imported_local_path, source) return source def _compute_affected_(self, progress=None, **kwargs): diff --git a/rapida/components/rwi/__init__.py b/rapida/components/rwi/__init__.py index 5a4c70cd..42e22eb8 100644 --- a/rapida/components/rwi/__init__.py +++ b/rapida/components/rwi/__init__.py @@ -168,7 +168,7 @@ def download(self, force=False, **kwargs): 'GDAL_HTTP_MERGE_CONSECUTIVE_RANGES':'YES', 'GDAL_CACHEMAX':'512', 'GDAL_NUMTHREADS':'4' - }): + }, thread_local=False): geo.import_raster( source=local_path, dst=self.local_path, target_srs=project.target_srs, crop_ds=project.geopackage_file_path, crop_layer_name=project.polygons_layer_name, diff --git a/rapida/connectivity/__init__.py b/rapida/connectivity/__init__.py index d1bcd5c2..aa525326 100644 --- a/rapida/connectivity/__init__.py +++ b/rapida/connectivity/__init__.py @@ -104,9 +104,10 @@ async def run_connectivity_analysis( url = f"/vsicurl/https://undpngddlsgeohubdev01.blob.core.windows.net/admin/cgaz/geoBoundariesCGAZ_ADM{stats_admin_level}.fgb" adm_gdf = gpd.read_file(url, bbox=bbox, engine="pyogrio") if clip_country: + if 'iso3' in isochrones_gdf.columns.tolist(): + isochrones_gdf.drop(columns=['iso3'], inplace=True) adm_gdf = adm_gdf[adm_gdf['iso3'] == clip_country] - if 'iso3' in isochrones_gdf.columns.tolist(): - adm_gdf.drop(columns=['iso3'], inplace=True) + isochrones_gdf['iso3'] = clip_country if isochrones_gdf.crs != adm_gdf.crs: adm_gdf.to_crs(isochrones_gdf.crs, inplace=True) results = [] @@ -131,7 +132,7 @@ async def run_connectivity_analysis( del isochrones_gdf del results - with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=isochrones_path, comment='temp project for conn isochrones') with click.Context(assess) as ctx: ctx.ensure_object(dict) @@ -146,12 +147,13 @@ async def run_connectivity_analysis( project=project.path, force=False ) - stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg') - pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population') - if not disjoint: - pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index] - pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326') + stat_gpkg_path = os.path.join(project_folder,'data', f'{project.name}.gpkg') + pop_stat_gdf = gpd.read_file(stat_gpkg_path, layer='stats.population') + if not disjoint: + pop_stat_gdf = pop_stat_gdf.iloc[pop_stat_gdf.geometry.area.sort_values(ascending=False).index] + pop_stat_gdf = pop_stat_gdf.to_crs('EPSG:4326') + if not stats_admin_level: pop_stat_gdf.to_file( filename=isochrones_path, driver="GeoJSON", @@ -161,6 +163,84 @@ async def run_connectivity_analysis( promote_to_multi=True, index=False ) + else: + if barriers_dataset is None: # do aggregation here only when no barriers were given + + logger.info('Aggregating zonal stats independently and pivoting to wide format...') + + pop_col_names = [f'{popv}_{year}' for popv in pop_vars] + admin_col_name = f'admin{stats_admin_level}_name' + assert admin_col_name in pop_stat_gdf.columns.tolist() + pop_stat_gdf['contour'] = pop_stat_gdf['contour'].astype(float).astype(int) + pop_stat_gdf[admin_col_name] = pop_stat_gdf[admin_col_name].astype(str).str.strip() + + # 1. Group and sum the base stats independently (vectorized aggregation) + base_agg = pop_stat_gdf.groupby([admin_col_name, 'contour'])[pop_col_names].sum().reset_index() + + + + # 2. Pivot to Wide Format (Option 2: contours become columns) + pivot_df = base_agg.pivot(index=admin_col_name, columns='contour', values=pop_col_names) + + # Flatten the MultiIndex columns (e.g., ('male_total_2026', 15.0) -> 'male_total_2026_15min') + pivot_df.columns = [f"{col[0]}_{int(col[1])}min" for col in pivot_df.columns] + wide_df = pivot_df.reset_index() + + # 6. Attach geometries from the original adm_gdf + logger.info('Merging aggregated stats back onto original admin boundaries...') + + # Handle case where CGAZ admin column might natively be 'shapeName' + adm_join_col = admin_col_name if admin_col_name in adm_gdf.columns else 'shapeName' + + final_gdf = adm_gdf.merge( + wide_df, + left_on=adm_join_col, + right_on=admin_col_name, + how='inner' # Use 'inner' to only keep admin units that actually had isochrones + ) + + gc.collect() + + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as admin_project_folder: + logger.info(f'Computing zonal stats for total population ') + adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') + adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") + admin_project = Project(path=admin_project_folder, polygons=adm_ds_path, + comment='temp project for admin stats') + with click.Context(assess) as ctx: + ctx.ensure_object(dict) + ctx.obj['progress'] = progress + # 2. Use invoke. Do NOT pass 'ctx' manually here. + # Click intercepts this and injects it as the first argument automatically. + ctx.invoke( + assess, + components=('population',), + variables=['total'], + year=year, + project=admin_project.path, + force=False + ) + + admin_stat_gpkg_path = os.path.join(admin_project_folder, 'data', f'{admin_project.name}.gpkg') + admin_pop_stat_gdf = gpd.read_file(admin_stat_gpkg_path, layer='stats.population') + final_gdf = final_gdf.merge(admin_pop_stat_gdf[[admin_col_name, f'total_{year}']], + on=admin_col_name) + + if os.path.exists(adm_ds_path): os.remove(adm_ds_path) + + admin_iso_stats = os.path.join(dest_dir, f"admin{stats_admin_level}_iso_stats.geojson") + logger.info(f"Writing final aggregated admin boundaries to {admin_iso_stats}") + + final_gdf.to_file( + filename=admin_iso_stats, # Fixed: this was pointing to barrier_isochrones_path + driver="GeoJSON", + engine="pyogrio", + mode="w", + layer=f"admin{stats_admin_level}_iso_stats", + promote_to_multi=True, + index=False + ) + if barriers_dataset is not None and pop_vars: @@ -239,7 +319,7 @@ async def run_connectivity_analysis( del results - with TemporaryDirectory(dir=dest_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=barrier_isochrones_path, comment='temp project for conn isochrones') with click.Context(assess) as ctx: ctx.ensure_object(dict) @@ -341,7 +421,7 @@ async def run_connectivity_analysis( gc.collect() - with TemporaryDirectory(dir=dest_dir, delete=True) as admin_project_folder: + with TemporaryDirectory(dir=dest_dir, delete=True,ignore_cleanup_errors=True ) as admin_project_folder: logger.info(f'Computing zonal stats for total population ') adm_ds_path = os.path.join(dest_dir, f'admin_{stats_admin_level}.fgb') adm_gdf.to_file(adm_ds_path, driver="FlatGeobuf", engine="pyogrio") diff --git a/rapida/connectivity/graph.py b/rapida/connectivity/graph.py index 01c6da9d..e27883c0 100644 --- a/rapida/connectivity/graph.py +++ b/rapida/connectivity/graph.py @@ -6,10 +6,8 @@ from valhalla.config import _sanitize_config, default_config from typing import Union from pathlib import Path -import re import logging import sys -import valhalla logger = logging.getLogger(__name__) DEFAULT_SPEEDS = { @@ -192,7 +190,7 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> } ] - with open(speeds_config_path, "w") as f: + with open(speeds_config_path, "w", encoding='utf-8') as f: json.dump(speed_schema, f, indent=4) valhalla_conf["mjolnir"]["default_speeds_config"] = speeds_config_path @@ -200,7 +198,23 @@ async def compile_valhalla_graph(pbf_path: str, dst_dir: str, progress=None) -> valhalla_conf["mjolnir"]["min_reachability"] = 0 - with open(config_path, "w") as f: + #valhalla_conf["mjolnir"]["data_processing"]["use_admin_db"] = False + admin_db_path = os.path.join(dst_dir, 'admin.sqlite') + valhalla_conf["mjolnir"]["admin"] = str(Path(admin_db_path)) + + # Enable file logging for the Valhalla C++ engine + valhalla_conf["logging"] = { + "type": "file", + "color": False, + "file_name": os.path.join(dst_dir, "valhalla_build.log") + } + + if sys.platform == "win32": + valhalla_conf["mjolnir"]["use_mmap"] = False + + + + with open(config_path, "w", encoding='utf-8') as f: json.dump(valhalla_conf, f, indent=4) diff --git a/rapida/connectivity/io.py b/rapida/connectivity/io.py index 31eb6a47..2e550e6c 100644 --- a/rapida/connectivity/io.py +++ b/rapida/connectivity/io.py @@ -261,7 +261,7 @@ async def prepare_osm_pbf(bbox: tuple[float, float, float, float], dst_dir: str elif len(extracted_chunks) == 1: # If there was only one valid chunk, just rename it to the final output target - os.rename(extracted_chunks[0], final_output_pbf) + os.replace(extracted_chunks[0], final_output_pbf) else: raise ValueError(f"No OSM data found in the provided bbox: {bbox}") @@ -300,7 +300,7 @@ async def extract_health_sites(pbf_path: str, dst_dir: str, progress=None) -> st # Step 3: Compute centroids and flatten properties in a background thread def process_geometries(): - with open(raw_geojson, "r") as f: + with open(raw_geojson, "r", encoding="utf-8") as f: data = json.load(f) processed_features = [] @@ -329,7 +329,7 @@ def process_geometries(): data["features"] = processed_features - with open(final_geojson, "w") as f: + with open(final_geojson, "w", encoding="utf-8") as f: json.dump(data, f) if progress: @@ -352,7 +352,7 @@ def extract_origins_from_geojson(geojson_path: str) -> list[tuple[float, float]] """ Extracts a list of (longitude, latitude) tuples from a GeoJSON FeatureCollection. """ - with open(geojson_path, "r") as f: + with open(geojson_path, "r", encoding="utf-8") as f: data = json.load(f) origins = [] @@ -613,7 +613,7 @@ async def extract_water_bodies(pbf_path: str, dst_dir: str, progress=None) -> st # Step 3: Process geometries and ensure clean polygon outputs in a background thread def process_water_geometries(): - with open(raw_geojson, "r") as f: + with open(raw_geojson, "r", encoding="utf-8") as f: data = json.load(f) processed_features = [] @@ -645,7 +645,7 @@ def process_water_geometries(): data["features"] = processed_features - with open(final_geojson, "w") as f: + with open(final_geojson, "w", encoding="utf-8") as f: json.dump(data, f) if progress: diff --git a/rapida/ntl/noaa/cmask.py b/rapida/ntl/noaa/cmask.py index 48c48add..9d5f779a 100644 --- a/rapida/ntl/noaa/cmask.py +++ b/rapida/ntl/noaa/cmask.py @@ -17,6 +17,7 @@ from shapely import concave_hull from shapely.ops import unary_union from itertools import combinations +import platform gdal.UseExceptions() @@ -250,8 +251,11 @@ def cloud_coverage_fast(hdf_url: str, bbox: Iterable[float], raise -def cloud_coverage(hdf_url: str, bbox: list) -> int: - +def cloud_coverage_winbug(hdf_url: str, bbox: list) -> int: + if platform.system() == "Windows": + gdal.SetConfigOption('GDAL_SKIP', 'netCDF') + else: + gdal.SetConfigOption('GDAL_SKIP', '') # 1. Initialize GDAL environment INSIDE the worker process gdal.UseExceptions() gdal.PushErrorHandler('CPLQuietErrorHandler') @@ -268,14 +272,18 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: if cc is not None:return cc lon_min, lat_min, lon_max, lat_max = bbox subdataset_str = f'NETCDF:"/vsicurl/{hdf_url}":CloudMaskBinary' - + # --- THE WARP FIX --- + # Open the dataset explicitly first to bypass the Warp string-parsing bug + src_ds = gdal.Open(subdataset_str) + if src_ds is None: + raise Exception(f'Failed to open subdataset for {hdf_url}') # 2. Warp directly from the subdataset string over the network # GDAL's C++ engine reads the global polygon, clips the exact byte blocks, # and handles the 750m resolution on-the-fly into a memory buffer. ds = gdal.Warp( '', # Output to RAM - subdataset_str, + src_ds, format='MEM', dstSRS='EPSG:4326', # works here because we are counting pixels not planar metrics outputBounds=[lon_min, lat_min, lon_max, lat_max], @@ -283,6 +291,7 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: dstNodata=-128, geoloc=True ) + src_ds = None if ds is None: raise Exception(f'Failed to compute cloud coverage for {hdf_url}') @@ -298,6 +307,96 @@ def cloud_coverage(hdf_url: str, bbox: list) -> int: +def cloud_coverage(hdf_url: str, bbox: list) -> int: + gdal.UseExceptions() + gdal.PushErrorHandler('CPLQuietErrorHandler') + + # --- 1. DYNAMIC DRIVER SELECTION --- + is_windows = platform.system() == "Windows" + + if is_windows: + gdal.SetConfigOption('GDAL_SKIP', 'netCDF') + # On Windows, we force the HDF5 driver and its specific string syntax + base_ds = f'HDF5:"/vsicurl/{hdf_url}"' + mask_str = f'{base_ds}://CloudMaskBinary' + lon_str = f'{base_ds}://Longitude' + lat_str = f'{base_ds}://Latitude' + else: + gdal.SetConfigOption('GDAL_SKIP', '') + # On Linux, we use the native NetCDF driver + base_ds = f'HDF:"/vsicurl/{hdf_url}"' + mask_str = f'{base_ds}:CloudMaskBinary' + lon_str = f'{base_ds}:Longitude' + lat_str = f'{base_ds}:Latitude' + + gdal.SetConfigOption('GDAL_HTTP_TIMEOUT', '600') + gdal.SetConfigOption('GDAL_HTTP_MULTIPLEX', 'YES') + gdal.SetConfigOption('GDAL_HTTP_VERSION', '2') + gdal.SetConfigOption('VSI_CACHE', 'TRUE') + gdal.SetConfigOption('VSI_CACHE_SIZE', '50000000') + gdal.SetConfigOption('GDAL_DISABLE_READDIR_ON_OPEN', 'EMPTY_DIR') + gdal.SetConfigOption('CPL_VSIL_CURL_ALLOWED_EXTENSIONS', '.nc') + + _, file_name = os.path.split(hdf_url) + + # (Assuming your cache object is available in scope) + cc = cache.fetch(key=file_name) + if cc is not None: + return cc + + lon_min, lat_min, lon_max, lat_max = bbox + + # --- 2. OPEN THE RAW DATASET --- + src_ds = gdal.Open(mask_str) + if src_ds is None: + raise Exception(f'Failed to open subdataset {mask_str}') + + # --- 3. THE GEOLOCATION FIX --- + # Wrap the dataset in an in-memory VRT (Virtual Raster) so we can modify its metadata safely. + # We explicitly tell GDAL exactly where the Lat/Lon datasets are located on the network. + vrt_ds = gdal.Translate('', src_ds, format='VRT') + vrt_ds.SetMetadata({ + 'X_DATASET': lon_str, + 'X_BAND': '1', + 'Y_DATASET': lat_str, + 'Y_BAND': '1', + 'PIXEL_OFFSET': '0', + 'LINE_OFFSET': '0', + 'PIXEL_STEP': '1', + 'LINE_STEP': '1' + }, 'GEOLOCATION') + + # --- 4. WARP --- + # Now warp the VRT wrapper. geoloc=True will read our injected metadata and work perfectly. + ds = gdal.Warp( + '', + vrt_ds, + format='MEM', + dstSRS='EPSG:4326', + outputBounds=[lon_min, lat_min, lon_max, lat_max], + xRes=0.00675, yRes=0.00675, + dstNodata=-128, + geoloc=True + ) + + if ds is None: + src_ds = vrt_ds = None # Release locks + raise Exception(f'Failed to compute cloud coverage for {hdf_url}') + + data = ds.GetRasterBand(1).ReadAsArray() + valid_data = data[(data == 0) | (data == 1)] + + # Clean up all C++ pointers for Windows file locking + src_ds = vrt_ds = ds = None + + if valid_data.size == 0: + raise Exception(f'Failed to compute cloud coverage for {hdf_url}. No valid data.') + + cc = int((np.count_nonzero(valid_data == 1) / valid_data.size) * 100) + + cache.store(key=file_name, value=cc) + return cc + def cloud_coverage_batch(urls: list[str], bbox: Iterable[float], max_threads: int = 5, progress: Progress = None): results = {} master_task = None diff --git a/rapida/ntl/outage.py b/rapida/ntl/outage.py index 35b54a19..fc28875b 100644 --- a/rapida/ntl/outage.py +++ b/rapida/ntl/outage.py @@ -224,7 +224,7 @@ async def detect_outage( geom_type=ogr.wkbPolygon ) - with TemporaryDirectory(dir=dst_dir, delete=True) as project_folder: + with TemporaryDirectory(dir=dst_dir, delete=True, ignore_cleanup_errors=True) as project_folder: project = Project(path=project_folder, polygons=outage_gpkg_path, comment='temp project for outage zonal stats') with click.Context(assess) as ctx: diff --git a/rapida/project/project.py b/rapida/project/project.py index acff8d85..6fabf052 100644 --- a/rapida/project/project.py +++ b/rapida/project/project.py @@ -159,7 +159,6 @@ def __init__(self, path: str,polygons: str = None, # 2. Project centroids to EPSG:4326 for the remote mask query centroids_4326 = centroids.to_crs(epsg=4326) - centroids_4326.to_file('/tmp/isocen.fgb', engine='pyogrio') # 3. Use 'mask' instead of 'bbox'. This asks GDAL to only download # features that intersect your specific points, ignoring empty space. @@ -656,4 +655,3 @@ def publish(self, target_layers=None, no_input=False): project_path = '/data/rap/bgd' p = Project(path=project_path) src_raster = '/data/rap/bgd/data/population/female_active/BGD_female_active_r.tif' - p.align_raster(source_raster=src_raster) \ No newline at end of file diff --git a/uv.lock b/uv.lock index f2bf0a29..28321ffe 100644 --- a/uv.lock +++ b/uv.lock @@ -3143,7 +3143,6 @@ dependencies = [ { name = "tensorflow", marker = "platform_machine == 'aarch64'" }, { name = "tensorflow-cpu", marker = "platform_machine == 'x86_64'" }, { name = "tqdm" }, - { name = "uvloop" }, ] [package.optional-dependencies] @@ -3203,7 +3202,6 @@ requires-dist = [ { name = "tensorflow", marker = "platform_machine == 'aarch64'", specifier = "==2.16.2" }, { name = "tensorflow-cpu", marker = "platform_machine == 'x86_64'", specifier = "==2.16.2" }, { name = "tqdm" }, - { name = "uvloop" }, ] provides-extras = ["dev"] @@ -4005,38 +4003,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3b/a4/0698f3e5c397442ec9323a537e48cc63b846288b6878d38efd04e91005e3/utm-0.8.1-py3-none-any.whl", hash = "sha256:e3d5e224082af138e40851dcaad08d7f99da1cc4b5c413a7de34eabee35f434a", size = 8613, upload-time = "2025-03-06T11:40:54.273Z" }, ] -[[package]] -name = "uvloop" -version = "0.22.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/06/f0/18d39dbd1971d6d62c4629cc7fa67f74821b0dc1f5a77af43719de7936a7/uvloop-0.22.1.tar.gz", hash = "sha256:6c84bae345b9147082b17371e3dd5d42775bddce91f885499017f4607fdaf39f", size = 2443250, upload-time = "2025-10-16T22:17:19.342Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/ff/7f72e8170be527b4977b033239a83a68d5c881cc4775fca255c677f7ac5d/uvloop-0.22.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:fe94b4564e865d968414598eea1a6de60adba0c040ba4ed05ac1300de402cd42", size = 1359936, upload-time = "2025-10-16T22:16:29.436Z" }, - { url = "https://files.pythonhosted.org/packages/c3/c6/e5d433f88fd54d81ef4be58b2b7b0cea13c442454a1db703a1eea0db1a59/uvloop-0.22.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:51eb9bd88391483410daad430813d982010f9c9c89512321f5b60e2cddbdddd6", size = 752769, upload-time = "2025-10-16T22:16:30.493Z" }, - { url = "https://files.pythonhosted.org/packages/24/68/a6ac446820273e71aa762fa21cdcc09861edd3536ff47c5cd3b7afb10eeb/uvloop-0.22.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:700e674a166ca5778255e0e1dc4e9d79ab2acc57b9171b79e65feba7184b3370", size = 4317413, upload-time = "2025-10-16T22:16:31.644Z" }, - { url = "https://files.pythonhosted.org/packages/5f/6f/e62b4dfc7ad6518e7eff2516f680d02a0f6eb62c0c212e152ca708a0085e/uvloop-0.22.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7b5b1ac819a3f946d3b2ee07f09149578ae76066d70b44df3fa990add49a82e4", size = 4426307, upload-time = "2025-10-16T22:16:32.917Z" }, - { url = "https://files.pythonhosted.org/packages/90/60/97362554ac21e20e81bcef1150cb2a7e4ffdaf8ea1e5b2e8bf7a053caa18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e047cc068570bac9866237739607d1313b9253c3051ad84738cbb095be0537b2", size = 4131970, upload-time = "2025-10-16T22:16:34.015Z" }, - { url = "https://files.pythonhosted.org/packages/99/39/6b3f7d234ba3964c428a6e40006340f53ba37993f46ed6e111c6e9141d18/uvloop-0.22.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:512fec6815e2dd45161054592441ef76c830eddaad55c8aa30952e6fe1ed07c0", size = 4296343, upload-time = "2025-10-16T22:16:35.149Z" }, - { url = "https://files.pythonhosted.org/packages/89/8c/182a2a593195bfd39842ea68ebc084e20c850806117213f5a299dfc513d9/uvloop-0.22.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:561577354eb94200d75aca23fbde86ee11be36b00e52a4eaf8f50fb0c86b7705", size = 1358611, upload-time = "2025-10-16T22:16:36.833Z" }, - { url = "https://files.pythonhosted.org/packages/d2/14/e301ee96a6dc95224b6f1162cd3312f6d1217be3907b79173b06785f2fe7/uvloop-0.22.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1cdf5192ab3e674ca26da2eada35b288d2fa49fdd0f357a19f0e7c4e7d5077c8", size = 751811, upload-time = "2025-10-16T22:16:38.275Z" }, - { url = "https://files.pythonhosted.org/packages/b7/02/654426ce265ac19e2980bfd9ea6590ca96a56f10c76e63801a2df01c0486/uvloop-0.22.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e2ea3d6190a2968f4a14a23019d3b16870dd2190cd69c8180f7c632d21de68d", size = 4288562, upload-time = "2025-10-16T22:16:39.375Z" }, - { url = "https://files.pythonhosted.org/packages/15/c0/0be24758891ef825f2065cd5db8741aaddabe3e248ee6acc5e8a80f04005/uvloop-0.22.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0530a5fbad9c9e4ee3f2b33b148c6a64d47bbad8000ea63704fa8260f4cf728e", size = 4366890, upload-time = "2025-10-16T22:16:40.547Z" }, - { url = "https://files.pythonhosted.org/packages/d2/53/8369e5219a5855869bcee5f4d317f6da0e2c669aecf0ef7d371e3d084449/uvloop-0.22.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bc5ef13bbc10b5335792360623cc378d52d7e62c2de64660616478c32cd0598e", size = 4119472, upload-time = "2025-10-16T22:16:41.694Z" }, - { url = "https://files.pythonhosted.org/packages/f8/ba/d69adbe699b768f6b29a5eec7b47dd610bd17a69de51b251126a801369ea/uvloop-0.22.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1f38ec5e3f18c8a10ded09742f7fb8de0108796eb673f30ce7762ce1b8550cad", size = 4239051, upload-time = "2025-10-16T22:16:43.224Z" }, - { url = "https://files.pythonhosted.org/packages/90/cd/b62bdeaa429758aee8de8b00ac0dd26593a9de93d302bff3d21439e9791d/uvloop-0.22.1-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:3879b88423ec7e97cd4eba2a443aa26ed4e59b45e6b76aabf13fe2f27023a142", size = 1362067, upload-time = "2025-10-16T22:16:44.503Z" }, - { url = "https://files.pythonhosted.org/packages/0d/f8/a132124dfda0777e489ca86732e85e69afcd1ff7686647000050ba670689/uvloop-0.22.1-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4baa86acedf1d62115c1dc6ad1e17134476688f08c6efd8a2ab076e815665c74", size = 752423, upload-time = "2025-10-16T22:16:45.968Z" }, - { url = "https://files.pythonhosted.org/packages/a3/94/94af78c156f88da4b3a733773ad5ba0b164393e357cc4bd0ab2e2677a7d6/uvloop-0.22.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:297c27d8003520596236bdb2335e6b3f649480bd09e00d1e3a99144b691d2a35", size = 4272437, upload-time = "2025-10-16T22:16:47.451Z" }, - { url = "https://files.pythonhosted.org/packages/b5/35/60249e9fd07b32c665192cec7af29e06c7cd96fa1d08b84f012a56a0b38e/uvloop-0.22.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c1955d5a1dd43198244d47664a5858082a3239766a839b2102a269aaff7a4e25", size = 4292101, upload-time = "2025-10-16T22:16:49.318Z" }, - { url = "https://files.pythonhosted.org/packages/02/62/67d382dfcb25d0a98ce73c11ed1a6fba5037a1a1d533dcbb7cab033a2636/uvloop-0.22.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b31dc2fccbd42adc73bc4e7cdbae4fc5086cf378979e53ca5d0301838c5682c6", size = 4114158, upload-time = "2025-10-16T22:16:50.517Z" }, - { url = "https://files.pythonhosted.org/packages/f0/7a/f1171b4a882a5d13c8b7576f348acfe6074d72eaf52cccef752f748d4a9f/uvloop-0.22.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:93f617675b2d03af4e72a5333ef89450dfaa5321303ede6e67ba9c9d26878079", size = 4177360, upload-time = "2025-10-16T22:16:52.646Z" }, - { url = "https://files.pythonhosted.org/packages/79/7b/b01414f31546caf0919da80ad57cbfe24c56b151d12af68cee1b04922ca8/uvloop-0.22.1-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:37554f70528f60cad66945b885eb01f1bb514f132d92b6eeed1c90fd54ed6289", size = 1454790, upload-time = "2025-10-16T22:16:54.355Z" }, - { url = "https://files.pythonhosted.org/packages/d4/31/0bb232318dd838cad3fa8fb0c68c8b40e1145b32025581975e18b11fab40/uvloop-0.22.1-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:b76324e2dc033a0b2f435f33eb88ff9913c156ef78e153fb210e03c13da746b3", size = 796783, upload-time = "2025-10-16T22:16:55.906Z" }, - { url = "https://files.pythonhosted.org/packages/42/38/c9b09f3271a7a723a5de69f8e237ab8e7803183131bc57c890db0b6bb872/uvloop-0.22.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:badb4d8e58ee08dad957002027830d5c3b06aea446a6a3744483c2b3b745345c", size = 4647548, upload-time = "2025-10-16T22:16:57.008Z" }, - { url = "https://files.pythonhosted.org/packages/c1/37/945b4ca0ac27e3dc4952642d4c900edd030b3da6c9634875af6e13ae80e5/uvloop-0.22.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b91328c72635f6f9e0282e4a57da7470c7350ab1c9f48546c0f2866205349d21", size = 4467065, upload-time = "2025-10-16T22:16:58.206Z" }, - { url = "https://files.pythonhosted.org/packages/97/cc/48d232f33d60e2e2e0b42f4e73455b146b76ebe216487e862700457fbf3c/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:daf620c2995d193449393d6c62131b3fbd40a63bf7b307a1527856ace637fe88", size = 4328384, upload-time = "2025-10-16T22:16:59.36Z" }, - { url = "https://files.pythonhosted.org/packages/e4/16/c1fd27e9549f3c4baf1dc9c20c456cd2f822dbf8de9f463824b0c0357e06/uvloop-0.22.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6cde23eeda1a25c75b2e07d39970f3374105d5eafbaab2a4482be82f272d5a5e", size = 4296730, upload-time = "2025-10-16T22:17:00.744Z" }, -] - [[package]] name = "werkzeug" version = "3.1.8"