diff --git a/activitysim/abm/models/park_and_ride_lot_choice.py b/activitysim/abm/models/park_and_ride_lot_choice.py index e8e0f6683..ca7cdddc2 100644 --- a/activitysim/abm/models/park_and_ride_lot_choice.py +++ b/activitysim/abm/models/park_and_ride_lot_choice.py @@ -3,6 +3,7 @@ # ActivitySim # See full license in LICENSE.txt. import logging +import math from typing import Literal import numpy as np @@ -11,6 +12,7 @@ from activitysim.abm.models.util import logsums from activitysim.abm.models.util.park_and_ride_capacity import ParkAndRideCapacity from activitysim.core import ( + chunk, config, estimation, expressions, @@ -175,82 +177,61 @@ def return_no_choices(state, choosers: pd.DataFrame, original_index=None) -> pd. return pd.Series(data=-1, index=index) -def run_park_and_ride_lot_choice( - state: workflow.State, - choosers: pd.DataFrame, - land_use: pd.DataFrame, - network_los: los.Network_LOS, - model_settings: ParkAndRideLotChoiceSettings | None = None, - choosers_dest_col_name: str = "destination", - choosers_origin_col_name: str = "home_zone_id", - estimator=None, - model_settings_file_name: str = "park_and_ride_lot_choice.yaml", - pnr_capacity_cls: ParkAndRideCapacity | None = None, - trace_label: str = "park_and_ride_lot_choice", -) -> pd.Series: +def _choosers_per_slice( + state: workflow.State, num_choosers: int, explicit_chunk_size: float +) -> int: """ - Run the park-and-ride lot choice model. + Number of choosers to process at a time, following the explicit_chunk setting. - Provides another entry point for the model which is useful for getting pnr locations while creating logsums. + Uses the same rules as explicit chunking in `activitysim.core.chunk`: a fraction + below 1 is a share of all choosers, otherwise the value is the total number of + choosers per chunk across all processes. When explicit chunking is not active, + all choosers are processed together. """ - - if model_settings is None: - model_settings = ParkAndRideLotChoiceSettings.read_settings_file( - state.filesystem, - model_settings_file_name, + if ( + state.settings.chunk_training_mode != chunk.MODE_EXPLICIT + or not explicit_chunk_size + or explicit_chunk_size < 0 + ): + return num_choosers + + if explicit_chunk_size < 1: + rows = math.ceil(num_choosers * explicit_chunk_size) + else: + num_processes = ( + state.get_injectable("num_processes", 1) + if state.settings.multiprocess + else 1 ) + rows = int(explicit_chunk_size / num_processes) - spec = state.filesystem.read_model_spec(file_name=model_settings.SPEC) - coefficients = state.filesystem.read_model_coefficients(model_settings) - model_spec = simulate.eval_coefficients(state, spec, coefficients, estimator) - locals_dict = state.get_global_constants() - locals_dict.update(model_settings.CONSTANTS) - - pnr_alts = land_use[land_use[model_settings.LANDUSE_PNR_SPACES_COLUMN] > 0] - pnr_alts["pnr_zone_id"] = pnr_alts.index.values + return min(max(rows, 1), num_choosers) - if pnr_alts.empty and model_settings.ALLOW_ZERO_PROBS: - logger.info( - "No park-and-ride lots are available. Returning -1 for all choosers." - ) - return pd.Series(data=-1, index=choosers.index) - # if we are running with capacitated pnr lots, we need to flag the lots that are over-capacitated - if pnr_capacity_cls is not None: - pnr_alts["pnr_lot_full"] = pnr_capacity_cls.flag_capacitated_pnr_zones(pnr_alts) - # if there are no available pnr lots left, we return a series of -1 - if (pnr_alts["pnr_lot_full"] == 1).all(): - logger.info( - "All park-and-ride lots are full. Returning -1 as park-and-ride lot choice." - ) - return pd.Series(data=-1, index=choosers.index) - else: - pnr_alts["pnr_lot_full"] = 0 +def _choose_lots_for_choosers( + state: workflow.State, + choosers: pd.DataFrame, + land_use: pd.DataFrame, + pnr_alts: pd.DataFrame, + network_los: los.Network_LOS, + model_settings: ParkAndRideLotChoiceSettings, + model_settings_file_name: str, + model_spec: pd.DataFrame, + locals_dict: dict, + choosers_dest_col_name: str, + choosers_origin_col_name: str, + estimator, + trace_label: str, +) -> pd.Series | None: + """ + Choose park-and-ride lots for choosers (with a unique index). - original_index = None - if not choosers.index.is_unique: - # non-unique index will crash interaction_simulate - # so we need to reset the index and add it to ActivitySim's rng - # this happens while the disaggregate accessibility model is running pnr lot choice - original_index = choosers.index - oi_name = original_index.name - oi_name = oi_name if oi_name else "index" - choosers = choosers.reset_index(drop=False) - idx_multiplier = choosers.groupby(oi_name).size().max() - # round to the nearest 10's place - idx_multiplier = int(np.ceil(idx_multiplier / 10.0) * 10) - choosers.index = ( - original_index * idx_multiplier + choosers.groupby(oi_name).cumcount() - ) - choosers.index.name = oi_name + "_pnr_lot_choice" - state.get_rn_generator().add_channel("pnr_lot_choice", choosers) - assert ( - choosers.index.is_unique - ), "The index of the choosers DataFrame must be unique after resetting the index." - assert len(choosers.index) == len(original_index), ( - f"The length of the choosers DataFrame must be equal to the original index length:" - f" {len(choosers.index)} != {len(original_index)}" - ) + Returns None if none of the choosers are eligible for park-and-ride. + """ + # the alternatives and locals are changed by preprocessing, so do not carry + # those changes into the next slice of choosers + pnr_alts = pnr_alts.copy() + locals_dict = locals_dict.copy() trn_accessible_choosers = filter_chooser_to_transit_accessible_destinations( state, @@ -263,7 +244,7 @@ def run_park_and_ride_lot_choice( ) if trn_accessible_choosers.empty: - return return_no_choices(state, choosers, original_index) + return None add_periods = False if "in_period" in trn_accessible_choosers.columns else True skims = logsums.setup_skims( @@ -312,7 +293,7 @@ def run_park_and_ride_lot_choice( ) if trn_accessible_choosers.empty: - return return_no_choices(state, choosers, original_index) + return None # preprocess alternatives expressions.annotate_preprocessors( @@ -345,7 +326,7 @@ def run_park_and_ride_lot_choice( estimator.write_choosers(trn_accessible_choosers) estimator.write_alternatives(pnr_alts) - choices = interaction_simulate( + return interaction_simulate( state, choosers=trn_accessible_choosers, alternatives=pnr_alts, @@ -362,7 +343,140 @@ def run_park_and_ride_lot_choice( zero_prob_choice_val=-1, ) - choices = choices.reindex(choosers.index, fill_value=-1) + +def run_park_and_ride_lot_choice( + state: workflow.State, + choosers: pd.DataFrame, + land_use: pd.DataFrame, + network_los: los.Network_LOS, + model_settings: ParkAndRideLotChoiceSettings | None = None, + choosers_dest_col_name: str = "destination", + choosers_origin_col_name: str = "home_zone_id", + estimator=None, + model_settings_file_name: str = "park_and_ride_lot_choice.yaml", + pnr_capacity_cls: ParkAndRideCapacity | None = None, + trace_label: str = "park_and_ride_lot_choice", +) -> pd.Series: + """ + Run the park-and-ride lot choice model. + + Provides another entry point for the model which is useful for getting pnr locations while creating logsums. + """ + + if model_settings is None: + model_settings = ParkAndRideLotChoiceSettings.read_settings_file( + state.filesystem, + model_settings_file_name, + ) + + spec = state.filesystem.read_model_spec(file_name=model_settings.SPEC) + coefficients = state.filesystem.read_model_coefficients(model_settings) + model_spec = simulate.eval_coefficients(state, spec, coefficients, estimator) + locals_dict = state.get_global_constants() + locals_dict.update(model_settings.CONSTANTS) + + pnr_alts = land_use[land_use[model_settings.LANDUSE_PNR_SPACES_COLUMN] > 0] + pnr_alts["pnr_zone_id"] = pnr_alts.index.values + + if pnr_alts.empty and model_settings.ALLOW_ZERO_PROBS: + logger.info( + "No park-and-ride lots are available. Returning -1 for all choosers." + ) + return pd.Series(data=-1, index=choosers.index) + + # if we are running with capacitated pnr lots, we need to flag the lots that are over-capacitated + if pnr_capacity_cls is not None: + pnr_alts["pnr_lot_full"] = pnr_capacity_cls.flag_capacitated_pnr_zones(pnr_alts) + # if there are no available pnr lots left, we return a series of -1 + if (pnr_alts["pnr_lot_full"] == 1).all(): + logger.info( + "All park-and-ride lots are full. Returning -1 as park-and-ride lot choice." + ) + return pd.Series(data=-1, index=choosers.index) + else: + pnr_alts["pnr_lot_full"] = 0 + + original_index = None + unique_index = None + if not choosers.index.is_unique: + # non-unique index will crash interaction_simulate + # so we need a unique index that is registered in ActivitySim's rng + # this happens while the disaggregate accessibility model is running pnr lot choice + # and while location choice logsums are computed. + # The unique index is computed once for the whole table so that random draws do not + # depend on how the choosers are sliced below; only the index is built up front + # (not a copy of the chooser table) to keep memory low. + original_index = choosers.index + oi_name = original_index.name + oi_name = oi_name if oi_name else "index" + by_index = choosers.groupby(level=0) + idx_multiplier = by_index.size().max() + # round to the nearest 10's place + idx_multiplier = int(np.ceil(idx_multiplier / 10.0) * 10) + unique_index = pd.Index( + original_index.values * idx_multiplier + by_index.cumcount().values, + name=oi_name + "_pnr_lot_choice", + ) + state.get_rn_generator().add_channel( + "pnr_lot_choice", pd.DataFrame(index=unique_index) + ) + assert ( + unique_index.is_unique + ), "The index of the choosers DataFrame must be unique after resetting the index." + assert len(unique_index) == len(original_index), ( + f"The length of the choosers DataFrame must be equal to the original index length:" + f" {len(unique_index)} != {len(original_index)}" + ) + + # Everything from the transit eligibility filter through the lot choice itself + # is run on slices of the choosers. Preprocessing and skim setup would otherwise + # create several copies of the entire chooser table, which is very large when + # computing location choice logsums (one row per person and sampled destination). + num_choosers = len(choosers) + if estimator is None: + rows_per_slice = _choosers_per_slice( + state, num_choosers, model_settings.explicit_chunk + ) + else: + # estimation writes the choosers once, so keep them together + rows_per_slice = num_choosers + + slice_choices = [] + for start in range(0, max(num_choosers, 1), max(rows_per_slice, 1)): + stop = min(start + rows_per_slice, num_choosers) + if unique_index is not None: + chooser_slice = choosers.iloc[start:stop].reset_index(drop=False) + chooser_slice.index = unique_index[start:stop] + elif rows_per_slice >= num_choosers: + chooser_slice = choosers + else: + chooser_slice = choosers.iloc[start:stop].copy() + + slice_result = _choose_lots_for_choosers( + state, + choosers=chooser_slice, + land_use=land_use, + pnr_alts=pnr_alts, + network_los=network_los, + model_settings=model_settings, + model_settings_file_name=model_settings_file_name, + model_spec=model_spec, + locals_dict=locals_dict, + choosers_dest_col_name=choosers_dest_col_name, + choosers_origin_col_name=choosers_origin_col_name, + estimator=estimator, + trace_label=trace_label, + ) + if slice_result is not None: + slice_choices.append(slice_result) + + if not slice_choices: + return return_no_choices(state, choosers, original_index) + + choices = pd.concat(slice_choices) + choices = choices.reindex( + choosers.index if unique_index is None else unique_index, fill_value=-1 + ) if original_index is not None: # set the choices index back to the original index diff --git a/activitysim/abm/models/util/test/test_park_and_ride_models.py b/activitysim/abm/models/util/test/test_park_and_ride_models.py index 0b0139174..07da959b9 100644 --- a/activitysim/abm/models/util/test/test_park_and_ride_models.py +++ b/activitysim/abm/models/util/test/test_park_and_ride_models.py @@ -399,6 +399,147 @@ def test_run_pnr_allows_no_lots(state, network_los, model_settings_base): assert choices.to_dict() == {100: -1} +def _run_pnr_with_slicing( + monkeypatch, state, network_los, model_settings, land_use, choosers, explicit_chunk +): + """Run lot choice with the given explicit_chunk, returning choices and slice sizes.""" + calls = [] + original = pnr_lot_choice._choose_lots_for_choosers + + def counting_choose(*args, **kwargs): + calls.append(len(kwargs["choosers"])) + return original(*args, **kwargs) + + monkeypatch.setattr(pnr_lot_choice, "_choose_lots_for_choosers", counting_choose) + model_settings.explicit_chunk = explicit_chunk + state.settings.chunk_training_mode = "explicit" if explicit_chunk else "disabled" + + choices = pnr_lot_choice.run_park_and_ride_lot_choice( + state=state, + choosers=choosers.copy(), + land_use=land_use, + network_los=network_los, + model_settings=model_settings, + ) + return choices, calls + + +@pytest.mark.parametrize("unique_index", [False, True]) +def test_run_pnr_slicing_does_not_change_choices( + monkeypatch, state, network_los, model_settings_base, unique_index +): + """Choosing lots in slices of choosers gives exactly the same choices as all at once.""" + model_settings_base.LANDUSE_COL_FOR_PNR_ELIGIBLE_DEST = "pnr_eligible" + land_use = pd.DataFrame( + {"pnr_spaces": [10, 20], "pnr_eligible": [True, True]}, + index=pd.Index([1, 2], name="zone_id"), + ) + + if unique_index: + index = pd.Index(np.arange(100, 150), name="tour_id") + else: + # repeated ids, like the person x sampled destination rows of location logsums + index = pd.Index( + np.repeat(np.arange(100, 120), [2, 3, 1, 4] * 5)[:50], name="person_id" + ) + assert not index.is_unique + n = len(index) + choosers = pd.DataFrame( + { + "destination": np.where(np.arange(n) % 3 == 0, 1, 2), + "home_zone_id": 1, + "in_period": 1, + }, + index=index, + ) + + rng = state.get_rn_generator() + rng.begin_step("test_run_pnr_slicing_does_not_change_choices") + if unique_index: + rng.add_channel("tours", choosers) + + expected, calls = _run_pnr_with_slicing( + monkeypatch, state, network_los, model_settings_base, land_use, choosers, 0 + ) + assert len(calls) == 1 + # equal utilities, so lots are picked at random and slicing would show up as a difference + assert expected.nunique() == 2 + + if unique_index: + rng.reset_offsets_for_df(choosers) + + sliced, calls = _run_pnr_with_slicing( + monkeypatch, state, network_los, model_settings_base, land_use, choosers, 8 + ) + rng.end_step("test_run_pnr_slicing_does_not_change_choices") + + assert calls == [8, 8, 8, 8, 8, 8, 2] + pd.testing.assert_series_equal(sliced, expected) + assert sliced.index.equals(choosers.index) + + +def test_run_pnr_slicing_keeps_zero_probability_choosers( + monkeypatch, state, network_los, model_settings_base +): + """Choosers with no available lot get -1 whichever slice they fall in.""" + model_settings_base.ALLOW_ZERO_PROBS = True + land_use = pd.DataFrame( + {"pnr_spaces": [10, 10]}, index=pd.Index([1, 2], name="zone_id") + ) + choosers = pd.DataFrame( + { + "destination": [1, 5, 2, 5, 1, 2], + "home_zone_id": 3, + "in_period": 1, + }, + index=pd.Index([100, 100, 102, 103, 103, 104], name="tour_id"), + ) + _write_model_files( + state, + "pnr_zero_prob_slice_spec.csv", + """ +Description,Expression,coefficient +availability,@-1000 * (df.destination != df.pnr_zone_id),coef_one +""", + "pnr_zero_prob_slice_coeffs.csv", + """ +coefficient_name,value,constrain +coef_one,1.0,F +""", + ) + model_settings_base.SPEC = "pnr_zero_prob_slice_spec.csv" + model_settings_base.COEFFICIENTS = "pnr_zero_prob_slice_coeffs.csv" + + state.get_rn_generator().begin_step("test_run_pnr_slicing_zero_probs") + choices, calls = _run_pnr_with_slicing( + monkeypatch, state, network_los, model_settings_base, land_use, choosers, 2 + ) + state.get_rn_generator().end_step("test_run_pnr_slicing_zero_probs") + + assert calls == [2, 2, 2] + assert choices.index.tolist() == [100, 100, 102, 103, 103, 104] + assert choices.tolist() == [1, -1, 2, -1, 1, 2] + + +def test_choosers_per_slice_follows_explicit_chunk_rules(state): + """Slice sizes follow the same rules as explicit chunking in the chunk module.""" + per_slice = pnr_lot_choice._choosers_per_slice + + state.settings.chunk_training_mode = "disabled" + assert per_slice(state, 1000, 100) == 1000 + + state.settings.chunk_training_mode = "explicit" + assert per_slice(state, 1000, 0) == 1000 + assert per_slice(state, 1000, 100) == 100 + assert per_slice(state, 1000, 5000) == 1000 + assert per_slice(state, 1000, 0.25) == 250 + + state.settings.multiprocess = True + state.add_injectable("num_processes", 4) + assert per_slice(state, 1000, 100) == 25 + assert per_slice(state, 1000, 2) == 1 + + def test_run_pnr_calculates_drive_transit_and_capacity_utilities( state, network_los, model_settings_base ):