diff --git a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue index c1b369556..9e26e077d 100644 --- a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue +++ b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue @@ -38,8 +38,8 @@ > Apply a mathematical formula to one or more input datastreams to produce a new output datastream. With more than one input, they're - matched by exact timestamp; if inputs stop lining up, the run stops - there until they're back in sync. + matched by exact timestamp; timestamps not shared by every input + stop the run or are skipped, depending on "Stop on input gaps".
-
Stop on no-data value
+
Stop on input gaps
- If an input is set to a no-data value, stop the run there - instead of writing the output's no-data value and continuing. + If an input has no value at a timestamp where the others do, stop + the run there and wait until the missing value arrives. When off, + those timestamps are skipped and the run continues. Runs never go + past the most recent value of the input that is furthest behind. + Inputs holding their no-data value always produce the output's + no-data value. (null) const outputDatastreamId = ref(null) const inputs = ref([makeRow('x')]) const formula = ref('') -const stopOnNoData = ref(true) +const stopOnGaps = ref(false) const stopOnError = ref(true) const selectedThingId = computed(() => props.initialThingId ?? null) @@ -547,7 +551,7 @@ async function loadExistingTask() { existingTransformationId.value = t.id outputDatastreamId.value = (t.outputDatastream as any)?.id ?? null formula.value = t.formula - stopOnNoData.value = t.stopOnNoData ?? true + stopOnGaps.value = t.stopOnGaps ?? false stopOnError.value = t.stopOnError ?? true if (t.inputDatastreams?.length) { @@ -617,7 +621,7 @@ async function onCreate( outputDatastreamId: outputDatastreamId.value!, inputDatastreams, formula: formula.value.trim(), - stopOnNoData: stopOnNoData.value, + stopOnGaps: stopOnGaps.value, stopOnError: stopOnError.value, } ) @@ -658,7 +662,7 @@ async function onUpdate( outputDatastreamId: outputDatastreamId.value!, inputDatastreams, formula: formula.value.trim(), - stopOnNoData: stopOnNoData.value, + stopOnGaps: stopOnGaps.value, stopOnError: stopOnError.value, } ) diff --git a/apps/docs/docs/references/data-dictionary.md b/apps/docs/docs/references/data-dictionary.md index 845b0798b..ac1bf5961 100644 --- a/apps/docs/docs/references/data-dictionary.md +++ b/apps/docs/docs/references/data-dictionary.md @@ -490,7 +490,7 @@ A DataProductTransformation describes how a single derived (output) Datastream i | M | transformation_type | The type of transformation: `rating_curve`, `derivation`, or `aggregation`. | String | | O | rating_curve_id | A foreign key identifier for the RatingCurve used when `transformation_type` is `rating_curve`. | UUID | | O | formula | A formula (expression) string used when `transformation_type` is `derivation`, referencing one or more input variable names. | Text | -| O | stop_on_no_data | For `derivation` transformations, whether a run stops when an input holds its no-data value (vs. filling the output and continuing). | Boolean | +| O | stop_on_gaps | For `derivation` transformations, whether a run stops at timestamps not shared by all inputs (vs. skipping them). Inputs holding their no-data value always produce the output's no-data value. | Boolean | | O | stop_on_error | For `derivation` transformations, whether a run stops when the formula produces a non-finite result (vs. filling and continuing). | Boolean | | O | output_interval_units | The interval unit for `aggregation` outputs: `minutes`, `hours`, `days`, `weeks`, or `months`. | String | | O | output_interval | The positive interval count corresponding to `output_interval_units`. | Integer | diff --git a/apps/docs/docs/user-guides/how-to/using-the-python-client.md b/apps/docs/docs/user-guides/how-to/using-the-python-client.md index ea19b5394..e1b92f131 100644 --- a/apps/docs/docs/user-guides/how-to/using-the-python-client.md +++ b/apps/docs/docs/user-guides/how-to/using-the-python-client.md @@ -667,7 +667,7 @@ hs_api.dataproducttransformations.delete_rating_curve(task_id, uid) #### Derivation Transformations -Combines one or more input datastreams using a formula. Each input datastream is assigned a variable name used in the formula; with a single input, this is equivalent to applying the formula to each observation independently. Multiple inputs are matched by exact timestamp — if they stop lining up, a run stops there until the inputs are back in sync. +Combines one or more input datastreams using a formula. Each input datastream is assigned a variable name used in the formula; with a single input, this is equivalent to applying the formula to each observation independently. Multiple inputs are matched by exact timestamp. Timestamps not shared by every input either stop the run there until the gap is filled or are skipped, depending on `stop_on_gaps`. Runs never go past the most recent value of the input that is furthest behind. If any input holds its no-data value at a timestamp, the output gets its own no-data value there. | Field | Type | Description | |---|---|---| @@ -675,7 +675,7 @@ Combines one or more input datastreams using a formula. Each input datastream is | `output_datastream_id` | `UUID` | | | `input_datastream_ids` | `List[UUID]` | | | `formula` | `str` | Python expression referencing the input variable name(s) | -| `stop_on_no_data` | `bool` | If `True`, stop the run when an input holds its no-data value; if `False`, fill the output with its no-data value and continue | +| `stop_on_gaps` | `bool` | If `True`, stop the run at the first timestamp not shared by all inputs; if `False`, skip those timestamps and continue. Inputs holding their no-data value always produce the output's no-data value | | `stop_on_error` | `bool` | If `True`, stop the run when the formula produces a non-finite result; if `False`, fill the output with its no-data value and continue | ```python @@ -683,11 +683,11 @@ hs_api.dataproducttransformations.list_derivation(task_id, output_datastream=Non hs_api.dataproducttransformations.get_derivation(task_id, uid) hs_api.dataproducttransformations.create_derivation( task_id, output_datastream, input_datastreams, formula, - stop_on_no_data=True, stop_on_error=True, uid=None + stop_on_gaps=False, stop_on_error=True, uid=None ) hs_api.dataproducttransformations.update_derivation( task_id, uid, input_datastreams=..., formula=..., - stop_on_no_data=..., stop_on_error=... + stop_on_gaps=..., stop_on_error=... ) hs_api.dataproducttransformations.delete_derivation(task_id, uid) ``` diff --git a/django/contracts/openapi/auth.openapi.json b/django/contracts/openapi/auth.openapi.json index 09d7ef8ba..c93c47834 100644 --- a/django/contracts/openapi/auth.openapi.json +++ b/django/contracts/openapi/auth.openapi.json @@ -678,7 +678,7 @@ "info": { "description": "", "title": "HydroServer User Authentication API", - "version": "1.12.1" + "version": "1.12.2b1" }, "openapi": "3.1.0", "paths": { diff --git a/django/contracts/openapi/data.openapi.json b/django/contracts/openapi/data.openapi.json index 6811d2fa2..a270c8214 100644 --- a/django/contracts/openapi/data.openapi.json +++ b/django/contracts/openapi/data.openapi.json @@ -3739,8 +3739,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3775,9 +3775,9 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "default": true, - "title": "Stoponnodata", + "stopOnGaps": { + "default": false, + "title": "Stopongaps", "type": "boolean" } }, @@ -3814,8 +3814,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3824,7 +3824,7 @@ "outputDatastream", "inputDatastreams", "formula", - "stopOnNoData", + "stopOnGaps", "stopOnError" ], "title": "DerivationTransformationResponse", @@ -3858,8 +3858,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3868,7 +3868,7 @@ "outputDatastreamId", "inputDatastreamIds", "formula", - "stopOnNoData", + "stopOnGaps", "stopOnError" ], "title": "DerivationTransformationSummaryResponse", @@ -11645,7 +11645,7 @@ "info": { "description": "", "title": "HydroServer Data Management API", - "version": "1.12.1" + "version": "1.12.2b1" }, "openapi": "3.1.0", "paths": { diff --git a/django/hydroserver/__init__.py b/django/hydroserver/__init__.py index 00a955d55..e578d9eea 100644 --- a/django/hydroserver/__init__.py +++ b/django/hydroserver/__init__.py @@ -1,4 +1,4 @@ from .celery import app as celery_app -__version__ = "1.12.1" +__version__ = "1.12.2b1" __all__ = ("celery_app", "__version__",) diff --git a/django/interfaces/api/schemas/products/transformation.py b/django/interfaces/api/schemas/products/transformation.py index cda3ad10a..9f006b7d9 100644 --- a/django/interfaces/api/schemas/products/transformation.py +++ b/django/interfaces/api/schemas/products/transformation.py @@ -70,7 +70,7 @@ class DerivationTransformationSummaryResponse(BaseGetResponse): output_datastream_id: uuid.UUID input_datastream_ids: list[uuid.UUID] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool @staticmethod @@ -120,7 +120,7 @@ class DerivationTransformationResponse(BaseGetResponse): output_datastream: DatastreamSummaryResponse input_datastreams: list[TransformationInputResponse] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool @staticmethod @@ -164,7 +164,7 @@ class RatingCurveTransformationPostBody(_TransformationPostBodyBase): class DerivationTransformationPostBody(_TransformationPostBodyBase): input_datastreams: list[TransformationInputPostBody] formula: str - stop_on_no_data: bool = True + stop_on_gaps: bool = False stop_on_error: bool = True @@ -190,7 +190,7 @@ class DerivationTransformationPatchBody(BasePatchBody): output_datastream: uuid.UUID = Field(alias="outputDatastreamId") input_datastreams: list[TransformationInputPostBody] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool diff --git a/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py b/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py new file mode 100644 index 000000000..9d7bc8d08 --- /dev/null +++ b/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.2 on 2026-10-19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('products', '0003_remove_dataproducttransformation_max_gap_interval_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='dataproducttransformation', + old_name='stop_on_no_data', + new_name='stop_on_gaps', + ), + migrations.AlterField( + model_name='dataproducttransformation', + name='stop_on_gaps', + field=models.BooleanField(blank=True, default=False), + ), + ] diff --git a/django/processing/products/models/transformation.py b/django/processing/products/models/transformation.py index f41e93bb4..2b69193c2 100644 --- a/django/processing/products/models/transformation.py +++ b/django/processing/products/models/transformation.py @@ -68,7 +68,7 @@ class DataProductTransformation(models.Model): blank=True, ) min_values = models.PositiveIntegerField(null=True, blank=True) - stop_on_no_data = models.BooleanField(default=True, blank=True) + stop_on_gaps = models.BooleanField(default=False, blank=True) stop_on_error = models.BooleanField(default=True, blank=True) class Meta: diff --git a/django/processing/products/services/transformation.py b/django/processing/products/services/transformation.py index 7ce2c2690..6d91e9bd6 100644 --- a/django/processing/products/services/transformation.py +++ b/django/processing/products/services/transformation.py @@ -174,7 +174,7 @@ def create( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> DataProductTransformation: """Create a transformation for a task.""" @@ -208,7 +208,7 @@ def create( timezone_type=timezone_type, timezone=timezone, min_values=min_values, - stop_on_no_data=stop_on_no_data, + stop_on_gaps=stop_on_gaps, stop_on_error=stop_on_error, ) @@ -225,7 +225,7 @@ def create( timezone_type=timezone_type if timezone_type is not Unset else None, timezone=timezone if timezone is not Unset else None, min_values=min_values if min_values is not Unset else None, - stop_on_no_data=stop_on_no_data if stop_on_no_data is not Unset else True, + stop_on_gaps=stop_on_gaps if stop_on_gaps is not Unset else False, stop_on_error=stop_on_error if stop_on_error is not Unset else True, ) @@ -250,7 +250,7 @@ def update( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> DataProductTransformation: """Update a transformation's parameters and inputs.""" @@ -300,9 +300,9 @@ def update( min_values if min_values is not Unset else (transformation.min_values if transformation_type == "aggregation" else Unset) ), - stop_on_no_data=( - stop_on_no_data if stop_on_no_data is not Unset - else (transformation.stop_on_no_data if transformation_type == "derivation" else Unset) + stop_on_gaps=( + stop_on_gaps if stop_on_gaps is not Unset + else (transformation.stop_on_gaps if transformation_type == "derivation" else Unset) ), stop_on_error=( stop_on_error if stop_on_error is not Unset @@ -320,7 +320,7 @@ def update( "timezone_type": timezone_type, "timezone": timezone, "min_values": min_values, - "stop_on_no_data": stop_on_no_data, + "stop_on_gaps": stop_on_gaps, "stop_on_error": stop_on_error, } for field, value in editable_fields.items(): @@ -380,7 +380,7 @@ def validate_transformation( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> None: """Validate transformation parameters and raise ValueError if invalid.""" @@ -443,11 +443,11 @@ def validate_transformation( raise ValueError("rating_curve is required for transformation_type 'rating_curve'.") if any(v is not Unset for v in ( formula, output_interval_units, output_interval, aggregation_method, min_values, - stop_on_no_data, stop_on_error, + stop_on_gaps, stop_on_error, )): raise ValueError( "formula, output_interval_units, output_interval, aggregation_method, min_values, " - "stop_on_no_data, and stop_on_error must not be set for transformation_type 'rating_curve'." + "stop_on_gaps, and stop_on_error must not be set for transformation_type 'rating_curve'." ) try: RatingCurve.objects.get( @@ -485,10 +485,10 @@ def validate_transformation( "transformation_type 'aggregation'." ) if any(v is not Unset for v in ( - rating_curve, formula, stop_on_no_data, stop_on_error, + rating_curve, formula, stop_on_gaps, stop_on_error, )): raise ValueError( - "rating_curve, formula, stop_on_no_data, and stop_on_error must not be set " + "rating_curve, formula, stop_on_gaps, and stop_on_error must not be set " "for transformation_type 'aggregation'." ) @@ -623,7 +623,7 @@ def run_derivation(self, transformation: DataProductTransformation) -> int: result_df = apply_derivation( inputs=inputs, formula=transformation.formula, - stop_on_no_data=transformation.stop_on_no_data, + stop_on_gaps=transformation.stop_on_gaps, stop_on_error=transformation.stop_on_error, input_no_data_values=input_no_data_values, output_no_data_value=output_ds.no_data_value, diff --git a/django/tests/fixtures/test_products_data.yaml b/django/tests/fixtures/test_products_data.yaml index 02fb1003c..9bcd4a1a7 100644 --- a/django/tests/fixtures/test_products_data.yaml +++ b/django/tests/fixtures/test_products_data.yaml @@ -118,7 +118,7 @@ timezone: null aggregation_method: null min_values: null - stop_on_no_data: true + stop_on_gaps: true stop_on_error: true - model: products.dataproducttransformationinput @@ -143,7 +143,7 @@ timezone: null aggregation_method: null min_values: null - stop_on_no_data: true + stop_on_gaps: true stop_on_error: true - model: products.dataproducttransformationinput diff --git a/django/tests/products/services/test_transformation.py b/django/tests/products/services/test_transformation.py index 7a56fdf8d..d418673e8 100644 --- a/django/tests/products/services/test_transformation.py +++ b/django/tests/products/services/test_transformation.py @@ -842,7 +842,7 @@ def test_run_derivation_stops_at_first_misaligned_timestamp(run_context): np.testing.assert_allclose(out_obs[0].result, 3.0) -def _run_offset_derivation(ctx, obs_a, obs_b): +def _run_offset_derivation(ctx, obs_a, obs_b, stop_on_gaps=Unset): """ Create inputs ds_a/ds_b from {hour: value} dicts (2025-02-10 UTC), derive "a + b" into an empty output, run once, and return (loaded, transformation, @@ -864,6 +864,7 @@ def _run_offset_derivation(ctx, obs_a, obs_b): transformation_type="derivation", output_datastream=ds_out.pk, formula="a + b", + stop_on_gaps=stop_on_gaps, input_datastreams=[ TransformationInput(datastream=ds_a.pk, variable_name="a"), TransformationInput(datastream=ds_b.pk, variable_name="b"), @@ -873,6 +874,14 @@ def _run_offset_derivation(ctx, obs_a, obs_b): return loaded, t, ds_a, ds_b, ds_out +def _append_obs(t, ds, obs): + """Add {hour: value} readings to ds, advance its end time, and reload t.""" + for hour, value in obs.items(): + _add_obs(ds, datetime(2025, 2, 10, hour, 0, tzinfo=dt_timezone.utc), value) + _set_end_time(ds, datetime(2025, 2, 10, max(obs), 0, tzinfo=dt_timezone.utc)) + return _load(str(t.pk)) + + def _output_results(ds_out): from core.sta.models.observation import Observation return list( @@ -924,11 +933,106 @@ def test_run_derivation_offset_start_stops_at_divergence(run_context): run_context, obs_a={8: 1.0, 10: 3.0, 11: 4.0, 12: 5.0}, obs_b={10: 10.0, 12: 30.0}, + stop_on_gaps=True, ) assert loaded == 1 assert _output_results(ds_out) == [(10, 13.0)] +def test_run_derivation_stop_on_gaps_true_waits_at_gap(run_context): + """With the toggle on, a gap holds the output back on every run until it's filled.""" + loaded, t, ds_a, ds_b, ds_out = _run_offset_derivation( + run_context, + obs_a={8: 1.0, 9: 2.0, 10: 3.0}, + obs_b={8: 10.0, 10: 30.0}, + stop_on_gaps=True, + ) + assert loaded == 1 + + t = _append_obs(t, ds_a, {11: 4.0}) + t = _append_obs(t, ds_b, {11: 40.0}) + assert transformation_service.run(t) == 0 + + _add_obs(ds_b, datetime(2025, 2, 10, 9, 0, tzinfo=dt_timezone.utc), 20.0) + assert transformation_service.run(t) == 3 + assert _output_results(ds_out) == [(8, 11.0), (9, 22.0), (10, 33.0), (11, 44.0)] + + +def test_run_derivation_skips_gaps_when_stop_on_gaps_false(run_context): + """With the toggle off, timestamps not shared by both inputs are skipped.""" + loaded, t, ds_a, ds_b, ds_out = _run_offset_derivation( + run_context, + obs_a={8: 1.0, 9: 2.0, 10: 3.0, 11: 4.0}, + obs_b={8: 10.0, 10: 30.0, 11: 40.0}, + stop_on_gaps=False, + ) + assert loaded == 3 + assert _output_results(ds_out) == [(8, 11.0), (10, 33.0), (11, 44.0)] + + # The incremental run picks up after the last output and skips new gaps too. + t = _append_obs(t, ds_a, {13: 6.0, 14: 7.0}) + t = _append_obs(t, ds_b, {12: 50.0, 14: 70.0}) + assert transformation_service.run(t) == 1 + assert _output_results(ds_out) == [(8, 11.0), (10, 33.0), (11, 44.0), (14, 77.0)] + + +def test_run_derivation_lagging_input_holds_output_back(run_context): + """ + ds_a falls behind ds_b. Output stops at ds_a's last reading rather than + skipping ds_b's newer readings, then catches up once ds_a reports again, + skipping the hour ds_a never reported. + """ + loaded, t, ds_a, ds_b, ds_out = _run_offset_derivation( + run_context, + obs_a={8: 1.0, 9: 2.0}, + obs_b={8: 10.0, 9: 20.0, 10: 30.0, 11: 40.0}, + stop_on_gaps=False, + ) + assert loaded == 2 + assert _output_results(ds_out) == [(8, 11.0), (9, 22.0)] + + t = _append_obs(t, ds_a, {11: 4.0}) + assert transformation_service.run(t) == 1 + assert _output_results(ds_out) == [(8, 11.0), (9, 22.0), (11, 44.0)] + + +def test_run_derivation_no_data_values_fill_output_and_continue(run_context): + """ + Each input is checked against its own no-data value, and those rows get the + output's no-data value. The run continues with the toggle at its default (off). + """ + ctx = run_context + kwargs = {k: ctx[k] for k in ("thing", "sensor", "observed_property", "processing_level", "unit")} + ds_a = _make_datastream(**kwargs) + ds_b = _make_datastream(**kwargs) + ds_out = _make_datastream(**kwargs) + for ds, ndv in [(ds_a, -9999), (ds_b, -1), (ds_out, -555)]: + ds.no_data_value = ndv + ds.save(update_fields=["no_data_value"]) + + # 09:00: ds_a at its no-data value. 10:00: ds_b at its no-data value. + # 11:00: each holds the *other's* no-data value, which is real data. + for ds, values in [(ds_a, (1.0, -9999.0, 3.0, -1.0)), (ds_b, (10.0, 20.0, -1.0, -9999.0))]: + for hour, value in zip((8, 9, 10, 11), values): + _add_obs(ds, datetime(2025, 2, 10, hour, 0, tzinfo=dt_timezone.utc), value) + _set_end_time(ds, datetime(2025, 2, 10, 11, 0, tzinfo=dt_timezone.utc)) + + t = transformation_service.create( + task=uuid.UUID(TASK1), + principal=ctx["owner"], + transformation_type="derivation", + output_datastream=ds_out.pk, + formula="a + b", + input_datastreams=[ + TransformationInput(datastream=ds_a.pk, variable_name="a"), + TransformationInput(datastream=ds_b.pk, variable_name="b"), + ], + ) + assert t.stop_on_gaps is False + assert transformation_service.run(t) == 4 + assert _output_results(ds_out) == [(8, 11.0), (9, -555.0), (10, -555.0), (11, -10000.0)] + + def test_run_derivation_latest_start_not_shared_loads_nothing(run_context): """ ds_a starts at 02:00 and ds_b at 01:00, but ds_b has no 02:00 reading; the @@ -938,6 +1042,7 @@ def test_run_derivation_latest_start_not_shared_loads_nothing(run_context): run_context, obs_a={2: 1.0, 3: 2.0}, obs_b={1: 10.0, 3: 20.0}, + stop_on_gaps=True, ) assert loaded == 0 assert _output_results(ds_out) == [] diff --git a/packages/hydroserver-ts/src/generated/data.types.d.ts b/packages/hydroserver-ts/src/generated/data.types.d.ts index 47d5b6060..e8409f242 100644 --- a/packages/hydroserver-ts/src/generated/data.types.d.ts +++ b/packages/hydroserver-ts/src/generated/data.types.d.ts @@ -3255,8 +3255,8 @@ export interface components { outputDatastreamId?: string; /** Stoponerror */ stopOnError?: boolean; - /** Stoponnodata */ - stopOnNoData?: boolean; + /** Stopongaps */ + stopOnGaps?: boolean; }; /** DerivationTransformationPostBody */ DerivationTransformationPostBody: { @@ -3280,10 +3280,10 @@ export interface components { */ stopOnError: boolean; /** - * Stoponnodata - * @default true + * Stopongaps + * @default false */ - stopOnNoData: boolean; + stopOnGaps: boolean; }; /** DerivationTransformationResponse */ DerivationTransformationResponse: { @@ -3299,8 +3299,8 @@ export interface components { outputDatastream: components["schemas"]["DatastreamSummaryResponse"]; /** Stoponerror */ stopOnError: boolean; - /** Stoponnodata */ - stopOnNoData: boolean; + /** Stopongaps */ + stopOnGaps: boolean; }; /** DerivationTransformationSummaryResponse */ DerivationTransformationSummaryResponse: { @@ -3320,8 +3320,8 @@ export interface components { outputDatastreamId: string; /** Stoponerror */ stopOnError: boolean; - /** Stoponnodata */ - stopOnNoData: boolean; + /** Stopongaps */ + stopOnGaps: boolean; }; /** EtlDataMappingPatchBody */ EtlDataMappingPatchBody: { diff --git a/packages/hydroserverpy/setup.cfg b/packages/hydroserverpy/setup.cfg index b3f2aef1b..161281c1c 100644 --- a/packages/hydroserverpy/setup.cfg +++ b/packages/hydroserverpy/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = hydroserverpy -version = 1.12.1 +version = 1.12.2b1 description = A Python client for managing HydroServer data long_description = file: README.md long_description_content_type = text/markdown diff --git a/packages/hydroserverpy/src/hydroserverpy/api/README.md b/packages/hydroserverpy/src/hydroserverpy/api/README.md index b5dd1f749..138a60852 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/README.md +++ b/packages/hydroserverpy/src/hydroserverpy/api/README.md @@ -1070,13 +1070,13 @@ new_transformation = hs_api.transformations.create_derivation( {'datastream_id': '00000000-0000-0000-0000-000000000001', 'variable_name': 'x'}, ], formula='x * 0.3048', # feet to meters - stop_on_no_data=True, # stop the run if an input holds its no-data value + stop_on_gaps=True, # stop the run at timestamps not shared by all inputs stop_on_error=True, # stop the run if the formula produces a non-finite result ) # A derivation with multiple inputs works the same way, just with more entries. -# Inputs are matched by exact timestamp; if they stop lining up, the run stops -# there until they're back in sync. +# Inputs are matched by exact timestamp; with stop_on_gaps=True a gap stops the +# run there until it's filled, and with False those timestamps are skipped. new_derived_transformation = hs_api.transformations.create_derivation( task_id='00000000-0000-0000-0000-000000000000', output_datastream='00000000-0000-0000-0000-000000000000', diff --git a/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py b/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py index dd1234c2f..633f2e26e 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py +++ b/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py @@ -21,7 +21,7 @@ class DerivationTransformation(BaseModel): output_datastream_id: uuid.UUID input_datastream_ids: List[uuid.UUID] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) diff --git a/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py b/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py index 1d03a4647..260990609 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py +++ b/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py @@ -157,7 +157,7 @@ def create_derivation( output_datastream: Union[UUID, str], input_datastreams: List[dict], formula: str, - stop_on_no_data: bool = True, + stop_on_gaps: bool = False, stop_on_error: bool = True, uid: Optional[UUID] = None, ) -> DerivationTransformation: @@ -177,7 +177,7 @@ def create_derivation( for inp in input_datastreams ], "formula": formula, - "stopOnNoData": stop_on_no_data, + "stopOnGaps": stop_on_gaps, "stopOnError": stop_on_error, } if uid is not None: @@ -191,7 +191,7 @@ def update_derivation( uid: Union[UUID, str], input_datastreams: List[dict] = ..., formula: str = ..., - stop_on_no_data: bool = ..., + stop_on_gaps: bool = ..., stop_on_error: bool = ..., ) -> DerivationTransformation: """ @@ -209,7 +209,7 @@ def update_derivation( for inp in input_datastreams ] if input_datastreams is not ... else ..., "formula": formula, - "stopOnNoData": stop_on_no_data, + "stopOnGaps": stop_on_gaps, "stopOnError": stop_on_error, } body = {k: v for k, v in body.items() if v is not ...} diff --git a/packages/hydroserverpy/src/hydroserverpy/products/derivation.py b/packages/hydroserverpy/src/hydroserverpy/products/derivation.py index 62533bc2d..33820501a 100644 --- a/packages/hydroserverpy/src/hydroserverpy/products/derivation.py +++ b/packages/hydroserverpy/src/hydroserverpy/products/derivation.py @@ -106,7 +106,7 @@ def apply_derivation( inputs: dict[str, pd.DataFrame], formula: str, *, - stop_on_no_data: bool, + stop_on_gaps: bool, stop_on_error: bool, input_no_data_values: dict[str, float] | None = None, output_no_data_value: float | None = None, @@ -114,23 +114,23 @@ def apply_derivation( """ Apply a mathematical formula to one or more input timeseries DataFrames. - Inputs are matched on exact timestamp (sorted defensively first). If they - diverge — one has a timestamp the others don't — output stops before that - point, and it's logged; nothing after is evaluated, even if they realign - later. + Inputs are matched on exact timestamp (sorted defensively first). + stop_on_gaps controls gaps, where one input has a timestamp the others + don't: True stops output before the first gap and logs it, so nothing + after is evaluated even if the inputs realign later; False skips every + timestamp not shared by all inputs and continues. - Within the aligned rows, stop_on_no_data and stop_on_error each - independently control a bad row (an input at its own no-data value, per - input_no_data_values, or a non-finite formula result): True truncates - there and logs it, False fills it with output_no_data_value and - continues silently. + Within the aligned rows, an input at its own no-data value (per + input_no_data_values) always produces output_no_data_value. stop_on_error + controls a non-finite formula result: True truncates there and logs it, + False fills it with output_no_data_value and continues silently. """ if not inputs: raise ValueError("At least one input DataFrame must be provided to run a derivation.") - if not stop_on_no_data and output_no_data_value is None: - raise ValueError("output_no_data_value is required when stop_on_no_data is False.") + if input_no_data_values and output_no_data_value is None: + raise ValueError("output_no_data_value is required when input_no_data_values are given.") if not stop_on_error and output_no_data_value is None: raise ValueError("output_no_data_value is required when stop_on_error is False.") @@ -144,9 +144,9 @@ def apply_derivation( } logger.debug( - "Evaluating derivation (formula=%r, variables=%r, stopOnNoData=%r, stopOnError=%r, " + "Evaluating derivation (formula=%r, variables=%r, stopOnGaps=%r, stopOnError=%r, " "inputNoDataValues=%r, outputNoDataValue=%r, rows=%r).", - formula, variables, stop_on_no_data, stop_on_error, input_no_data_values, output_no_data_value, + formula, variables, stop_on_gaps, stop_on_error, input_no_data_values, output_no_data_value, {var: len(df) for var, df in validated.items()}, ) @@ -161,6 +161,13 @@ def apply_derivation( combined_df = combined_df.merge(other, on=TIMESTAMP_COL, how="inner") combined_df = combined_df.sort_values(TIMESTAMP_COL).reset_index(drop=True) + if divergence_timestamp is not None and not stop_on_gaps: + logger.info( + "Derivation skipped %d timestamp(s) not shared by all inputs, starting at %s.", + len(divergent_timestamps), divergence_timestamp, + ) + divergence_timestamp = None + if divergence_timestamp is not None: combined_df = combined_df[combined_df[TIMESTAMP_COL] < divergence_timestamp].reset_index(drop=True) @@ -194,32 +201,23 @@ def apply_derivation( except Exception as e: raise ValueError(f"Formula evaluation failed: {e}") from e - # Excludes no-data rows, which stop_on_no_data already accounts for. + # Excludes no-data rows, which always get output_no_data_value instead. error_mask = ~ndv_mask & ~np.isfinite(result_array) - ndv_stop_idx = int(np.argmax(ndv_mask)) if stop_on_no_data and ndv_mask.any() else None - error_stop_idx = int(np.argmax(error_mask)) if stop_on_error and error_mask.any() else None - - stop_idx = None - stop_reason = None - if ndv_stop_idx is not None and (error_stop_idx is None or ndv_stop_idx <= error_stop_idx): - stop_idx, stop_reason = ndv_stop_idx, "input(s) contained a no-data value" - elif error_stop_idx is not None: - stop_idx, stop_reason = error_stop_idx, "formula produced a non-finite result" - - if stop_idx is not None: + if stop_on_error and error_mask.any(): + stop_idx = int(np.argmax(error_mask)) stop_timestamp = combined_df[TIMESTAMP_COL].iloc[stop_idx] combined_df = combined_df.iloc[:stop_idx].reset_index(drop=True) result_array = result_array[:stop_idx] ndv_mask = ndv_mask[:stop_idx] error_mask = error_mask[:stop_idx] - logger.warning("Derivation stopped at %s: %s.", stop_timestamp, stop_reason) + logger.warning("Derivation stopped at %s: formula produced a non-finite result.", stop_timestamp) elif divergence_timestamp is not None: logger.warning( "Derivation stopped at %s: input timestamps are not aligned.", divergence_timestamp, ) - if not stop_on_no_data and ndv_mask.any(): + if ndv_mask.any(): result_array = np.where(ndv_mask, output_no_data_value, result_array) if not stop_on_error and error_mask.any(): diff --git a/packages/hydroserverpy/tests/products/test_derivation.py b/packages/hydroserverpy/tests/products/test_derivation.py index 49c366901..ef3d96526 100644 --- a/packages/hydroserverpy/tests/products/test_derivation.py +++ b/packages/hydroserverpy/tests/products/test_derivation.py @@ -107,70 +107,78 @@ class TestApplyDerivation: def test_empty_inputs_raises(self): with pytest.raises(ValueError, match="At least one"): - apply_derivation({}, formula="x", stop_on_no_data=True, stop_on_error=True) + apply_derivation({}, formula="x", stop_on_gaps=True, stop_on_error=True) - def test_stop_on_no_data_false_requires_output_no_data_value(self): + def test_input_no_data_values_require_output_no_data_value(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) with pytest.raises(ValueError, match="output_no_data_value"): - apply_derivation({"x": df}, formula="x", stop_on_no_data=False, stop_on_error=True) + apply_derivation( + {"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True, + input_no_data_values={"x": -9999.0}, + ) + + def test_stop_on_gaps_false_does_not_require_output_no_data_value(self): + df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=False, stop_on_error=True) + assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_stop_on_error_false_requires_output_no_data_value(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) with pytest.raises(ValueError, match="output_no_data_value"): - apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=False) + apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=False) def test_single_input_passthrough_preserves_timestamps(self): ts = _hourly(3) df = _make_df(ts, [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert len(result) == 3 assert result[TIMESTAMP_COL].tolist() == df[TIMESTAMP_COL].tolist() def test_single_input_sorted_defensively(self): ts = list(reversed(_hourly(3))) df = _make_df(ts, [3.0, 2.0, 1.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) assert result[TIMESTAMP_COL].is_monotonic_increasing def test_multiply_by_scalar(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x * 2", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x * 2", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([2.0, 4.0, 6.0]) def test_add_constant(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x + 10", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x + 10", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([11.0, 12.0, 13.0]) def test_subtract_constant(self): df = _make_df(_hourly(3), [5.0, 10.0, 15.0]) - result = apply_derivation({"x": df}, formula="x - 5", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x - 5", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([0.0, 5.0, 10.0]) def test_divide_by_constant(self): df = _make_df(_hourly(3), [2.0, 4.0, 6.0]) - result = apply_derivation({"x": df}, formula="x / 2", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x / 2", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_sqrt_function(self): df = _make_df(_hourly(3), [1.0, 4.0, 9.0]) - result = apply_derivation({"x": df}, formula="sqrt(x)", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="sqrt(x)", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_abs_function(self): df = _make_df(_hourly(3), [-1.0, 0.0, 1.0]) - result = apply_derivation({"x": df}, formula="abs(x)", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="abs(x)", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 0.0, 1.0]) def test_output_result_dtype_is_float64(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert pd.api.types.is_float_dtype(result[RESULT_COL]) def test_output_timestamp_dtype_matches_canonical_schema(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert pd.api.types.is_datetime64_any_dtype(result[TIMESTAMP_COL]) # -- Multi-input alignment -------------------------------------------- @@ -180,7 +188,7 @@ def test_multiple_inputs_addition_on_matching_timestamps(self): df_x = _make_df(ts, [1.0, 2.0, 3.0]) df_y = _make_df(ts, [4.0, 5.0, 6.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([5.0, 7.0, 9.0]) @@ -189,7 +197,7 @@ def test_multiple_inputs_formula_with_division(self): df_x = _make_df(ts, [2.0, 4.0, 6.0]) df_y = _make_df(ts, [1.0, 2.0, 3.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x / y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([2.0, 2.0, 2.0]) @@ -198,7 +206,7 @@ def test_inputs_sorted_defensively_before_joining(self): df_x = _make_df(list(reversed(ts)), [3.0, 2.0, 1.0]) df_y = _make_df(ts, [10.0, 20.0, 30.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([11.0, 22.0, 33.0]) @@ -210,7 +218,7 @@ def test_misalignment_stops_before_first_divergent_timestamp(self, caplog): df_y = _make_df(ts_y, [10.0, 30.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) # Hour 2 matched on both sides, but comes after the hour-1 divergence, # so it's excluded even though it would otherwise be a valid match. @@ -218,13 +226,76 @@ def test_misalignment_stops_before_first_divergent_timestamp(self, caplog): assert result[RESULT_COL].tolist() == pytest.approx([11.0]) assert "not aligned" in caplog.text + def test_misalignment_skipped_when_stop_on_gaps_false(self, caplog): + # Same inputs as above, but hour 1 is skipped and hour 2 still matches. + ts_x = [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 1), _utc(2024, 1, 1, 2)] + ts_y = [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 2)] + df_x = _make_df(ts_x, [1.0, 2.0, 3.0]) + df_y = _make_df(ts_y, [10.0, 30.0]) + with caplog.at_level(logging.INFO): + result = apply_derivation( + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True + ) + assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 2)] + assert result[RESULT_COL].tolist() == pytest.approx([11.0, 33.0]) + assert "skipped 1 timestamp" in caplog.text + assert "stopped" not in caplog.text + + def test_misalignment_skipped_in_both_directions(self): + # x is missing hour 1 and y is missing hour 3; both are skipped. + ts_x = [_utc(2024, 1, 1, h) for h in (0, 2, 3, 4)] + ts_y = [_utc(2024, 1, 1, h) for h in (0, 1, 2, 4)] + df_x = _make_df(ts_x, [1.0, 3.0, 4.0, 5.0]) + df_y = _make_df(ts_y, [10.0, 20.0, 30.0, 50.0]) + result = apply_derivation( + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True + ) + assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, h) for h in (0, 2, 4)] + assert result[RESULT_COL].tolist() == pytest.approx([11.0, 33.0, 55.0]) + + def test_skipped_gap_and_no_data_value_handled_independently(self): + # Hour 1 is a gap (skipped); hour 2 has x at its no-data value (filled). + ts_x = [_utc(2024, 1, 1, h) for h in (0, 1, 2, 3)] + ts_y = [_utc(2024, 1, 1, h) for h in (0, 2, 3)] + df_x = _make_df(ts_x, [1.0, 2.0, -9999.0, 4.0]) + df_y = _make_df(ts_y, [10.0, 30.0, 40.0]) + result = apply_derivation( + {"x": df_x, "y": df_y}, formula="x + y", + stop_on_gaps=False, stop_on_error=True, + input_no_data_values={"x": -9999.0, "y": -9999.0}, output_no_data_value=-1.0, + ) + assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, h) for h in (0, 2, 3)] + assert result[RESULT_COL].tolist() == pytest.approx([11.0, -1.0, 44.0]) + + def test_stop_on_error_still_applies_after_skipped_gap(self, caplog): + ts_x = [_utc(2024, 1, 1, h) for h in (0, 1, 2, 3)] + ts_y = [_utc(2024, 1, 1, h) for h in (0, 2, 3)] + df_x = _make_df(ts_x, [1.0, 2.0, 3.0, 4.0]) + df_y = _make_df(ts_y, [1.0, 0.0, 4.0]) + with caplog.at_level(logging.WARNING): + result = apply_derivation( + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=False, stop_on_error=True + ) + assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, 0)] + assert "non-finite" in caplog.text + + def test_completely_disjoint_timestamps_return_empty_when_skipping(self, caplog): + df_x = _make_df([_utc(2024, 1, 1, 0)], [1.0]) + df_y = _make_df([_utc(2024, 1, 1, 1)], [2.0]) + with caplog.at_level(logging.WARNING): + result = apply_derivation( + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True + ) + assert len(result) == 0 + assert caplog.text == "" + def test_fully_aligned_inputs_produce_no_warning(self, caplog): ts = _hourly(3) df_x = _make_df(ts, [1.0, 2.0, 3.0]) df_y = _make_df(ts, [4.0, 5.0, 6.0]) with caplog.at_level(logging.WARNING): apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert caplog.text == "" @@ -232,38 +303,28 @@ def test_completely_disjoint_timestamps_return_empty(self): df_x = _make_df([_utc(2024, 1, 1, 0)], [1.0]) df_y = _make_df([_utc(2024, 1, 1, 1)], [2.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert len(result) == 0 - # -- stop_on_no_data (no-data values in aligned inputs) ----------------- + # -- No-data values in aligned inputs ----------------------------------- - def test_stop_on_no_data_false_fills_and_continues(self): - ts = _hourly(3) - df_x = _make_df(ts, [1.0, -9999.0, 3.0]) - df_y = _make_df(ts, [10.0, 20.0, 30.0]) - result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=False, stop_on_error=True, - input_no_data_values={"x": -9999.0, "y": -9999.0}, output_no_data_value=-9999.0, - ) - assert len(result) == 3 - assert result[RESULT_COL].tolist() == pytest.approx([11.0, -9999.0, 33.0]) - - def test_stop_on_no_data_true_truncates_at_first_no_data_value(self, caplog): + @pytest.mark.parametrize("stop_on_gaps", [True, False]) + def test_no_data_value_always_fills_and_continues(self, stop_on_gaps, caplog): ts = _hourly(3) df_x = _make_df(ts, [1.0, -9999.0, 3.0]) df_y = _make_df(ts, [10.0, 20.0, 30.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=True, stop_on_error=True, - input_no_data_values={"x": -9999.0, "y": -9999.0}, + stop_on_gaps=stop_on_gaps, stop_on_error=True, + input_no_data_values={"x": -9999.0, "y": -9999.0}, output_no_data_value=-9999.0, ) - assert result[RESULT_COL].tolist() == pytest.approx([11.0]) - assert "no-data value" in caplog.text + assert len(result) == 3 + assert result[RESULT_COL].tolist() == pytest.approx([11.0, -9999.0, 33.0]) + assert caplog.text == "" - def test_stop_on_no_data_only_checks_that_inputs_own_sentinel(self): + def test_no_data_value_only_checks_that_inputs_own_sentinel(self): # x's sentinel (-9999) never appears in y, and y's sentinel (-777) # never appears in x, so neither input's real values are misdetected # as no-data, and both real no-data rows are still caught. @@ -272,7 +333,7 @@ def test_stop_on_no_data_only_checks_that_inputs_own_sentinel(self): df_y = _make_df(ts, [10.0, 20.0, -777.0]) result = apply_derivation( {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=False, stop_on_error=True, + stop_on_gaps=False, stop_on_error=True, input_no_data_values={"x": -9999.0, "y": -777.0}, output_no_data_value=-1.0, ) assert result[RESULT_COL].tolist() == pytest.approx([11.0, -1.0, -1.0]) @@ -285,7 +346,7 @@ def test_stop_on_error_false_fills_and_continues(self): df_y = _make_df(ts, [1.0, 0.0, 3.0]) result = apply_derivation( {"x": df_x, "y": df_y}, formula="x / y", - stop_on_no_data=True, stop_on_error=False, output_no_data_value=-9999.0, + stop_on_gaps=True, stop_on_error=False, output_no_data_value=-9999.0, ) assert len(result) == 3 assert result[RESULT_COL].tolist() == pytest.approx([1.0, -9999.0, 1.0]) @@ -296,20 +357,20 @@ def test_stop_on_error_true_truncates_at_first_non_finite_result(self, caplog): df_y = _make_df(ts, [1.0, 0.0, 3.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x / y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([1.0]) assert "non-finite" in caplog.text def test_no_data_value_rows_are_not_also_flagged_as_errors(self): # A no-data sentinel fed into the formula could itself produce a - # non-finite result (e.g. via log); stop_on_no_data must claim that - # row, not stop_on_error, even when stop_on_error is True. + # non-finite result (e.g. via log); it must be filled as no-data, not + # flagged by stop_on_error, even when stop_on_error is True. ts = _hourly(2) df_x = _make_df(ts, [-9999.0, 4.0]) result = apply_derivation( {"x": df_x}, formula="log(x)", - stop_on_no_data=False, stop_on_error=True, + stop_on_gaps=False, stop_on_error=True, input_no_data_values={"x": -9999.0}, output_no_data_value=-9999.0, ) assert len(result) == 2