From ecd461e685aac94d1feaafca7e5272a53eee7440 Mon Sep 17 00:00:00 2001 From: Ken Lippold Date: Fri, 16 Oct 2026 13:24:25 -0700 Subject: [PATCH 1/2] Updated gap handlign in derivation tasks --- .../data-products/DerivationForm.vue | 14 ++- apps/docs/docs/references/data-dictionary.md | 2 +- .../how-to/using-the-python-client.md | 4 +- django/contracts/openapi/auth.openapi.json | 2 +- django/contracts/openapi/data.openapi.json | 2 +- django/hydroserver/__init__.py | 2 +- .../products/services/test_transformation.py | 104 ++++++++++++++++- packages/hydroserverpy/setup.cfg | 2 +- .../src/hydroserverpy/api/README.md | 6 +- .../src/hydroserverpy/products/derivation.py | 50 ++++---- .../tests/products/test_derivation.py | 107 ++++++++++++++---- 11 files changed, 230 insertions(+), 65 deletions(-) diff --git a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue index c1b369556..6292e8070 100644 --- a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue +++ b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue @@ -38,8 +38,8 @@ > Apply a mathematical formula to one or more input datastreams to produce a new output datastream. With more than one input, they're - matched by exact timestamp; if inputs stop lining up, the run stops - there until they're back in sync. + matched by exact timestamp; timestamps not shared by every input + stop the run or are skipped, depending on "Stop on input gaps".
-
Stop on no-data value
+
Stop on input gaps
- If an input is set to a no-data value, stop the run there - instead of writing the output's no-data value and continuing. + If an input has no value at a timestamp where the others do, stop + the run there and wait until the missing value arrives. When off, + those timestamps are skipped and the run continues. Runs never go + past the most recent value of the input that is furthest behind. + Inputs holding their no-data value always produce the output's + no-data value. Date: Mon, 19 Oct 2026 10:24:05 -0700 Subject: [PATCH 2/2] Rename stop_on_no_data to stop_on_gaps --- .../data-products/DerivationForm.vue | 10 +-- apps/docs/docs/references/data-dictionary.md | 2 +- .../how-to/using-the-python-client.md | 8 +-- django/contracts/openapi/data.openapi.json | 22 +++--- .../api/schemas/products/transformation.py | 8 +-- ..._dataproducttransformation_stop_on_gaps.py | 23 +++++++ .../products/models/transformation.py | 2 +- .../products/services/transformation.py | 28 ++++---- django/tests/fixtures/test_products_data.yaml | 4 +- .../products/services/test_transformation.py | 19 +++--- .../src/generated/data.types.d.ts | 18 ++--- .../src/hydroserverpy/api/README.md | 4 +- .../api/models/products/transformation.py | 2 +- .../api/services/products/transformation.py | 8 +-- .../src/hydroserverpy/products/derivation.py | 10 +-- .../tests/products/test_derivation.py | 68 +++++++++---------- 16 files changed, 131 insertions(+), 105 deletions(-) create mode 100644 django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py diff --git a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue index 6292e8070..9e26e077d 100644 --- a/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue +++ b/apps/data-management/src/components/Orchestration/data-products/DerivationForm.vue @@ -213,7 +213,7 @@ (null) const outputDatastreamId = ref(null) const inputs = ref([makeRow('x')]) const formula = ref('') -const stopOnNoData = ref(true) +const stopOnGaps = ref(false) const stopOnError = ref(true) const selectedThingId = computed(() => props.initialThingId ?? null) @@ -551,7 +551,7 @@ async function loadExistingTask() { existingTransformationId.value = t.id outputDatastreamId.value = (t.outputDatastream as any)?.id ?? null formula.value = t.formula - stopOnNoData.value = t.stopOnNoData ?? true + stopOnGaps.value = t.stopOnGaps ?? false stopOnError.value = t.stopOnError ?? true if (t.inputDatastreams?.length) { @@ -621,7 +621,7 @@ async function onCreate( outputDatastreamId: outputDatastreamId.value!, inputDatastreams, formula: formula.value.trim(), - stopOnNoData: stopOnNoData.value, + stopOnGaps: stopOnGaps.value, stopOnError: stopOnError.value, } ) @@ -662,7 +662,7 @@ async function onUpdate( outputDatastreamId: outputDatastreamId.value!, inputDatastreams, formula: formula.value.trim(), - stopOnNoData: stopOnNoData.value, + stopOnGaps: stopOnGaps.value, stopOnError: stopOnError.value, } ) diff --git a/apps/docs/docs/references/data-dictionary.md b/apps/docs/docs/references/data-dictionary.md index 4f2d3f81f..ac1bf5961 100644 --- a/apps/docs/docs/references/data-dictionary.md +++ b/apps/docs/docs/references/data-dictionary.md @@ -490,7 +490,7 @@ A DataProductTransformation describes how a single derived (output) Datastream i | M | transformation_type | The type of transformation: `rating_curve`, `derivation`, or `aggregation`. | String | | O | rating_curve_id | A foreign key identifier for the RatingCurve used when `transformation_type` is `rating_curve`. | UUID | | O | formula | A formula (expression) string used when `transformation_type` is `derivation`, referencing one or more input variable names. | Text | -| O | stop_on_no_data | For `derivation` transformations, whether a run stops at timestamps not shared by all inputs (vs. skipping them). Inputs holding their no-data value always produce the output's no-data value. | Boolean | +| O | stop_on_gaps | For `derivation` transformations, whether a run stops at timestamps not shared by all inputs (vs. skipping them). Inputs holding their no-data value always produce the output's no-data value. | Boolean | | O | stop_on_error | For `derivation` transformations, whether a run stops when the formula produces a non-finite result (vs. filling and continuing). | Boolean | | O | output_interval_units | The interval unit for `aggregation` outputs: `minutes`, `hours`, `days`, `weeks`, or `months`. | String | | O | output_interval | The positive interval count corresponding to `output_interval_units`. | Integer | diff --git a/apps/docs/docs/user-guides/how-to/using-the-python-client.md b/apps/docs/docs/user-guides/how-to/using-the-python-client.md index 1166d2113..e1b92f131 100644 --- a/apps/docs/docs/user-guides/how-to/using-the-python-client.md +++ b/apps/docs/docs/user-guides/how-to/using-the-python-client.md @@ -667,7 +667,7 @@ hs_api.dataproducttransformations.delete_rating_curve(task_id, uid) #### Derivation Transformations -Combines one or more input datastreams using a formula. Each input datastream is assigned a variable name used in the formula; with a single input, this is equivalent to applying the formula to each observation independently. Multiple inputs are matched by exact timestamp. Timestamps not shared by every input either stop the run there until the gap is filled or are skipped, depending on `stop_on_no_data`. Runs never go past the most recent value of the input that is furthest behind. If any input holds its no-data value at a timestamp, the output gets its own no-data value there. +Combines one or more input datastreams using a formula. Each input datastream is assigned a variable name used in the formula; with a single input, this is equivalent to applying the formula to each observation independently. Multiple inputs are matched by exact timestamp. Timestamps not shared by every input either stop the run there until the gap is filled or are skipped, depending on `stop_on_gaps`. Runs never go past the most recent value of the input that is furthest behind. If any input holds its no-data value at a timestamp, the output gets its own no-data value there. | Field | Type | Description | |---|---|---| @@ -675,7 +675,7 @@ Combines one or more input datastreams using a formula. Each input datastream is | `output_datastream_id` | `UUID` | | | `input_datastream_ids` | `List[UUID]` | | | `formula` | `str` | Python expression referencing the input variable name(s) | -| `stop_on_no_data` | `bool` | If `True`, stop the run at the first timestamp not shared by all inputs; if `False`, skip those timestamps and continue. Inputs holding their no-data value always produce the output's no-data value | +| `stop_on_gaps` | `bool` | If `True`, stop the run at the first timestamp not shared by all inputs; if `False`, skip those timestamps and continue. Inputs holding their no-data value always produce the output's no-data value | | `stop_on_error` | `bool` | If `True`, stop the run when the formula produces a non-finite result; if `False`, fill the output with its no-data value and continue | ```python @@ -683,11 +683,11 @@ hs_api.dataproducttransformations.list_derivation(task_id, output_datastream=Non hs_api.dataproducttransformations.get_derivation(task_id, uid) hs_api.dataproducttransformations.create_derivation( task_id, output_datastream, input_datastreams, formula, - stop_on_no_data=True, stop_on_error=True, uid=None + stop_on_gaps=False, stop_on_error=True, uid=None ) hs_api.dataproducttransformations.update_derivation( task_id, uid, input_datastreams=..., formula=..., - stop_on_no_data=..., stop_on_error=... + stop_on_gaps=..., stop_on_error=... ) hs_api.dataproducttransformations.delete_derivation(task_id, uid) ``` diff --git a/django/contracts/openapi/data.openapi.json b/django/contracts/openapi/data.openapi.json index c6842eacf..a270c8214 100644 --- a/django/contracts/openapi/data.openapi.json +++ b/django/contracts/openapi/data.openapi.json @@ -3739,8 +3739,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3775,9 +3775,9 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "default": true, - "title": "Stoponnodata", + "stopOnGaps": { + "default": false, + "title": "Stopongaps", "type": "boolean" } }, @@ -3814,8 +3814,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3824,7 +3824,7 @@ "outputDatastream", "inputDatastreams", "formula", - "stopOnNoData", + "stopOnGaps", "stopOnError" ], "title": "DerivationTransformationResponse", @@ -3858,8 +3858,8 @@ "title": "Stoponerror", "type": "boolean" }, - "stopOnNoData": { - "title": "Stoponnodata", + "stopOnGaps": { + "title": "Stopongaps", "type": "boolean" } }, @@ -3868,7 +3868,7 @@ "outputDatastreamId", "inputDatastreamIds", "formula", - "stopOnNoData", + "stopOnGaps", "stopOnError" ], "title": "DerivationTransformationSummaryResponse", diff --git a/django/interfaces/api/schemas/products/transformation.py b/django/interfaces/api/schemas/products/transformation.py index cda3ad10a..9f006b7d9 100644 --- a/django/interfaces/api/schemas/products/transformation.py +++ b/django/interfaces/api/schemas/products/transformation.py @@ -70,7 +70,7 @@ class DerivationTransformationSummaryResponse(BaseGetResponse): output_datastream_id: uuid.UUID input_datastream_ids: list[uuid.UUID] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool @staticmethod @@ -120,7 +120,7 @@ class DerivationTransformationResponse(BaseGetResponse): output_datastream: DatastreamSummaryResponse input_datastreams: list[TransformationInputResponse] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool @staticmethod @@ -164,7 +164,7 @@ class RatingCurveTransformationPostBody(_TransformationPostBodyBase): class DerivationTransformationPostBody(_TransformationPostBodyBase): input_datastreams: list[TransformationInputPostBody] formula: str - stop_on_no_data: bool = True + stop_on_gaps: bool = False stop_on_error: bool = True @@ -190,7 +190,7 @@ class DerivationTransformationPatchBody(BasePatchBody): output_datastream: uuid.UUID = Field(alias="outputDatastreamId") input_datastreams: list[TransformationInputPostBody] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool diff --git a/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py b/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py new file mode 100644 index 000000000..9d7bc8d08 --- /dev/null +++ b/django/processing/products/migrations/0004_rename_stop_on_no_data_dataproducttransformation_stop_on_gaps.py @@ -0,0 +1,23 @@ +# Generated by Django 5.2.2 on 2026-10-19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('products', '0003_remove_dataproducttransformation_max_gap_interval_and_more'), + ] + + operations = [ + migrations.RenameField( + model_name='dataproducttransformation', + old_name='stop_on_no_data', + new_name='stop_on_gaps', + ), + migrations.AlterField( + model_name='dataproducttransformation', + name='stop_on_gaps', + field=models.BooleanField(blank=True, default=False), + ), + ] diff --git a/django/processing/products/models/transformation.py b/django/processing/products/models/transformation.py index f41e93bb4..2b69193c2 100644 --- a/django/processing/products/models/transformation.py +++ b/django/processing/products/models/transformation.py @@ -68,7 +68,7 @@ class DataProductTransformation(models.Model): blank=True, ) min_values = models.PositiveIntegerField(null=True, blank=True) - stop_on_no_data = models.BooleanField(default=True, blank=True) + stop_on_gaps = models.BooleanField(default=False, blank=True) stop_on_error = models.BooleanField(default=True, blank=True) class Meta: diff --git a/django/processing/products/services/transformation.py b/django/processing/products/services/transformation.py index 7ce2c2690..6d91e9bd6 100644 --- a/django/processing/products/services/transformation.py +++ b/django/processing/products/services/transformation.py @@ -174,7 +174,7 @@ def create( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> DataProductTransformation: """Create a transformation for a task.""" @@ -208,7 +208,7 @@ def create( timezone_type=timezone_type, timezone=timezone, min_values=min_values, - stop_on_no_data=stop_on_no_data, + stop_on_gaps=stop_on_gaps, stop_on_error=stop_on_error, ) @@ -225,7 +225,7 @@ def create( timezone_type=timezone_type if timezone_type is not Unset else None, timezone=timezone if timezone is not Unset else None, min_values=min_values if min_values is not Unset else None, - stop_on_no_data=stop_on_no_data if stop_on_no_data is not Unset else True, + stop_on_gaps=stop_on_gaps if stop_on_gaps is not Unset else False, stop_on_error=stop_on_error if stop_on_error is not Unset else True, ) @@ -250,7 +250,7 @@ def update( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> DataProductTransformation: """Update a transformation's parameters and inputs.""" @@ -300,9 +300,9 @@ def update( min_values if min_values is not Unset else (transformation.min_values if transformation_type == "aggregation" else Unset) ), - stop_on_no_data=( - stop_on_no_data if stop_on_no_data is not Unset - else (transformation.stop_on_no_data if transformation_type == "derivation" else Unset) + stop_on_gaps=( + stop_on_gaps if stop_on_gaps is not Unset + else (transformation.stop_on_gaps if transformation_type == "derivation" else Unset) ), stop_on_error=( stop_on_error if stop_on_error is not Unset @@ -320,7 +320,7 @@ def update( "timezone_type": timezone_type, "timezone": timezone, "min_values": min_values, - "stop_on_no_data": stop_on_no_data, + "stop_on_gaps": stop_on_gaps, "stop_on_error": stop_on_error, } for field, value in editable_fields.items(): @@ -380,7 +380,7 @@ def validate_transformation( timezone_type: str | None | Unset = Unset, timezone: str | None | Unset = Unset, min_values: int | None | Unset = Unset, - stop_on_no_data: bool | Unset = Unset, + stop_on_gaps: bool | Unset = Unset, stop_on_error: bool | Unset = Unset, ) -> None: """Validate transformation parameters and raise ValueError if invalid.""" @@ -443,11 +443,11 @@ def validate_transformation( raise ValueError("rating_curve is required for transformation_type 'rating_curve'.") if any(v is not Unset for v in ( formula, output_interval_units, output_interval, aggregation_method, min_values, - stop_on_no_data, stop_on_error, + stop_on_gaps, stop_on_error, )): raise ValueError( "formula, output_interval_units, output_interval, aggregation_method, min_values, " - "stop_on_no_data, and stop_on_error must not be set for transformation_type 'rating_curve'." + "stop_on_gaps, and stop_on_error must not be set for transformation_type 'rating_curve'." ) try: RatingCurve.objects.get( @@ -485,10 +485,10 @@ def validate_transformation( "transformation_type 'aggregation'." ) if any(v is not Unset for v in ( - rating_curve, formula, stop_on_no_data, stop_on_error, + rating_curve, formula, stop_on_gaps, stop_on_error, )): raise ValueError( - "rating_curve, formula, stop_on_no_data, and stop_on_error must not be set " + "rating_curve, formula, stop_on_gaps, and stop_on_error must not be set " "for transformation_type 'aggregation'." ) @@ -623,7 +623,7 @@ def run_derivation(self, transformation: DataProductTransformation) -> int: result_df = apply_derivation( inputs=inputs, formula=transformation.formula, - stop_on_no_data=transformation.stop_on_no_data, + stop_on_gaps=transformation.stop_on_gaps, stop_on_error=transformation.stop_on_error, input_no_data_values=input_no_data_values, output_no_data_value=output_ds.no_data_value, diff --git a/django/tests/fixtures/test_products_data.yaml b/django/tests/fixtures/test_products_data.yaml index 02fb1003c..9bcd4a1a7 100644 --- a/django/tests/fixtures/test_products_data.yaml +++ b/django/tests/fixtures/test_products_data.yaml @@ -118,7 +118,7 @@ timezone: null aggregation_method: null min_values: null - stop_on_no_data: true + stop_on_gaps: true stop_on_error: true - model: products.dataproducttransformationinput @@ -143,7 +143,7 @@ timezone: null aggregation_method: null min_values: null - stop_on_no_data: true + stop_on_gaps: true stop_on_error: true - model: products.dataproducttransformationinput diff --git a/django/tests/products/services/test_transformation.py b/django/tests/products/services/test_transformation.py index d0cf6e005..d418673e8 100644 --- a/django/tests/products/services/test_transformation.py +++ b/django/tests/products/services/test_transformation.py @@ -842,7 +842,7 @@ def test_run_derivation_stops_at_first_misaligned_timestamp(run_context): np.testing.assert_allclose(out_obs[0].result, 3.0) -def _run_offset_derivation(ctx, obs_a, obs_b, stop_on_no_data=Unset): +def _run_offset_derivation(ctx, obs_a, obs_b, stop_on_gaps=Unset): """ Create inputs ds_a/ds_b from {hour: value} dicts (2025-02-10 UTC), derive "a + b" into an empty output, run once, and return (loaded, transformation, @@ -864,7 +864,7 @@ def _run_offset_derivation(ctx, obs_a, obs_b, stop_on_no_data=Unset): transformation_type="derivation", output_datastream=ds_out.pk, formula="a + b", - stop_on_no_data=stop_on_no_data, + stop_on_gaps=stop_on_gaps, input_datastreams=[ TransformationInput(datastream=ds_a.pk, variable_name="a"), TransformationInput(datastream=ds_b.pk, variable_name="b"), @@ -933,17 +933,19 @@ def test_run_derivation_offset_start_stops_at_divergence(run_context): run_context, obs_a={8: 1.0, 10: 3.0, 11: 4.0, 12: 5.0}, obs_b={10: 10.0, 12: 30.0}, + stop_on_gaps=True, ) assert loaded == 1 assert _output_results(ds_out) == [(10, 13.0)] -def test_run_derivation_stop_on_no_data_true_waits_at_gap(run_context): +def test_run_derivation_stop_on_gaps_true_waits_at_gap(run_context): """With the toggle on, a gap holds the output back on every run until it's filled.""" loaded, t, ds_a, ds_b, ds_out = _run_offset_derivation( run_context, obs_a={8: 1.0, 9: 2.0, 10: 3.0}, obs_b={8: 10.0, 10: 30.0}, + stop_on_gaps=True, ) assert loaded == 1 @@ -956,13 +958,13 @@ def test_run_derivation_stop_on_no_data_true_waits_at_gap(run_context): assert _output_results(ds_out) == [(8, 11.0), (9, 22.0), (10, 33.0), (11, 44.0)] -def test_run_derivation_skips_gaps_when_stop_on_no_data_false(run_context): +def test_run_derivation_skips_gaps_when_stop_on_gaps_false(run_context): """With the toggle off, timestamps not shared by both inputs are skipped.""" loaded, t, ds_a, ds_b, ds_out = _run_offset_derivation( run_context, obs_a={8: 1.0, 9: 2.0, 10: 3.0, 11: 4.0}, obs_b={8: 10.0, 10: 30.0, 11: 40.0}, - stop_on_no_data=False, + stop_on_gaps=False, ) assert loaded == 3 assert _output_results(ds_out) == [(8, 11.0), (10, 33.0), (11, 44.0)] @@ -984,7 +986,7 @@ def test_run_derivation_lagging_input_holds_output_back(run_context): run_context, obs_a={8: 1.0, 9: 2.0}, obs_b={8: 10.0, 9: 20.0, 10: 30.0, 11: 40.0}, - stop_on_no_data=False, + stop_on_gaps=False, ) assert loaded == 2 assert _output_results(ds_out) == [(8, 11.0), (9, 22.0)] @@ -997,7 +999,7 @@ def test_run_derivation_lagging_input_holds_output_back(run_context): def test_run_derivation_no_data_values_fill_output_and_continue(run_context): """ Each input is checked against its own no-data value, and those rows get the - output's no-data value. The run continues even with the toggle at its default (on). + output's no-data value. The run continues with the toggle at its default (off). """ ctx = run_context kwargs = {k: ctx[k] for k in ("thing", "sensor", "observed_property", "processing_level", "unit")} @@ -1026,7 +1028,7 @@ def test_run_derivation_no_data_values_fill_output_and_continue(run_context): TransformationInput(datastream=ds_b.pk, variable_name="b"), ], ) - assert t.stop_on_no_data is True + assert t.stop_on_gaps is False assert transformation_service.run(t) == 4 assert _output_results(ds_out) == [(8, 11.0), (9, -555.0), (10, -555.0), (11, -10000.0)] @@ -1040,6 +1042,7 @@ def test_run_derivation_latest_start_not_shared_loads_nothing(run_context): run_context, obs_a={2: 1.0, 3: 2.0}, obs_b={1: 10.0, 3: 20.0}, + stop_on_gaps=True, ) assert loaded == 0 assert _output_results(ds_out) == [] diff --git a/packages/hydroserver-ts/src/generated/data.types.d.ts b/packages/hydroserver-ts/src/generated/data.types.d.ts index 47d5b6060..e8409f242 100644 --- a/packages/hydroserver-ts/src/generated/data.types.d.ts +++ b/packages/hydroserver-ts/src/generated/data.types.d.ts @@ -3255,8 +3255,8 @@ export interface components { outputDatastreamId?: string; /** Stoponerror */ stopOnError?: boolean; - /** Stoponnodata */ - stopOnNoData?: boolean; + /** Stopongaps */ + stopOnGaps?: boolean; }; /** DerivationTransformationPostBody */ DerivationTransformationPostBody: { @@ -3280,10 +3280,10 @@ export interface components { */ stopOnError: boolean; /** - * Stoponnodata - * @default true + * Stopongaps + * @default false */ - stopOnNoData: boolean; + stopOnGaps: boolean; }; /** DerivationTransformationResponse */ DerivationTransformationResponse: { @@ -3299,8 +3299,8 @@ export interface components { outputDatastream: components["schemas"]["DatastreamSummaryResponse"]; /** Stoponerror */ stopOnError: boolean; - /** Stoponnodata */ - stopOnNoData: boolean; + /** Stopongaps */ + stopOnGaps: boolean; }; /** DerivationTransformationSummaryResponse */ DerivationTransformationSummaryResponse: { @@ -3320,8 +3320,8 @@ export interface components { outputDatastreamId: string; /** Stoponerror */ stopOnError: boolean; - /** Stoponnodata */ - stopOnNoData: boolean; + /** Stopongaps */ + stopOnGaps: boolean; }; /** EtlDataMappingPatchBody */ EtlDataMappingPatchBody: { diff --git a/packages/hydroserverpy/src/hydroserverpy/api/README.md b/packages/hydroserverpy/src/hydroserverpy/api/README.md index 601ba657b..138a60852 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/README.md +++ b/packages/hydroserverpy/src/hydroserverpy/api/README.md @@ -1070,12 +1070,12 @@ new_transformation = hs_api.transformations.create_derivation( {'datastream_id': '00000000-0000-0000-0000-000000000001', 'variable_name': 'x'}, ], formula='x * 0.3048', # feet to meters - stop_on_no_data=True, # stop the run at timestamps not shared by all inputs + stop_on_gaps=True, # stop the run at timestamps not shared by all inputs stop_on_error=True, # stop the run if the formula produces a non-finite result ) # A derivation with multiple inputs works the same way, just with more entries. -# Inputs are matched by exact timestamp; with stop_on_no_data=True a gap stops the +# Inputs are matched by exact timestamp; with stop_on_gaps=True a gap stops the # run there until it's filled, and with False those timestamps are skipped. new_derived_transformation = hs_api.transformations.create_derivation( task_id='00000000-0000-0000-0000-000000000000', diff --git a/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py b/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py index dd1234c2f..633f2e26e 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py +++ b/packages/hydroserverpy/src/hydroserverpy/api/models/products/transformation.py @@ -21,7 +21,7 @@ class DerivationTransformation(BaseModel): output_datastream_id: uuid.UUID input_datastream_ids: List[uuid.UUID] formula: str - stop_on_no_data: bool + stop_on_gaps: bool stop_on_error: bool model_config = ConfigDict(alias_generator=to_camel, populate_by_name=True) diff --git a/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py b/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py index 1d03a4647..260990609 100644 --- a/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py +++ b/packages/hydroserverpy/src/hydroserverpy/api/services/products/transformation.py @@ -157,7 +157,7 @@ def create_derivation( output_datastream: Union[UUID, str], input_datastreams: List[dict], formula: str, - stop_on_no_data: bool = True, + stop_on_gaps: bool = False, stop_on_error: bool = True, uid: Optional[UUID] = None, ) -> DerivationTransformation: @@ -177,7 +177,7 @@ def create_derivation( for inp in input_datastreams ], "formula": formula, - "stopOnNoData": stop_on_no_data, + "stopOnGaps": stop_on_gaps, "stopOnError": stop_on_error, } if uid is not None: @@ -191,7 +191,7 @@ def update_derivation( uid: Union[UUID, str], input_datastreams: List[dict] = ..., formula: str = ..., - stop_on_no_data: bool = ..., + stop_on_gaps: bool = ..., stop_on_error: bool = ..., ) -> DerivationTransformation: """ @@ -209,7 +209,7 @@ def update_derivation( for inp in input_datastreams ] if input_datastreams is not ... else ..., "formula": formula, - "stopOnNoData": stop_on_no_data, + "stopOnGaps": stop_on_gaps, "stopOnError": stop_on_error, } body = {k: v for k, v in body.items() if v is not ...} diff --git a/packages/hydroserverpy/src/hydroserverpy/products/derivation.py b/packages/hydroserverpy/src/hydroserverpy/products/derivation.py index 071971faf..33820501a 100644 --- a/packages/hydroserverpy/src/hydroserverpy/products/derivation.py +++ b/packages/hydroserverpy/src/hydroserverpy/products/derivation.py @@ -106,7 +106,7 @@ def apply_derivation( inputs: dict[str, pd.DataFrame], formula: str, *, - stop_on_no_data: bool, + stop_on_gaps: bool, stop_on_error: bool, input_no_data_values: dict[str, float] | None = None, output_no_data_value: float | None = None, @@ -115,7 +115,7 @@ def apply_derivation( Apply a mathematical formula to one or more input timeseries DataFrames. Inputs are matched on exact timestamp (sorted defensively first). - stop_on_no_data controls gaps, where one input has a timestamp the others + stop_on_gaps controls gaps, where one input has a timestamp the others don't: True stops output before the first gap and logs it, so nothing after is evaluated even if the inputs realign later; False skips every timestamp not shared by all inputs and continues. @@ -144,9 +144,9 @@ def apply_derivation( } logger.debug( - "Evaluating derivation (formula=%r, variables=%r, stopOnNoData=%r, stopOnError=%r, " + "Evaluating derivation (formula=%r, variables=%r, stopOnGaps=%r, stopOnError=%r, " "inputNoDataValues=%r, outputNoDataValue=%r, rows=%r).", - formula, variables, stop_on_no_data, stop_on_error, input_no_data_values, output_no_data_value, + formula, variables, stop_on_gaps, stop_on_error, input_no_data_values, output_no_data_value, {var: len(df) for var, df in validated.items()}, ) @@ -161,7 +161,7 @@ def apply_derivation( combined_df = combined_df.merge(other, on=TIMESTAMP_COL, how="inner") combined_df = combined_df.sort_values(TIMESTAMP_COL).reset_index(drop=True) - if divergence_timestamp is not None and not stop_on_no_data: + if divergence_timestamp is not None and not stop_on_gaps: logger.info( "Derivation skipped %d timestamp(s) not shared by all inputs, starting at %s.", len(divergent_timestamps), divergence_timestamp, diff --git a/packages/hydroserverpy/tests/products/test_derivation.py b/packages/hydroserverpy/tests/products/test_derivation.py index e797e9be6..ef3d96526 100644 --- a/packages/hydroserverpy/tests/products/test_derivation.py +++ b/packages/hydroserverpy/tests/products/test_derivation.py @@ -107,78 +107,78 @@ class TestApplyDerivation: def test_empty_inputs_raises(self): with pytest.raises(ValueError, match="At least one"): - apply_derivation({}, formula="x", stop_on_no_data=True, stop_on_error=True) + apply_derivation({}, formula="x", stop_on_gaps=True, stop_on_error=True) def test_input_no_data_values_require_output_no_data_value(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) with pytest.raises(ValueError, match="output_no_data_value"): apply_derivation( - {"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True, + {"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True, input_no_data_values={"x": -9999.0}, ) - def test_stop_on_no_data_false_does_not_require_output_no_data_value(self): + def test_stop_on_gaps_false_does_not_require_output_no_data_value(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=False, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=False, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_stop_on_error_false_requires_output_no_data_value(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) with pytest.raises(ValueError, match="output_no_data_value"): - apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=False) + apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=False) def test_single_input_passthrough_preserves_timestamps(self): ts = _hourly(3) df = _make_df(ts, [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert len(result) == 3 assert result[TIMESTAMP_COL].tolist() == df[TIMESTAMP_COL].tolist() def test_single_input_sorted_defensively(self): ts = list(reversed(_hourly(3))) df = _make_df(ts, [3.0, 2.0, 1.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) assert result[TIMESTAMP_COL].is_monotonic_increasing def test_multiply_by_scalar(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x * 2", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x * 2", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([2.0, 4.0, 6.0]) def test_add_constant(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x + 10", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x + 10", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([11.0, 12.0, 13.0]) def test_subtract_constant(self): df = _make_df(_hourly(3), [5.0, 10.0, 15.0]) - result = apply_derivation({"x": df}, formula="x - 5", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x - 5", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([0.0, 5.0, 10.0]) def test_divide_by_constant(self): df = _make_df(_hourly(3), [2.0, 4.0, 6.0]) - result = apply_derivation({"x": df}, formula="x / 2", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x / 2", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_sqrt_function(self): df = _make_df(_hourly(3), [1.0, 4.0, 9.0]) - result = apply_derivation({"x": df}, formula="sqrt(x)", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="sqrt(x)", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 2.0, 3.0]) def test_abs_function(self): df = _make_df(_hourly(3), [-1.0, 0.0, 1.0]) - result = apply_derivation({"x": df}, formula="abs(x)", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="abs(x)", stop_on_gaps=True, stop_on_error=True) assert result[RESULT_COL].tolist() == pytest.approx([1.0, 0.0, 1.0]) def test_output_result_dtype_is_float64(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert pd.api.types.is_float_dtype(result[RESULT_COL]) def test_output_timestamp_dtype_matches_canonical_schema(self): df = _make_df(_hourly(3), [1.0, 2.0, 3.0]) - result = apply_derivation({"x": df}, formula="x", stop_on_no_data=True, stop_on_error=True) + result = apply_derivation({"x": df}, formula="x", stop_on_gaps=True, stop_on_error=True) assert pd.api.types.is_datetime64_any_dtype(result[TIMESTAMP_COL]) # -- Multi-input alignment -------------------------------------------- @@ -188,7 +188,7 @@ def test_multiple_inputs_addition_on_matching_timestamps(self): df_x = _make_df(ts, [1.0, 2.0, 3.0]) df_y = _make_df(ts, [4.0, 5.0, 6.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([5.0, 7.0, 9.0]) @@ -197,7 +197,7 @@ def test_multiple_inputs_formula_with_division(self): df_x = _make_df(ts, [2.0, 4.0, 6.0]) df_y = _make_df(ts, [1.0, 2.0, 3.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x / y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([2.0, 2.0, 2.0]) @@ -206,7 +206,7 @@ def test_inputs_sorted_defensively_before_joining(self): df_x = _make_df(list(reversed(ts)), [3.0, 2.0, 1.0]) df_y = _make_df(ts, [10.0, 20.0, 30.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([11.0, 22.0, 33.0]) @@ -218,7 +218,7 @@ def test_misalignment_stops_before_first_divergent_timestamp(self, caplog): df_y = _make_df(ts_y, [10.0, 30.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) # Hour 2 matched on both sides, but comes after the hour-1 divergence, # so it's excluded even though it would otherwise be a valid match. @@ -226,7 +226,7 @@ def test_misalignment_stops_before_first_divergent_timestamp(self, caplog): assert result[RESULT_COL].tolist() == pytest.approx([11.0]) assert "not aligned" in caplog.text - def test_misalignment_skipped_when_stop_on_no_data_false(self, caplog): + def test_misalignment_skipped_when_stop_on_gaps_false(self, caplog): # Same inputs as above, but hour 1 is skipped and hour 2 still matches. ts_x = [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 1), _utc(2024, 1, 1, 2)] ts_y = [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 2)] @@ -234,7 +234,7 @@ def test_misalignment_skipped_when_stop_on_no_data_false(self, caplog): df_y = _make_df(ts_y, [10.0, 30.0]) with caplog.at_level(logging.INFO): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=False, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True ) assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, 0), _utc(2024, 1, 1, 2)] assert result[RESULT_COL].tolist() == pytest.approx([11.0, 33.0]) @@ -248,7 +248,7 @@ def test_misalignment_skipped_in_both_directions(self): df_x = _make_df(ts_x, [1.0, 3.0, 4.0, 5.0]) df_y = _make_df(ts_y, [10.0, 20.0, 30.0, 50.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=False, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True ) assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, h) for h in (0, 2, 4)] assert result[RESULT_COL].tolist() == pytest.approx([11.0, 33.0, 55.0]) @@ -261,7 +261,7 @@ def test_skipped_gap_and_no_data_value_handled_independently(self): df_y = _make_df(ts_y, [10.0, 30.0, 40.0]) result = apply_derivation( {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=False, stop_on_error=True, + stop_on_gaps=False, stop_on_error=True, input_no_data_values={"x": -9999.0, "y": -9999.0}, output_no_data_value=-1.0, ) assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, h) for h in (0, 2, 3)] @@ -274,7 +274,7 @@ def test_stop_on_error_still_applies_after_skipped_gap(self, caplog): df_y = _make_df(ts_y, [1.0, 0.0, 4.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x / y", stop_on_no_data=False, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=False, stop_on_error=True ) assert result[TIMESTAMP_COL].tolist() == [_utc(2024, 1, 1, 0)] assert "non-finite" in caplog.text @@ -284,7 +284,7 @@ def test_completely_disjoint_timestamps_return_empty_when_skipping(self, caplog) df_y = _make_df([_utc(2024, 1, 1, 1)], [2.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=False, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=False, stop_on_error=True ) assert len(result) == 0 assert caplog.text == "" @@ -295,7 +295,7 @@ def test_fully_aligned_inputs_produce_no_warning(self, caplog): df_y = _make_df(ts, [4.0, 5.0, 6.0]) with caplog.at_level(logging.WARNING): apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert caplog.text == "" @@ -303,21 +303,21 @@ def test_completely_disjoint_timestamps_return_empty(self): df_x = _make_df([_utc(2024, 1, 1, 0)], [1.0]) df_y = _make_df([_utc(2024, 1, 1, 1)], [2.0]) result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x + y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x + y", stop_on_gaps=True, stop_on_error=True ) assert len(result) == 0 # -- No-data values in aligned inputs ----------------------------------- - @pytest.mark.parametrize("stop_on_no_data", [True, False]) - def test_no_data_value_always_fills_and_continues(self, stop_on_no_data, caplog): + @pytest.mark.parametrize("stop_on_gaps", [True, False]) + def test_no_data_value_always_fills_and_continues(self, stop_on_gaps, caplog): ts = _hourly(3) df_x = _make_df(ts, [1.0, -9999.0, 3.0]) df_y = _make_df(ts, [10.0, 20.0, 30.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=stop_on_no_data, stop_on_error=True, + stop_on_gaps=stop_on_gaps, stop_on_error=True, input_no_data_values={"x": -9999.0, "y": -9999.0}, output_no_data_value=-9999.0, ) assert len(result) == 3 @@ -333,7 +333,7 @@ def test_no_data_value_only_checks_that_inputs_own_sentinel(self): df_y = _make_df(ts, [10.0, 20.0, -777.0]) result = apply_derivation( {"x": df_x, "y": df_y}, formula="x + y", - stop_on_no_data=False, stop_on_error=True, + stop_on_gaps=False, stop_on_error=True, input_no_data_values={"x": -9999.0, "y": -777.0}, output_no_data_value=-1.0, ) assert result[RESULT_COL].tolist() == pytest.approx([11.0, -1.0, -1.0]) @@ -346,7 +346,7 @@ def test_stop_on_error_false_fills_and_continues(self): df_y = _make_df(ts, [1.0, 0.0, 3.0]) result = apply_derivation( {"x": df_x, "y": df_y}, formula="x / y", - stop_on_no_data=True, stop_on_error=False, output_no_data_value=-9999.0, + stop_on_gaps=True, stop_on_error=False, output_no_data_value=-9999.0, ) assert len(result) == 3 assert result[RESULT_COL].tolist() == pytest.approx([1.0, -9999.0, 1.0]) @@ -357,7 +357,7 @@ def test_stop_on_error_true_truncates_at_first_non_finite_result(self, caplog): df_y = _make_df(ts, [1.0, 0.0, 3.0]) with caplog.at_level(logging.WARNING): result = apply_derivation( - {"x": df_x, "y": df_y}, formula="x / y", stop_on_no_data=True, stop_on_error=True + {"x": df_x, "y": df_y}, formula="x / y", stop_on_gaps=True, stop_on_error=True ) assert result[RESULT_COL].tolist() == pytest.approx([1.0]) assert "non-finite" in caplog.text @@ -370,7 +370,7 @@ def test_no_data_value_rows_are_not_also_flagged_as_errors(self): df_x = _make_df(ts, [-9999.0, 4.0]) result = apply_derivation( {"x": df_x}, formula="log(x)", - stop_on_no_data=False, stop_on_error=True, + stop_on_gaps=False, stop_on_error=True, input_no_data_values={"x": -9999.0}, output_no_data_value=-9999.0, ) assert len(result) == 2