diff --git a/aurora/pipelines/process_mth5.py b/aurora/pipelines/process_mth5.py index 85ee9040..aa496bcd 100644 --- a/aurora/pipelines/process_mth5.py +++ b/aurora/pipelines/process_mth5.py @@ -50,7 +50,6 @@ ) from aurora.transfer_function.TTFZ import TTFZ - SUPPORTED_PROCESSINGS = [ "legacy", ] @@ -188,6 +187,13 @@ def process_mth5_legacy( tfk.apply_clock_zero(dec_level_config) stfts = get_spectrograms(tfk, i_dec_level, units=units) + if stfts is None: + logger.warning( + "Skipping transfer-function estimation for decimation level {} " + "because no complete STFT set is available", + i_dec_level, + ) + continue local_merged_stft_obj, remote_merged_stft_obj = merge_stfts(stfts, tfk) diff --git a/aurora/time_series/spectrogram_helpers.py b/aurora/time_series/spectrogram_helpers.py index 3cbd0019..cf775917 100644 --- a/aurora/time_series/spectrogram_helpers.py +++ b/aurora/time_series/spectrogram_helpers.py @@ -391,7 +391,7 @@ def save_fourier_coefficients( def get_spectrograms( tfk: TransferFunctionKernel, i_dec_level: int, units: str = "MT" -) -> dict: +) -> dict | None: """ Given a decimation level id, loads a dictianary of all spectragrams from information in tfk. TODO: Make this a method of TFK @@ -409,9 +409,10 @@ def get_spectrograms( Returns ------- - stfts: dict + stfts: dict or None The short time fourier transforms for the decimation level as a dictionary. Keys are "local" and "remote". Values are lists, one (element) xr.Dataset per run + or None when a valid run cannot produce any STFT windows. """ stfts = {} @@ -438,13 +439,24 @@ def get_spectrograms( # Musgraves workaround for old MT data triage_run_id(row.run, run_obj) - stft_obj = make_stft_objects( - tfk.config, - i_dec_level, - run_obj, - run_xrds, - units, - ) + try: + stft_obj = make_stft_objects( + tfk.config, + i_dec_level, + run_obj, + run_xrds, + units, + ) + except ValueError as error: + logger.warning( + "Skipping decimation level {} because station {}, run {} " + "cannot produce STFT windows: {}", + i_dec_level, + row.station, + row.run, + error, + ) + return None # TODO: Cast stft_obj to a Spectrogram here or in make_stft_objects # Pack FCs into h5 @@ -491,7 +503,7 @@ def run_ts_to_stft( run_xrds, dt=1.0 / decimation_obj.decimation.sample_rate ) if not np.prod(windowed_obj.to_array().data.shape): - raise ValueError + raise ValueError("STFT windowing produced no samples") windowed_obj = WindowedTimeSeries.detrend(data=windowed_obj, detrend_type="linear") tapered_obj = WindowedTimeSeries.apply_taper( diff --git a/tests/synthetic/test_processing_pytest.py b/tests/synthetic/test_processing_pytest.py index ddf60308..df543a50 100644 --- a/tests/synthetic/test_processing_pytest.py +++ b/tests/synthetic/test_processing_pytest.py @@ -4,9 +4,14 @@ """ import logging +from unittest.mock import Mock, patch +import numpy as np +import pandas as pd import pytest +import xarray as xr +from aurora.time_series.spectrogram_helpers import get_spectrograms from aurora.test_utils.synthetic.processing_helpers import ( process_synthetic_1, process_synthetic_1r2, @@ -21,6 +26,56 @@ def setup_logging(): logging.getLogger("matplotlib.ticker").disabled = True +def test_empty_stft_skips_decimation_without_saving_fourier_coefficients(): + """A valid run that yields no windows skips its entire decimation level.""" + run_object = Mock() + mth5_object = Mock() + mth5_object.from_reference.return_value = run_object + run_dataarray = xr.DataArray( + np.arange(4.0)[np.newaxis, :], + dims=["channel", "time"], + coords={ + "channel": ["hx"], + "time": np.datetime64("2020-01-01T00:00:00") + + np.arange(4).astype("timedelta64[s]"), + }, + ) + tfk = Mock() + tfk.dataset_df = pd.DataFrame( + [ + { + "mth5_obj": mth5_object, + "run_hdf5_reference": "run-reference", + "fc": False, + "run_dataarray": run_dataarray, + "run": "001", + "station": "test1", + "remote": False, + } + ] + ) + tfk.is_valid_dataset.return_value = True + + with ( + patch("aurora.time_series.spectrogram_helpers.triage_run_id"), + patch( + "aurora.time_series.spectrogram_helpers.make_stft_objects", + side_effect=ValueError("STFT windowing produced no samples"), + ), + patch( + "aurora.time_series.spectrogram_helpers.save_fourier_coefficients" + ) as save_fourier_coefficients, + patch("aurora.time_series.spectrogram_helpers.logger.warning") as log_warning, + ): + stfts = get_spectrograms(tfk, i_dec_level=2) + + assert stfts is None + save_fourier_coefficients.assert_not_called() + log_warning.assert_called_once() + assert "Skipping decimation level" in log_warning.call_args.args[0] + assert log_warning.call_args.args[1] == 2 + + @pytest.mark.skip( reason="mt_metadata pydantic branch has issue with provenance.archive.comments.value being None" )