From 1bbdef350663fe38c8be0468e3239e1a61c8d7b4 Mon Sep 17 00:00:00 2001 From: Shahid Shamim Shah Date: Sun, 1 Nov 2026 01:45:19 +0530 Subject: [PATCH 1/4] fix: honor fast step three for individual color channels --- ledsa/core/parser_arguments_run.py | 2 +- .../UnitTests/test_parser_arguments_run.py | 34 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 ledsa/tests/UnitTests/test_parser_arguments_run.py diff --git a/ledsa/core/parser_arguments_run.py b/ledsa/core/parser_arguments_run.py index 8e5fa95..27bb1b6 100644 --- a/ledsa/core/parser_arguments_run.py +++ b/ledsa/core/parser_arguments_run.py @@ -57,7 +57,7 @@ def run_data_extraction_arguments(args: argparse.Namespace) -> None: if len(channels) == 0: channels.append(0) - if args.red or args.green or args.blue or args.rgb and not args.step_3_fast: + if (args.red or args.green or args.blue or args.rgb) and not args.step_3_fast: args.step_3 = True if args.step_1 or args.step_2: diff --git a/ledsa/tests/UnitTests/test_parser_arguments_run.py b/ledsa/tests/UnitTests/test_parser_arguments_run.py new file mode 100644 index 0000000..de0726a --- /dev/null +++ b/ledsa/tests/UnitTests/test_parser_arguments_run.py @@ -0,0 +1,34 @@ +"""Channel flags select the requested step 3 mode without a second analysis.""" +import argparse +from unittest.mock import Mock + +import pytest + +from ledsa.core.parser_arguments_declaration import add_parser_arguments_data_extraction +from ledsa.core import parser_arguments_run + + +@pytest.mark.parametrize('flags, channels', [ + (['-r'], [0]), + (['-g'], [1]), + (['-b'], [2]), + (['-rgb'], [0, 1, 2]), + (['-r', '-b'], [0, 2]), +]) +@pytest.mark.parametrize('fast', [False, True]) +def test_channel_flags_run_step_three_once(tmp_path, monkeypatch, flags, channels, fast): + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.ini').touch() + extractor = Mock() + monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor) + parser = add_parser_arguments_data_extraction(argparse.ArgumentParser()) + args = parser.parse_args(flags + (['-s3_fast'] if fast else [])) + + parser_arguments_run.run_data_extraction_arguments(args) + + kwargs = {'build_experiment_infos': True, 'channels': channels} + if fast: + kwargs['fit_leds'] = False + extractor.assert_called_once_with(**kwargs) + extractor.return_value.setup_step3.assert_called_once_with() + extractor.return_value.process_image_data.assert_called_once_with() From f2e3eadc33a1d523964a477795839edac977a443 Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 12:46:54 +0100 Subject: [PATCH 2/4] let a colour flag pick the channels of -re instead of running step 3 A colour flag without a step runs step 3 with the LED fits. The guard against that checked -s3_fast only, so -re -r, -re -g and -re -rgb first ran the full step 3 on every image, which overwrote the partial results, and the restart then found nothing left to analyse. Plain -re resumes the red channel only, so an interrupted green, blue or RGB run could not be resumed at all. The guard now also leaves out -re, and the colour flags only select the channels of the restart. -re alone, an explicit -s3 next to -re and a colour flag next to -s1, -s2 or -coord behave as before. The CLI documentation listed --r, which argparse rejects as ambiguous, and no channel flags for -s3_fast and -re. The help of -re now says to give the channel flags of the interrupted run. Co-Authored-By: Claude Opus 5.5 --- doc/cli_documentation.rst | 6 ++--- ledsa/core/parser_arguments_declaration.py | 3 ++- ledsa/core/parser_arguments_run.py | 3 ++- .../UnitTests/test_parser_arguments_run.py | 24 +++++++++++++++++++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/doc/cli_documentation.rst b/doc/cli_documentation.rst index 02d371c..d3dd1b7 100644 --- a/doc/cli_documentation.rst +++ b/doc/cli_documentation.rst @@ -32,13 +32,13 @@ Data Extraction - -- * - ``-s3``, ``--step_3`, ``--analyse_photo`` - Step 3: Analyse all images to extract pixel values inside the search areas and fit them to an LED model (SLOW!). - - ``--r`` (Use the red channel, default), ``--g`` (Use the green channel), ``--b`` (Use the blue channel), ``-rgb`` (Run for each channel) + - ``-r`` (Use the red channel, default), ``-g`` (Use the green channel), ``-b`` (Use the blue channel), ``-rgb`` (Run for each channel) * - ``-s3_fast``, ``--step_3_fast`, ``--analyse_photo_fast`` - Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). Besides the plain sum, the background-corrected sums ``bgsub_sum_col_val``, ``bgsub2_sum_col_val`` and ``sidebg_sum_col_val`` are written (see ``reference_property`` in the analysis configuration). - - -- + - Channel flags as for ``-s3`` * - ``-re``, ``--restart`` - Restart step 3 if it was previously interrupted. Only the images that have not been analysed are taken into account. - - -- + - The channel flags of the interrupted run, as for ``-s3`` Analysis diff --git a/ledsa/core/parser_arguments_declaration.py b/ledsa/core/parser_arguments_declaration.py index 8143c75..6186f87 100644 --- a/ledsa/core/parser_arguments_declaration.py +++ b/ledsa/core/parser_arguments_declaration.py @@ -23,7 +23,8 @@ def add_parser_arguments_data_extraction(parser: argparse.ArgumentParser) -> arg help='creates the default configuration file. optional arguments are are: img_directory, ' 'reference_img, number_of_cores.') parser.add_argument('-re', '--restart', action='store_true', - help='Restarts step 3 of the analysis after the program was terminated before it finished.') + help='Restarts step 3 of the analysis after the program was terminated before it finished. ' + 'Give the channel flags of the interrupted run, e.g. -re -rgb.') parser.add_argument('-r', '--red', action='store_true', help='Use the red channel for step3. Default value.') parser.add_argument('-g', '--green', action='store_true', diff --git a/ledsa/core/parser_arguments_run.py b/ledsa/core/parser_arguments_run.py index 27bb1b6..8f26163 100644 --- a/ledsa/core/parser_arguments_run.py +++ b/ledsa/core/parser_arguments_run.py @@ -57,7 +57,8 @@ def run_data_extraction_arguments(args: argparse.Namespace) -> None: if len(channels) == 0: channels.append(0) - if (args.red or args.green or args.blue or args.rgb) and not args.step_3_fast: + # A colour flag on its own runs step 3. With -s3_fast or -re it only selects the channels of that run. + if (args.red or args.green or args.blue or args.rgb) and not (args.step_3_fast or args.restart): args.step_3 = True if args.step_1 or args.step_2: diff --git a/ledsa/tests/UnitTests/test_parser_arguments_run.py b/ledsa/tests/UnitTests/test_parser_arguments_run.py index de0726a..5fe7744 100644 --- a/ledsa/tests/UnitTests/test_parser_arguments_run.py +++ b/ledsa/tests/UnitTests/test_parser_arguments_run.py @@ -32,3 +32,27 @@ def test_channel_flags_run_step_three_once(tmp_path, monkeypatch, flags, channel extractor.assert_called_once_with(**kwargs) extractor.return_value.setup_step3.assert_called_once_with() extractor.return_value.process_image_data.assert_called_once_with() + + +@pytest.mark.parametrize('flags, channels', [ + ([], [0]), + (['-r'], [0]), + (['-g'], [1]), + (['-b'], [2]), + (['-rgb'], [0, 1, 2]), + (['-g', '-b'], [1, 2]), +]) +def test_restart_resumes_the_selected_channels_without_step_three(tmp_path, monkeypatch, flags, channels): + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.ini').touch() + extractor = Mock() + monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor) + parser = add_parser_arguments_data_extraction(argparse.ArgumentParser()) + args = parser.parse_args(['-re'] + flags) + + parser_arguments_run.run_data_extraction_arguments(args) + + extractor.assert_called_once_with(build_experiment_infos=False, channels=channels, fit_leds=False) + extractor.return_value.setup_restart.assert_called_once_with() + extractor.return_value.setup_step3.assert_not_called() + extractor.return_value.process_image_data.assert_called_once_with() From 355158fb9fa7c563d6dabe9528f25d7a5c43d63b Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 13:12:19 +0100 Subject: [PATCH 3/4] resume -re in the fit mode of the interrupted run The restart always analysed without the LED fits. After an interrupted -s3 run, the resumed images therefore got the -s3_fast columns, and the analysis refused the channel because its result files were written with different column layouts. -s3 is the slow run, so it is the one most likely to be interrupted. The restart now reads the mode from the headers of the result files of its channels: only a file written with fits names the fit columns, fit_success among them, in every version of the header. The number of columns is not used, because it changes between LEDSA versions. If no channel has a result file yet, the restart runs without fits as before. If the files of a channel, or those of different channels, were written in both modes, it stops and says how to go on instead of guessing, because one restart writes all its channels in one mode. Empty files, which an interruption can leave, are ignored. The help of -re and the CLI documentation describe the mode. Co-Authored-By: Claude Opus 5.5 --- doc/cli_documentation.rst | 2 +- ledsa/core/parser_arguments_declaration.py | 4 +- ledsa/core/parser_arguments_run.py | 7 +- ledsa/data_extraction/step_3_functions.py | 100 +++++++++++++++- .../UnitTests/test_parser_arguments_run.py | 109 +++++++++++++++++- 5 files changed, 211 insertions(+), 11 deletions(-) diff --git a/doc/cli_documentation.rst b/doc/cli_documentation.rst index d3dd1b7..c90d082 100644 --- a/doc/cli_documentation.rst +++ b/doc/cli_documentation.rst @@ -37,7 +37,7 @@ Data Extraction - Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). Besides the plain sum, the background-corrected sums ``bgsub_sum_col_val``, ``bgsub2_sum_col_val`` and ``sidebg_sum_col_val`` are written (see ``reference_property`` in the analysis configuration). - Channel flags as for ``-s3`` * - ``-re``, ``--restart`` - - Restart step 3 if it was previously interrupted. Only the images that have not been analysed are taken into account. + - Restart step 3 if it was previously interrupted. Only the images that have not been analysed are taken into account. The restart resumes in the mode of the interrupted run, with the LED fits after ``-s3`` and without them after ``-s3_fast``, as the headers of its result files tell. If there are no result files yet, it runs without the fits. If the result files were written in both modes, it stops, because it cannot tell which mode to resume. - The channel flags of the interrupted run, as for ``-s3`` diff --git a/ledsa/core/parser_arguments_declaration.py b/ledsa/core/parser_arguments_declaration.py index 6186f87..059f67d 100644 --- a/ledsa/core/parser_arguments_declaration.py +++ b/ledsa/core/parser_arguments_declaration.py @@ -24,7 +24,9 @@ def add_parser_arguments_data_extraction(parser: argparse.ArgumentParser) -> arg 'reference_img, number_of_cores.') parser.add_argument('-re', '--restart', action='store_true', help='Restarts step 3 of the analysis after the program was terminated before it finished. ' - 'Give the channel flags of the interrupted run, e.g. -re -rgb.') + 'Give the channel flags of the interrupted run, e.g. -re -rgb. It resumes with the LED ' + 'fits after -s3 and without them after -s3_fast, as the result files of the interrupted ' + 'run tell.') parser.add_argument('-r', '--red', action='store_true', help='Use the red channel for step3. Default value.') parser.add_argument('-g', '--green', action='store_true', diff --git a/ledsa/core/parser_arguments_run.py b/ledsa/core/parser_arguments_run.py index 8f26163..d2a99d2 100644 --- a/ledsa/core/parser_arguments_run.py +++ b/ledsa/core/parser_arguments_run.py @@ -15,6 +15,7 @@ from ledsa.core.ConfigData import ConfigData from ledsa.core.file_handling import get_extinction_coefficients_file_path from ledsa.data_extraction.DataExtractor import DataExtractor +from ledsa.data_extraction.step_3_functions import find_fit_leds_for_restart @@ -79,8 +80,10 @@ def run_data_extraction_arguments(args: argparse.Namespace) -> None: de.process_image_data() if args.restart: - # Use the channels determined earlier in the function - de = DataExtractor(build_experiment_infos=False, channels=channels, fit_leds=False) + # Resume in the mode of the interrupted run, which the headers of its result files tell: with the LED fits + # after -s3, without them after -s3_fast. The analysis cannot load a channel with the columns of both modes. + fit_leds = find_fit_leds_for_restart(channels) + de = DataExtractor(build_experiment_infos=False, channels=channels, fit_leds=fit_leds) de.setup_restart() de.process_image_data() diff --git a/ledsa/data_extraction/step_3_functions.py b/ledsa/data_extraction/step_3_functions.py index 4bf9138..c0ab284 100644 --- a/ledsa/data_extraction/step_3_functions.py +++ b/ledsa/data_extraction/step_3_functions.py @@ -285,6 +285,56 @@ def find_and_save_not_analysed_imgs(channel: int) -> None: _save_list_of_remaining_imgs_needed_to_be_processed(remaining_imgs) +def find_fit_leds_for_restart(channels: Sequence[int]) -> bool: + """ + Find whether the restart of an interrupted step 3 fits the LEDs: if the interrupted run did (-s3), not if it + did not (-s3_fast). The analysis loads all result files of a channel with the same columns, so the resumed + images need the columns of the images analysed before. The mode is read from the headers of the result files, + not from their number of columns, which differs between LEDSA versions. If none of the channels has a result + file yet, the LEDs are not fitted. + + :param channels: Channels to be resumed. + :type channels: Sequence[int] + :return: True if the restart fits the LEDs, False if not. + :rtype: bool + :raises ValueError: If the result files of a channel, or those of different channels, were written both with + and without the fits. + """ + fit_leds_of_channels = {} + for channel in channels: + files_by_fit_leds = {True: [], False: []} + for _, file_path in _find_result_files(channel): + has_fits = _result_file_has_fits(file_path) + if has_fits is not None: + files_by_fit_leds[has_fits].append(file_path) + fitted_files, unfitted_files = files_by_fit_leds[True], files_by_fit_leds[False] + if fitted_files and unfitted_files: + raise ValueError(f'The result files of channel {channel} were written both with the LED fits (-s3, ' + f'{len(fitted_files)} files, e.g. {fitted_files[0]}) and without them (-s3_fast, ' + f'{len(unfitted_files)} files, e.g. {unfitted_files[0]}), so the restart cannot tell ' + f'in which mode to resume. Delete the files of the mode you do not want and run -re ' + f'again, or run -s3 or -s3_fast for the channel again.') + if fitted_files or unfitted_files: + fit_leds_of_channels[channel] = bool(fitted_files) + + if len(set(fit_leds_of_channels.values())) > 1: + modes = ', '.join(f'channel {channel} {"with" if fit_leds else "without"}' + for channel, fit_leds in fit_leds_of_channels.items()) + raise ValueError(f'The channels were analysed with and without the LED fits ({modes}), but a restart ' + f'resumes all of its channels in one mode. Restart these channels separately, selecting ' + f'them with -r, -g or -b.') + fit_leds = True in fit_leds_of_channels.values() + if not fit_leds_of_channels: + print('There are no result files of an interrupted run yet, so step 3 runs without the LED fits, as by ' + '-s3_fast.') + elif fit_leds: + print('The result files of the interrupted run have LED fits, so step 3 is resumed with them, as by -s3.') + else: + print('The result files of the interrupted run have no LED fits, so step 3 is resumed without them, as by ' + '-s3_fast.') + return fit_leds + + def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray, debug: bool, iled: int, img_filename: str, led_array_idx: int, search_areas: np.ndarray, search_area_radius: int, fit_leds: bool = True, channel_mask: Optional[np.ndarray] = None, pixel_noise: Optional[np.ndarray] = None, @@ -506,20 +556,58 @@ def _save_results_in_file(channel: int, img_data: LEDAnalysisData, img_filename: def _find_analysed_img_ids(channel): """ - Find and save filenames of images that have not yet been analyzed. + Find the IDs of the images that have a result file in a channel. :param channel: Channel to check for analyzed images. :return: List of IDs of analyzed images. :rtype: List[int] """ - processed_imgs = [] + return [img_id for img_id, _ in _find_result_files(channel)] + + +def _find_result_files(channel: int) -> List[Tuple[int, str]]: + """ + Find the result files of step 3 in a channel. + + :param channel: Channel whose result files are searched. + :type channel: int + :return: Image ID and path of every result file, sorted by image ID. Empty if the channel has no directory yet. + :rtype: List[Tuple[int, str]] + """ dir_path = os.path.join('analysis', f'channel{channel}') - directory_content = os.listdir(dir_path) - for file_name in directory_content: + if not os.path.isdir(dir_path): + return [] + result_files = [] + for file_name in os.listdir(dir_path): img = re.search(r"(\d+)_led_positions.csv", file_name) if img is not None: - processed_imgs.append(int(img.group(1))) - return processed_imgs + result_files.append((int(img.group(1)), os.path.join(dir_path, file_name))) + return sorted(result_files) + + +def _result_file_has_fits(file_path: str) -> Optional[bool]: + """ + Tell from its header whether a result file of step 3 was written with the LED fits. + + Only a file written with the fits names their columns in the header line of the column names, see + _create_header. fit_success is one of them in every version of the header. + + :param file_path: Path of the result file. + :type file_path: str + :return: True if the file was written with the fits, False if without them, None if it has no header line of + the column names, for example because it is empty. + :rtype: Optional[bool] + """ + # only the column names are of interest, a directory or image name in another encoding must not stop the restart + with open(file_path, 'r', encoding='utf-8', errors='replace') as result_file: + for line in result_file: + if not line.startswith('#'): + break + # the fit columns are followed by a comment after // + column_names = [name.strip() for name in line.lstrip('#').split('//')[0].split(',')] + if column_names[0] == 'id': + return 'fit_success' in column_names + return None def _save_list_of_remaining_imgs_needed_to_be_processed(remaining_imgs): # TODO: Why extra function needed? diff --git a/ledsa/tests/UnitTests/test_parser_arguments_run.py b/ledsa/tests/UnitTests/test_parser_arguments_run.py index 5fe7744..8de4a42 100644 --- a/ledsa/tests/UnitTests/test_parser_arguments_run.py +++ b/ledsa/tests/UnitTests/test_parser_arguments_run.py @@ -1,11 +1,14 @@ -"""Channel flags select the requested step 3 mode without a second analysis.""" +"""Channel flags select the requested step 3 mode without a second analysis, and a restart resumes the mode of the +interrupted run.""" import argparse +from pathlib import Path from unittest.mock import Mock import pytest from ledsa.core.parser_arguments_declaration import add_parser_arguments_data_extraction from ledsa.core import parser_arguments_run +from ledsa.data_extraction.step_3_functions import _create_header @pytest.mark.parametrize('flags, channels', [ @@ -56,3 +59,107 @@ def test_restart_resumes_the_selected_channels_without_step_three(tmp_path, monk extractor.return_value.setup_restart.assert_called_once_with() extractor.return_value.setup_step3.assert_not_called() extractor.return_value.process_image_data.assert_called_once_with() + + +# Header lines of the result files as LEDSA wrote them before the background-corrected sums were added, with fewer +# columns in both modes than the current version +HEADER_WITH_FEWER_COLUMNS = ('# image root = experiment, photo file name = IMG_0001.CR2, channel = 0, time[s] = 0.0\n' + '# id,line,sum_col_value,average_col_value,max_col_value') +FIT_COLUMNS_OF_HEADER = (',led_center_x, led_center_y,x,y,dx,dy,A,alpha,wx,wy,fit_success,fit_fun,fit_nfev,fit_time' + '// all spatial quantities in pixel coordinates\n') + + +@pytest.fixture +def extractor(tmp_path, monkeypatch): + """Mock of the DataExtractor, run in an experiment directory with a config.ini.""" + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.ini').touch() + extractor = Mock() + monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor) + return extractor + + +def run_data_extraction(flags): + parser = add_parser_arguments_data_extraction(argparse.ArgumentParser()) + parser_arguments_run.run_data_extraction_arguments(parser.parse_args(flags)) + + +def step_3_header(channel, img_id, fit_leds): + """Header that step 3 writes into the result file of an image.""" + img_infos = [['', '', '', '0.0']] * img_id + return _create_header(channel, img_id, f'IMG_{img_id:04d}.CR2', img_infos, 'experiment', fit_leds) + + +def write_result_file(channel, img_id, header): + """Write the result file of step 3 of an image with a single LED, all of its values 0.""" + channel_dir = Path('analysis', f'channel{channel}') + channel_dir.mkdir(parents=True, exist_ok=True) + num_columns = header.splitlines()[-1].count(',') + 1 + row = ','.join(['0'] * num_columns) + (channel_dir / f'{img_id}_led_positions.csv').write_text(f'{header}{row}\n', encoding='utf-8') + + +@pytest.mark.parametrize('flags, channels', [ + ([], [0]), + (['-g'], [1]), + (['-rgb'], [0, 1, 2]), +]) +@pytest.mark.parametrize('fit_leds', [True, False]) +def test_restart_resumes_in_the_mode_of_the_interrupted_run(extractor, flags, channels, fit_leds): + for channel in channels: + for img_id in (1, 2): + write_result_file(channel, img_id, step_3_header(channel, img_id, fit_leds)) + + run_data_extraction(['-re'] + flags) + + extractor.assert_called_once_with(build_experiment_infos=False, channels=channels, fit_leds=fit_leds) + extractor.return_value.setup_restart.assert_called_once_with() + extractor.return_value.process_image_data.assert_called_once_with() + + +@pytest.mark.parametrize('fit_leds', [True, False]) +def test_restart_reads_the_mode_from_the_column_names_not_their_number(extractor, fit_leds): + write_result_file(0, 1, HEADER_WITH_FEWER_COLUMNS + (FIT_COLUMNS_OF_HEADER if fit_leds else '\n')) + + run_data_extraction(['-re']) + + extractor.assert_called_once_with(build_experiment_infos=False, channels=[0], fit_leds=fit_leds) + + +def test_restart_takes_the_mode_from_the_channels_with_result_files(extractor): + # interrupted in the first image, after its red channel was written + write_result_file(0, 1, step_3_header(0, 1, fit_leds=True)) + + run_data_extraction(['-re', '-rgb']) + + extractor.assert_called_once_with(build_experiment_infos=False, channels=[0, 1, 2], fit_leds=True) + + +def test_restart_ignores_an_empty_result_file(extractor): + # killed after the file of image 2 was created, before its buffered contents reached it + write_result_file(0, 1, step_3_header(0, 1, fit_leds=True)) + Path('analysis', 'channel0', '2_led_positions.csv').touch() + + run_data_extraction(['-re']) + + extractor.assert_called_once_with(build_experiment_infos=False, channels=[0], fit_leds=True) + + +def test_restart_stops_if_the_result_files_of_a_channel_disagree(extractor): + write_result_file(0, 1, step_3_header(0, 1, fit_leds=True)) + write_result_file(0, 2, step_3_header(0, 2, fit_leds=False)) + + with pytest.raises(ValueError, match='channel 0 were written both with the LED fits'): + run_data_extraction(['-re']) + + extractor.assert_not_called() + + +def test_restart_stops_if_the_channels_disagree(extractor): + write_result_file(0, 1, step_3_header(0, 1, fit_leds=True)) + write_result_file(1, 1, step_3_header(1, 1, fit_leds=False)) + + with pytest.raises(ValueError, match='resumes all of its channels in one mode'): + run_data_extraction(['-re', '-r', '-g']) + + extractor.assert_not_called() From 63a4830a4481e06392bae86bfc0d91c7f7e3b662 Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 13:13:56 +0100 Subject: [PATCH 4/4] take the images of -re from all of its channels The restart took the images still to analyse from the first of its channels only. An image is analysed for one channel after the other, so an interrupted multi-channel run usually stops after an image was written for some channels but not for the others, and with several cores this happens for several images. The restart skipped them, and the analysis then left them out of the other channels. An image is now analysed again, for all channels of the restart, unless every one of them has its result file. The commented-out exit for restarts of several channels is removed. The images still to analyse are taken from image_infos_analysis.csv in its order, instead of looking up the name of every analysed image in that file, which read it once per image and wrote them in set order. Co-Authored-By: Claude Opus 5.5 --- doc/cli_documentation.rst | 2 +- ledsa/data_extraction/DataExtractor.py | 8 ++-- ledsa/data_extraction/step_3_functions.py | 18 +++---- ledsa/tests/UnitTests/test_restart.py | 57 +++++++++++++++++++++++ 4 files changed, 71 insertions(+), 14 deletions(-) create mode 100644 ledsa/tests/UnitTests/test_restart.py diff --git a/doc/cli_documentation.rst b/doc/cli_documentation.rst index c90d082..8cce105 100644 --- a/doc/cli_documentation.rst +++ b/doc/cli_documentation.rst @@ -37,7 +37,7 @@ Data Extraction - Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). Besides the plain sum, the background-corrected sums ``bgsub_sum_col_val``, ``bgsub2_sum_col_val`` and ``sidebg_sum_col_val`` are written (see ``reference_property`` in the analysis configuration). - Channel flags as for ``-s3`` * - ``-re``, ``--restart`` - - Restart step 3 if it was previously interrupted. Only the images that have not been analysed are taken into account. The restart resumes in the mode of the interrupted run, with the LED fits after ``-s3`` and without them after ``-s3_fast``, as the headers of its result files tell. If there are no result files yet, it runs without the fits. If the result files were written in both modes, it stops, because it cannot tell which mode to resume. + - Restart step 3 if it was previously interrupted. Only the images that lack the result file of one of the selected channels are analysed, in all of these channels. The restart resumes in the mode of the interrupted run, with the LED fits after ``-s3`` and without them after ``-s3_fast``, as the headers of its result files tell. If there are no result files yet, it runs without the fits. If the result files were written in both modes, it stops, because it cannot tell which mode to resume. - The channel flags of the interrupted run, as for ``-s3`` diff --git a/ledsa/data_extraction/DataExtractor.py b/ledsa/data_extraction/DataExtractor.py index fb7aceb..f669fc3 100644 --- a/ledsa/data_extraction/DataExtractor.py +++ b/ledsa/data_extraction/DataExtractor.py @@ -307,9 +307,7 @@ def setup_step3(self) -> None: def setup_restart(self) -> None: """ - Setup a restart in case the data extraction process was interrupted earlier. + Setup a restart in case the data extraction process was interrupted earlier. Every image that misses the + result file of one of the channels is analysed again, for all of them. """ - # if len(self.channels) > 1: #TODO: deactivated for testing - # print('Restart of a run currently only supports one channel. \nExiting...') - # exit(1) - ledsa.data_extraction.step_3_functions.find_and_save_not_analysed_imgs(self.channels[0]) + ledsa.data_extraction.step_3_functions.find_and_save_not_analysed_imgs(self.channels) diff --git a/ledsa/data_extraction/step_3_functions.py b/ledsa/data_extraction/step_3_functions.py index c0ab284..d9e9d86 100644 --- a/ledsa/data_extraction/step_3_functions.py +++ b/ledsa/data_extraction/step_3_functions.py @@ -268,19 +268,21 @@ def create_imgs_to_process_file() -> None: out_file.close() -def find_and_save_not_analysed_imgs(channel: int) -> None: +def find_and_save_not_analysed_imgs(channels: Sequence[int]) -> None: """ - Find and save filenames of images that have not yet been analyzed. + Find and save filenames of images that have not yet been analyzed in all of the channels. - :param channel: Channel to check for analyzed images. - :type channel: int + An image is analysed for one channel after the other, so an interrupted run can leave it with the result files + of only some of its channels. Such an image is analysed again for all of them. + + :param channels: Channels to check for analyzed images. + :type channels: Sequence[int] """ file_path = os.path.join('analysis', 'image_infos_analysis.csv') image_infos = read_table(file_path, dtype='str', delim=',', atleast_2d=True) - all_imgs = image_infos[:, 1] - processed_img_ids = _find_analysed_img_ids(channel) - processed_imgs = np.frompyfunc(get_img_name, 1, 1)(processed_img_ids) - remaining_imgs = set(all_imgs)-set(processed_imgs) + analysed_img_ids = set.intersection(*(set(_find_analysed_img_ids(channel)) for channel in channels)) + remaining_imgs = [img_filename for img_id, img_filename in image_infos[:, :2] + if int(img_id) not in analysed_img_ids] _save_list_of_remaining_imgs_needed_to_be_processed(remaining_imgs) diff --git a/ledsa/tests/UnitTests/test_restart.py b/ledsa/tests/UnitTests/test_restart.py new file mode 100644 index 0000000..4e02d28 --- /dev/null +++ b/ledsa/tests/UnitTests/test_restart.py @@ -0,0 +1,57 @@ +"""A restart of step 3 analyses an image again unless every channel of the restart has its result file.""" +from pathlib import Path + +import pytest + +from ledsa.data_extraction.DataExtractor import DataExtractor + + +def write_image_infos(num_imgs): + analysis_dir = Path('analysis') + analysis_dir.mkdir(exist_ok=True) + lines = ['#ID,Name,Time[s],Experiment_Time[s]'] + lines += [f'{img_id},IMG_{img_id:04d}.CR2,12:00:{img_id:02d}.000,{float(img_id)}' for img_id in + range(1, num_imgs + 1)] + (analysis_dir / 'image_infos_analysis.csv').write_text('\n'.join(lines) + '\n', encoding='utf-8') + + +def write_result_files(channel, img_ids): + """Create the result files of the images, the restart looks at their names only.""" + channel_dir = Path('analysis', f'channel{channel}') + channel_dir.mkdir(parents=True, exist_ok=True) + for img_id in img_ids: + (channel_dir / f'{img_id}_led_positions.csv').touch() + + +def restart(channels): + """Set up the restart of the channels and return the images it analyses.""" + # the configuration that the constructor reads is not needed to find the images + extractor = DataExtractor.__new__(DataExtractor) + extractor.channels = channels + extractor.setup_restart() + return Path('images_to_process.csv').read_text(encoding='utf-8').split() + + +@pytest.mark.parametrize('analysed_img_ids, remaining_img_ids', [ + # interrupted in image 3 after its red channel was written + ({0: [1, 2, 3], 1: [1, 2], 2: [1, 2]}, [3, 4, 5]), + # several cores finish the images out of order and each image one channel after the other + ({0: [1, 2, 4], 1: [1, 4], 2: [1]}, [2, 3, 4, 5]), +]) +def test_restart_analyses_an_image_again_that_misses_one_of_the_channels(tmp_path, monkeypatch, analysed_img_ids, + remaining_img_ids): + monkeypatch.chdir(tmp_path) + write_image_infos(5) + for channel, img_ids in analysed_img_ids.items(): + write_result_files(channel, img_ids) + + assert restart([0, 1, 2]) == [f'IMG_{img_id:04d}.CR2' for img_id in remaining_img_ids] + + +def test_restart_of_one_channel_ignores_the_others(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + write_image_infos(5) + write_result_files(0, [1, 2, 3]) + write_result_files(1, [1]) + + assert restart([0]) == ['IMG_0004.CR2', 'IMG_0005.CR2']