Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions doc/cli_documentation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,13 @@ Data Extraction
- --
* - ``-s3``, ``--step_3`, ``--analyse_photo``
- Step 3: Analyse all images to extract pixel values inside the search areas and fit them to an LED model (SLOW!).
- ``--r`` (Use the red channel, default), ``--g`` (Use the green channel), ``--b`` (Use the blue channel), ``-rgb`` (Run for each channel)
- ``-r`` (Use the red channel, default), ``-g`` (Use the green channel), ``-b`` (Use the blue channel), ``-rgb`` (Run for each channel)
* - ``-s3_fast``, ``--step_3_fast`, ``--analyse_photo_fast``
- Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). Besides the plain sum, the background-corrected sums ``bgsub_sum_col_val``, ``bgsub2_sum_col_val`` and ``sidebg_sum_col_val`` are written (see ``reference_property`` in the analysis configuration).
- --
- Channel flags as for ``-s3``
* - ``-re``, ``--restart``
- Restart step 3 if it was previously interrupted. Only the images that have not been analysed are taken into account.
- --
- Restart step 3 if it was previously interrupted. Only the images that lack the result file of one of the selected channels are analysed, in all of these channels. The restart resumes in the mode of the interrupted run, with the LED fits after ``-s3`` and without them after ``-s3_fast``, as the headers of its result files tell. If there are no result files yet, it runs without the fits. If the result files were written in both modes, it stops, because it cannot tell which mode to resume.
- The channel flags of the interrupted run, as for ``-s3``


Analysis
Expand Down
5 changes: 4 additions & 1 deletion ledsa/core/parser_arguments_declaration.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,10 @@ def add_parser_arguments_data_extraction(parser: argparse.ArgumentParser) -> arg
help='creates the default configuration file. optional arguments are are: img_directory, '
'reference_img, number_of_cores.')
parser.add_argument('-re', '--restart', action='store_true',
help='Restarts step 3 of the analysis after the program was terminated before it finished.')
help='Restarts step 3 of the analysis after the program was terminated before it finished. '
'Give the channel flags of the interrupted run, e.g. -re -rgb. It resumes with the LED '
'fits after -s3 and without them after -s3_fast, as the result files of the interrupted '
'run tell.')
parser.add_argument('-r', '--red', action='store_true',
help='Use the red channel for step3. Default value.')
parser.add_argument('-g', '--green', action='store_true',
Expand Down
10 changes: 7 additions & 3 deletions ledsa/core/parser_arguments_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from ledsa.core.ConfigData import ConfigData
from ledsa.core.file_handling import get_extinction_coefficients_file_path
from ledsa.data_extraction.DataExtractor import DataExtractor
from ledsa.data_extraction.step_3_functions import find_fit_leds_for_restart



Expand Down Expand Up @@ -57,7 +58,8 @@ def run_data_extraction_arguments(args: argparse.Namespace) -> None:
if len(channels) == 0:
channels.append(0)

if args.red or args.green or args.blue or args.rgb and not args.step_3_fast:
# A colour flag on its own runs step 3. With -s3_fast or -re it only selects the channels of that run.
if (args.red or args.green or args.blue or args.rgb) and not (args.step_3_fast or args.restart):
args.step_3 = True

if args.step_1 or args.step_2:
Expand All @@ -78,8 +80,10 @@ def run_data_extraction_arguments(args: argparse.Namespace) -> None:
de.process_image_data()

if args.restart:
# Use the channels determined earlier in the function
de = DataExtractor(build_experiment_infos=False, channels=channels, fit_leds=False)
# Resume in the mode of the interrupted run, which the headers of its result files tell: with the LED fits
# after -s3, without them after -s3_fast. The analysis cannot load a channel with the columns of both modes.
fit_leds = find_fit_leds_for_restart(channels)
de = DataExtractor(build_experiment_infos=False, channels=channels, fit_leds=fit_leds)
de.setup_restart()
de.process_image_data()

Expand Down
8 changes: 3 additions & 5 deletions ledsa/data_extraction/DataExtractor.py
Original file line number Diff line number Diff line change
Expand Up @@ -307,9 +307,7 @@ def setup_step3(self) -> None:

def setup_restart(self) -> None:
"""
Setup a restart in case the data extraction process was interrupted earlier.
Setup a restart in case the data extraction process was interrupted earlier. Every image that misses the
result file of one of the channels is analysed again, for all of them.
"""
# if len(self.channels) > 1: #TODO: deactivated for testing
# print('Restart of a run currently only supports one channel. \nExiting...')
# exit(1)
ledsa.data_extraction.step_3_functions.find_and_save_not_analysed_imgs(self.channels[0])
ledsa.data_extraction.step_3_functions.find_and_save_not_analysed_imgs(self.channels)
118 changes: 104 additions & 14 deletions ledsa/data_extraction/step_3_functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,23 +268,75 @@ def create_imgs_to_process_file() -> None:
out_file.close()


def find_and_save_not_analysed_imgs(channel: int) -> None:
def find_and_save_not_analysed_imgs(channels: Sequence[int]) -> None:
"""
Find and save filenames of images that have not yet been analyzed.
Find and save filenames of images that have not yet been analyzed in all of the channels.

:param channel: Channel to check for analyzed images.
:type channel: int
An image is analysed for one channel after the other, so an interrupted run can leave it with the result files
of only some of its channels. Such an image is analysed again for all of them.

:param channels: Channels to check for analyzed images.
:type channels: Sequence[int]
"""
file_path = os.path.join('analysis', 'image_infos_analysis.csv')
image_infos = read_table(file_path, dtype='str', delim=',', atleast_2d=True)
all_imgs = image_infos[:, 1]
processed_img_ids = _find_analysed_img_ids(channel)
processed_imgs = np.frompyfunc(get_img_name, 1, 1)(processed_img_ids)
remaining_imgs = set(all_imgs)-set(processed_imgs)
analysed_img_ids = set.intersection(*(set(_find_analysed_img_ids(channel)) for channel in channels))
remaining_imgs = [img_filename for img_id, img_filename in image_infos[:, :2]
if int(img_id) not in analysed_img_ids]

_save_list_of_remaining_imgs_needed_to_be_processed(remaining_imgs)


def find_fit_leds_for_restart(channels: Sequence[int]) -> bool:
"""
Find whether the restart of an interrupted step 3 fits the LEDs: if the interrupted run did (-s3), not if it
did not (-s3_fast). The analysis loads all result files of a channel with the same columns, so the resumed
images need the columns of the images analysed before. The mode is read from the headers of the result files,
not from their number of columns, which differs between LEDSA versions. If none of the channels has a result
file yet, the LEDs are not fitted.

:param channels: Channels to be resumed.
:type channels: Sequence[int]
:return: True if the restart fits the LEDs, False if not.
:rtype: bool
:raises ValueError: If the result files of a channel, or those of different channels, were written both with
and without the fits.
"""
fit_leds_of_channels = {}
for channel in channels:
files_by_fit_leds = {True: [], False: []}
for _, file_path in _find_result_files(channel):
has_fits = _result_file_has_fits(file_path)
if has_fits is not None:
files_by_fit_leds[has_fits].append(file_path)
fitted_files, unfitted_files = files_by_fit_leds[True], files_by_fit_leds[False]
if fitted_files and unfitted_files:
raise ValueError(f'The result files of channel {channel} were written both with the LED fits (-s3, '
f'{len(fitted_files)} files, e.g. {fitted_files[0]}) and without them (-s3_fast, '
f'{len(unfitted_files)} files, e.g. {unfitted_files[0]}), so the restart cannot tell '
f'in which mode to resume. Delete the files of the mode you do not want and run -re '
f'again, or run -s3 or -s3_fast for the channel again.')
if fitted_files or unfitted_files:
fit_leds_of_channels[channel] = bool(fitted_files)

if len(set(fit_leds_of_channels.values())) > 1:
modes = ', '.join(f'channel {channel} {"with" if fit_leds else "without"}'
for channel, fit_leds in fit_leds_of_channels.items())
raise ValueError(f'The channels were analysed with and without the LED fits ({modes}), but a restart '
f'resumes all of its channels in one mode. Restart these channels separately, selecting '
f'them with -r, -g or -b.')
fit_leds = True in fit_leds_of_channels.values()
if not fit_leds_of_channels:
print('There are no result files of an interrupted run yet, so step 3 runs without the LED fits, as by '
'-s3_fast.')
elif fit_leds:
print('The result files of the interrupted run have LED fits, so step 3 is resumed with them, as by -s3.')
else:
print('The result files of the interrupted run have no LED fits, so step 3 is resumed without them, as by '
'-s3_fast.')
return fit_leds


def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray, debug: bool, iled: int, img_filename: str, led_array_idx: int, search_areas: np.ndarray, search_area_radius: int, fit_leds: bool = True,
channel_mask: Optional[np.ndarray] = None,
pixel_noise: Optional[np.ndarray] = None,
Expand Down Expand Up @@ -506,20 +558,58 @@ def _save_results_in_file(channel: int, img_data: LEDAnalysisData, img_filename:

def _find_analysed_img_ids(channel):
"""
Find and save filenames of images that have not yet been analyzed.
Find the IDs of the images that have a result file in a channel.

:param channel: Channel to check for analyzed images.
:return: List of IDs of analyzed images.
:rtype: List[int]
"""
processed_imgs = []
return [img_id for img_id, _ in _find_result_files(channel)]


def _find_result_files(channel: int) -> List[Tuple[int, str]]:
"""
Find the result files of step 3 in a channel.

:param channel: Channel whose result files are searched.
:type channel: int
:return: Image ID and path of every result file, sorted by image ID. Empty if the channel has no directory yet.
:rtype: List[Tuple[int, str]]
"""
dir_path = os.path.join('analysis', f'channel{channel}')
directory_content = os.listdir(dir_path)
for file_name in directory_content:
if not os.path.isdir(dir_path):
return []
result_files = []
for file_name in os.listdir(dir_path):
img = re.search(r"(\d+)_led_positions.csv", file_name)
if img is not None:
processed_imgs.append(int(img.group(1)))
return processed_imgs
result_files.append((int(img.group(1)), os.path.join(dir_path, file_name)))
return sorted(result_files)


def _result_file_has_fits(file_path: str) -> Optional[bool]:
"""
Tell from its header whether a result file of step 3 was written with the LED fits.

Only a file written with the fits names their columns in the header line of the column names, see
_create_header. fit_success is one of them in every version of the header.

:param file_path: Path of the result file.
:type file_path: str
:return: True if the file was written with the fits, False if without them, None if it has no header line of
the column names, for example because it is empty.
:rtype: Optional[bool]
"""
# only the column names are of interest, a directory or image name in another encoding must not stop the restart
with open(file_path, 'r', encoding='utf-8', errors='replace') as result_file:
for line in result_file:
if not line.startswith('#'):
break
# the fit columns are followed by a comment after //
column_names = [name.strip() for name in line.lstrip('#').split('//')[0].split(',')]
if column_names[0] == 'id':
return 'fit_success' in column_names
return None


def _save_list_of_remaining_imgs_needed_to_be_processed(remaining_imgs): # TODO: Why extra function needed?
Expand Down
165 changes: 165 additions & 0 deletions ledsa/tests/UnitTests/test_parser_arguments_run.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
"""Channel flags select the requested step 3 mode without a second analysis, and a restart resumes the mode of the
interrupted run."""
import argparse
from pathlib import Path
from unittest.mock import Mock

import pytest

from ledsa.core.parser_arguments_declaration import add_parser_arguments_data_extraction
from ledsa.core import parser_arguments_run
from ledsa.data_extraction.step_3_functions import _create_header


@pytest.mark.parametrize('flags, channels', [
(['-r'], [0]),
(['-g'], [1]),
(['-b'], [2]),
(['-rgb'], [0, 1, 2]),
(['-r', '-b'], [0, 2]),
])
@pytest.mark.parametrize('fast', [False, True])
def test_channel_flags_run_step_three_once(tmp_path, monkeypatch, flags, channels, fast):
monkeypatch.chdir(tmp_path)
(tmp_path / 'config.ini').touch()
extractor = Mock()
monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor)
parser = add_parser_arguments_data_extraction(argparse.ArgumentParser())
args = parser.parse_args(flags + (['-s3_fast'] if fast else []))

parser_arguments_run.run_data_extraction_arguments(args)

kwargs = {'build_experiment_infos': True, 'channels': channels}
if fast:
kwargs['fit_leds'] = False
extractor.assert_called_once_with(**kwargs)
extractor.return_value.setup_step3.assert_called_once_with()
extractor.return_value.process_image_data.assert_called_once_with()


@pytest.mark.parametrize('flags, channels', [
([], [0]),
(['-r'], [0]),
(['-g'], [1]),
(['-b'], [2]),
(['-rgb'], [0, 1, 2]),
(['-g', '-b'], [1, 2]),
])
def test_restart_resumes_the_selected_channels_without_step_three(tmp_path, monkeypatch, flags, channels):
monkeypatch.chdir(tmp_path)
(tmp_path / 'config.ini').touch()
extractor = Mock()
monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor)
parser = add_parser_arguments_data_extraction(argparse.ArgumentParser())
args = parser.parse_args(['-re'] + flags)

parser_arguments_run.run_data_extraction_arguments(args)

extractor.assert_called_once_with(build_experiment_infos=False, channels=channels, fit_leds=False)
extractor.return_value.setup_restart.assert_called_once_with()
extractor.return_value.setup_step3.assert_not_called()
extractor.return_value.process_image_data.assert_called_once_with()


# Header lines of the result files as LEDSA wrote them before the background-corrected sums were added, with fewer
# columns in both modes than the current version
HEADER_WITH_FEWER_COLUMNS = ('# image root = experiment, photo file name = IMG_0001.CR2, channel = 0, time[s] = 0.0\n'
'# id,line,sum_col_value,average_col_value,max_col_value')
FIT_COLUMNS_OF_HEADER = (',led_center_x, led_center_y,x,y,dx,dy,A,alpha,wx,wy,fit_success,fit_fun,fit_nfev,fit_time'
'// all spatial quantities in pixel coordinates\n')


@pytest.fixture
def extractor(tmp_path, monkeypatch):
"""Mock of the DataExtractor, run in an experiment directory with a config.ini."""
monkeypatch.chdir(tmp_path)
(tmp_path / 'config.ini').touch()
extractor = Mock()
monkeypatch.setattr(parser_arguments_run, 'DataExtractor', extractor)
return extractor


def run_data_extraction(flags):
parser = add_parser_arguments_data_extraction(argparse.ArgumentParser())
parser_arguments_run.run_data_extraction_arguments(parser.parse_args(flags))


def step_3_header(channel, img_id, fit_leds):
"""Header that step 3 writes into the result file of an image."""
img_infos = [['', '', '', '0.0']] * img_id
return _create_header(channel, img_id, f'IMG_{img_id:04d}.CR2', img_infos, 'experiment', fit_leds)


def write_result_file(channel, img_id, header):
"""Write the result file of step 3 of an image with a single LED, all of its values 0."""
channel_dir = Path('analysis', f'channel{channel}')
channel_dir.mkdir(parents=True, exist_ok=True)
num_columns = header.splitlines()[-1].count(',') + 1
row = ','.join(['0'] * num_columns)
(channel_dir / f'{img_id}_led_positions.csv').write_text(f'{header}{row}\n', encoding='utf-8')


@pytest.mark.parametrize('flags, channels', [
([], [0]),
(['-g'], [1]),
(['-rgb'], [0, 1, 2]),
])
@pytest.mark.parametrize('fit_leds', [True, False])
def test_restart_resumes_in_the_mode_of_the_interrupted_run(extractor, flags, channels, fit_leds):
for channel in channels:
for img_id in (1, 2):
write_result_file(channel, img_id, step_3_header(channel, img_id, fit_leds))

run_data_extraction(['-re'] + flags)

extractor.assert_called_once_with(build_experiment_infos=False, channels=channels, fit_leds=fit_leds)
extractor.return_value.setup_restart.assert_called_once_with()
extractor.return_value.process_image_data.assert_called_once_with()


@pytest.mark.parametrize('fit_leds', [True, False])
def test_restart_reads_the_mode_from_the_column_names_not_their_number(extractor, fit_leds):
write_result_file(0, 1, HEADER_WITH_FEWER_COLUMNS + (FIT_COLUMNS_OF_HEADER if fit_leds else '\n'))

run_data_extraction(['-re'])

extractor.assert_called_once_with(build_experiment_infos=False, channels=[0], fit_leds=fit_leds)


def test_restart_takes_the_mode_from_the_channels_with_result_files(extractor):
# interrupted in the first image, after its red channel was written
write_result_file(0, 1, step_3_header(0, 1, fit_leds=True))

run_data_extraction(['-re', '-rgb'])

extractor.assert_called_once_with(build_experiment_infos=False, channels=[0, 1, 2], fit_leds=True)


def test_restart_ignores_an_empty_result_file(extractor):
# killed after the file of image 2 was created, before its buffered contents reached it
write_result_file(0, 1, step_3_header(0, 1, fit_leds=True))
Path('analysis', 'channel0', '2_led_positions.csv').touch()

run_data_extraction(['-re'])

extractor.assert_called_once_with(build_experiment_infos=False, channels=[0], fit_leds=True)


def test_restart_stops_if_the_result_files_of_a_channel_disagree(extractor):
write_result_file(0, 1, step_3_header(0, 1, fit_leds=True))
write_result_file(0, 2, step_3_header(0, 2, fit_leds=False))

with pytest.raises(ValueError, match='channel 0 were written both with the LED fits'):
run_data_extraction(['-re'])

extractor.assert_not_called()


def test_restart_stops_if_the_channels_disagree(extractor):
write_result_file(0, 1, step_3_header(0, 1, fit_leds=True))
write_result_file(1, 1, step_3_header(1, 1, fit_leds=False))

with pytest.raises(ValueError, match='resumes all of its channels in one mode'):
run_data_extraction(['-re', '-r', '-g'])

extractor.assert_not_called()
Loading
Loading