From 9cf52b3d47808046db1788e17b0d3bbdd817a383 Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 11:57:03 +0100 Subject: [PATCH 1/6] name the config options a release has removed when a config sets them A config file keeps the options of the version that wrote it, and the parser reads every option a file sets whether or not LEDSA uses it. An option a release removes would therefore stop taking effect without a word, and the result of an analysis would change unnoticed: a config_analysis.ini of v0.12.0 sets snr_full_weight = 10.0, which v0.13.0 no longer has. report_removed_options names every such option once per process, with what became of it. config.ini and config_analysis.ini each get a REMOVED_OPTIONS registry, empty here; the commits that remove options add them, so that reverting one of those commits takes its entries with it. Co-Authored-By: Claude Opus 5.5 --- ledsa/analysis/ConfigDataAnalysis.py | 8 +- ledsa/core/ConfigData.py | 8 +- ledsa/core/config_file.py | 36 +++++++++ .../UnitTests/test_removed_config_options.py | 80 +++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 ledsa/tests/UnitTests/test_removed_config_options.py diff --git a/ledsa/analysis/ConfigDataAnalysis.py b/ledsa/analysis/ConfigDataAnalysis.py index c787d2f..88ebaa9 100644 --- a/ledsa/analysis/ConfigDataAnalysis.py +++ b/ledsa/analysis/ConfigDataAnalysis.py @@ -2,11 +2,16 @@ from typing import Iterable, List, Optional, Union from ledsa.analysis.temperature_correction import TEMPERATURE_CORRECTION_CHANNELS -from ledsa.core.config_file import save_config +from ledsa.core.config_file import report_removed_options, save_config from ledsa.analysis.ExtinctionCoefficients import DEFAULT_SNR_FULL_WEIGHT DEFAULT_REFERENCE_PROPERTY = 'sum_col_val' +# Options of config_analysis.ini that a release of LEDSA has removed, with what became of them. A +# config written by an older version still sets them, and loading it names them. +REMOVED_OPTIONS = { +} + def parse_reference_properties(value: str) -> List[str]: """ @@ -205,6 +210,7 @@ def load(self) -> None: 'config_analysis.ini not found in working directory! Please create it with argument "--config_analysis".') exit(1) print('config_analysis.ini loaded!') + report_removed_options(self, 'config_analysis.ini', REMOVED_OPTIONS) def save(self) -> None: """ diff --git a/ledsa/core/ConfigData.py b/ledsa/core/ConfigData.py index b889c3e..8fa8162 100644 --- a/ledsa/core/ConfigData.py +++ b/ledsa/core/ConfigData.py @@ -5,9 +5,14 @@ import numpy as np -from ledsa.core.config_file import save_config +from ledsa.core.config_file import report_removed_options, save_config from ledsa.core.image_reading import AUTO_ROTATION, get_exif_datetime +# Options of config.ini that a release of LEDSA has removed, with what became of them. A config +# written by an older version still sets them, and loading it names them. +REMOVED_OPTIONS = { +} + class ConfigData(cp.ConfigParser): """ @@ -221,6 +226,7 @@ def load(self) -> None: except FileNotFoundError: print('config.ini not found in working directory! Please create it with argument "--config".') print('config.ini loaded') + report_removed_options(self, 'config.ini', REMOVED_OPTIONS) def save(self) -> None: """ diff --git a/ledsa/core/config_file.py b/ledsa/core/config_file.py index cc81a3c..3f38def 100644 --- a/ledsa/core/config_file.py +++ b/ledsa/core/config_file.py @@ -9,6 +9,9 @@ only the values of the options are replaced with the ones the parser holds now. Options and sections the parser has gained in the meantime are appended, options it no longer holds are dropped. + +:func:`report_removed_options` names the options of a loaded file that a release of LEDSA has +removed, which a config written by an older version still sets. """ import os @@ -16,6 +19,39 @@ COMMENT_PREFIXES = ('#', ';') +# The removed options already named in this process, as (file name, option) pairs: a run loads +# its config files several times, and the note is needed once. +_reported_removed_options = set() + + +def report_removed_options(parser, file_name: str, removed_options: Dict[str, str]) -> List[str]: + """Name the options of a loaded config file that LEDSA no longer has. + + A config file keeps the options of the version it was written with, and the parser reads + every option the file sets, whether or not LEDSA uses it. An option a release has removed + would therefore stop taking effect without a word, and with it the result of an analysis + would change unnoticed. Each such option is named once per process, with what became of it. + + :param parser: The loaded configuration. + :type parser: configparser.RawConfigParser + :param file_name: Name of the config file, used in the note. + :type file_name: str + :param removed_options: What became of every removed option, by option name. + :type removed_options: Dict[str, str] + :return: The removed options the file still sets, in alphabetical order. + :rtype: List[str] + """ + set_options = set(parser.defaults()) + for section in parser.sections(): + set_options.update(parser[section]) + found = sorted(name for name in removed_options if parser.optionxform(name) in set_options) + for name in found: + if (file_name, name) not in _reported_removed_options: + _reported_removed_options.add((file_name, name)) + print(f'Note: {file_name} sets {name}, which is ignored. {removed_options[name]} ' + f'Delete the option to silence this note.') + return found + def save_config(parser, file_path: str) -> None: """Write *parser* to *file_path*, keeping the comments and the layout of an existing file. diff --git a/ledsa/tests/UnitTests/test_removed_config_options.py b/ledsa/tests/UnitTests/test_removed_config_options.py new file mode 100644 index 0000000..a85d7e8 --- /dev/null +++ b/ledsa/tests/UnitTests/test_removed_config_options.py @@ -0,0 +1,80 @@ +"""Unit tests for the note on config options that LEDSA no longer has. + +A config file written by an older version keeps setting the options a release has removed. The +parser reads them without complaint and LEDSA ignores them, so without the note an analysis would +change without the user noticing. +""" +import configparser +import importlib + +import pytest + +from ledsa.analysis.ConfigDataAnalysis import ConfigDataAnalysis +from ledsa.core import config_file +from ledsa.core.ConfigData import ConfigData + +# the modules, not the classes of the same name +config_data_module = importlib.import_module('ledsa.core.ConfigData') +config_data_analysis_module = importlib.import_module('ledsa.analysis.ConfigDataAnalysis') + + +@pytest.fixture(autouse=True) +def fresh_process(monkeypatch): + """Every test starts like a new process, with no option named yet.""" + monkeypatch.setattr(config_file, '_reported_removed_options', set()) + + +def parser_of(text: str) -> configparser.ConfigParser: + parser = configparser.ConfigParser(allow_no_value=True) + parser.read_string(text) + return parser + + +class TestReportRemovedOptions: + def test_an_option_the_file_still_sets_is_named_with_what_became_of_it(self, capsys): + parser = parser_of('[analyse_photo]\n old_option = 3\n num_skip_leds = 0\n') + found = config_file.report_removed_options(parser, 'config.ini', {'old_option': 'It is gone.'}) + assert found == ['old_option'] + assert 'config.ini sets old_option, which is ignored. It is gone.' in capsys.readouterr().out + + def test_an_option_of_the_default_section_is_found(self): + parser = parser_of('[DEFAULT]\nold_option = 3\n') + assert config_file.report_removed_options(parser, 'x.ini', {'old_option': 'Gone.'}) == ['old_option'] + + def test_a_file_without_removed_options_gives_no_note(self, capsys): + parser = parser_of('[analyse_photo]\nnum_skip_leds = 0\n') + assert config_file.report_removed_options(parser, 'config.ini', {'old_option': 'Gone.'}) == [] + assert capsys.readouterr().out == '' + + def test_the_note_is_given_once_per_process(self, capsys): + parser = parser_of('[DEFAULT]\nold_option = 3\n') + config_file.report_removed_options(parser, 'x.ini', {'old_option': 'Gone.'}) + config_file.report_removed_options(parser, 'x.ini', {'old_option': 'Gone.'}) + assert capsys.readouterr().out.count('old_option') == 1 + + +class TestLoadingAConfigFile: + def test_config_ini_names_its_removed_options(self, tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.setitem(config_data_module.REMOVED_OPTIONS, 'old_option', 'Gone.') + (tmp_path / 'config.ini').write_text('[analyse_photo]\n old_option = 3\n', encoding='utf-8') + ConfigData() + assert 'config.ini sets old_option, which is ignored. Gone.' in capsys.readouterr().out + + def test_config_analysis_ini_names_its_removed_options(self, tmp_path, monkeypatch, capsys): + monkeypatch.chdir(tmp_path) + monkeypatch.setitem(config_data_analysis_module.REMOVED_OPTIONS, 'old_option', 'Gone.') + (tmp_path / 'config_analysis.ini').write_text('[DEFAULT]\n old_option = 3\n', encoding='utf-8') + ConfigDataAnalysis() + assert 'config_analysis.ini sets old_option, which is ignored. Gone.' in capsys.readouterr().out + + @pytest.mark.parametrize('module, file_name', [(config_data_module, 'config.ini'), + (config_data_analysis_module, 'config_analysis.ini')]) + def test_every_removed_option_is_named(self, module, file_name, capsys): + """The options the releases have removed are all found where the old templates wrote them.""" + if not module.REMOVED_OPTIONS: + pytest.skip(f'no option of {file_name} has been removed') + section = 'analyse_photo' if file_name == 'config.ini' else 'DEFAULT' + parser = parser_of(f'[{section}]\n' + ''.join(f' {name} = 1\n' for name in module.REMOVED_OPTIONS)) + assert config_file.report_removed_options(parser, file_name, module.REMOVED_OPTIONS) == \ + sorted(module.REMOVED_OPTIONS) From cae109c2bddda5f0f3bf631772a6714a1df47d6f Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 11:59:42 +0100 Subject: [PATCH 2/6] remove the moving average of the reference property num_moving_avg_images smoothed the reference property of every LED with a centred moving average over neighbouring images. It came with the SNR weighting and has not been validated on the data of an experiment, so v0.13.0 leaves it out; develop keeps it. average_images with n_summarize, which summarizes consecutive images into one, stays. A config_analysis.ini that still sets num_moving_avg_images is told that the option is ignored. Reverting this commit brings the option back as it was. Co-Authored-By: Claude Opus 5.5 --- ledsa/analysis/ConfigDataAnalysis.py | 13 ++-- ledsa/analysis/ExperimentData.py | 6 -- ledsa/analysis/ExtinctionCoefficients.py | 48 +-------------- .../analysis/ExtinctionCoefficientsLinear.py | 8 +-- ledsa/core/parser_arguments_run.py | 3 +- ledsa/tests/UnitTests/test_led_weighting.py | 59 ------------------- .../UnitTests/test_sideband_background.py | 3 +- 7 files changed, 10 insertions(+), 130 deletions(-) diff --git a/ledsa/analysis/ConfigDataAnalysis.py b/ledsa/analysis/ConfigDataAnalysis.py index 88ebaa9..e97a933 100644 --- a/ledsa/analysis/ConfigDataAnalysis.py +++ b/ledsa/analysis/ConfigDataAnalysis.py @@ -10,6 +10,9 @@ # Options of config_analysis.ini that a release of LEDSA has removed, with what became of them. A # config written by an older version still sets them, and loading it names them. REMOVED_OPTIONS = { + 'num_moving_avg_images': 'LEDSA 0.13.0 removed the moving average of the reference property until ' + 'it is validated; it is kept on the develop branch of the repository. ' + 'average_images with n_summarize averages consecutive images instead.', } @@ -65,8 +68,7 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d average_images=False, n_summarize=2, solver='linear', weighting_preference=-6e-3, weighting_curvature=1e-2, num_iterations=1000, lambda_reg=1e-3, snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, - min_layer_weight=0.5, - num_moving_avg_images=1, reduction_files=None): + min_layer_weight=0.5, reduction_files=None): """ :param load_config_file: Determines whether to load the config file on initialization. Defaults to True. :type load_config_file: bool @@ -116,10 +118,6 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d :param min_layer_weight: Linear solver with snr_full_weight only: a layer whose path length weighted mean LED weight stays below this value is set to NaN. Defaults to 0.5. :type min_layer_weight: float - :param num_moving_avg_images: Linear solver only: number of consecutive images over which the - reference property is smoothed by a centred moving average, every image keeping its own time. - 1 leaves it as it is and is the default. Cannot be combined with average_images. - :type num_moving_avg_images: int :param reduction_files: Path of the reduction factor file of the temperature correction per camera channel. Channels without an entry are analysed without the correction, which is the default for every channel. @@ -160,9 +158,6 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d self['DEFAULT'][' snr_full_weight'] = str(snr_full_weight) self.set('DEFAULT', ' # Layers whose mean LED weight stays below this value become NaN') self['DEFAULT'][' min_layer_weight'] = str(min_layer_weight) - self.set('DEFAULT', ' # Images for a centred moving average of the reference property, 1 is off.') - self.set('DEFAULT', ' # Every image keeps its own time, unlike average_images with n_summarize.') - self['DEFAULT'][' num_moving_avg_images'] = str(num_moving_avg_images) self['experiment_geometry'] = {} self.set('experiment_geometry', '# Global X Y Z position of the camera ') diff --git a/ledsa/analysis/ExperimentData.py b/ledsa/analysis/ExperimentData.py index 88a1302..8b7b110 100644 --- a/ledsa/analysis/ExperimentData.py +++ b/ledsa/analysis/ExperimentData.py @@ -45,9 +45,6 @@ class ExperimentData: :ivar min_layer_weight: Only with snr_full_weight: a layer whose path length weighted mean LED weight stays below this value is set to NaN. :type min_layer_weight: float - :ivar num_moving_avg_images: Number of consecutive images for a centred moving average of the reference - property. 1 leaves it as it is. - :type num_moving_avg_images: int :ivar average_images: Intensities are computed as an average from consecutive images. :type average_images: bool :ivar n_summarize: Number of consecutive images summarized into one when averaging. @@ -80,7 +77,6 @@ def __init__(self, load_config_file=True): self.lambda_reg = None self.snr_full_weight = DEFAULT_SNR_FULL_WEIGHT self.min_layer_weight = 0.5 - self.num_moving_avg_images = 1 self.reference_properties = None self.reduction_files = {} self.merge_led_arrays = None @@ -133,8 +129,6 @@ def load_config_parameters(self) -> None: self.snr_full_weight = None if entry.lower() == 'none' else float(entry) if config_analysis.has_option('DEFAULT', 'min_layer_weight'): self.min_layer_weight = float(config_analysis['DEFAULT']['min_layer_weight']) - if config_analysis.has_option('DEFAULT', 'num_moving_avg_images'): - self.num_moving_avg_images = int(config_analysis['DEFAULT']['num_moving_avg_images']) self.reference_properties = config_analysis.get_reference_properties() self.reduction_files = {channel: config_analysis.get_reduction_file(channel) for channel in (self.channels or [])} diff --git a/ledsa/analysis/ExtinctionCoefficients.py b/ledsa/analysis/ExtinctionCoefficients.py index a12b215..655b460 100644 --- a/ledsa/analysis/ExtinctionCoefficients.py +++ b/ledsa/analysis/ExtinctionCoefficients.py @@ -65,8 +65,7 @@ class ExtinctionCoefficients(ABC): def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, ref_img_indices=None, average_images=False, n_summarize=NUM_IMGS_PER_AVERAGE, - temperature_correction=None, snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, - num_moving_avg_images=1): + temperature_correction=None, snr_full_weight=DEFAULT_SNR_FULL_WEIGHT): """ :param experiment: Object representing the experimental setup. :type experiment: Experiment @@ -89,19 +88,7 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 data; where either is missing it is switched off with a note, so that the default does not stop an analysis of data that cannot carry it. :type snr_full_weight: float or None - :param num_moving_avg_images: Number of consecutive images over which the reference property is - smoothed by a centred moving average before the normalisation. 1 leaves it as it is. Unlike - average_images this keeps every image with its own time and does not renumber the images. - :type num_moving_avg_images: int - :raises ValueError: If snr_full_weight is given for a reference property without a noise column, or - if average_images and num_moving_avg_images are combined. """ - if average_images and int(num_moving_avg_images) > 1: - raise ValueError('average_images and num_moving_avg_images cannot be combined. ' - 'average_images summarizes n_summarize images into one and numbers the ' - 'result from 1 again, while num_moving_avg_images smooths the reference ' - 'property over neighbouring images and keeps every image with its own ' - 'time. Pick the one that suits the time resolution you need.') self.coefficients_per_image_and_layer = [] self.experiment = experiment self.reference_property = reference_property @@ -120,7 +107,6 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 self.solver = None self.lambda_reg = None self.snr_full_weight = snr_full_weight - self.num_moving_avg_images = int(num_moving_avg_images) self.noise_property = None if snr_full_weight is not None: if reference_property in NOISE_COLUMNS: @@ -132,7 +118,7 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 def __str__(self): out = str(self.experiment) + \ - f'Reference_property: {self.reference_property}, Ref_img_indices: {self.ref_img_indices}, Snr_full_weight: {self.snr_full_weight}, Noise_property: {self.noise_property}, Min_layer_weight: {getattr(self, "min_layer_weight", None) if self.noise_property is not None else None}, Num_moving_avg_images: {self.num_moving_avg_images}, Solver: {self.solver}, Lambda_reg: {self.lambda_reg}, Temperature_correction: {self.temperature_correction_info()}, LEDSA {version("ledsa")}\n' + f'Reference_property: {self.reference_property}, Ref_img_indices: {self.ref_img_indices}, Snr_full_weight: {self.snr_full_weight}, Noise_property: {self.noise_property}, Min_layer_weight: {getattr(self, "min_layer_weight", None) if self.noise_property is not None else None}, Solver: {self.solver}, Lambda_reg: {self.lambda_reg}, Temperature_correction: {self.temperature_correction_info()}, LEDSA {version("ledsa")}\n' return out def temperature_correction_info(self) -> str: @@ -278,36 +264,6 @@ def load_img_data(self) -> None: # gapless range. They are kept to pair every computed coefficient with the time of its own image. self.analysed_img_ids = sorted( self.calculated_img_data.index.get_level_values('img_id').unique().tolist()) - if self.num_moving_avg_images > 1: - self.apply_moving_average() - - def apply_moving_average(self) -> None: - """ - Replace the reference property of every LED by its centred moving average over - num_moving_avg_images consecutive analysed images, fewer at the start and the end of the series. - The noise, where it is used, becomes the standard deviation of that mean, so the root of the summed - variances divided by the number of images. - - Every image keeps its own time and its own row, so the series is smoothed rather than decimated: - the time resolution becomes num_moving_avg_images times the image interval while the number of - extinction coefficients stays the same. average_images with n_summarize is the other way round. - """ - n = self.num_moving_avg_images - columns = [self.reference_property] + ([self.noise_property] if self.noise_property else []) - averaged = {} - for column in columns: - wide = self.calculated_img_data[column].unstack('led_id') - if column == self.noise_property: - count = wide.notna().astype(float).rolling(n, center=True, min_periods=1).sum() - wide = np.sqrt((wide ** 2).rolling(n, center=True, min_periods=1).sum()) / count - else: - wide = wide.rolling(n, center=True, min_periods=1).mean() - averaged[column] = wide.stack() - data = self.calculated_img_data.copy() - for column, values in averaged.items(): - data[column] = values.reindex(data.index).to_numpy() - self.calculated_img_data = data - print(f'Reference property smoothed over {n} consecutive images (centred moving average).') def apply_temperature_correction(self) -> None: """ diff --git a/ledsa/analysis/ExtinctionCoefficientsLinear.py b/ledsa/analysis/ExtinctionCoefficientsLinear.py index 8c054a3..ab27513 100644 --- a/ledsa/analysis/ExtinctionCoefficientsLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsLinear.py @@ -20,8 +20,7 @@ class ExtinctionCoefficientsLinear(ExtinctionCoefficients): """ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, ref_img_indices=None, lambda_reg=1e-3, n_summarize=2, temperature_correction=None, - snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, min_layer_weight=0.5, - num_moving_avg_images=1): + snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, min_layer_weight=0.5): """ Initialize the ExtinctionCoefficientsLinear object. @@ -48,13 +47,10 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 :param min_layer_weight: Only with snr_full_weight: a layer is set to NaN if the mean weight of the LEDs crossing it, weighted with their path length in the layer, stays below this value. :type min_layer_weight: float - :param num_moving_avg_images: Number of consecutive images for a centred moving average of the - reference property. - :type num_moving_avg_images: int """ self.min_layer_weight = min_layer_weight super().__init__(experiment, reference_property, num_ref_imgs, ref_img_indices, average_images, - n_summarize, temperature_correction, snr_full_weight, num_moving_avg_images) + n_summarize, temperature_correction, snr_full_weight) self.lambda_reg = lambda_reg self.solver = 'linear' diff --git a/ledsa/core/parser_arguments_run.py b/ledsa/core/parser_arguments_run.py index d2a99d2..b9c67db 100644 --- a/ledsa/core/parser_arguments_run.py +++ b/ledsa/core/parser_arguments_run.py @@ -291,8 +291,7 @@ def extionction_coefficient_calculation(args) -> None: temperature_correction=temperature_corrections.get(channel), lambda_reg=ex_data.lambda_reg, snr_full_weight=ex_data.snr_full_weight, - min_layer_weight=ex_data.min_layer_weight, - num_moving_avg_images=ex_data.num_moving_avg_images) + min_layer_weight=ex_data.min_layer_weight) else: raise ValueError(f"Invalid solver type '{solver}'. Must be 'linear' or 'nonlinear'.") if ex_data.n_cpus > 1: diff --git a/ledsa/tests/UnitTests/test_led_weighting.py b/ledsa/tests/UnitTests/test_led_weighting.py index 1a308a7..acf453f 100644 --- a/ledsa/tests/UnitTests/test_led_weighting.py +++ b/ledsa/tests/UnitTests/test_led_weighting.py @@ -60,19 +60,6 @@ def test_the_header_says_whether_the_weighting_was_used(self): assert 'Min_layer_weight: None' in str(unweighted) -class TestSmoothingVersusAveraging: - def test_the_two_ways_of_averaging_cannot_be_combined(self): - # one summarizes images into fewer rows and renumbers them, the other smooths and keeps the rows - experiment = SimpleNamespace(channel=0, path='.', led_array=0) - with pytest.raises(ValueError, match=r'average_images and num_moving_avg_images cannot be ' - r'combined'): - ExtinctionCoefficientsLinear(experiment, average_images=True, num_moving_avg_images=3) - - def test_smoothing_alone_is_allowed(self): - experiment = SimpleNamespace(channel=0, path='.', led_array=0) - assert ExtinctionCoefficientsLinear(experiment, num_moving_avg_images=3).num_moving_avg_images == 3 - - class TestLedWeights: def test_equal_weights_reproduce_the_unweighted_solution(self): solver = make_solver() @@ -134,51 +121,6 @@ def test_without_the_limit_nothing_is_discarded(self): assert np.isfinite(make_solver().calc_coefficients_of_img(REL_INTENSITIES, self.SNR)).all() -class TestMovingAverage: - NUM_IMAGES, NUM_LEDS = 5, 2 - - def make_coefficients(self, values, noise=None): - coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) - coefficients.reference_property = 'sum_col_val' - coefficients.noise_property = 'noise_sum_col_val' if noise is not None else None - coefficients.num_moving_avg_images = 3 - index = pd.MultiIndex.from_product([range(1, self.NUM_IMAGES + 1), range(self.NUM_LEDS)], - names=['img_id', 'led_id']) - data = {'led_array_id': 0, 'sum_col_val': values} - if noise is not None: - data['noise_sum_col_val'] = noise - coefficients.calculated_img_data = pd.DataFrame(data, index=index) - return coefficients - - def test_every_image_keeps_its_own_row(self): - coefficients = self.make_coefficients(np.arange(self.NUM_IMAGES * self.NUM_LEDS, dtype=float)) - coefficients.apply_moving_average() - assert len(coefficients.calculated_img_data) == self.NUM_IMAGES * self.NUM_LEDS - assert coefficients.calculated_img_data.index.get_level_values('img_id').nunique() == self.NUM_IMAGES - - def test_a_spike_is_smoothed_over_its_neighbours(self): - values = np.zeros(self.NUM_IMAGES * self.NUM_LEDS) - values[2 * self.NUM_LEDS] = 30.0 # led 0 of the middle image - coefficients = self.make_coefficients(values) - coefficients.apply_moving_average() - smoothed = coefficients.calculated_img_data['sum_col_val'].unstack('led_id')[0].to_numpy() - np.testing.assert_allclose(smoothed, [0.0, 10.0, 10.0, 10.0, 0.0]) - - def test_the_ends_average_over_fewer_images(self): - coefficients = self.make_coefficients(np.ones(self.NUM_IMAGES * self.NUM_LEDS)) - coefficients.apply_moving_average() - assert coefficients.calculated_img_data['sum_col_val'].to_numpy() == pytest.approx(1.0) - - def test_the_noise_becomes_the_one_of_the_mean(self): - coefficients = self.make_coefficients(np.ones(self.NUM_IMAGES * self.NUM_LEDS), - noise=np.full(self.NUM_IMAGES * self.NUM_LEDS, 3.0)) - coefficients.apply_moving_average() - noise = coefficients.calculated_img_data['noise_sum_col_val'].unstack('led_id')[0].to_numpy() - # three images in the middle, two at the ends: sqrt(n * sigma^2) / n = sigma / sqrt(n) - np.testing.assert_allclose(noise, [3 / np.sqrt(2), 3 / np.sqrt(3), 3 / np.sqrt(3), - 3 / np.sqrt(3), 3 / np.sqrt(2)]) - - class TestTheWeightingFallsBackAtLoadTime: """Data without a usable noise column switch the weighting off instead of stopping the analysis.""" @@ -198,7 +140,6 @@ def build(noise): coefficients.average_images = False coefficients.reference_property = 'sum_col_val' coefficients.noise_property = 'noise_sum_col_val' - coefficients.num_moving_avg_images = 1 coefficients.ref_img_ids_of_loaded_data = None coefficients.temperature_correction = None coefficients.temperature_corrected = False diff --git a/ledsa/tests/UnitTests/test_sideband_background.py b/ledsa/tests/UnitTests/test_sideband_background.py index 35e3597..86cf05e 100644 --- a/ledsa/tests/UnitTests/test_sideband_background.py +++ b/ledsa/tests/UnitTests/test_sideband_background.py @@ -296,9 +296,8 @@ def coefficients(self, monkeypatch): coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) coefficients.experiment = SimpleNamespace(channel=0, path='.', led_array=0) coefficients.average_images = False - # set in __init__: the property analysed here has no noise column and is not smoothed + # set in __init__: the property analysed here has no noise column coefficients.noise_property = None - coefficients.num_moving_avg_images = 1 return coefficients def test_nan_values_stop_the_analysis(self, coefficients): From bfc5c49b3ca5136f7f0a6375748eb9eab0b28c65 Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 12:06:04 +0100 Subject: [PATCH 3/6] remove the SNR weighting and the read noise of the sums The linear solver weighted every LED with min(1, SNR / snr_full_weight) and set a layer to NaN when the path weighted mean weight of its LEDs stayed below min_layer_weight. The SNR came from noise_sum_col_val, the read noise of the sum, which step 3 estimated from the standard deviation of the optical black pixels. Neither has been validated beyond the Warsaw tests V008 and V009, so v0.13.0 leaves them out; develop keeps them. What stays from the same change is the unclipped data: the black level of every Bayer color is still taken from the optical black, and the linear solver still leaves out an LED whose relative intensity is at or below zero. Without the weighting that rule alone decides, as it already did in the stacked analysis. Its known limitation is documented with the solver: where the transmission reaches the read noise, the LEDs that are left are those whose noise happened to be positive, and the coefficients come out low (14 to 31 % at 1.24 and 1.64 1/m in V008 and V009, where the weighting gave NaN). The read noise goes from the RAW reader as well: _optical_black_mean and read_channel_data_and_mask_from_img return to the names and the results they had before the noise was added, and ImageChannels builds the data and the mask of a channel. Step 3 writes the layouts of v0.11.0 again (10 names, 24 with the fit), so the reader names their columns as before; the v0.12.0 layouts with noise_sum_col_val are still read. config_analysis.ini files that set snr_full_weight or min_layer_weight are told that the options are ignored. test_led_weighting.py goes; the rule for LEDs at or below zero moves to test_extinction_coefficients_linear.py. Co-Authored-By: Claude Opus 5.5 --- ledsa/analysis/ConfigDataAnalysis.py | 25 +-- ledsa/analysis/ExperimentData.py | 16 -- ledsa/analysis/ExtinctionCoefficients.py | 57 +----- .../analysis/ExtinctionCoefficientsLinear.py | 57 ++---- .../ExtinctionCoefficientsNonLinear.py | 3 +- ledsa/core/file_handling.py | 6 +- ledsa/core/image_reading.py | 110 +++++------- ledsa/core/parser_arguments_run.py | 4 +- ledsa/data_extraction/LEDAnalysisData.py | 6 - ledsa/data_extraction/step_3_functions.py | 21 +-- ledsa/tests/UnitTests/test_black_level.py | 71 ++------ .../test_extinction_coefficients_linear.py | 42 +++++ ledsa/tests/UnitTests/test_image_channels.py | 39 ++-- .../tests/UnitTests/test_image_orientation.py | 8 +- ledsa/tests/UnitTests/test_led_weighting.py | 167 ------------------ .../UnitTests/test_sideband_background.py | 15 +- .../UnitTests/test_stacked_search_areas.py | 12 +- 17 files changed, 167 insertions(+), 492 deletions(-) create mode 100644 ledsa/tests/UnitTests/test_extinction_coefficients_linear.py delete mode 100644 ledsa/tests/UnitTests/test_led_weighting.py diff --git a/ledsa/analysis/ConfigDataAnalysis.py b/ledsa/analysis/ConfigDataAnalysis.py index e97a933..f19d6e3 100644 --- a/ledsa/analysis/ConfigDataAnalysis.py +++ b/ledsa/analysis/ConfigDataAnalysis.py @@ -3,13 +3,18 @@ from ledsa.analysis.temperature_correction import TEMPERATURE_CORRECTION_CHANNELS from ledsa.core.config_file import report_removed_options, save_config -from ledsa.analysis.ExtinctionCoefficients import DEFAULT_SNR_FULL_WEIGHT DEFAULT_REFERENCE_PROPERTY = 'sum_col_val' # Options of config_analysis.ini that a release of LEDSA has removed, with what became of them. A # config written by an older version still sets them, and loading it names them. REMOVED_OPTIONS = { + 'snr_full_weight': 'LEDSA 0.13.0 removed the weighting of the LEDs by their signal-to-noise ratio ' + 'until it is validated; it is kept on the develop branch of the repository. The ' + 'linear solver weights all LEDs equally and leaves out those at or below zero.', + 'min_layer_weight': 'It belonged to the weighting of the LEDs by their signal-to-noise ratio, which ' + 'LEDSA 0.13.0 removed until it is validated; it is kept on the develop branch of ' + 'the repository.', 'num_moving_avg_images': 'LEDSA 0.13.0 removed the moving average of the reference property until ' 'it is validated; it is kept on the develop branch of the repository. ' 'average_images with n_summarize averages consecutive images instead.', @@ -67,8 +72,7 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d reference_property='sum_col_val', average_images=False, n_summarize=2, solver='linear', weighting_preference=-6e-3, weighting_curvature=1e-2, - num_iterations=1000, lambda_reg=1e-3, snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, - min_layer_weight=0.5, reduction_files=None): + num_iterations=1000, lambda_reg=1e-3, reduction_files=None): """ :param load_config_file: Determines whether to load the config file on initialization. Defaults to True. :type load_config_file: bool @@ -109,15 +113,6 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d :type num_iterations: int :param lambda_reg: Regularization parameter for the linear solver. Defaults to 1e-3. :type lambda_reg: float - :param snr_full_weight: Linear solver only: signal-to-noise ratio from which on an LED gets the - full weight. Below it the weight falls linearly to zero at a signal of zero. None weights all - LEDs equally. The weighting needs a reference property with a noise column, which only - sum_col_val has, and noise values in the data; where either is missing it is switched off with - a note. - :type snr_full_weight: float or None - :param min_layer_weight: Linear solver with snr_full_weight only: a layer whose path length - weighted mean LED weight stays below this value is set to NaN. Defaults to 0.5. - :type min_layer_weight: float :param reduction_files: Path of the reduction factor file of the temperature correction per camera channel. Channels without an entry are analysed without the correction, which is the default for every channel. @@ -152,12 +147,6 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d self['DEFAULT'][' num_iterations'] = str(num_iterations) self.set('DEFAULT', ' # Options for linear solver (ignored when nonlinear solver is used) ') self['DEFAULT'][' lambda_reg'] = str(lambda_reg) - self.set('DEFAULT', ' # SNR from which on an LED gets the full weight, None for equal weights.') - self.set('DEFAULT', ' # Needs a reference property with a noise column (sum_col_val) and') - self.set('DEFAULT', ' # noise values in the data, otherwise the LEDs are weighted equally.') - self['DEFAULT'][' snr_full_weight'] = str(snr_full_weight) - self.set('DEFAULT', ' # Layers whose mean LED weight stays below this value become NaN') - self['DEFAULT'][' min_layer_weight'] = str(min_layer_weight) self['experiment_geometry'] = {} self.set('experiment_geometry', '# Global X Y Z position of the camera ') diff --git a/ledsa/analysis/ExperimentData.py b/ledsa/analysis/ExperimentData.py index 8b7b110..7a47836 100644 --- a/ledsa/analysis/ExperimentData.py +++ b/ledsa/analysis/ExperimentData.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from ledsa.analysis.ConfigDataAnalysis import ConfigDataAnalysis -from ledsa.analysis.ExtinctionCoefficients import DEFAULT_SNR_FULL_WEIGHT from ledsa.analysis.Experiment import Camera, Layers # Todo: Import init_function for request of missing data from ledsa.core.ConfigData import ConfigData @@ -38,13 +37,6 @@ class ExperimentData: :type num_ref_images: int :ivar ref_img_indices: Indices of reference images to use. If None, use num_ref_imgs. :type ref_img_indices: list[int] or None - :ivar snr_full_weight: Signal-to-noise ratio from which on an LED gets the full weight in the linear - solver. None weights all LEDs equally. Configurations written before the option existed do not - hold it and therefore get the default. - :type snr_full_weight: float or None - :ivar min_layer_weight: Only with snr_full_weight: a layer whose path length weighted mean LED weight - stays below this value is set to NaN. - :type min_layer_weight: float :ivar average_images: Intensities are computed as an average from consecutive images. :type average_images: bool :ivar n_summarize: Number of consecutive images summarized into one when averaging. @@ -75,8 +67,6 @@ def __init__(self, load_config_file=True): self.average_images = False self.n_summarize = 2 self.lambda_reg = None - self.snr_full_weight = DEFAULT_SNR_FULL_WEIGHT - self.min_layer_weight = 0.5 self.reference_properties = None self.reduction_files = {} self.merge_led_arrays = None @@ -123,12 +113,6 @@ def load_config_parameters(self) -> None: self.num_iterations = int(config_analysis['DEFAULT']['num_iterations']) elif self.solver == 'linear': self.lambda_reg = float(config_analysis['DEFAULT']['lambda_reg']) - if config_analysis.has_option('DEFAULT', 'snr_full_weight'): - # an explicit None switches the weighting off, which the default no longer does - entry = config_analysis['DEFAULT']['snr_full_weight'].strip() - self.snr_full_weight = None if entry.lower() == 'none' else float(entry) - if config_analysis.has_option('DEFAULT', 'min_layer_weight'): - self.min_layer_weight = float(config_analysis['DEFAULT']['min_layer_weight']) self.reference_properties = config_analysis.get_reference_properties() self.reduction_files = {channel: config_analysis.get_reduction_file(channel) for channel in (self.channels or [])} diff --git a/ledsa/analysis/ExtinctionCoefficients.py b/ledsa/analysis/ExtinctionCoefficients.py index 655b460..14996dc 100644 --- a/ledsa/analysis/ExtinctionCoefficients.py +++ b/ledsa/analysis/ExtinctionCoefficients.py @@ -8,7 +8,7 @@ from ledsa.analysis.Experiment import Experiment, Layers, Camera from ledsa.analysis.temperature_correction import TemperatureCorrection -from ledsa.core.file_handling import (NOISE_COLUMNS, NUM_IMGS_PER_AVERAGE, avg_img_ids_of_ref_imgs, +from ledsa.core.file_handling import (NUM_IMGS_PER_AVERAGE, avg_img_ids_of_ref_imgs, create_analysis_infos_avg, extend_hdf, get_extinction_coefficients_file_path, read_analysis_img_ids, read_experiment_times, @@ -18,12 +18,6 @@ from importlib.metadata import version -# Signal-to-noise ratio from which on an LED gets the full weight. Measured on the Warsaw tests V008 and -# V009: at the fog stages that still carry signal the weighting changes the extinction coefficients by -# less than a percent, and where the transmission approaches the read noise it is the only thing that -# separates "no signal" from "small signal". -DEFAULT_SNR_FULL_WEIGHT = 10.0 - class ExtinctionCoefficients(ABC): """ Parent class for the calculation of the Extinction Coefficients. @@ -65,7 +59,7 @@ class ExtinctionCoefficients(ABC): def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, ref_img_indices=None, average_images=False, n_summarize=NUM_IMGS_PER_AVERAGE, - temperature_correction=None, snr_full_weight=DEFAULT_SNR_FULL_WEIGHT): + temperature_correction=None): """ :param experiment: Object representing the experimental setup. :type experiment: Experiment @@ -82,12 +76,6 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 :param temperature_correction: Reduction factors of the LED intensities of the channel. None leaves the intensities as they were extracted. :type temperature_correction: TemperatureCorrection or None - :param snr_full_weight: Signal-to-noise ratio from which on an LED gets the full weight in the - solver. Below it the weight falls linearly to zero at a signal of zero. None weights all LEDs - equally. The weighting needs a reference property with a noise column and noise values in the - data; where either is missing it is switched off with a note, so that the default does not - stop an analysis of data that cannot carry it. - :type snr_full_weight: float or None """ self.coefficients_per_image_and_layer = [] self.experiment = experiment @@ -106,19 +94,10 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 self.temperature_corrected = False self.solver = None self.lambda_reg = None - self.snr_full_weight = snr_full_weight - self.noise_property = None - if snr_full_weight is not None: - if reference_property in NOISE_COLUMNS: - self.noise_property = NOISE_COLUMNS[reference_property] - else: - print(f'The LEDs are weighted equally: {reference_property} has no noise column, so the ' - f'signal-to-noise ratio of an LED is unknown. Columns with one: ' - f'{", ".join(sorted(NOISE_COLUMNS))}.') def __str__(self): out = str(self.experiment) + \ - f'Reference_property: {self.reference_property}, Ref_img_indices: {self.ref_img_indices}, Snr_full_weight: {self.snr_full_weight}, Noise_property: {self.noise_property}, Min_layer_weight: {getattr(self, "min_layer_weight", None) if self.noise_property is not None else None}, Solver: {self.solver}, Lambda_reg: {self.lambda_reg}, Temperature_correction: {self.temperature_correction_info()}, LEDSA {version("ledsa")}\n' + f'Reference_property: {self.reference_property}, Ref_img_indices: {self.ref_img_indices}, Solver: {self.solver}, Lambda_reg: {self.lambda_reg}, Temperature_correction: {self.temperature_correction_info()}, LEDSA {version("ledsa")}\n' return out def temperature_correction_info(self) -> str: @@ -145,11 +124,7 @@ def calc_and_set_coefficients(self) -> None: rel_intensities = single_img_array / self.ref_intensities # Calculate the extinction coefficients depending on child class used - if self.noise_property is None: - sigmas = self.calc_coefficients_of_img(rel_intensities) - else: - snr = single_img_array / single_img_data[self.noise_property].to_numpy() - sigmas = self.calc_coefficients_of_img(rel_intensities, snr) + sigmas = self.calc_coefficients_of_img(rel_intensities) self.coefficients_per_image_and_layer.append(sigmas) def calc_and_set_coefficients_mp(self, cores=4) -> None: @@ -166,12 +141,7 @@ def calc_and_set_coefficients_mp(self, cores=4) -> None: # Calculate the extinction coefficients depending on child class used pool = Pool(processes=cores) - if self.noise_property is None: - sigmas = pool.map(self.calc_coefficients_of_img, rel_intensities) - else: - snr = img_property_array / multiindex_series_to_nparray( - self.calculated_img_data[self.noise_property]) - sigmas = pool.starmap(self.calc_coefficients_of_img, zip(rel_intensities, snr)) + sigmas = pool.map(self.calc_coefficients_of_img, rel_intensities) pool.close() self.coefficients_per_image_and_layer = sigmas @@ -230,16 +200,7 @@ def load_img_data(self) -> None: raise ValueError(f'{self.reference_property} is not among the columns of the binary data of ' f'channel {self.experiment.channel} ' f'({", ".join(map(str, img_data.columns))}). {hint}') - columns = ['led_array_id', self.reference_property] - if self.noise_property is not None: - if self.noise_property not in img_data.columns: - print(f'The LEDs are weighted equally: {self.noise_property} is not among the columns of ' - f'the binary data of channel {self.experiment.channel}. Run step 3 again with a ' - f'LEDSA version that writes it to weight them by their signal-to-noise ratio.') - self.noise_property = None - else: - columns.append(self.noise_property) - img_data_cropped = img_data[columns] + img_data_cropped = img_data[['led_array_id', self.reference_property]] self.calculated_img_data = img_data_cropped[img_data_cropped['led_array_id'] == self.experiment.led_array] if self.calculated_img_data.empty: exit(f"Apparently there are no intensity values for led array {self.experiment.led_array}!") @@ -253,12 +214,6 @@ def load_img_data(self) -> None: f'{self.experiment.led_array} are NaN (led ids {led_ids_text}). For sidebg_sum_col_val ' f'this means that no side band pixels were found for these LEDs, see sideband_distances ' f'in config.ini.') - if self.noise_property is not None and self.calculated_img_data[self.noise_property].isna().all(): - # JPG and PNG images and sensors without an optical black margin have no read noise - print(f'The LEDs are weighted equally: {self.noise_property} is NaN for every LED of led ' - f'array {self.experiment.led_array}, so the images carry no read noise.') - self.calculated_img_data = self.calculated_img_data.drop(columns=[self.noise_property]) - self.noise_property = None # Images without analysis results are missing from the binary data, so the IDs are not necessarily a # gapless range. They are kept to pair every computed coefficient with the time of its own image. diff --git a/ledsa/analysis/ExtinctionCoefficientsLinear.py b/ledsa/analysis/ExtinctionCoefficientsLinear.py index ab27513..0d9229e 100644 --- a/ledsa/analysis/ExtinctionCoefficientsLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsLinear.py @@ -2,7 +2,7 @@ from scipy.optimize import nnls from ledsa.analysis.Experiment import Experiment, Layers, Camera -from ledsa.analysis.ExtinctionCoefficients import DEFAULT_SNR_FULL_WEIGHT, ExtinctionCoefficients +from ledsa.analysis.ExtinctionCoefficients import ExtinctionCoefficients class ExtinctionCoefficientsLinear(ExtinctionCoefficients): """ @@ -19,8 +19,7 @@ class ExtinctionCoefficientsLinear(ExtinctionCoefficients): :vartype solver: str """ - def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, ref_img_indices=None, lambda_reg=1e-3, n_summarize=2, temperature_correction=None, - snr_full_weight=DEFAULT_SNR_FULL_WEIGHT, min_layer_weight=0.5): + def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10, average_images=False, ref_img_indices=None, lambda_reg=1e-3, n_summarize=2, temperature_correction=None): """ Initialize the ExtinctionCoefficientsLinear object. @@ -41,43 +40,29 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 :param temperature_correction: Reduction factors of the LED intensities of the channel. None leaves the intensities as they were extracted. :type temperature_correction: TemperatureCorrection or None - :param snr_full_weight: Signal-to-noise ratio from which on an LED gets the full weight. Below it - the weight falls linearly to zero at a signal of zero. If None, all LEDs are weighted equally. - :type snr_full_weight: float or None - :param min_layer_weight: Only with snr_full_weight: a layer is set to NaN if the mean weight of the - LEDs crossing it, weighted with their path length in the layer, stays below this value. - :type min_layer_weight: float """ - self.min_layer_weight = min_layer_weight super().__init__(experiment, reference_property, num_ref_imgs, ref_img_indices, average_images, - n_summarize, temperature_correction, snr_full_weight) + n_summarize, temperature_correction) self.lambda_reg = lambda_reg self.solver = 'linear' - def calc_coefficients_of_img(self, rel_intensities: np.ndarray, snr: np.ndarray = None) -> np.ndarray: + def calc_coefficients_of_img(self, rel_intensities: np.ndarray) -> np.ndarray: """ Calculate the extinction coefficients for a single image using a linearized approach. This method solves the linear system derived from the Beer-Lambert law using non-negative least squares with Tikhonov regularization. - LEDs whose relative intensity is at or below zero are left out: their signal has vanished in the - noise of the sensor, so they carry no information on the extinction. If fewer than one LED with - full weight is left, the image yields NaN. An LED whose noise is unknown keeps the full weight, - because a missing noise estimate is no reason to drop a measured intensity. + LEDs whose relative intensity is at or below zero, or not finite, are left out: the intensities + are not clipped at zero, so the signal of such an LED has vanished in the noise of the sensor, + and it carries no information on the extinction. If no LED is left, the image yields NaN. - If snr is given, every LED is weighted with min(1, snr / snr_full_weight). The standard deviation - of the optical depth -ln(I_e/I_0) is about 1/snr, so the weight is the inverse standard deviation, - capped so that well measured LEDs keep the weight 1 and the meaning of lambda_reg does not change. - LEDs with a signal at or below zero get the weight 0. If the weights add up to less than one LED - with full weight, no coefficients can be computed and NaN is returned. Layers in which the path - length weighted mean weight of the crossing LEDs stays below min_layer_weight are set to NaN as - well: their coefficients would rest on LEDs near the noise level, where the linearised - Beer-Lambert law is biased. + Known limitation: where the transmission approaches the read noise of the sensor, the LEDs that + are left are those whose noise happened to be positive, so the extinction coefficients come out + too low. On the Warsaw tests V008 and V009 this was 14 to 31 % at 1.24 and 1.64 1/m, where the + transmission of the red LEDs was 0.1 to 0.2 % of I0. :param rel_intensities: Array of relative (normalized) LED intensities (I_e/I_0). :type rel_intensities: np.ndarray - :param snr: Signal-to-noise ratio of every LED, or None for equal weights. - :type snr: np.ndarray or None :return: Array of the computed extinction coefficients (sigmas). :rtype: np.ndarray """ @@ -94,20 +79,11 @@ def calc_coefficients_of_img(self, rel_intensities: np.ndarray, snr: np.ndarray # A relative intensity at or below zero is not a measurement: the LED has vanished in the noise, # and the clamp above turns it into an optical depth of about 23, which would dominate the solve. - # Such an LED is discarded by its weight instead, the same way a vanishing SNR discards it. + # Such an LED is discarded by a weight of 0, which zeroes its row of the system. measured = np.isfinite(rel_intensities) & (np.asarray(rel_intensities) > 0.0) - - # Weights of the LEDs: the inverse standard deviation of the optical depth, capped at 1 - if snr is None: - weights = measured.astype(float) - else: - ratio = np.asarray(snr, dtype=float) / self.snr_full_weight - weights = np.clip(ratio, 0.0, 1.0) - # an LED without a noise estimate is not downweighted, only one with a known small SNR is - weights[~np.isfinite(ratio)] = 1.0 - weights = weights * measured - if weights.sum() < 1.0: + if not measured.any(): return np.full(n_layers, np.nan) + weights = measured.astype(float) # Build finite-difference matrix L for second derivative (curvature penalty) # This matrix approximates the second derivative and is used for regularization @@ -136,11 +112,6 @@ def calc_coefficients_of_img(self, rel_intensities: np.ndarray, snr: np.ndarray sigmas, residuals = nnls(A_aug, b_aug) # sigmas, residuals, rank, s = np.linalg.lstsq(A_aug, b_aug, rcond=None) - if snr is not None and self.min_layer_weight is not None: - path = self.distances_per_led_and_layer - layer_weight = (weights[:, None] * path).sum(axis=0) / np.maximum(path.sum(axis=0), 1e-12) - sigmas = np.where(layer_weight < self.min_layer_weight, np.nan, sigmas) - return sigmas # For investigation diff --git a/ledsa/analysis/ExtinctionCoefficientsNonLinear.py b/ledsa/analysis/ExtinctionCoefficientsNonLinear.py index 76d2fa7..3743bfd 100644 --- a/ledsa/analysis/ExtinctionCoefficientsNonLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsNonLinear.py @@ -67,9 +67,8 @@ def __init__(self, experiment, reference_property='sum_col_val', num_ref_imgs=10 :type temperature_correction: TemperatureCorrection or None """ - # the nonlinear solver has no weighting, so it is switched off whatever the default is super().__init__(experiment, reference_property, num_ref_imgs, ref_img_indices, average_images, - n_summarize, temperature_correction, snr_full_weight=None) + n_summarize, temperature_correction) self.bounds = [(0, 10) for _ in range(self.experiment.layers.amount)] self.weighting_preference = weighting_preference self.weighting_curvature = weighting_curvature diff --git a/ledsa/core/file_handling.py b/ledsa/core/file_handling.py index 4a3c8c0..5356660 100644 --- a/ledsa/core/file_handling.py +++ b/ledsa/core/file_handling.py @@ -700,14 +700,12 @@ def create_binary_data(channel: int, path: str = '.') -> None: BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit, before the optical black len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS): - BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS, # -s3_fast + BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS, # -s3_fast of v0.12.0 len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS + LED_CENTER_COLUMNS - + FIT_COLUMNS, # -s3 with fit + + FIT_COLUMNS, # -s3 with fit of v0.12.0 } -# noise column of a reference property, used to weight the LEDs in the linear solver -NOISE_COLUMNS = {"sum_col_val": "noise_sum_col_val"} # Fitted files written before the delimiter between the main and the fit part was added: max_col_value and # led_center_x are glued into one field, so the file has one column less and the names are shifted from there on. LEGACY_FIT_COLUMNS = BASE_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS[:-1] diff --git a/ledsa/core/image_reading.py b/ledsa/core/image_reading.py index fb6e1a1..34d52cf 100644 --- a/ledsa/core/image_reading.py +++ b/ledsa/core/image_reading.py @@ -72,20 +72,15 @@ def read_channel_data_from_img(filename: str, channel: int, rotation: str = AUTO return _read_or_raise(filename, reader, channel, orientation) -def read_channel_data_mask_and_noise_from_img( - filename: str, channel: int, - rotation: str = AUTO_ROTATION) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: +def read_channel_data_and_mask_from_img(filename: str, channel: int, + rotation: str = AUTO_ROTATION) -> Tuple[np.ndarray, np.ndarray]: """ Returns the same 2D channel array as read_channel_data_from_img together with a boolean mask of the - pixels that belong to the selected color channel and, for RAW images with an optical black margin, the - read noise of every pixel. + pixels that belong to the selected color channel. For RAW files the mask is taken from the Bayer filter pattern (raw_colors_visible) and not from the pixel - values, because pixels of the selected channel can be 0 as well (noise clipped at 0 after the black level - subtraction). For JPG/PNG files every pixel belongs to the channel. - - The noise carries 0 outside the channel, like the channel array itself, so that a sum over a search - area and the noise of that sum need the mask no further. + values, because pixels of the selected channel can be 0 or negative as well: the values are not clipped + after the black level subtraction. For JPG/PNG files every pixel belongs to the channel. :param filename: The path of the image file to read. :type filename: str @@ -94,14 +89,12 @@ def read_channel_data_mask_and_noise_from_img( :param rotation: Degrees the stored array is turned clockwise to be upright, 'auto' to take the rotation from the metadata of the image. :type rotation: str or int - :return: The channel array, the boolean channel mask and the read noise of every pixel, all of the - same shape. The noise is None for JPG and PNG images and for sensors without an optical black - margin. - :rtype: tuple[np.ndarray, np.ndarray, np.ndarray or None] + :return: The channel array and the boolean channel mask of the same shape. + :rtype: tuple[np.ndarray, np.ndarray] :raises CorruptedImageError: If the image cannot be read or has an unsupported file extension. :raises ValueError: If rotation is neither 'auto' nor one of 0, 90, 180 and 270. """ - return ImageChannels(filename, rotation).channel_data_mask_and_noise(channel) + return ImageChannels(filename, rotation).channel_data_and_mask(channel) class ImageChannels: @@ -109,14 +102,13 @@ class ImageChannels: An image read once for the analysis of several color channels. Decoding a RAW file takes most of the time of reading it, and nothing the decoding yields depends on the - channel, because the black level and the read noise are taken per Bayer color. The file is therefore read - when the object is created, and channel_data_mask_and_noise builds the arrays of a channel from it. They - are built for one channel at a time, so that besides the decoded image only the channel in work is held in - memory. + channel, because the black level is taken per Bayer color. The file is therefore read when the object is + created, and channel_data_and_mask builds the arrays of a channel from it. They are built for one channel + at a time, so that besides the decoded image only the channel in work is held in memory. - The stored values of a RAW image are kept as they are, and the black level, the white level and the read - noise are applied to the pixels that are asked for. Step 3 asks only for its search areas and side bands, - a few percent of the sensor, which saves applying them to the whole image. + The stored values of a RAW image are kept as they are, and the black level and the white level are + applied to the pixels that are asked for. Step 3 asks only for its search areas and side bands, a few + percent of the sensor, which saves applying them to the whole image. """ def __init__(self, filename: str, rotation: str = AUTO_ROTATION): @@ -156,13 +148,12 @@ def shape(self) -> Tuple[int, int]: """The number of rows and columns of the upright image.""" return (self._img_array if self._raw is None else self._raw.data).shape[:2] - def channel_data_mask_and_noise( - self, channel: int, - pixels: Optional[Tuple[np.ndarray, np.ndarray]] = None - ) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: + def channel_data_and_mask(self, channel: int, + pixels: Optional[Tuple[np.ndarray, np.ndarray]] = None + ) -> Tuple[np.ndarray, np.ndarray]: """ - The channel array, the boolean channel mask and the read noise of every pixel of one color channel, as - read_channel_data_mask_and_noise_from_img returns them, of the whole image or of selected pixels. + The channel array and the boolean channel mask of one color channel, as + read_channel_data_and_mask_from_img returns them, of the whole image or of selected pixels. The calibrated pixels of a RAW image do not depend on the channel. They are kept for the next call that passes the same selection, the same object, as the analysis of the next channel of an image does. @@ -171,39 +162,34 @@ def channel_data_mask_and_noise( :type channel: int :param pixels: Row and column indices of the selected pixels in the upright image, None for all pixels. :type pixels: tuple[np.ndarray, np.ndarray] or None - :return: The channel array, the boolean channel mask and the read noise of every pixel, all of the - shape of the image or of the index arrays. The noise is None for JPG and PNG images and for sensors - without an optical black margin. - :rtype: tuple[np.ndarray, np.ndarray, np.ndarray or None] + :return: The channel array and the boolean channel mask, both of the shape of the image or of the index + arrays. + :rtype: tuple[np.ndarray, np.ndarray] :raises CorruptedImageError: If the image does not have the channel. """ try: if self._raw is None: - # every pixel of a JPG or PNG belongs to every channel, and there is no optical black to take the - # read noise from + # every pixel of a JPG or PNG belongs to every channel channel_array = self._img_array[:, :, channel] if pixels is not None: channel_array = channel_array[pixels] - return channel_array, np.ones(channel_array.shape, dtype=bool), None - img_array, filter_array, pixel_noise = self._calibrate(pixels) + return channel_array, np.ones(channel_array.shape, dtype=bool) + img_array, filter_array = self._calibrate(pixels) channel_mask = get_bayer_channel_mask(filter_array, channel) - return (np.where(channel_mask, img_array, 0), channel_mask, - None if pixel_noise is None else np.where(channel_mask, pixel_noise, 0.0)) + return np.where(channel_mask, img_array, 0), channel_mask except Exception as error: raise CorruptedImageError(self.filename, error) from error - def _calibrate(self, pixels: Optional[Tuple[np.ndarray, np.ndarray]] - ) -> Tuple[np.ndarray, np.ndarray, Optional[np.ndarray]]: + def _calibrate(self, pixels: Optional[Tuple[np.ndarray, np.ndarray]]) -> Tuple[np.ndarray, np.ndarray]: """ - The values with the black level subtracted and scaled to the white level, the Bayer colors and the read - noise of the selected pixels of the RAW image, as _channel_data and _pixel_read_noise yield them for the - whole image. + The values with the black level subtracted and scaled to the white level and the Bayer colors of the + selected pixels of the RAW image, as _channel_data yields them for the whole image. """ if self._calibrated is None or self._calibrated[0] is not pixels: raw = self._raw if pixels is None else self._raw._replace(data=self._raw.data[pixels], filter_array=self._raw.filter_array[pixels]) # the selection is kept with the result, so that its identity cannot pass to another selection - self._calibrated = (pixels, (_channel_data(raw), raw.filter_array, _pixel_read_noise(raw))) + self._calibrated = (pixels, (_channel_data(raw), raw.filter_array)) return self._calibrated[1] @@ -347,28 +333,25 @@ def _read_channel_data_from_raw_file(filename: str, channel: int, orientation: O MIN_OPTICAL_BLACK_WIDTH = 8 -def _optical_black_statistics(raw) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: +def _optical_black_mean(raw) -> Optional[np.ndarray]: """ - Mean and standard deviation of the optical black pixels of a RAW image for every color of the Bayer - pattern (index as in raw_colors). The optical black pixels are the masked pixels at the left margin of - the sensor, which is read out with the image itself, so the mean is the black level of this image and - the standard deviation is the read noise of a single pixel. + Mean of the optical black pixels of a RAW image for every color of the Bayer pattern (index as in + raw_colors). The optical black pixels are the masked pixels at the left margin of the sensor, which is + read out with the image itself, so the mean is the black level of this image. :param raw: The opened RAW image (rawpy). - :return: Mean and standard deviation per Bayer color, (None, None) if the sensor has no optical black - margin of at least MIN_OPTICAL_BLACK_WIDTH pixels. - :rtype: tuple[np.ndarray or None, np.ndarray or None] + :return: Mean per Bayer color, None if the sensor has no optical black margin of at least + MIN_OPTICAL_BLACK_WIDTH pixels. + :rtype: np.ndarray or None """ sizes = raw.sizes if sizes.left_margin < MIN_OPTICAL_BLACK_WIDTH: - return None, None + return None rows = np.s_[sizes.top_margin:sizes.top_margin + sizes.height] margin = raw.raw_image[rows, :sizes.left_margin].astype(np.float64) colors = raw.raw_colors[rows, :sizes.left_margin] num_colors = int(raw.raw_colors_visible.max()) + 1 - per_color = [margin[colors == color] for color in range(num_colors)] - return (np.array([values.mean() for values in per_color]), - np.array([values.std() for values in per_color])) + return np.array([margin[colors == color].mean() for color in range(num_colors)]) class RawImage(NamedTuple): @@ -379,7 +362,6 @@ class RawImage(NamedTuple): black_levels: np.ndarray # per Bayer color white_level: float orientation: int # the one the camera recorded in its maker notes - read_noise: Optional[np.ndarray] # per Bayer color, None without an optical black margin @property def scales(self) -> np.ndarray: @@ -395,12 +377,12 @@ def _read_raw_image(filename: str) -> RawImage: sensors without an optical black margin, from the RAW metadata. Both sources hold one value per color. """ with rawpy.imread(filename) as raw: - optical_black_mean, read_noise = _optical_black_statistics(raw) + optical_black_mean = _optical_black_mean(raw) black_levels = (np.asarray(raw.black_level_per_channel, dtype=np.float64) if optical_black_mean is None else optical_black_mean) return RawImage(raw.raw_image_visible.copy(), raw.raw_colors_visible.copy(), black_levels, float(raw.white_level), - RAW_FLIP_ORIENTATIONS.get(raw.sizes.flip, UPRIGHT_ORIENTATION), read_noise) + RAW_FLIP_ORIENTATIONS.get(raw.sizes.flip, UPRIGHT_ORIENTATION)) def _channel_data(raw: RawImage) -> np.ndarray: @@ -419,16 +401,6 @@ def _channel_data(raw: RawImage) -> np.ndarray: return (raw.data.astype(np.float32) - levels) * scales -def _pixel_read_noise(raw: RawImage) -> Optional[np.ndarray]: - """ - The read noise of every single pixel, taken from the optical black pixels of its own Bayer color and - scaled like the data. None for sensors without an optical black margin. - """ - if raw.read_noise is None: - return None - return (raw.read_noise.astype(np.float32) * raw.scales.astype(np.float32))[raw.filter_array] - - def _read_img_array_from_raw_file(filename: str, channel: int, orientation: Optional[int] = None) -> Tuple[np.ndarray, np.ndarray]: # channel is kept for the signature of the readers; the black level is taken per Bayer color diff --git a/ledsa/core/parser_arguments_run.py b/ledsa/core/parser_arguments_run.py index b9c67db..ff10a83 100644 --- a/ledsa/core/parser_arguments_run.py +++ b/ledsa/core/parser_arguments_run.py @@ -289,9 +289,7 @@ def extionction_coefficient_calculation(args) -> None: average_images=ex_data.average_images, n_summarize=ex_data.n_summarize, temperature_correction=temperature_corrections.get(channel), - lambda_reg=ex_data.lambda_reg, - snr_full_weight=ex_data.snr_full_weight, - min_layer_weight=ex_data.min_layer_weight) + lambda_reg=ex_data.lambda_reg) else: raise ValueError(f"Invalid solver type '{solver}'. Must be 'linear' or 'nonlinear'.") if ex_data.n_cpus > 1: diff --git a/ledsa/data_extraction/LEDAnalysisData.py b/ledsa/data_extraction/LEDAnalysisData.py index ce27e46..7c19af8 100644 --- a/ledsa/data_extraction/LEDAnalysisData.py +++ b/ledsa/data_extraction/LEDAnalysisData.py @@ -31,10 +31,6 @@ class LEDAnalysisData: over the channel pixels (including 0) of the border ring of the search area, at least a fraction of the plain sum (background_floor). :vartype bgsub2_sum_color_value: float - :ivar noise_sum_color_value: Standard deviation of sum_color_value from the read noise of the sensor, - estimated from the optical black pixels (NaN for JPG and PNG images and for sensors without an - optical black margin). - :vartype noise_sum_color_value: float :ivar fit_results: Fit results after fitting. :vartype fit_results: OptimizeResult :ivar fit_time: Time taken for fitting. @@ -62,7 +58,6 @@ def __init__(self, led_id, led_array, fit_leds): self.sidebg_sum_color_value = None self.sidebg_pedestal = None self.bgsub2_sum_color_value = None - self.noise_sum_color_value = float('nan') self.fit_leds = fit_leds self.fit_results = None self.fit_time = None @@ -93,7 +88,6 @@ def get_main_data_string(self) -> str: out_str += f',{self.bgsub_sum_color_value:10.4e}' out_str += f',{self.sidebg_sum_color_value:10.4e},{self.sidebg_pedestal:10.4e}' out_str += f',{self.bgsub2_sum_color_value:10.4e}' - out_str += f',{self.noise_sum_color_value:10.4e}' return out_str def get_fit_data_string(self) -> str: diff --git a/ledsa/data_extraction/step_3_functions.py b/ledsa/data_extraction/step_3_functions.py index d9e9d86..7241e81 100644 --- a/ledsa/data_extraction/step_3_functions.py +++ b/ledsa/data_extraction/step_3_functions.py @@ -52,7 +52,7 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda search_area_radius = int(conf['find_search_areas']['search_area_radius']) if debug: - data, _, _ = img_channels.channel_data_mask_and_noise(channel) + data, _ = img_channels.channel_data_and_mask(channel) analysis_res = _generate_led_analysis_data(conf, channel, data, debug, debug_led, img_filename, 0, search_areas, search_area_radius, fit_leds) return analysis_res @@ -71,7 +71,7 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda # only the pixels of the search areas and side bands are read, which are a few percent of the image analysis_pixels = _get_analysis_pixels(search_areas, led_ids, sideband_pixels, img_channels.shape, search_area_radius) - data, channel_mask, pixel_noise = img_channels.channel_data_mask_and_noise(channel, analysis_pixels.pixels) + data, channel_mask = img_channels.channel_data_and_mask(channel, analysis_pixels.pixels) sidebands = np.s_[analysis_pixels.num_box_pixels:] sideband_backgrounds = estimate_sideband_backgrounds(data[sidebands], channel_mask[sidebands], analysis_pixels.num_sideband_pixels) @@ -84,7 +84,6 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda conf, channel, img_filename, led_ids[positions], led_array_ids[positions], analysis_pixels.centers[positions], search_area_radius, fit_leds, data[boxes].reshape(stack_shape), channel_mask[boxes].reshape(stack_shape), - None if pixel_noise is None else pixel_noise[boxes].reshape(stack_shape), sideband_backgrounds[positions], background_floor) for position, led_analysis_data in zip(positions, group_data): img_analysis_data[position] = led_analysis_data @@ -339,7 +338,6 @@ def find_fit_leds_for_restart(channels: Sequence[int]) -> bool: def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray, debug: bool, iled: int, img_filename: str, led_array_idx: int, search_areas: np.ndarray, search_area_radius: int, fit_leds: bool = True, channel_mask: Optional[np.ndarray] = None, - pixel_noise: Optional[np.ndarray] = None, sideband_pixels: Optional[Tuple[np.ndarray, np.ndarray]] = None, background_floor: float = DEFAULT_BACKGROUND_FLOOR) -> LEDAnalysisData: """ @@ -366,9 +364,6 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray :type search_area_radius: int :param fit_leds: If True, the LED is fitted to a model function. :type fit_leds: bool - :param pixel_noise: Read noise of every pixel, None if the image has no optical black margin. If - None, the noise of the sum stays NaN. - :type pixel_noise: np.ndarray or None :param channel_mask: Boolean mask of the pixels belonging to the color channel (Bayer pattern for raw files). None if all pixels belong to the channel. :type channel_mask: np.ndarray or None @@ -385,7 +380,6 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray box_mask = np.ones(data[search_area].shape, dtype=bool) if channel_mask is None else channel_mask[search_area] led_data, = _analyse_search_areas(conf, channel, img_filename, [iled], [led_array_idx], center, search_area_radius, fit_leds, data[search_area][None], box_mask[None], - None if pixel_noise is None else pixel_noise[search_area][None], [estimate_sideband_background(data, channel_mask, sideband_pixels)], background_floor) if debug and fit_leds: @@ -395,7 +389,7 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led_ids: Sequence[int], led_array_ids: Sequence[int], centers: np.ndarray, search_area_radius: int, fit_leds: bool, - boxes: np.ndarray, box_masks: np.ndarray, box_noise: Optional[np.ndarray], + boxes: np.ndarray, box_masks: np.ndarray, sideband_backgrounds: Sequence[float], background_floor: float = DEFAULT_BACKGROUND_FLOOR) -> List[LEDAnalysisData]: """ @@ -425,9 +419,6 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led :type boxes: np.ndarray :param box_masks: Whether each pixel of the search areas belongs to the channel, of the shape of boxes. :type box_masks: np.ndarray - :param box_noise: Read noise of each pixel of the search areas, 0 outside the channel, None if the image has - no optical black margin. If None, the noise of the sums stays NaN. - :type box_noise: np.ndarray or None :param sideband_backgrounds: Mean pedestal per channel pixel in the side bands of every LED, see estimate_sideband_background. :type sideband_backgrounds: Sequence[float] @@ -445,9 +436,6 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led ring_medians, ring_means = _estimate_ring_backgrounds(boxes, box_masks) bgsub_sums = np.sum(boxes.astype(np.float64) - ring_medians[:, None, None] * box_masks, axis=(1, 2)) - # the noise is 0 outside the channel, so the variance of the sum needs no mask - noise_sums = None if box_noise is None else np.sqrt(np.sum(box_noise.astype(np.float64) ** 2, axis=(1, 2))) - # the pedestals below are subtracted as N * b with N the number of channel pixels in the box num_channel_pixels = np.count_nonzero(box_masks, axis=(1, 2)) @@ -470,8 +458,6 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led led_data.sum_color_value = sums[i] led_data.max_color_value = maxima[i] led_data.bgsub_sum_color_value = float(bgsub_sums[i]) - if noise_sums is not None: - led_data.noise_sum_color_value = float(noise_sums[i]) # border ring as for bgsub, but with the estimator of the side bands: mean over the channel pixels including 0; # same lower limit as for the side band correction @@ -711,7 +697,6 @@ def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.n out_str += f"LEDSA {version('ledsa')}\n" out_str += "# id,line,sum_col_value,average_col_value,max_col_value,bgsub_sum_col_value" out_str += ",sidebg_sum_col_value,sidebg_pedestal,bgsub2_sum_col_value" - out_str += ",noise_sum_col_value" if fit_leds: out_str += ",led_center_x, led_center_y" out_str += ",x,y,dx,dy,A,alpha,wx,wy,fit_success,fit_fun,fit_nfev,fit_time" diff --git a/ledsa/tests/UnitTests/test_black_level.py b/ledsa/tests/UnitTests/test_black_level.py index da2e37c..1a7f549 100644 --- a/ledsa/tests/UnitTests/test_black_level.py +++ b/ledsa/tests/UnitTests/test_black_level.py @@ -5,7 +5,7 @@ import ledsa.core.image_reading as image_reading from ledsa.core.image_reading import (MIN_OPTICAL_BLACK_WIDTH, ImageChannels, _channel_data, - _optical_black_statistics, _pixel_read_noise, _read_raw_image) + _optical_black_mean, _read_raw_image) # RGGB pattern as in rawpy raw_colors: 0 red, 1 and 3 green, 2 blue BAYER_TILE = np.array([[0, 1], [3, 2]]) @@ -20,8 +20,8 @@ def make_raw(left_margin: int, noise: float = 0.0, visible: float = None) -> Sim """ The part of a rawpy RawPy object that LEDSA reads: 4 x 4 visible pixels behind a top and a left margin, with a known black level per Bayer color in the margin. With noise the two halves of the margin columns - are offset by -noise and +noise, so the standard deviation of every color is noise. With visible the - visible pixels are set to that value instead of the black level of their color. + are offset by -noise and +noise, which leaves the mean of every color at its black level. With visible + the visible pixels are set to that value instead of the black level of their color. """ top_margin, height, width = 2, 4, 4 shape = (top_margin + height, left_margin + width) @@ -61,23 +61,21 @@ def __exit__(self, *args): return install -class TestOpticalBlackStatistics: - def test_mean_and_standard_deviation_per_bayer_color(self): - mean, std = _optical_black_statistics(make_raw(MIN_OPTICAL_BLACK_WIDTH, noise=3.0)) +class TestOpticalBlackMean: + def test_mean_per_bayer_color(self): + mean = _optical_black_mean(make_raw(MIN_OPTICAL_BLACK_WIDTH, noise=3.0)) assert mean == pytest.approx(OPTICAL_BLACK_LEVELS) - assert std == pytest.approx([3.0] * len(OPTICAL_BLACK_LEVELS)) - def test_only_the_margin_enters_the_statistics(self): - # the visible pixels of a bright image must not raise the black level or the noise - mean, std = _optical_black_statistics(make_raw(MIN_OPTICAL_BLACK_WIDTH, visible=9000.0)) + def test_only_the_margin_enters_the_mean(self): + # the visible pixels of a bright image must not raise the black level + mean = _optical_black_mean(make_raw(MIN_OPTICAL_BLACK_WIDTH, visible=9000.0)) assert mean == pytest.approx(OPTICAL_BLACK_LEVELS) - assert std == pytest.approx([0.0] * len(OPTICAL_BLACK_LEVELS)) def test_margin_narrower_than_the_minimum_is_not_used(self): - assert _optical_black_statistics(make_raw(MIN_OPTICAL_BLACK_WIDTH - 2)) == (None, None) + assert _optical_black_mean(make_raw(MIN_OPTICAL_BLACK_WIDTH - 2)) is None def test_sensor_without_a_margin(self): - assert _optical_black_statistics(make_raw(0)) == (None, None) + assert _optical_black_mean(make_raw(0)) is None class TestBlackLevelsOfARawImage: @@ -87,13 +85,11 @@ def test_the_optical_black_is_preferred_over_the_metadata(self, fake_imread): fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH, noise=2.0)) raw = _read_raw_image('img.CR3') assert raw.black_levels == pytest.approx(OPTICAL_BLACK_LEVELS) - assert raw.read_noise == pytest.approx([2.0] * len(OPTICAL_BLACK_LEVELS)) def test_the_metadata_are_the_fallback_and_are_per_color_too(self, fake_imread): fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH - 1)) raw = _read_raw_image('img.CR3') assert raw.black_levels == pytest.approx([float(level) for level in METADATA_BLACK_LEVELS]) - assert raw.read_noise is None def test_every_color_is_scaled_by_its_own_black_level(self, fake_imread): fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH)) @@ -125,30 +121,17 @@ def test_the_data_are_float_because_the_black_level_is_a_fraction_of_a_count(sel assert (data < 0).all() and (data > -1.0).all() -class TestPixelReadNoise: - def test_a_pixel_carries_the_noise_of_its_own_color(self, fake_imread): - fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH, noise=4.0)) - raw = _read_raw_image('img.CR3') - scales = np.array([WHITE_LEVEL / (WHITE_LEVEL - level) for level in OPTICAL_BLACK_LEVELS]) - assert _pixel_read_noise(raw) == pytest.approx(4.0 * scales[raw.filter_array], rel=1e-5) - - def test_there_is_no_noise_without_an_optical_black_margin(self, fake_imread): - fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH - 1)) - assert _pixel_read_noise(_read_raw_image('img.CR3')) is None - - class TestTheReaderStepThreeUses: - def test_the_orientation_reaches_the_data_the_filter_and_the_noise(self, fake_imread): + def test_the_orientation_reaches_the_data_and_the_filter(self, fake_imread): fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH, noise=4.0, visible=5000.0)) - upright = ImageChannels('img.CR3', rotation=0).channel_data_mask_and_noise(0) - turned = ImageChannels('img.CR3', rotation=90).channel_data_mask_and_noise(0) + upright = ImageChannels('img.CR3', rotation=0).channel_data_and_mask(0) + turned = ImageChannels('img.CR3', rotation=90).channel_data_and_mask(0) for upright_array, turned_array in zip(upright, turned): np.testing.assert_array_equal(turned_array, np.rot90(upright_array, -1)) - def test_without_a_margin_only_the_noise_is_missing(self, fake_imread): + def test_an_image_without_a_margin_is_read_with_the_black_level_of_the_metadata(self, fake_imread): fake_imread(make_raw(MIN_OPTICAL_BLACK_WIDTH - 1, visible=5000.0)) - data, channel_mask, noise = ImageChannels('img.CR3').channel_data_mask_and_noise(0) - assert noise is None + data, channel_mask = ImageChannels('img.CR3').channel_data_and_mask(0) assert data.shape == channel_mask.shape and np.isfinite(data).all() @@ -156,12 +139,11 @@ class TestSumsOfAnLed: SEARCH_AREA_RADIUS = 6 CENTER = (20, 20) - def _extract(self, data, pixel_noise=None): + def _extract(self, data): from ledsa.data_extraction.step_3_functions import _generate_led_analysis_data search_areas = np.array([[0, self.CENTER[0], self.CENTER[1]]]) return _generate_led_analysis_data(None, 0, data, False, 0, 'img', 0, search_areas, - self.SEARCH_AREA_RADIUS, fit_leds=False, - pixel_noise=pixel_noise) + self.SEARCH_AREA_RADIUS, fit_leds=False) @property def _box(self): @@ -174,20 +156,3 @@ def test_a_sum_below_the_black_level_stays_negative(self): data = np.full((40, 40), -2.0) led_data = self._extract(data) assert led_data.sum_color_value == pytest.approx(-2.0 * data[self._box].size) - - def test_without_an_optical_black_margin_the_noise_stays_nan(self): - assert np.isnan(self._extract(np.full((40, 40), 5.0)).noise_sum_color_value) - - def test_the_noise_of_the_sum_adds_up_quadratically(self): - noise = np.zeros((40, 40), dtype=np.float32) - noise[self._box] = 2.0 - led_data = self._extract(np.zeros((40, 40)), noise) - assert led_data.noise_sum_color_value == pytest.approx(2.0 * np.sqrt(noise[self._box].size)) - - def test_pixels_outside_the_channel_carry_no_noise(self): - # the reader zeroes the noise outside the channel, so a Bayer image sums only its own pixels - noise = np.zeros((40, 40), dtype=np.float32) - noise[::2, ::2] = 2.0 - led_data = self._extract(np.zeros((40, 40)), noise) - assert led_data.noise_sum_color_value == pytest.approx( - 2.0 * np.sqrt(np.count_nonzero(noise[self._box]))) diff --git a/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py b/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py new file mode 100644 index 0000000..39b91ac --- /dev/null +++ b/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py @@ -0,0 +1,42 @@ +"""Unit tests for the linear solver of the extinction coefficients. + +The intensities are not clipped at zero, so the relative intensity of an LED that has vanished in the noise +of the sensor can be zero or negative. Such an LED is no measurement and must not reach the coefficients. +""" +import numpy as np + +from ledsa.analysis.ExtinctionCoefficientsLinear import ExtinctionCoefficientsLinear + +# 4 LEDs over 3 layers: the first three cross one layer each, the fourth crosses all of them +DISTANCES = np.array([[1.0, 0.0, 0.0], + [0.0, 1.0, 0.0], + [0.0, 0.0, 1.0], + [1.0, 1.0, 1.0]]) +SIGMAS = np.array([0.1, 0.2, 0.3]) +REL_INTENSITIES = np.exp(-DISTANCES @ SIGMAS) + + +def make_solver(distances=DISTANCES) -> ExtinctionCoefficientsLinear: + """A linear solver with the geometry above, built without __init__ as the config is not needed.""" + solver = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) + solver.distances_per_led_and_layer = distances + solver.lambda_reg = 1e-6 + return solver + + +class TestLinearSolver: + def test_the_solution_recovers_the_coefficients(self): + np.testing.assert_allclose(make_solver().calc_coefficients_of_img(REL_INTENSITIES), SIGMAS, atol=1e-3) + + def test_an_led_at_or_below_zero_is_left_out(self): + # whatever it reports, the result is that of the other LEDs alone + without_the_led = make_solver(DISTANCES[1:]).calc_coefficients_of_img(REL_INTENSITIES[1:]) + for value in [0.0, -0.3]: + rel_intensities = REL_INTENSITIES.copy() + rel_intensities[0] = value + np.testing.assert_allclose(make_solver().calc_coefficients_of_img(rel_intensities), without_the_led, + atol=1e-9) + + def test_an_image_without_any_measured_led_gives_nan(self): + rel_intensities = np.array([0.0, -0.1, 0.0, -2.0]) + assert np.isnan(make_solver().calc_coefficients_of_img(rel_intensities)).all() diff --git a/ledsa/tests/UnitTests/test_image_channels.py b/ledsa/tests/UnitTests/test_image_channels.py index 9ad47ea..c3be4eb 100644 --- a/ledsa/tests/UnitTests/test_image_channels.py +++ b/ledsa/tests/UnitTests/test_image_channels.py @@ -20,8 +20,6 @@ # RGGB pattern as in rawpy raw_colors_visible: 0 red, 1 and 3 green, 2 blue FILTER_ARRAY = np.tile(np.array([[0, 1], [3, 2]]), (2, 3)) CHANNEL_COLORS = {0: [0], 1: [1, 3], 2: [2]} -# different per Bayer color, so that a test can tell the noise of one channel from that of another -READ_NOISE = np.array([1.0, 2.0, 3.0, 4.0]) SIDEWAYS = 6 @@ -41,7 +39,7 @@ def install(orientation=image_reading.UPRIGHT_ORIENTATION): def read_raw_image(filename): reads.append(filename) - return RawImage(stored, FILTER_ARRAY, np.zeros(4), 255.0, orientation, READ_NOISE) + return RawImage(stored, FILTER_ARRAY, np.zeros(4), 255.0, orientation) monkeypatch.setattr(image_reading, '_read_raw_image', read_raw_image) return reads @@ -55,26 +53,25 @@ def test_raw_channels_are_built_from_one_read(self, raw_reads, stored, orientati img_channels = ImageChannels('img.CR3') upright = image_reading.ORIENTATION_TRANSFORMATIONS[orientation] for channel, colors in CHANNEL_COLORS.items(): - channel_array, channel_mask, pixel_noise = img_channels.channel_data_mask_and_noise(channel) + channel_array, channel_mask = img_channels.channel_data_and_mask(channel) mask = upright(np.isin(FILTER_ARRAY, colors)) np.testing.assert_array_equal(channel_mask, mask) np.testing.assert_array_equal(channel_array, np.where(mask, upright(stored), 0)) - np.testing.assert_array_equal(pixel_noise, np.where(mask, upright(READ_NOISE[FILTER_ARRAY]), 0)) assert reads == ['img.CR3'] def test_raw_channels_are_those_of_a_read_per_channel(self, raw_reads): raw_reads(SIDEWAYS) img_channels = ImageChannels('img.CR3') for channel in CHANNEL_COLORS: - for once, per_channel in zip(img_channels.channel_data_mask_and_noise(channel), - image_reading.read_channel_data_mask_and_noise_from_img('img.CR3', channel)): + for once, per_channel in zip(img_channels.channel_data_and_mask(channel), + image_reading.read_channel_data_and_mask_from_img('img.CR3', channel)): np.testing.assert_array_equal(once, per_channel) - np.testing.assert_array_equal(img_channels.channel_data_mask_and_noise(channel)[0], + np.testing.assert_array_equal(img_channels.channel_data_and_mask(channel)[0], image_reading.read_channel_data_from_img('img.CR3', channel)) def test_rotation_option_overrides_the_raw_metadata(self, raw_reads, stored): raw_reads(SIDEWAYS) - channel_array, _, _ = ImageChannels('img.CR3', rotation=0).channel_data_mask_and_noise(0) + channel_array, _ = ImageChannels('img.CR3', rotation=0).channel_data_and_mask(0) np.testing.assert_array_equal(channel_array, np.where(FILTER_ARRAY == 0, stored, 0)) def test_png_channels_are_the_color_planes(self, tmp_path): @@ -82,17 +79,16 @@ def test_png_channels_are_the_color_planes(self, tmp_path): plt.imsave(path, np.random.default_rng(0).random((6, 8, 3))) img_channels = ImageChannels(path) for channel in CHANNEL_COLORS: - channel_array, channel_mask, pixel_noise = img_channels.channel_data_mask_and_noise(channel) + channel_array, channel_mask = img_channels.channel_data_and_mask(channel) np.testing.assert_array_equal(channel_array, image_reading.read_channel_data_from_img(path, channel)) assert channel_mask.shape == channel_array.shape and channel_mask.all() - assert pixel_noise is None # a PNG has no optical black def test_missing_channel_is_reported_as_corrupted_image(self, tmp_path): path = str(tmp_path / 'gray.png') Image.fromarray(np.zeros((6, 8), dtype=np.uint8)).save(path) # read as a 2D array without color planes img_channels = ImageChannels(path) with pytest.raises(CorruptedImageError, match='gray.png'): - img_channels.channel_data_mask_and_noise(1) + img_channels.channel_data_and_mask(1) def test_unreadable_image_raises_when_it_is_read(self, tmp_path): with pytest.raises(CorruptedImageError, match='missing.CR3'): @@ -110,8 +106,8 @@ def test_raw_pixels_are_those_of_the_whole_image(self, raw_reads, orientation): img_channels = ImageChannels('img.CR3') pixels = (np.array([0, 3, 1, 1]), np.array([2, 0, 3, 3])) # in the upright image, one of them twice for channel in CHANNEL_COLORS: - for selected, whole in zip(img_channels.channel_data_mask_and_noise(channel, pixels), - img_channels.channel_data_mask_and_noise(channel)): + for selected, whole in zip(img_channels.channel_data_and_mask(channel, pixels), + img_channels.channel_data_and_mask(channel)): np.testing.assert_array_equal(selected, whole[pixels]) def test_png_pixels_are_those_of_the_color_planes(self, tmp_path): @@ -120,10 +116,9 @@ def test_png_pixels_are_those_of_the_color_planes(self, tmp_path): img_channels = ImageChannels(path) pixels = (np.array([[0, 5], [2, 2]]), np.array([[7, 0], [3, 4]])) # index arrays of any shape for channel in CHANNEL_COLORS: - channel_array, channel_mask, pixel_noise = img_channels.channel_data_mask_and_noise(channel, pixels) - np.testing.assert_array_equal(channel_array, img_channels.channel_data_mask_and_noise(channel)[0][pixels]) + channel_array, channel_mask = img_channels.channel_data_and_mask(channel, pixels) + np.testing.assert_array_equal(channel_array, img_channels.channel_data_and_mask(channel)[0][pixels]) assert channel_mask.shape == channel_array.shape == (2, 2) and channel_mask.all() - assert pixel_noise is None def test_the_calibration_is_kept_for_the_same_selection_only(self, raw_reads, monkeypatch): raw_reads() @@ -134,12 +129,12 @@ def test_the_calibration_is_kept_for_the_same_selection_only(self, raw_reads, mo img_channels = ImageChannels('img.CR3') pixels = (np.array([0, 1]), np.array([2, 3])) for channel in CHANNEL_COLORS: - img_channels.channel_data_mask_and_noise(channel, pixels) + img_channels.channel_data_and_mask(channel, pixels) assert calibrated == [(2,)] # another selection, even of the same pixels, and the whole image are calibrated anew - img_channels.channel_data_mask_and_noise(0, (pixels[0].copy(), pixels[1].copy())) - img_channels.channel_data_mask_and_noise(0) - img_channels.channel_data_mask_and_noise(1) + img_channels.channel_data_and_mask(0, (pixels[0].copy(), pixels[1].copy())) + img_channels.channel_data_and_mask(0) + img_channels.channel_data_and_mask(1) assert calibrated == [(2,), (2,), FILTER_ARRAY.shape] def test_shape_is_that_of_the_upright_image(self, raw_reads): @@ -170,7 +165,7 @@ def test_all_channels_are_analysed_from_one_read(self, extractor, raw_reads, mon def generate_analysis_data(img_filename, channel, *args, img_channels=None, **kwargs): analysed.append(channel) - return img_channels.channel_data_mask_and_noise(channel) + return img_channels.channel_data_and_mask(channel) monkeypatch.setattr(step_3_functions, 'generate_analysis_data', generate_analysis_data) monkeypatch.setattr(step_3_functions, 'create_fit_result_file', diff --git a/ledsa/tests/UnitTests/test_image_orientation.py b/ledsa/tests/UnitTests/test_image_orientation.py index 9659f0e..a7538c5 100644 --- a/ledsa/tests/UnitTests/test_image_orientation.py +++ b/ledsa/tests/UnitTests/test_image_orientation.py @@ -13,7 +13,7 @@ from ledsa.core import image_reading from ledsa.core.image_reading import (ORIENTATION_TRANSFORMATIONS, _apply_orientation, _orientation_from_rotation, get_bayer_channel_mask, - read_channel_data_mask_and_noise_from_img, + read_channel_data_and_mask_from_img, read_channel_data_from_img, read_img_array_from_img) from ledsa.data_extraction.step_1_functions import find_search_areas @@ -119,7 +119,7 @@ def test_all_channels_are_read_upright(self, tmp_path): def test_channel_mask_fits_the_upright_array(self, tmp_path): stored = np.rot90(led_strip_img_array()) path = write_img(tmp_path / 'sideways.png', stored, orientation=SIDEWAYS) - channel_array, channel_mask, _ = read_channel_data_mask_and_noise_from_img(path, channel=0) + channel_array, channel_mask = read_channel_data_and_mask_from_img(path, channel=0) assert channel_mask.shape == channel_array.shape and channel_mask.all() def test_rotation_option_overrides_the_metadata(self, tmp_path): @@ -194,8 +194,8 @@ def test_sideways_raw_is_read_upright(self, monkeypatch, stored): def test_bayer_pattern_is_turned_with_the_pixels(self, monkeypatch, stored): """The mask has to keep naming the color of every pixel, or the sums mix the channels.""" self.fake_imread(monkeypatch, stored, flip=6) - channel_array, channel_mask, _ = \ - read_channel_data_mask_and_noise_from_img('img.CR3', channel=1) + channel_array, channel_mask = \ + read_channel_data_and_mask_from_img('img.CR3', channel=1) np.testing.assert_array_equal(channel_mask, np.rot90(get_bayer_channel_mask(self.FILTER_ARRAY, 1), -1)) np.testing.assert_array_equal(channel_array, np.where(channel_mask, np.rot90(stored, -1), 0)) diff --git a/ledsa/tests/UnitTests/test_led_weighting.py b/ledsa/tests/UnitTests/test_led_weighting.py deleted file mode 100644 index acf453f..0000000 --- a/ledsa/tests/UnitTests/test_led_weighting.py +++ /dev/null @@ -1,167 +0,0 @@ -from types import SimpleNamespace - -import numpy as np -import pandas as pd -import pytest - -from ledsa.analysis.ExtinctionCoefficients import DEFAULT_SNR_FULL_WEIGHT -from ledsa.analysis.ExtinctionCoefficientsLinear import ExtinctionCoefficientsLinear - -# 4 LEDs over 3 layers: the first three cross one layer each, the fourth crosses all of them -DISTANCES = np.array([[1.0, 0.0, 0.0], - [0.0, 1.0, 0.0], - [0.0, 0.0, 1.0], - [1.0, 1.0, 1.0]]) -SIGMAS = np.array([0.1, 0.2, 0.3]) -REL_INTENSITIES = np.exp(-DISTANCES @ SIGMAS) -SNR_FULL_WEIGHT = 10.0 - - -def make_solver(min_layer_weight=None) -> ExtinctionCoefficientsLinear: - """A linear solver with the geometry above, built without __init__ as the config is not needed.""" - solver = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) - solver.distances_per_led_and_layer = DISTANCES - solver.lambda_reg = 1e-6 - solver.snr_full_weight = SNR_FULL_WEIGHT - solver.min_layer_weight = min_layer_weight - return solver - - -class TestWhenTheWeightingIsUsed: - """The weighting is on by default and switched off wherever the data cannot carry it.""" - - @staticmethod - def experiment(): - return SimpleNamespace(channel=0, path='.', led_array=0) - - def test_the_default_weights_the_unclipped_sum_by_its_noise(self): - coefficients = ExtinctionCoefficientsLinear(self.experiment(), reference_property='sum_col_val') - assert coefficients.snr_full_weight == DEFAULT_SNR_FULL_WEIGHT - assert coefficients.noise_property == 'noise_sum_col_val' - - def test_a_property_without_a_noise_column_is_weighted_equally(self, capsys): - # the default must not stop an analysis of a property that cannot carry the weighting - coefficients = ExtinctionCoefficientsLinear(self.experiment(), - reference_property='sidebg_sum_col_val') - assert coefficients.noise_property is None - assert 'weighted equally' in capsys.readouterr().out - - def test_the_option_can_be_switched_off(self): - coefficients = ExtinctionCoefficientsLinear(self.experiment(), reference_property='sum_col_val', - snr_full_weight=None) - assert coefficients.noise_property is None - - def test_the_header_says_whether_the_weighting_was_used(self): - weighted = ExtinctionCoefficientsLinear(self.experiment(), reference_property='sum_col_val') - unweighted = ExtinctionCoefficientsLinear(self.experiment(), reference_property='sum_col_val', - snr_full_weight=None) - assert 'Noise_property: noise_sum_col_val' in str(weighted) - assert 'Noise_property: None' in str(unweighted) - assert 'Min_layer_weight: None' in str(unweighted) - - -class TestLedWeights: - def test_equal_weights_reproduce_the_unweighted_solution(self): - solver = make_solver() - snr = np.full(DISTANCES.shape[0], 5 * SNR_FULL_WEIGHT) # every LED well above the full weight - np.testing.assert_allclose(solver.calc_coefficients_of_img(REL_INTENSITIES, snr), - solver.calc_coefficients_of_img(REL_INTENSITIES), atol=1e-9) - - def test_the_unweighted_solution_recovers_the_coefficients(self): - np.testing.assert_allclose(make_solver().calc_coefficients_of_img(REL_INTENSITIES), SIGMAS, - atol=1e-3) - - def test_an_led_without_signal_does_not_influence_the_result(self): - # its weight is 0, so whatever it reports must not reach the coefficients - solver = make_solver() - snr = np.array([0.0, 5 * SNR_FULL_WEIGHT, 5 * SNR_FULL_WEIGHT, 5 * SNR_FULL_WEIGHT]) - first, second = REL_INTENSITIES.copy(), REL_INTENSITIES.copy() - first[0], second[0] = 0.9, 0.01 - np.testing.assert_allclose(solver.calc_coefficients_of_img(first, snr), - solver.calc_coefficients_of_img(second, snr), atol=1e-9) - - def test_an_unknown_noise_keeps_the_full_weight(self): - # a missing noise estimate is no reason to drop a measured intensity; only a known small SNR is - solver = make_solver() - snr = np.array([np.nan, 5 * SNR_FULL_WEIGHT, 5 * SNR_FULL_WEIGHT, 5 * SNR_FULL_WEIGHT]) - np.testing.assert_allclose(solver.calc_coefficients_of_img(REL_INTENSITIES, snr), - solver.calc_coefficients_of_img(REL_INTENSITIES), atol=1e-9) - - def test_an_led_with_a_known_small_ratio_is_downweighted(self): - solver = make_solver() - weak = np.array([0.1 * SNR_FULL_WEIGHT] + [5 * SNR_FULL_WEIGHT] * 3) - corrupted = REL_INTENSITIES.copy() - corrupted[0] = 0.5 - # the corrupted LED moves the result less than it would with the full weight - weighted = solver.calc_coefficients_of_img(corrupted, weak) - unweighted = solver.calc_coefficients_of_img(corrupted) - clean = solver.calc_coefficients_of_img(REL_INTENSITIES) - assert np.abs(weighted - clean).max() < np.abs(unweighted - clean).max() - - def test_too_little_signal_in_the_whole_image_gives_nan(self): - # the weights add up to less than one LED with full weight, so nothing can be computed - solver = make_solver() - snr = np.full(DISTANCES.shape[0], 0.1 * SNR_FULL_WEIGHT) - assert np.isnan(solver.calc_coefficients_of_img(REL_INTENSITIES, snr)).all() - - -class TestLayersRestingOnNoise: - SNR = np.array([0.0, 0.0, 0.0, 2 * SNR_FULL_WEIGHT]) # only the LED crossing every layer is measured - - def test_layers_below_the_limit_become_nan(self): - # every layer carries the path weighted mean weight 0.5, from the one LED that crosses all of them - assert np.isnan(make_solver(min_layer_weight=0.6).calc_coefficients_of_img(REL_INTENSITIES, - self.SNR)).all() - - def test_layers_above_the_limit_are_kept(self): - assert np.isfinite(make_solver(min_layer_weight=0.4).calc_coefficients_of_img(REL_INTENSITIES, - self.SNR)).all() - - def test_without_the_limit_nothing_is_discarded(self): - assert np.isfinite(make_solver().calc_coefficients_of_img(REL_INTENSITIES, self.SNR)).all() - - -class TestTheWeightingFallsBackAtLoadTime: - """Data without a usable noise column switch the weighting off instead of stopping the analysis.""" - - @pytest.fixture - def coefficients_for(self, monkeypatch): - import ledsa.analysis.ExtinctionCoefficients as extinction_coefficients - - def build(noise): - index = pd.MultiIndex.from_product([[1, 2], [0, 1]], names=['img_id', 'led_id']) - data = {'led_array_id': 0, 'sum_col_val': 100.0} - if noise is not None: - data['noise_sum_col_val'] = noise - img_data = pd.DataFrame(data, index=index) - monkeypatch.setattr(extinction_coefficients, 'read_hdf', lambda channel, path: img_data) - coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) - coefficients.experiment = SimpleNamespace(channel=0, path='.', led_array=0) - coefficients.average_images = False - coefficients.reference_property = 'sum_col_val' - coefficients.noise_property = 'noise_sum_col_val' - coefficients.ref_img_ids_of_loaded_data = None - coefficients.temperature_correction = None - coefficients.temperature_corrected = False - return coefficients - return build - - def test_a_binary_without_the_noise_column_is_weighted_equally(self, coefficients_for, capsys): - coefficients = coefficients_for(None) - coefficients.load_img_data() - assert coefficients.noise_property is None - out = capsys.readouterr().out - assert 'weighted equally' in out and 'Run step 3 again' in out - - def test_a_noise_column_of_only_nan_is_weighted_equally(self, coefficients_for, capsys): - # JPG and PNG images and sensors without an optical black margin carry no read noise - coefficients = coefficients_for(float('nan')) - coefficients.load_img_data() - assert coefficients.noise_property is None - assert 'carry no read noise' in capsys.readouterr().out - - def test_a_usable_noise_column_is_kept(self, coefficients_for): - coefficients = coefficients_for(2.0) - coefficients.load_img_data() - assert coefficients.noise_property == 'noise_sum_col_val' - assert 'noise_sum_col_val' in coefficients.calculated_img_data.columns diff --git a/ledsa/tests/UnitTests/test_sideband_background.py b/ledsa/tests/UnitTests/test_sideband_background.py index 86cf05e..296be70 100644 --- a/ledsa/tests/UnitTests/test_sideband_background.py +++ b/ledsa/tests/UnitTests/test_sideband_background.py @@ -296,8 +296,6 @@ def coefficients(self, monkeypatch): coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) coefficients.experiment = SimpleNamespace(channel=0, path='.', led_array=0) coefficients.average_images = False - # set in __init__: the property analysed here has no noise column - coefficients.noise_property = None return coefficients def test_nan_values_stop_the_analysis(self, coefficients): @@ -326,10 +324,10 @@ def test_raw_mask_keeps_channel_pixels_with_value_zero(self, monkeypatch): # mocked at the one pass over the RAW file, so that every reader derived from it stays consistent. # A black level of 0 and a white level of 255 leave the stored values as they are. raw = image_reading.RawImage(data, self.FILTER_ARRAY, np.zeros(4), 255.0, - image_reading.UPRIGHT_ORIENTATION, None) + image_reading.UPRIGHT_ORIENTATION) monkeypatch.setattr(image_reading, '_read_raw_image', lambda filename: raw) - channel_array, mask, _ = \ - image_reading.read_channel_data_mask_and_noise_from_img('img.CR3', 1) + channel_array, mask = \ + image_reading.read_channel_data_and_mask_from_img('img.CR3', 1) np.testing.assert_array_equal(mask, np.isin(self.FILTER_ARRAY, [1, 3])) np.testing.assert_array_equal(channel_array, np.where(mask, data, 0)) np.testing.assert_array_equal(channel_array, image_reading._read_channel_data_from_raw_file('img.CR3', 1)) @@ -338,13 +336,12 @@ def test_all_pixels_of_a_png_belong_to_the_channel(self, tmp_path): import matplotlib.pyplot as plt path = tmp_path / 'img.png' plt.imsave(path, np.random.default_rng(0).random((6, 8, 3))) - channel_array, mask, pixel_noise = \ - image_reading.read_channel_data_mask_and_noise_from_img(str(path), 1) - assert pixel_noise is None # a PNG has no optical black + channel_array, mask = \ + image_reading.read_channel_data_and_mask_from_img(str(path), 1) np.testing.assert_array_equal(channel_array, image_reading.read_channel_data_from_img(str(path), 1)) assert mask.shape == channel_array.shape and mask.all() def test_unsupported_format_raises(self): from ledsa.core.image_integrity import CorruptedImageError with pytest.raises(CorruptedImageError, match='unsupported file extension'): - image_reading.read_channel_data_mask_and_noise_from_img('img.xyz', 0) + image_reading.read_channel_data_and_mask_from_img('img.xyz', 0) diff --git a/ledsa/tests/UnitTests/test_stacked_search_areas.py b/ledsa/tests/UnitTests/test_stacked_search_areas.py index 35b41b6..3abee3b 100644 --- a/ledsa/tests/UnitTests/test_stacked_search_areas.py +++ b/ledsa/tests/UnitTests/test_stacked_search_areas.py @@ -26,7 +26,7 @@ SEARCH_AREAS = np.array([[0, 8, 20], [1, 16, 20], [2, 24, 20], [3, 32, 20], [4, 38, 20], [5, 20, 38]]) LINE_INDICES = [np.array([0, 1, 2, 3, 4]), np.array([5])] QUANTITIES = ['sum_color_value', 'mean_color_value', 'max_color_value', 'bgsub_sum_color_value', - 'sidebg_sum_color_value', 'sidebg_pedestal', 'bgsub2_sum_color_value', 'noise_sum_color_value'] + 'sidebg_sum_color_value', 'sidebg_pedestal', 'bgsub2_sum_color_value'] def make_config(): @@ -46,17 +46,16 @@ def sideband_pixels_of(img_channels, config): @pytest.fixture(params=[image_reading.UPRIGHT_ORIENTATION, SIDEWAYS]) def img_channels(request, monkeypatch): - """A RAW image with a black level and a read noise of its own for every Bayer color.""" + """A RAW image with a black level of its own for every Bayer color.""" stored = np.random.default_rng(0).integers(1900, 4000, IMG_SHAPE).astype(np.uint16) - raw = RawImage(stored, FILTER_ARRAY, np.array([2040.3, 2041.7, 2042.1, 2043.9]), 16383.0, request.param, - np.array([1.5, 2.5, 3.5, 4.5])) + raw = RawImage(stored, FILTER_ARRAY, np.array([2040.3, 2041.7, 2042.1, 2043.9]), 16383.0, request.param) monkeypatch.setattr(image_reading, '_read_raw_image', lambda filename: raw) return ImageChannels('img.CR3') def quantities_by_definition(img_channels, channel, row, col, sideband_pixels): """The quantities of one LED, from their definitions on the arrays of the whole image.""" - data, mask, noise = img_channels.channel_data_mask_and_noise(channel) + data, mask = img_channels.channel_data_and_mask(channel) box = np.s_[row - SEARCH_AREA_RADIUS:row + SEARCH_AREA_RADIUS, col - SEARCH_AREA_RADIUS:col + SEARCH_AREA_RADIUS] values, in_channel = data[box], mask[box] ring = np.ones(values.shape, dtype=bool) @@ -71,8 +70,7 @@ def quantities_by_definition(img_channels, channel, row, col, sideband_pixels): 'bgsub_sum_color_value': np.sum(values.astype(np.float64) - np.median(ring_pixels) * in_channel), 'sidebg_sum_color_value': box_sum - num_channel_pixels * sideband_background, 'sidebg_pedestal': num_channel_pixels * sideband_background, - 'bgsub2_sum_color_value': box_sum - num_channel_pixels * np.mean(ring_pixels, dtype=np.float64), - 'noise_sum_color_value': np.sqrt(np.sum(noise[box].astype(np.float64) ** 2))} + 'bgsub2_sum_color_value': box_sum - num_channel_pixels * np.mean(ring_pixels, dtype=np.float64)} class TestStackedAnalysis: From 3302faf6fede9fc5bcd62093aa3f093052141f1f Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 12:06:25 +0100 Subject: [PATCH 4/6] leave out an LED without a finite relative intensity as intended The linear solver marks an LED whose relative intensity is not finite as not measured, but it zeroed the row of such an LED by multiplying it with its weight of 0, and the optical depth of a NaN intensity is NaN, which the product keeps. nnls then stopped with "array must not contain infs or NaNs" instead of leaving the LED out. An LED whose reference intensity is 0 gives such a value. The optical depth of a left out LED is now set to 0 with np.where. For finite intensities the system is the same as before. Co-Authored-By: Claude Opus 5.5 --- ledsa/analysis/ExtinctionCoefficientsLinear.py | 5 +++-- .../UnitTests/test_extinction_coefficients_linear.py | 11 ++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ledsa/analysis/ExtinctionCoefficientsLinear.py b/ledsa/analysis/ExtinctionCoefficientsLinear.py index 0d9229e..ed7b28e 100644 --- a/ledsa/analysis/ExtinctionCoefficientsLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsLinear.py @@ -105,8 +105,9 @@ def calc_coefficients_of_img(self, rel_intensities: np.ndarray) -> np.ndarray: # A_aug = [A; sqrt(lambda)*L], where A is the distance matrix A_aug = np.vstack((self.distances_per_led_and_layer * weights[:, None], sqrt_lambda * L)) - # Augment the right-hand side: b_aug = [b; 0] - b_aug = np.hstack((optical_depths * weights, np.zeros(L.shape[0]))) + # Augment the right-hand side: b_aug = [b; 0]. The optical depth of a left out LED is set to 0 rather + # than multiplied by its weight: that of an LED without a finite value is NaN, and NaN * 0 is NaN. + b_aug = np.hstack((np.where(measured, optical_depths, 0.0), np.zeros(L.shape[0]))) # Solve the non-negative least squares problem: min ||A_aug*x - b_aug||^2 subject to x >= 0 sigmas, residuals = nnls(A_aug, b_aug) diff --git a/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py b/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py index 39b91ac..56ccdf1 100644 --- a/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py +++ b/ledsa/tests/UnitTests/test_extinction_coefficients_linear.py @@ -37,6 +37,15 @@ def test_an_led_at_or_below_zero_is_left_out(self): np.testing.assert_allclose(make_solver().calc_coefficients_of_img(rel_intensities), without_the_led, atol=1e-9) + def test_an_led_without_a_finite_value_is_left_out(self): + # an LED whose reference intensity is 0 gives NaN or an infinite relative intensity + without_the_led = make_solver(DISTANCES[1:]).calc_coefficients_of_img(REL_INTENSITIES[1:]) + for value in [np.nan, np.inf, -np.inf]: + rel_intensities = REL_INTENSITIES.copy() + rel_intensities[0] = value + np.testing.assert_allclose(make_solver().calc_coefficients_of_img(rel_intensities), without_the_led, + atol=1e-9) + def test_an_image_without_any_measured_led_gives_nan(self): - rel_intensities = np.array([0.0, -0.1, 0.0, -2.0]) + rel_intensities = np.array([0.0, -0.1, np.nan, -2.0]) assert np.isnan(make_solver().calc_coefficients_of_img(rel_intensities)).all() From fae2ed77b048f1686f13210a987b4a985411d116 Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 12:13:38 +0100 Subject: [PATCH 5/6] remove the background corrections of step 3 Step 3 wrote three background-corrected sums besides the plain sum: bgsub_sum_col_val (median of a border ring of the search area), bgsub2_sum_col_val (mean of that ring) and sidebg_sum_col_val (mean of side bands next to the LED strip, with sidebg_pedestal). They have not been validated on the data of an experiment, so v0.13.0 leaves them out; develop keeps them. Step 3 writes sum_col_val, mean_col_val and max_col_val again, plus the fit columns with -s3. The fast layout is the one of the versions before v0.11.0; the fitted one has 20 names and is new, because those versions glued max_col_val to led_center_x. The reader still knows every earlier layout, so result files of v0.11.0 and v0.12.0 can be analysed, including their background-corrected columns. The search areas keep being stacked by their shape and calibrated once for all channels; only the side band pixels leave the selection. The options sideband_distances, background_floor and sideband_exclusion_margin leave config.ini, and a config.ini that still sets them is told that they are ignored. The tests of the corrections go: test_background_subtraction.py, test_ring_background_mean.py, test_sideband_background.py and the acceptance suite 10. What they held beyond the corrections moves: - the column layouts to test_file_handling.py, now with the layouts of v0.13.0 and v0.12.0, - the Bayer channel mask to test_image_channels.py, - the NaN check of the analysed quantity to test_analysis_quantities.py, - the acceptance cases for several quantities per run (now sum_col_val and mean_col_val) and for fitted step 3 results to suite 08, renamed to 08_test_analysis_quantities.robot. The README section and the documentation of the corrections go as well. Co-Authored-By: Claude Opus 5.5 --- README.md | 17 +- doc/cli_documentation.rst | 2 +- doc/data_extraction.rst | 8 - ledsa/analysis/ConfigDataAnalysis.py | 8 +- ledsa/analysis/ConfigDataStacked.py | 2 +- ledsa/analysis/ExtinctionCoefficients.py | 9 +- ledsa/core/ConfigData.py | 27 +- ledsa/core/file_handling.py | 13 +- ledsa/core/image_reading.py | 4 +- ledsa/data_extraction/LEDAnalysisData.py | 20 - ledsa/data_extraction/sideband_background.py | 227 ------------ ledsa/data_extraction/step_3_functions.py | 209 +---------- .../08_test_analysis_quantities.robot | 68 ++++ .../08_test_background_subtraction.robot | 136 ------- .../10_test_sideband_background.robot | 92 ----- .../AcceptanceTests/LedsaATestLibrary.py | 37 +- .../UnitTests/test_analysis_quantities.py | 69 +++- .../UnitTests/test_background_subtraction.py | 240 ------------ .../UnitTests/test_config_file_saving.py | 4 +- ledsa/tests/UnitTests/test_file_handling.py | 98 +++++ ledsa/tests/UnitTests/test_image_channels.py | 35 +- .../UnitTests/test_ring_background_mean.py | 146 -------- .../UnitTests/test_sideband_background.py | 347 ------------------ .../UnitTests/test_stacked_search_areas.py | 55 +-- 24 files changed, 313 insertions(+), 1560 deletions(-) delete mode 100644 ledsa/data_extraction/sideband_background.py create mode 100644 ledsa/tests/AcceptanceTests/08_test_analysis_quantities.robot delete mode 100644 ledsa/tests/AcceptanceTests/08_test_background_subtraction.robot delete mode 100644 ledsa/tests/AcceptanceTests/10_test_sideband_background.robot delete mode 100644 ledsa/tests/UnitTests/test_background_subtraction.py delete mode 100644 ledsa/tests/UnitTests/test_ring_background_mean.py delete mode 100644 ledsa/tests/UnitTests/test_sideband_background.py diff --git a/README.md b/README.md index 52a3914..6a909ef 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ another, each solver and quantity writing into a directory of its own: ```ini [DEFAULT] -reference_property = sum_col_val bgsub_sum_col_val +reference_property = sum_col_val mean_col_val ``` ``` @@ -135,21 +135,6 @@ individual runs. Since image IDs are counted per run, `num_ref_images` and `ref_ be set per `[simulation_N]` section; pick the reference images of all cameras from the same time window before the fire. -### Background correction (optional) - -Besides the plain sum `sum_col_val`, step 3 writes background-corrected sums that can be selected -as `reference_property`: - -| `reference_property` | Background estimate | -|---|---| -| `sum_col_val` (default) | none | -| `bgsub_sum_col_val` | median of a 2 px border ring of the search area, subtracted pixel-wise | -| `bgsub2_sum_col_val` | mean of the same border ring, subtracted per pixel of the search area | -| `sidebg_sum_col_val` | mean of side bands next to the LED strip, subtracted per pixel of the search area | - -The side bands lie `sideband_distances` (default 25 to 60 px) across the strip in `config.ini`; -raise `sideband_exclusion_margin` if light of a neighbouring strip reaches them. - ### Temperature correction (optional) LEDs lose radiant power as they heat up, which would otherwise be read as extinction. Reduction diff --git a/doc/cli_documentation.rst b/doc/cli_documentation.rst index 8cce105..1f2484e 100644 --- a/doc/cli_documentation.rst +++ b/doc/cli_documentation.rst @@ -34,7 +34,7 @@ Data Extraction - Step 3: Analyse all images to extract pixel values inside the search areas and fit them to an LED model (SLOW!). - ``-r`` (Use the red channel, default), ``-g`` (Use the green channel), ``-b`` (Use the blue channel), ``-rgb`` (Run for each channel) * - ``-s3_fast``, ``--step_3_fast`, ``--analyse_photo_fast`` - - Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). Besides the plain sum, the background-corrected sums ``bgsub_sum_col_val``, ``bgsub2_sum_col_val`` and ``sidebg_sum_col_val`` are written (see ``reference_property`` in the analysis configuration). + - Step 3: Analyse all images to extract pixel values and calculate the accumulated values inside the search areas (FAST!). - Channel flags as for ``-s3`` * - ``-re``, ``--restart`` - Restart step 3 if it was previously interrupted. Only the images that lack the result file of one of the selected channels are analysed, in all of these channels. The restart resumes in the mode of the interrupted run, with the LED fits after ``-s3`` and without them after ``-s3_fast``, as the headers of its result files tell. If there are no result files yet, it runs without the fits. If the result files were written in both modes, it stops, because it cannot tell which mode to resume. diff --git a/doc/data_extraction.rst b/doc/data_extraction.rst index 2dae0da..b1416d1 100644 --- a/doc/data_extraction.rst +++ b/doc/data_extraction.rst @@ -7,14 +7,6 @@ Data Extraction :show-inheritance: .. automodule:: ledsa.data_extraction.LEDAnalysisData - :members: - :undoc-members: - :show-inheritance: - -Side Band Pedestal ------------------- - -.. automodule:: ledsa.data_extraction.sideband_background :members: :undoc-members: :show-inheritance: \ No newline at end of file diff --git a/ledsa/analysis/ConfigDataAnalysis.py b/ledsa/analysis/ConfigDataAnalysis.py index f19d6e3..5087c7b 100644 --- a/ledsa/analysis/ConfigDataAnalysis.py +++ b/ledsa/analysis/ConfigDataAnalysis.py @@ -44,7 +44,7 @@ def parse_reference_properties(value: str) -> List[str]: properties.append(name) if not properties: raise ValueError(f"The reference_property option names no quantity ({value!r}). Give one or " - f"more quantities separated by spaces, e.g. 'sum_col_val bgsub_sum_col_val'.") + f"more quantities separated by spaces, e.g. 'sum_col_val mean_col_val'.") return properties @@ -93,10 +93,8 @@ def __init__(self, load_config_file=True, camera_position=None, num_layers=20, d :param num_cores: Number of CPU cores for (multicore) processing. If greater than 1, multicore processing is applied. Defaults to 1. :type num_cores: int :param reference_property: Quantity or quantities the extinction coefficients are computed - from. Defaults to 'sum_col_val'. Use 'bgsub_sum_col_val' to base the analysis on - intensities with the local background subtracted, estimated from the border of each LED - search area. Several quantities, given as a list or separated by spaces, are analysed one - after another, each into a directory of its own. + from. Defaults to 'sum_col_val'. Several quantities, given as a list or separated by + spaces, are analysed one after another, each into a directory of its own. :type reference_property: Union[str, Iterable[str]] :param average_images: Determines if intensities are computed as an average from two consecutive images. Defaults to False. :type average_images: bool diff --git a/ledsa/analysis/ConfigDataStacked.py b/ledsa/analysis/ConfigDataStacked.py index 66465c9..16ce23e 100644 --- a/ledsa/analysis/ConfigDataStacked.py +++ b/ledsa/analysis/ConfigDataStacked.py @@ -26,7 +26,7 @@ class ConfigDataStacked(cp.ConfigParser): camera_channels = 0 1 2 num_layers = 20 domain_bounds = 0.0 3.0 - reference_property = sum_col_val bgsub_sum_col_val + reference_property = sum_col_val mean_col_val num_ref_images = 10 ref_img_indices = None average_images = False diff --git a/ledsa/analysis/ExtinctionCoefficients.py b/ledsa/analysis/ExtinctionCoefficients.py index 14996dc..39394ba 100644 --- a/ledsa/analysis/ExtinctionCoefficients.py +++ b/ledsa/analysis/ExtinctionCoefficients.py @@ -204,16 +204,15 @@ def load_img_data(self) -> None: self.calculated_img_data = img_data_cropped[img_data_cropped['led_array_id'] == self.experiment.led_array] if self.calculated_img_data.empty: exit(f"Apparently there are no intensity values for led array {self.experiment.led_array}!") - # The solvers cannot handle NaN: the linear one stops with an unspecific error, the nonlinear one returns - # zeros without a warning. sidebg_sum_col_val is NaN for LEDs without side band pixels, for example. + # A NaN is no measurement, and the solvers would hide it: the nonlinear one returns zeros without a + # warning, the linear one leaves the LED out without a word. nan_values = self.calculated_img_data[self.reference_property].isna() if nan_values.any(): nan_led_ids = sorted(self.calculated_img_data[nan_values].index.get_level_values('led_id').unique()) led_ids_text = ', '.join(map(str, nan_led_ids[:10])) + (', ...' if len(nan_led_ids) > 10 else '') raise ValueError(f'{int(nan_values.sum())} values of {self.reference_property} of led array ' - f'{self.experiment.led_array} are NaN (led ids {led_ids_text}). For sidebg_sum_col_val ' - f'this means that no side band pixels were found for these LEDs, see sideband_distances ' - f'in config.ini.') + f'{self.experiment.led_array} are NaN (led ids {led_ids_text}). Check the step 3 ' + f'results of these LEDs or analyse another quantity.') # Images without analysis results are missing from the binary data, so the IDs are not necessarily a # gapless range. They are kept to pair every computed coefficient with the time of its own image. diff --git a/ledsa/core/ConfigData.py b/ledsa/core/ConfigData.py index 8fa8162..b0325bd 100644 --- a/ledsa/core/ConfigData.py +++ b/ledsa/core/ConfigData.py @@ -11,6 +11,10 @@ # Options of config.ini that a release of LEDSA has removed, with what became of them. A config # written by an older version still sets them, and loading it names them. REMOVED_OPTIONS = { + option: 'It belonged to the background corrections of step 3 (bgsub_sum_col_val, bgsub2_sum_col_val, ' + 'sidebg_sum_col_val), which LEDSA 0.13.0 removed until they are validated; they are kept on the ' + 'develop branch of the repository.' + for option in ('sideband_distances', 'background_floor', 'sideband_exclusion_margin') } @@ -26,8 +30,7 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius first_img_experiment_id=None, last_img_experiment_id=None, ref_img_id=None, ignore_led_indices=None, led_array_edge_indices=None, led_array_edge_coordinates=None, first_img_analysis_id=None, last_img_analysis_id=None, num_skip_imgs=0, ref_img_experiment_ids=None, num_skip_leds=0, - merge_led_array_indices=None, - sideband_distances='25 60', background_floor=None, sideband_exclusion_margin=None): # TODO: merge LED arrays + merge_led_array_indices=None): # TODO: merge LED arrays """ :param load_config_file: Determines whether to load the config file on initialization. Defaults to True. :type load_config_file: bool @@ -90,17 +93,6 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius :type num_skip_leds: int :param merge_led_array_indices: Flag to merge LED arrays for analysis. Defaults to None. TODO: not a flag but list of arrays to merge? :type merge_led_array_indices: bool or None - :param sideband_distances: Inner and outer distance in pixels across the LED strip of the side bands used to - estimate the pedestal for sidebg_sum_col_val. Defaults to '25 60'. - :type sideband_distances: str - :param background_floor: Lower limit of sidebg_sum_col_val and bgsub2_sum_col_val as fraction of - sum_col_val. None, the default, applies no limit: a corrected sum at or below zero says that - the signal of the LED has vanished, and the solver discards such an LED by its weight. - Defaults to None. - :type background_floor: float or None - :param sideband_exclusion_margin: Side band pixels within search_area_radius + margin of any other search area - are excluded. Defaults to None, i.e. margin = search_area_radius. - :type sideband_exclusion_margin: int or None """ cp.ConfigParser.__init__(self, allow_no_value=True) if load_config_file: @@ -185,15 +177,6 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius self['analyse_photo'][' ref_img_experiment_ids'] = str(ref_img_experiment_ids) self.set('analyse_photo', ' # In the analysis, only LEDs are considered with id dividable by skip_leds + 1.') self['analyse_photo'][' num_skip_leds'] = str(num_skip_leds) - self.set('analyse_photo', ' # Pedestal estimation in side bands parallel to the LED strip (sidebg_sum_col_val):') - self.set('analyse_photo', ' # inner and outer distance of the side bands across the strip in pixels,') - self['analyse_photo'][' sideband_distances'] = str(sideband_distances) - self.set('analyse_photo', ' # lower limit of the corrected sums sidebg_sum_col_val and bgsub2_sum_col_val as fraction of the plain sum.') - self.set('analyse_photo', ' # None applies no limit, so that an LED whose signal has vanished stays recognisable by its weight,') - self['analyse_photo'][' background_floor'] = str(background_floor) - self.set('analyse_photo', ' # pixels within search_area_radius + margin of other search areas are excluded, None: margin = radius.') - self.set('analyse_photo', ' # Raise the margin above the halo of the LEDs if the strips are close, otherwise their light enters the side bands.') - self['analyse_photo'][' sideband_exclusion_margin'] = str(sideband_exclusion_margin) with open('config.ini', 'w', encoding='utf-8') as configfile: self.write(configfile) diff --git a/ledsa/core/file_handling.py b/ledsa/core/file_handling.py index 5356660..d784443 100644 --- a/ledsa/core/file_handling.py +++ b/ledsa/core/file_handling.py @@ -689,16 +689,19 @@ def create_binary_data(channel: int, path: str = '.') -> None: LED_CENTER_COLUMNS = ["led_center_x", "led_center_y"] FIT_COLUMNS = ["x", "y", "dx", "dy", "A", "alpha", "wx", "wy", "fit_success", "fit_fun", "fit_nfev", "fit_time"] -# known layouts by the number of names (including img_id) +# Known layouts by the number of names (including img_id). Step 3 writes the first two; the others are those of +# earlier versions, whose background-corrected sums and read noise v0.13.0 no longer computes but still reads. COLUMN_LAYOUTS = { - len(BASE_COLUMNS): BASE_COLUMNS, # -s3_fast, before the local background subtraction + len(BASE_COLUMNS): BASE_COLUMNS, # -s3_fast, before v0.11.0 and since v0.13.0 + len(BASE_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): + BASE_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit, since v0.13.0 len(BASE_COLUMNS + BGSUB_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS, # -s3_fast, before the side band pedestal - len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS, # -s3_fast + len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS): + BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS, # -s3_fast of v0.11.0 len(BASE_COLUMNS + BGSUB_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit, before the side band pedestal len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): - BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, - # -s3 with fit, before the optical black + BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit of v0.11.0 len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS, # -s3_fast of v0.12.0 len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS + NOISE_SUM_COLUMNS + LED_CENTER_COLUMNS diff --git a/ledsa/core/image_reading.py b/ledsa/core/image_reading.py index 34d52cf..ff2b9ff 100644 --- a/ledsa/core/image_reading.py +++ b/ledsa/core/image_reading.py @@ -107,8 +107,8 @@ class ImageChannels: at a time, so that besides the decoded image only the channel in work is held in memory. The stored values of a RAW image are kept as they are, and the black level and the white level are - applied to the pixels that are asked for. Step 3 asks only for its search areas and side bands, a few - percent of the sensor, which saves applying them to the whole image. + applied to the pixels that are asked for. Step 3 asks only for its search areas, a few percent of the + sensor, which saves applying them to the whole image. """ def __init__(self, filename: str, rotation: str = AUTO_ROTATION): diff --git a/ledsa/data_extraction/LEDAnalysisData.py b/ledsa/data_extraction/LEDAnalysisData.py index 7c19af8..0e405ad 100644 --- a/ledsa/data_extraction/LEDAnalysisData.py +++ b/ledsa/data_extraction/LEDAnalysisData.py @@ -18,19 +18,6 @@ class LEDAnalysisData: :vartype sum_color_value: float :ivar max_color_value: Maximum color value observed for the LED. :vartype max_color_value: float - :ivar bgsub_sum_color_value: Integrated color value of the LED over the search area after subtraction of the - local background estimated from the border of the search area. - :vartype bgsub_sum_color_value: float - :ivar sidebg_sum_color_value: Integrated color value of the LED over the search area after subtraction of the - pedestal estimated in side bands next to the LED strip, at least a fraction of the plain sum. - :vartype sidebg_sum_color_value: float - :ivar sidebg_pedestal: Pedestal N * B subtracted for sidebg_sum_color_value (N channel pixels of the search - area, B mean per channel pixel in the side bands). - :vartype sidebg_pedestal: float - :ivar bgsub2_sum_color_value: Integrated color value of the LED over the search area minus N * b, with b the mean - over the channel pixels (including 0) of the border ring of the search area, at least a fraction of the - plain sum (background_floor). - :vartype bgsub2_sum_color_value: float :ivar fit_results: Fit results after fitting. :vartype fit_results: OptimizeResult :ivar fit_time: Time taken for fitting. @@ -54,10 +41,6 @@ def __init__(self, led_id, led_array, fit_leds): self.mean_color_value = None self.sum_color_value = None self.max_color_value = None - self.bgsub_sum_color_value = None - self.sidebg_sum_color_value = None - self.sidebg_pedestal = None - self.bgsub2_sum_color_value = None self.fit_leds = fit_leds self.fit_results = None self.fit_time = None @@ -85,9 +68,6 @@ def get_main_data_string(self) -> str: """ out_str = f'{self.led_id:4d},{self.led_array:2d},' out_str += f'{self.sum_color_value:10.4e},{self.mean_color_value:10.4e},{self.max_color_value}' - out_str += f',{self.bgsub_sum_color_value:10.4e}' - out_str += f',{self.sidebg_sum_color_value:10.4e},{self.sidebg_pedestal:10.4e}' - out_str += f',{self.bgsub2_sum_color_value:10.4e}' return out_str def get_fit_data_string(self) -> str: diff --git a/ledsa/data_extraction/sideband_background.py b/ledsa/data_extraction/sideband_background.py deleted file mode 100644 index 4108362..0000000 --- a/ledsa/data_extraction/sideband_background.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -Pedestal (additive background) of the LED search areas, estimated in side bands next to the LED strips. - -The search area of an LED contains, besides the LED itself, an additive pedestal: stray light, light emitted or -scattered by the smoke and noise clipped at 0 after the black level subtraction of raw images. Under dense smoke the -LED dims while the pedestal remains, so the plain box sum (sum_col_val) overestimates the transmission. The border -ring of the search area (bgsub_sum_col_val) is not suited to measure the pedestal because it lies in the halo of the -LED itself. The side bands used here run parallel to the strip, at a distance of typically 25 to 60 px across the -strip and over +-R along the strip (R = search_area_radius), so they sample the scene next to the LED but not the LED. - -The strip direction is estimated locally for each LED from its nearest neighbours in the same LED array, so vertical, -horizontal and oblique strips are supported. Pixels close to any other search area are excluded from the bands, so -neighbouring strips do not enter the estimate. - -The exclusion is purely geometric and does not know the halo of the LEDs, so for strips closer than the outer band -distance and for wide halos the light of a neighbouring strip can still reach the bands and raise the estimated -pedestal. Most of that light cancels in the ratio to the reference image, because it is LED light and dims with the -same transmission as the LED itself; what remains is a bias only if the two strips lie at different depths in the -smoke, which is removed by raising sideband_exclusion_margin above the visible halo. -""" -from typing import List, Optional, Sequence, Tuple - -import numpy as np - -DEFAULT_SIDEBAND_DISTANCES = (25, 60) -# No lower limit by default. It was introduced because the clipping at zero made the pedestal of a -# vanished LED unavoidable, so that the optical depth ran away where the LED disappeared in it. With -# unclipped data the limit does the opposite of what is wanted: it fabricates a positive sum exactly -# where the signal of the LED has vanished, and the solver then weights it like a measurement instead of -# discarding it by its signal-to-noise ratio. A fraction of the plain sum is still accepted for the -# behaviour of v0.11.0. -DEFAULT_BACKGROUND_FLOOR = None -NUM_DIRECTION_NEIGHBOURS = 4 - -# geometry of the side bands, cached per process because it only depends on the LED positions -_sideband_cache = {} - - -def estimate_strip_directions(search_areas: np.ndarray, led_arrays: Sequence[Sequence[int]], - num_neighbours: int = NUM_DIRECTION_NEIGHBOURS) -> np.ndarray: - """ - Estimate the local direction of the LED strip at each LED from its nearest neighbours in the same LED array. - - The direction is the principal axis of the positions of the LED and its num_neighbours nearest neighbours of the - same array. It is oriented such that the row component is positive (or, for horizontal strips, the column - component). - - :param search_areas: Array with rows (LED id, pixel row, pixel column), the row index equals the LED id. - :type search_areas: np.ndarray - :param led_arrays: LED ids of each LED array. - :type led_arrays: Sequence[Sequence[int]] - :param num_neighbours: Number of neighbours used to estimate the direction. - :type num_neighbours: int - :return: Array of shape (number of LEDs, 2) with the unit vector (row, column) along the strip, NaN for LEDs - that are not part of an array with at least two LEDs. - :rtype: np.ndarray - """ - directions = np.full((search_areas.shape[0], 2), np.nan) - for led_ids in led_arrays: - led_ids = np.atleast_1d(np.asarray(led_ids, dtype=int)) - if led_ids.size < 2: - continue - positions = search_areas[led_ids, 1:3].astype(np.float64) - num_points = min(num_neighbours, led_ids.size - 1) + 1 - for i, led_id in enumerate(led_ids): - distances = np.hypot(*(positions - positions[i]).T) - nearest = np.argsort(distances, kind='stable')[:num_points] - points = positions[nearest] - positions[nearest].mean(axis=0) - direction = np.linalg.svd(points, full_matrices=False)[2][0] - if direction[0] < 0 or (direction[0] == 0 and direction[1] < 0): - direction = -direction - directions[led_id] = direction - return directions - - -def build_sideband_pixels(search_areas: np.ndarray, directions: np.ndarray, img_shape: Tuple[int, int], - search_area_radius: int, distances: Tuple[float, float] = DEFAULT_SIDEBAND_DISTANCES, - exclusion_margin: Optional[int] = None) -> List[Optional[Tuple[np.ndarray, np.ndarray]]]: - """ - Determine the pixels of the two side bands of each LED. - - With t the coordinate along the strip and s the coordinate across the strip, both relative to the center of the - search area, a pixel belongs to the side bands if ``-R <= t < R`` and ``lo <= abs(s) < hi`` (with the same - half-open convention as the search area, i.e. ``lo <= s < hi`` or ``-hi <= s < -lo``). For a vertical strip this is - rows r-R..r+R-1 and columns c+lo..c+hi-1 and c-hi..c-lo-1. Pixels outside the image and pixels within - R + exclusion_margin (Chebyshev distance) of the center of any other search area are excluded. - - :param search_areas: Array with rows (LED id, pixel row, pixel column), the row index equals the LED id. - :type search_areas: np.ndarray - :param directions: Unit vectors along the strip per LED, see estimate_strip_directions. - :type directions: np.ndarray - :param img_shape: Shape of the image. - :type img_shape: tuple[int, int] - :param search_area_radius: Radius R of the search areas. - :type search_area_radius: int - :param distances: Inner and outer distance (lo, hi) of the side bands across the strip in pixels. - :type distances: tuple[float, float] - :param exclusion_margin: Margin added to R for the exclusion of other search areas. Defaults to R. - :type exclusion_margin: int or None - :return: Per LED id a tuple of row and column indices of the side band pixels, None if the strip direction of - the LED is unknown. - :rtype: list[tuple[np.ndarray, np.ndarray] or None] - """ - radius = int(search_area_radius) - inner, outer = float(distances[0]), float(distances[1]) - if exclusion_margin is None: - exclusion_margin = radius - exclusion = radius + int(exclusion_margin) - reach = int(np.ceil(np.hypot(outer, radius))) + 1 - d_row, d_col = np.mgrid[-reach:reach + 1, -reach:reach + 1] - centers = search_areas[:, 1:3].astype(np.int64) - sideband_pixels = [None] * search_areas.shape[0] - for led_id in range(search_areas.shape[0]): - along = directions[led_id] - if not np.all(np.isfinite(along)): - continue - across = np.array([-along[1], along[0]]) - # rounding avoids that pixels exactly on a band border flip due to floating point errors - t = np.round(d_row * along[0] + d_col * along[1], 6) - s = np.round(d_row * across[0] + d_col * across[1], 6) - in_band = (t >= -radius) & (t < radius) & (((s >= inner) & (s < outer)) | ((s >= -outer) & (s < -inner))) - rows = centers[led_id, 0] + d_row[in_band] - cols = centers[led_id, 1] + d_col[in_band] - keep = (rows >= 0) & (rows < img_shape[0]) & (cols >= 0) & (cols < img_shape[1]) - nearby = np.abs(centers - centers[led_id]).max(axis=1) <= reach + exclusion - nearby[led_id] = False - for row_other, col_other in centers[nearby]: - keep &= ~((rows >= row_other - exclusion) & (rows < row_other + exclusion) & - (cols >= col_other - exclusion) & (cols < col_other + exclusion)) - sideband_pixels[led_id] = (rows[keep].astype(np.intp), cols[keep].astype(np.intp)) - return sideband_pixels - - -def get_sideband_pixels(search_areas: np.ndarray, led_arrays: Sequence[Sequence[int]], img_shape: Tuple[int, int], - search_area_radius: int, distances: Tuple[float, float] = DEFAULT_SIDEBAND_DISTANCES, - exclusion_margin: Optional[int] = None) -> List[Optional[Tuple[np.ndarray, np.ndarray]]]: - """ - Cached combination of estimate_strip_directions and build_sideband_pixels. The geometry only depends on the - LED positions and is therefore computed once per process and reused for all images. - - :return: See build_sideband_pixels. - """ - key = (search_areas.tobytes(), search_areas.shape, - tuple(tuple(int(i) for i in np.atleast_1d(led_ids)) for led_ids in led_arrays), - tuple(img_shape), int(search_area_radius), tuple(float(d) for d in distances), exclusion_margin) - if key not in _sideband_cache: - _sideband_cache.clear() - directions = estimate_strip_directions(search_areas, led_arrays) - _sideband_cache[key] = build_sideband_pixels(search_areas, directions, img_shape, search_area_radius, - distances, exclusion_margin) - return _sideband_cache[key] - - -def estimate_sideband_background(data: np.ndarray, channel_mask: Optional[np.ndarray], - sideband_pixels: Optional[Tuple[np.ndarray, np.ndarray]]) -> float: - """ - Estimate the pedestal per channel pixel as the mean over the channel pixels in the side bands. - - Pixels of the channel with value 0 are included: the black level subtraction clips noise at 0 and the box sum - contains these clipped pixels in the same way, so the mean is the matching estimator for the pedestal in the box. - Foreign Bayer channels are excluded via the channel mask, not via the pixel value. - - :param data: Channel array of the image. - :type data: np.ndarray - :param channel_mask: Boolean mask of the channel pixels, None if all pixels belong to the channel. - :type channel_mask: np.ndarray or None - :param sideband_pixels: Row and column indices of the side band pixels of the LED. - :type sideband_pixels: tuple[np.ndarray, np.ndarray] or None - :return: Mean pedestal per channel pixel, NaN if the side bands contain no channel pixel. - :rtype: float - """ - if sideband_pixels is None: - return np.nan - values = data[sideband_pixels] - in_channel = np.ones(values.shape, dtype=bool) if channel_mask is None else channel_mask[sideband_pixels] - return float(estimate_sideband_backgrounds(values, in_channel, [values.size])[0]) - - -def estimate_sideband_backgrounds(values: np.ndarray, channel_mask: np.ndarray, - num_pixels: Sequence[int]) -> np.ndarray: - """ - estimate_sideband_background for many LEDs at once, from the side band pixels of all of them. - - :param values: Channel values of the side band pixels of all LEDs, the pixels of one LED after those of the - previous one. - :type values: np.ndarray - :param channel_mask: Whether each of these pixels belongs to the channel. - :type channel_mask: np.ndarray - :param num_pixels: Number of side band pixels of every LED. - :type num_pixels: Sequence[int] - :return: Mean pedestal per channel pixel of every LED, NaN for an LED whose side bands contain no channel pixel. - :rtype: np.ndarray - """ - num_pixels = np.asarray(num_pixels, dtype=np.intp) - backgrounds = np.full(num_pixels.size, np.nan) - # reduceat cannot sum an empty segment, and the segments left are contiguous without the empty ones - has_pixels = num_pixels > 0 - if not has_pixels.any(): - return backgrounds - starts = (np.cumsum(num_pixels) - num_pixels)[has_pixels] - sums = np.add.reduceat(np.where(channel_mask, values, 0), starts, dtype=np.float64) - counts = np.add.reduceat(channel_mask, starts, dtype=np.intp) - with np.errstate(invalid='ignore'): - backgrounds[has_pixels] = sums / counts # 0 / 0 is NaN - return backgrounds - - -def subtract_pedestal(sum_color_value: float, pedestal: float, floor: float = DEFAULT_BACKGROUND_FLOOR) -> float: - """ - Subtract the pedestal N * B from the box sum S, with the optional lower limit max(S - N * B, - floor * S). - - Without the lower limit the optical depth would be unbounded where the LED signal vanishes in the pedestal; - values where the limit applies carry no information on the extinction and should be treated as invalid. - - :param sum_color_value: Box sum S. - :type sum_color_value: float - :param pedestal: Pedestal N * B of the box. - :type pedestal: float - :param floor: Lower limit as fraction of S. - :type floor: float - :return: Pedestal corrected box sum, NaN if the pedestal is NaN. - :rtype: float - """ - if floor is None: - return float(sum_color_value - pedestal) - return float(np.maximum(sum_color_value - pedestal, floor * sum_color_value)) diff --git a/ledsa/data_extraction/step_3_functions.py b/ledsa/data_extraction/step_3_functions.py index 7241e81..0fd8052 100644 --- a/ledsa/data_extraction/step_3_functions.py +++ b/ledsa/data_extraction/step_3_functions.py @@ -13,9 +13,6 @@ from ledsa.core.image_reading import ImageChannels from ledsa.data_extraction.LEDAnalysisData import LEDAnalysisData from ledsa.data_extraction.model import target_function -from ledsa.data_extraction.sideband_background import (DEFAULT_SIDEBAND_DISTANCES, DEFAULT_BACKGROUND_FLOOR, - estimate_sideband_background, estimate_sideband_backgrounds, - get_sideband_pixels, subtract_pedestal) def generate_analysis_data(img_filename: str, channel: int, search_areas: np.ndarray, line_indices: List[List[int]], @@ -57,24 +54,15 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda search_area_radius, fit_leds) return analysis_res - sideband_distances, background_floor, sideband_exclusion_margin = get_sideband_parameters(conf) - sideband_pixels = get_sideband_pixels(search_areas, _get_unmerged_led_arrays(conf, line_indices), - img_channels.shape, search_area_radius, sideband_distances, - sideband_exclusion_margin) - num_skip_leds = int(conf['analyse_photo']['num_skip_leds']) leds = [(int(iled), led_array_idx) for led_array_idx, led_array in enumerate(line_indices) for iled in led_array if iled % (num_skip_leds + 1) == 0] led_ids = np.array([iled for iled, _ in leds], dtype=int) led_array_ids = np.array([led_array_idx for _, led_array_idx in leds], dtype=int) - # only the pixels of the search areas and side bands are read, which are a few percent of the image - analysis_pixels = _get_analysis_pixels(search_areas, led_ids, sideband_pixels, img_channels.shape, - search_area_radius) - data, channel_mask = img_channels.channel_data_and_mask(channel, analysis_pixels.pixels) - sidebands = np.s_[analysis_pixels.num_box_pixels:] - sideband_backgrounds = estimate_sideband_backgrounds(data[sidebands], channel_mask[sidebands], - analysis_pixels.num_sideband_pixels) + # only the pixels of the search areas are read, which are a few percent of the image + analysis_pixels = _get_analysis_pixels(search_areas, led_ids, img_channels.shape, search_area_radius) + data, _ = img_channels.channel_data_and_mask(channel, analysis_pixels.pixels) img_analysis_data = [None] * len(leds) for positions, shape, start in analysis_pixels.box_groups: @@ -82,9 +70,7 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda stack_shape = (positions.size,) + shape group_data = _analyse_search_areas( conf, channel, img_filename, led_ids[positions], led_array_ids[positions], - analysis_pixels.centers[positions], search_area_radius, fit_leds, - data[boxes].reshape(stack_shape), channel_mask[boxes].reshape(stack_shape), - sideband_backgrounds[positions], background_floor) + analysis_pixels.centers[positions], search_area_radius, fit_leds, data[boxes].reshape(stack_shape)) for position, led_analysis_data in zip(positions, group_data): img_analysis_data[position] = led_analysis_data return img_analysis_data @@ -93,7 +79,7 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda class AnalysisPixels(NamedTuple): """ The pixels step 3 reads from an image, in one selection so that the image calibrates them once for all - channels: the search areas of the analysed LEDs, stacked by their shape, followed by their side bands. + channels: the search areas of the analysed LEDs, stacked by their shape. """ pixels: Tuple[np.ndarray, np.ndarray] # row and column indices @@ -101,17 +87,14 @@ class AnalysisPixels(NamedTuple): # per shape of the search areas: the positions of their LEDs in the analysis, the shape and the index of the # first pixel of the stack in pixels box_groups: List[Tuple[np.ndarray, Tuple[int, int], int]] - num_box_pixels: int # the side band pixels follow the search areas, LED after LED - num_sideband_pixels: np.ndarray # number of side band pixels of every analysed LED # pixels of the analysis, cached per process because they only depend on the LED positions _analysis_pixels_cache = {} -def _get_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, - sideband_pixels: List[Optional[Tuple[np.ndarray, np.ndarray]]], - img_shape: Tuple[int, int], search_area_radius: int) -> AnalysisPixels: +def _get_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_shape: Tuple[int, int], + search_area_radius: int) -> AnalysisPixels: """ Cached _build_analysis_pixels. The same object is returned for every image and channel, which lets the image keep the pixels it calibrated for the next channel. @@ -119,20 +102,16 @@ def _get_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, :return: See _build_analysis_pixels. """ key = (search_areas.tobytes(), search_areas.shape, led_ids.tobytes(), tuple(img_shape), search_area_radius) - # the side band pixels come from a cache of their own, which builds a new list when its key changes; the - # list is kept with the entry, so that its identity cannot pass to another one - if key not in _analysis_pixels_cache or _analysis_pixels_cache[key][0] is not sideband_pixels: + if key not in _analysis_pixels_cache: _analysis_pixels_cache.clear() - _analysis_pixels_cache[key] = (sideband_pixels, _build_analysis_pixels( - search_areas, led_ids, sideband_pixels, img_shape, search_area_radius)) - return _analysis_pixels_cache[key][1] + _analysis_pixels_cache[key] = _build_analysis_pixels(search_areas, led_ids, img_shape, search_area_radius) + return _analysis_pixels_cache[key] -def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, - sideband_pixels: List[Optional[Tuple[np.ndarray, np.ndarray]]], - img_shape: Tuple[int, int], search_area_radius: int) -> AnalysisPixels: +def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_shape: Tuple[int, int], + search_area_radius: int) -> AnalysisPixels: """ - Collect the pixels of the search areas and side bands of the analysed LEDs. + Collect the pixels of the search areas of the analysed LEDs. The search area is cut from the image as data[row - R:row + R, col - R:col + R], so an area at the border of the image is cut short as the slice is. Its shape differs from that of the others, and the search areas @@ -142,13 +121,11 @@ def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, :type search_areas: np.ndarray :param led_ids: Ids of the analysed LEDs, in the order of the analysis. :type led_ids: np.ndarray - :param sideband_pixels: Per LED id the row and column indices of its side band pixels, see get_sideband_pixels. - :type sideband_pixels: list[tuple[np.ndarray, np.ndarray] or None] :param img_shape: Shape of the image. :type img_shape: tuple[int, int] :param search_area_radius: Radius R of the search areas. :type search_area_radius: int - :return: The pixels and where the analysis finds the search area and the side bands of every LED in them. + :return: The pixels and where the analysis finds the search area of every LED in them. :rtype: AnalysisPixels """ centers = search_areas[led_ids, 1:3].astype(int) @@ -170,69 +147,7 @@ def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, cols.append(np.broadcast_to(first_cols[:, None, None] + np.arange(shape[1]), stack_shape).ravel()) box_groups.append((positions, shape, start)) start += positions.size * shape[0] * shape[1] - - num_sideband_pixels = np.zeros(led_ids.size, dtype=np.intp) - for position, led_id in enumerate(led_ids): - if sideband_pixels[led_id] is not None: - rows.append(sideband_pixels[led_id][0]) - cols.append(sideband_pixels[led_id][1]) - num_sideband_pixels[position] = sideband_pixels[led_id][0].size - return AnalysisPixels((np.concatenate(rows), np.concatenate(cols)), centers, box_groups, start, - num_sideband_pixels) - - -def get_sideband_parameters(conf: ConfigData) -> Tuple[Tuple[float, float], float, Optional[int]]: - """ - Read the parameters of the side band pedestal estimation from the config. Config files written before the - side band estimation was added do not contain the options, the defaults are used then. - - :param conf: Configuration data. - :type conf: ConfigData - :return: Distances (inner, outer) of the side bands across the strip in pixels, lower limit of the corrected - sum as fraction of the box sum and margin for the exclusion of other search areas (None: R). - :rtype: tuple[tuple[float, float], float, int or None] - """ - distances = DEFAULT_SIDEBAND_DISTANCES - floor = DEFAULT_BACKGROUND_FLOOR - exclusion_margin = None - if conf is not None: - if conf.get('analyse_photo', 'sideband_distances') != 'None': - distances = tuple(float(d) for d in conf.get('analyse_photo', 'sideband_distances').split()) - if len(distances) != 2 or not 0 <= distances[0] < distances[1]: - raise ValueError(f'sideband_distances must be two distances 0 <= inner < outer, got {distances}.') - if conf.get('analyse_photo', 'background_floor') != 'None': - floor = float(conf.get('analyse_photo', 'background_floor')) - if conf.get('analyse_photo', 'sideband_exclusion_margin') != 'None': - exclusion_margin = int(conf.get('analyse_photo', 'sideband_exclusion_margin')) - return distances, floor, exclusion_margin - - -def _get_unmerged_led_arrays(conf: ConfigData, line_indices: List[List[int]]) -> List[List[int]]: - """ - Return the LED arrays as found in step 2. If LED arrays are merged for the analysis, the merged arrays may - contain several strips, which would spoil the estimate of the local strip direction. - - :param conf: Configuration data. - :type conf: ConfigData - :param line_indices: LED ids of the (possibly merged) LED arrays used in the analysis. - :type line_indices: List[List[int]] - :return: LED ids of the LED arrays as found in step 2. - :rtype: List[List[int]] - """ - if conf is None or conf['analyse_positions']['merge_led_array_indices'] == 'None': - return line_indices - led_arrays = [] - for i in range(int(conf['analyse_positions']['num_arrays'])): - file_path = os.path.join('analysis', f'led_array_indices_{i:03}.csv') - # read_table stays silent, it would report every file of every image; its error message is unspecific - try: - led_arrays.append(read_table(file_path, dtype='int', silent=True)) - except OSError as error: - raise OSError(f'{file_path} could not be read ({error}). The LED arrays of step 2 are needed to ' - f'estimate the strip direction of sidebg_sum_col_val, because merge_led_array_indices ' - f'merges LED arrays that may contain several strips. Run step 2 again to write the ' - f'files, or unset merge_led_array_indices.') from error - return led_arrays + return AnalysisPixels((np.concatenate(rows), np.concatenate(cols)), centers, box_groups) def create_fit_result_file(img_data: List[LEDAnalysisData], img_id: int, channel: int) -> None: # TODO: rename because misleading @@ -336,10 +251,7 @@ def find_fit_leds_for_restart(channels: Sequence[int]) -> bool: return fit_leds -def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray, debug: bool, iled: int, img_filename: str, led_array_idx: int, search_areas: np.ndarray, search_area_radius: int, fit_leds: bool = True, - channel_mask: Optional[np.ndarray] = None, - sideband_pixels: Optional[Tuple[np.ndarray, np.ndarray]] = None, - background_floor: float = DEFAULT_BACKGROUND_FLOOR) -> LEDAnalysisData: +def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray, debug: bool, iled: int, img_filename: str, led_array_idx: int, search_areas: np.ndarray, search_area_radius: int, fit_leds: bool = True) -> LEDAnalysisData: """ Generate analysis data for a specific LED from the arrays of the whole image, as _analyse_search_areas does for many. @@ -364,24 +276,14 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray :type search_area_radius: int :param fit_leds: If True, the LED is fitted to a model function. :type fit_leds: bool - :param channel_mask: Boolean mask of the pixels belonging to the color channel (Bayer pattern for raw files). - None if all pixels belong to the channel. - :type channel_mask: np.ndarray or None - :param sideband_pixels: Row and column indices of the side band pixels of the LED, None if not available. - :type sideband_pixels: tuple[np.ndarray, np.ndarray] or None - :param background_floor: Lower limit of the side band and ring corrected sums as fraction of the box sum. - :type background_floor: float :return: Analysis data for the LED. :rtype: LEDAnalysisData """ center = np.array([[int(search_areas[iled, 1]), int(search_areas[iled, 2])]]) search_area = np.index_exp[center[0, 0] - search_area_radius:center[0, 0] + search_area_radius, center[0, 1] - search_area_radius:center[0, 1] + search_area_radius] - box_mask = np.ones(data[search_area].shape, dtype=bool) if channel_mask is None else channel_mask[search_area] led_data, = _analyse_search_areas(conf, channel, img_filename, [iled], [led_array_idx], center, - search_area_radius, fit_leds, data[search_area][None], box_mask[None], - [estimate_sideband_background(data, channel_mask, sideband_pixels)], - background_floor) + search_area_radius, fit_leds, data[search_area][None]) if debug and fit_leds: return led_data.fit_results.x return led_data @@ -389,9 +291,7 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led_ids: Sequence[int], led_array_ids: Sequence[int], centers: np.ndarray, search_area_radius: int, fit_leds: bool, - boxes: np.ndarray, box_masks: np.ndarray, - sideband_backgrounds: Sequence[float], - background_floor: float = DEFAULT_BACKGROUND_FLOOR) -> List[LEDAnalysisData]: + boxes: np.ndarray) -> List[LEDAnalysisData]: """ Generate the analysis data of the LEDs of a stack of search areas of one shape. @@ -417,13 +317,6 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led :param boxes: Channel values of the search areas, 0 outside the channel, of shape (number of LEDs, rows, columns). :type boxes: np.ndarray - :param box_masks: Whether each pixel of the search areas belongs to the channel, of the shape of boxes. - :type box_masks: np.ndarray - :param sideband_backgrounds: Mean pedestal per channel pixel in the side bands of every LED, see - estimate_sideband_background. - :type sideband_backgrounds: Sequence[float] - :param background_floor: Lower limit of the side band and ring corrected sums as fraction of the box sum. - :type background_floor: float :return: Analysis data for every LED. :rtype: List[LEDAnalysisData] """ @@ -431,14 +324,6 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led sums = np.sum(boxes, axis=(1, 2)) maxima = np.amax(boxes, axis=(1, 2)) - # The median of the border ring, subtracted from every channel pixel of the box. It used to be - # clipped at zero pixel by pixel, which puts back the offset that the unclipped data remove. - ring_medians, ring_means = _estimate_ring_backgrounds(boxes, box_masks) - bgsub_sums = np.sum(boxes.astype(np.float64) - ring_medians[:, None, None] * box_masks, axis=(1, 2)) - - # the pedestals below are subtracted as N * b with N the number of channel pixels in the box - num_channel_pixels = np.count_nonzero(box_masks, axis=(1, 2)) - img_analysis_data = [] for i, led_id in enumerate(led_ids): led_data = LEDAnalysisData(led_id, int(led_array_ids[i]), fit_leds) @@ -457,65 +342,10 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led led_data.mean_color_value = means[i] led_data.sum_color_value = sums[i] led_data.max_color_value = maxima[i] - led_data.bgsub_sum_color_value = float(bgsub_sums[i]) - - # border ring as for bgsub, but with the estimator of the side bands: mean over the channel pixels including 0; - # same lower limit as for the side band correction - led_data.bgsub2_sum_color_value = subtract_pedestal(float(sums[i]), num_channel_pixels[i] * ring_means[i], - background_floor) - - # b = mean over the channel pixels in the side bands next to the LED strip - led_data.sidebg_pedestal = float(num_channel_pixels[i] * sideband_backgrounds[i]) - led_data.sidebg_sum_color_value = subtract_pedestal(float(sums[i]), led_data.sidebg_pedestal, - background_floor) img_analysis_data.append(led_data) return img_analysis_data -def _estimate_ring_backgrounds(boxes: np.ndarray, box_masks: Optional[np.ndarray] = None, - border_width: int = 2) -> Tuple[np.ndarray, np.ndarray]: - """ - Estimate the local background of every search area of a stack from the channel pixels of its border ring, - as their median and as their mean. - - The LED sits at the center of the search area, so the outer border ring samples the scene background - (stray light, smoke path radiance). Pixels of foreign Bayer colors are excluded through the channel - mask and no longer by their value: the data are not clipped at zero any more, so a pixel of the - channel may well be 0 or negative, and dropping those would raise the median. - - The median is subtracted from the box pixel by pixel (bgsub), the mean as N * b from the box sum (N channel - pixels in the box, bgsub2), which is linear in the pixel values like the side band correction. Note that the - border ring lies in the halo of the LED for small search areas. - - :param boxes: Search areas of one shape, stacked along the first axis. - :type boxes: np.ndarray - :param box_masks: Boolean mask of the channel pixels of the search areas, None if all pixels belong to the - channel. - :type box_masks: np.ndarray or None - :param border_width: Width of the border ring in pixels. - :type border_width: int - :return: The median of every search area, 0 if its ring contains no channel pixel, and the mean, NaN then. - :rtype: tuple[np.ndarray, np.ndarray] - """ - medians = np.zeros(boxes.shape[0]) - means = np.full(boxes.shape[0], np.nan) - if boxes.shape[1] <= 2 * border_width or boxes.shape[2] <= 2 * border_width: - return medians, means - in_ring = np.ones(boxes.shape, dtype=bool) - in_ring[:, border_width:-border_width, border_width:-border_width] = False - if box_masks is not None: - in_ring &= box_masks - num_ring_pixels = np.count_nonzero(in_ring, axis=(1, 2)) - # The rings of a Bayer pattern hold the same number of channel pixels wherever the search area lies, so - # the search areas are grouped by that number and the estimates are taken along the axis of a group. - for num in np.unique(num_ring_pixels[num_ring_pixels > 0]): - group = num_ring_pixels == num - ring_pixels = boxes[group][in_ring[group]].reshape(-1, num) - medians[group] = np.median(ring_pixels, axis=1) - means[group] = np.mean(ring_pixels, axis=1, dtype=np.float64) - return medians, means - - def _save_results_in_file(channel: int, img_data: LEDAnalysisData, img_filename: str, img_id: str, img_infos: np.ndarray, basename: str) -> None: """ Save analysis results to a file. @@ -695,8 +525,7 @@ def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.n out_str += f"channel = {channel}, " out_str += f"time[s] = {img_infos[int(img_id) - 1][3]}, " out_str += f"LEDSA {version('ledsa')}\n" - out_str += "# id,line,sum_col_value,average_col_value,max_col_value,bgsub_sum_col_value" - out_str += ",sidebg_sum_col_value,sidebg_pedestal,bgsub2_sum_col_value" + out_str += "# id,line,sum_col_value,average_col_value,max_col_value" if fit_leds: out_str += ",led_center_x, led_center_y" out_str += ",x,y,dx,dy,A,alpha,wx,wy,fit_success,fit_fun,fit_nfev,fit_time" diff --git a/ledsa/tests/AcceptanceTests/08_test_analysis_quantities.robot b/ledsa/tests/AcceptanceTests/08_test_analysis_quantities.robot new file mode 100644 index 0000000..068ffbc --- /dev/null +++ b/ledsa/tests/AcceptanceTests/08_test_analysis_quantities.robot @@ -0,0 +1,68 @@ +*** Settings *** +Resource global_keywords.resource + +Suite Setup Extract Step 3 Results + +Force Tags analysis + +*** Variables *** +${BINARY} analysis${/}channel0${/}all_parameters.h5 +# Every analysed quantity gets a directory of its own below the one of the solver +${EXTINCTION_DIR} analysis${/}extinction_coefficients${/}linear +${SUM_RESULT} ${EXTINCTION_DIR}${/}sum_col_val${/}extinction_coefficients_linear_channel_0_sum_col_val_led_array_0.csv +${MEAN_RESULT} ${EXTINCTION_DIR}${/}mean_col_val${/}extinction_coefficients_linear_channel_0_mean_col_val_led_array_0.csv + +*** Test Cases *** +Every Configured Quantity Is Analysed In One Run + [Documentation] config_analysis.ini can name several quantities, which one analysis run + ... computes one after another, each into a directory of its own instead of + ... overwriting the results of the quantity before it. The mean of a search area + ... is its sum divided by its number of pixels, so it has to reproduce the ground + ... truth as well. + Change Directory ${WORKDIR} + Remove Directory ${EXTINCTION_DIR} recursive=True + Create And Fill Config Analysis linear sum_col_val mean_col_val + Execute Ledsa --analysis + File Should Exist ${SUM_RESULT} + File Should Exist ${MEAN_RESULT} + Check Both Quantities Of One Run 2 + Check Both Quantities Of One Run 3 + Check Both Quantities Of One Run 4 + +Fitted Step 3 Output Can Be Analysed + [Documentation] Step 3 with fits writes the main and the fit columns into one row; the analysis must + ... read these files and reproduce the ground truth. + Create Directory ${WORKDIR}${/}fitted + Change Directory ${WORKDIR}${/}fitted + Create Test Data + Create Config + Execute Ledsa -s1 + Execute Ledsa -s2 + Execute Ledsa --coordinates + Execute Ledsa -s3 + Create And Fill Config Analysis linear + Execute Ledsa --analysis + Check Fitted Results 2 + Check Fitted Results 3 + Check Fitted Results 4 + +*** Keywords *** +Extract Step 3 Results + [Documentation] The suites before may leave the work directory without step 3 results (suite 07 empties + ... analysis/channel0), so extract them here and drop a binary of an earlier run. + Change Directory ${WORKDIR} + Execute Ledsa -s3_fast + Remove File ${BINARY} + +Check Both Quantities Of One Run + [Arguments] ${image_id} + ${rmse_sum} = Check Input Vs Computed Extinction Coefficients ${image_id} linear + ${rmse_mean} = Check Input Vs Computed Extinction Coefficients ${image_id} linear + ... reference_property=mean_col_val + Should Be True ${rmse_sum} < 0.05 + Should Be True ${rmse_mean} < 0.05 + +Check Fitted Results + [Arguments] ${image_id} + ${rmse} = Check Input Vs Computed Extinction Coefficients ${image_id} linear + Should Be True ${rmse} < 0.05 diff --git a/ledsa/tests/AcceptanceTests/08_test_background_subtraction.robot b/ledsa/tests/AcceptanceTests/08_test_background_subtraction.robot deleted file mode 100644 index 18c731d..0000000 --- a/ledsa/tests/AcceptanceTests/08_test_background_subtraction.robot +++ /dev/null @@ -1,136 +0,0 @@ -*** Settings *** -Resource global_keywords.resource - -Suite Setup Extract Step 3 Results - -Force Tags analysis background_subtraction - -*** Variables *** -${BINARY} analysis${/}channel0${/}all_parameters.h5 -# Every analysed quantity gets a directory of its own below the one of the solver -${EXTINCTION_DIR} analysis${/}extinction_coefficients${/}linear -${SUM_DIR} ${EXTINCTION_DIR}${/}sum_col_val -${BGSUB_DIR} ${EXTINCTION_DIR}${/}bgsub_sum_col_val -${SUM_RESULT} ${SUM_DIR}${/}extinction_coefficients_linear_channel_0_sum_col_val_led_array_0.csv -${SUM_REFERENCE} ${SUM_DIR}${/}extinction_coefficients_linear_channel_0_sum_col_val_led_array_0_before_strip.csv -${BGSUB_RESULT} ${BGSUB_DIR}${/}extinction_coefficients_linear_channel_0_bgsub_sum_col_val_led_array_0.csv - -*** Test Cases *** -Every Configured Quantity Is Analysed In One Run - [Documentation] config_analysis.ini can name several quantities, which one analysis run - ... computes one after another, each into a directory of its own instead of - ... overwriting the results of the quantity before it. - Change Directory ${WORKDIR} - Remove Directory ${EXTINCTION_DIR} recursive=True - Create And Fill Config Analysis linear sum_col_val bgsub_sum_col_val - Execute Ledsa --analysis - File Should Exist ${SUM_RESULT} - File Should Exist ${BGSUB_RESULT} - Check Both Quantities Of One Run 2 - Check Both Quantities Of One Run 3 - Check Both Quantities Of One Run 4 - -Bgsub Reference Property Matches Ground Truth On Clean Images - [Documentation] On background-free test images there is nothing to subtract, so the - ... opt-in bgsub property must reproduce the ground truth just like the - ... default property does. - Change Directory ${WORKDIR} - Create And Fill Config Analysis linear bgsub_sum_col_val - Execute Ledsa --analysis - Check Bgsub Results 2 - Check Bgsub Results 3 - Check Bgsub Results 4 - -Default Analysis Is Not Changed By The Bgsub Column - [Documentation] Results with the default reference property must be identical whether - ... the step 3 files contain the new bgsub column or not (old data format). - Change Directory ${WORKDIR} - Create And Fill Config Analysis linear - # the binary is only created from the step 3 files if it does not exist - Remove File ${BINARY} - Execute Ledsa --analysis - Copy File ${SUM_RESULT} ${SUM_REFERENCE} - Strip Bgsub Column From Led Position Files - Remove File ${BINARY} - Execute Ledsa --analysis - ${identical} = Check Extinction Coefficient Files Are Identical ${SUM_RESULT} ${SUM_REFERENCE} - Should Be True ${identical} - -Background Light Biases Sum But Not Bgsub - [Documentation] With a constant additive background in the images the default property - ... underestimates the extinction while the bgsub property stays close to - ... the ground truth. - Create Directory ${WORKDIR}${/}background - Change Directory ${WORKDIR}${/}background - Create Test Data background=15 - Create Config - Execute Ledsa -s1 - Execute Ledsa -s2 - Execute Ledsa --coordinates - Execute Ledsa -s3_fast - Create And Fill Config Analysis linear - Execute Ledsa --analysis - Create And Fill Config Analysis linear bgsub_sum_col_val - Execute Ledsa --analysis - Check Background Results 2 - Check Background Results 3 - Check Background Results 4 - -Fitted Step 3 Output Can Be Analysed - [Documentation] Step 3 with fits writes the main and the fit columns into one row; the analysis must - ... read these files (both reference properties) and reproduce the ground truth. - Create Directory ${WORKDIR}${/}fitted - Change Directory ${WORKDIR}${/}fitted - Create Test Data - Create Config - Execute Ledsa -s1 - Execute Ledsa -s2 - Execute Ledsa --coordinates - Execute Ledsa -s3 - Create And Fill Config Analysis linear - Execute Ledsa --analysis - Create And Fill Config Analysis linear bgsub_sum_col_val - Execute Ledsa --analysis - Check Fitted Results 2 - Check Fitted Results 3 - Check Fitted Results 4 - -*** Keywords *** -Check Both Quantities Of One Run - [Arguments] ${image_id} - ${rmse_sum} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ${rmse_bgsub} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=bgsub_sum_col_val - Should Be True ${rmse_sum} < 0.05 - Should Be True ${rmse_bgsub} < 0.06 - -Extract Step 3 Results - [Documentation] The suites before may leave the work directory without step 3 results (suite 07 empties - ... analysis/channel0), so extract them here and drop a binary of an earlier run. - Change Directory ${WORKDIR} - Execute Ledsa -s3_fast - Remove File ${BINARY} - -Check Fitted Results - [Arguments] ${image_id} - ${rmse_sum} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ${rmse_bgsub} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=bgsub_sum_col_val - Should Be True ${rmse_sum} < 0.05 - Should Be True ${rmse_bgsub} < 0.06 - -Check Bgsub Results - [Arguments] ${image_id} - ${rmse} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=bgsub_sum_col_val - # slightly more slack than the 0.05 of the default property: the border ring of the - # synthetic JPGs contains faint LED tails and compression artefacts that get subtracted - Should Be True ${rmse} < 0.06 - -Check Background Results - [Arguments] ${image_id} - ${rmse_sum} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ${rmse_bgsub} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=bgsub_sum_col_val - Should Be True ${rmse_sum} > 0.05 - Should Be True ${rmse_bgsub} < 0.05 diff --git a/ledsa/tests/AcceptanceTests/10_test_sideband_background.robot b/ledsa/tests/AcceptanceTests/10_test_sideband_background.robot deleted file mode 100644 index fe14962..0000000 --- a/ledsa/tests/AcceptanceTests/10_test_sideband_background.robot +++ /dev/null @@ -1,92 +0,0 @@ -*** Settings *** -Resource global_keywords.resource - -Force Tags analysis background_subtraction sideband - -*** Variables *** -${BINARY} analysis${/}channel0${/}all_parameters.h5 -# Every analysed quantity gets a directory of its own below the one of the solver -${EXTINCTION_DIR} analysis${/}extinction_coefficients${/}linear -${SUM_DIR} ${EXTINCTION_DIR}${/}sum_col_val -${BGSUB_DIR} ${EXTINCTION_DIR}${/}bgsub_sum_col_val -${SUM_RESULT} ${SUM_DIR}${/}extinction_coefficients_linear_channel_0_sum_col_val_led_array_0.csv -${SUM_REFERENCE} ${SUM_DIR}${/}extinction_coefficients_linear_channel_0_sum_col_val_led_array_0_sidebg_ref.csv -${BGSUB_RESULT} ${BGSUB_DIR}${/}extinction_coefficients_linear_channel_0_bgsub_sum_col_val_led_array_0.csv -${BGSUB_REFERENCE} ${BGSUB_DIR}${/}extinction_coefficients_linear_channel_0_bgsub_sum_col_val_led_array_0_sidebg_ref.csv - -*** Test Cases *** -Default Results Are Not Changed By The Sideband Columns - [Documentation] Results with the existing reference properties sum_col_val and bgsub_sum_col_val must be - ... identical whether the step 3 files contain the side band columns or not (data format - ... before the side band pedestal was added). The binary is removed before each comparison - ... run, because it is only created from the step 3 files if it does not exist. - Change Directory ${WORKDIR} - Execute Ledsa -s3_fast - Remove File ${BINARY} - Create And Fill Config Analysis linear - Execute Ledsa --analysis - Copy File ${SUM_RESULT} ${SUM_REFERENCE} - Create And Fill Config Analysis linear bgsub_sum_col_val - Execute Ledsa --analysis - Copy File ${BGSUB_RESULT} ${BGSUB_REFERENCE} - Strip Bgsub Column From Led Position Files num_columns=6 - Remove File ${BINARY} - Execute Ledsa --analysis - ${identical} = Check Extinction Coefficient Files Are Identical ${BGSUB_RESULT} ${BGSUB_REFERENCE} - Should Be True ${identical} - Create And Fill Config Analysis linear - Execute Ledsa --analysis - ${identical} = Check Extinction Coefficient Files Are Identical ${SUM_RESULT} ${SUM_REFERENCE} - Should Be True ${identical} - -Sidebg Reference Property Matches Ground Truth On Clean Images - [Documentation] Without background the side bands contain nothing to subtract and the sidebg property - ... reproduces the ground truth like the default property. - Run Sideband Analysis sideband_clean 0 - Check Sidebg Results 2 - Check Sidebg Results 3 - Check Sidebg Results 4 - -Background Light Biases Sum But Not Sidebg - [Documentation] With a constant additive background the default property underestimates the extinction - ... while the sidebg property stays close to the ground truth. - Run Sideband Analysis sideband_background 15 - Check Background Results 2 - Check Background Results 3 - Check Background Results 4 - -*** Keywords *** -Run Sideband Analysis - [Arguments] ${dir} ${background} - [Documentation] The test images are only 50 px wide, so the side bands are placed 12 to 20 px across - ... the strip instead of the default 25 to 60 px. - Create Directory ${WORKDIR}${/}${dir} - Change Directory ${WORKDIR}${/}${dir} - Create Test Data background=${background} - Create Config - Set Config Option analyse_photo sideband_distances 12 20 - Execute Ledsa -s1 - Execute Ledsa -s2 - Execute Ledsa --coordinates - Execute Ledsa -s3_fast - Create And Fill Config Analysis linear - Execute Ledsa --analysis - Create And Fill Config Analysis linear sidebg_sum_col_val - Execute Ledsa --analysis - -Check Sidebg Results - [Arguments] ${image_id} - ${rmse} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=sidebg_sum_col_val - Should Be True ${rmse} < 0.05 - -Check Background Results - [Arguments] ${image_id} - ${rmse_sum} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ${rmse_sidebg} = Check Input Vs Computed Extinction Coefficients ${image_id} linear - ... reference_property=sidebg_sum_col_val - # same slack as for bgsub in suite 08: the test data are random and the background clips the LED peaks of - # the 8 bit images at 255; in addition the side band correction has to remove most of the bias of the sum - Should Be True ${rmse_sum} > 0.05 - Should Be True ${rmse_sidebg} < 0.06 - Should Be True ${rmse_sidebg} < 0.5 * ${rmse_sum} diff --git a/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py b/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py index 90b25ab..962d9e0 100644 --- a/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py +++ b/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py @@ -39,7 +39,7 @@ def change_dir(self, new_dir): os.chdir(new_dir) @keyword - def create_test_data(self, num_of_leds=100, num_of_layers=20, bottom_border=0, top_border=3, background=0): + def create_test_data(self, num_of_leds=100, num_of_layers=20, bottom_border=0, top_border=3): # Create test_data directory if it doesn't exist if not os.path.exists('test_data'): os.makedirs('test_data') @@ -81,7 +81,7 @@ def extco_quad(z): for z in np.linspace(bottom_border + 0.05, top_border - 0.05, num_of_leds): ex.add_led(0, 4, z) ex.set_extinction_coefficients(extinction_coefficients) - create_test_image(image_id, ex, background=int(background)) + create_test_image(image_id, ex) # ------------------------------------------------------------------ # Stacked / multi-camera keywords @@ -486,31 +486,6 @@ def create_cc_matrix_file(self): file.write("2,3,4\n1,2,7\n3,4,5") file.close() - @keyword - def strip_bgsub_column_from_led_position_files(self, channel=0, num_columns=5): - """Keep only the first num_columns columns of the step 3 output files to simulate data extracted with - an older ledsa version (5: prior to the local background subtraction). The column header is shortened - accordingly.""" - num_columns = int(num_columns) - pattern = os.path.join('analysis', f'channel{channel}', '*_led_positions.csv') - for path in glob.glob(pattern): - with open(path, encoding='utf-8') as file: - lines = file.readlines() - with open(path, 'w', encoding='utf-8') as file: - for line in lines: - if line.startswith('# id,line,'): - file.write(','.join(line.strip().split(',')[:num_columns]) + '\n') - elif not line.strip() or line.lstrip().startswith('#'): - file.write(line) - else: - file.write(','.join(line.strip().split(',')[:num_columns]) + '\n') - - @keyword - def check_extinction_coefficient_files_are_identical(self, file_a, file_b): - data_a = np.loadtxt(file_a, delimiter=',') - data_b = np.loadtxt(file_b, delimiter=',') - return bool(np.array_equal(data_a, data_b)) - @keyword def corrupt_image(self, img_name, mode='truncate'): """Damage a test image and keep a backup, so that it can be restored afterwards. @@ -779,7 +754,7 @@ def load_extinction_coefficients_computed(solver, channel, led_array, reference_ extinction_coefficients_computed = data[:, 1:] return time, extinction_coefficients_computed -def create_test_image(image_id, experiment, img_dir='test_data', background=0): +def create_test_image(image_id, experiment, img_dir='test_data'): """ Creates three test images with black and gray pixels representing 3 leds and sets the exif data needed The first image has 100% transmission on all LEDs, the second image has 50% transmission on all LEDs, the third has 50%, 70% and 80% transmission on the top, middle and bottom LEDs. @@ -793,7 +768,7 @@ def create_test_image(image_id, experiment, img_dir='test_data', background=0): transmissions = experiment.calc_all_led_transmissions() # Reverse transmissions because images are created from top down - img_array = create_img_array(num_of_leds, list(reversed(transmissions)), background=background) + img_array = create_img_array(num_of_leds, list(reversed(transmissions))) img = Image.fromarray(img_array, 'RGB') # Save image without EXIF data @@ -831,12 +806,10 @@ def turn_img_sideways(path, write_orientation=True): turned_img.writeMetadata() -def create_img_array(num_of_leds, transmissions, background=0): +def create_img_array(num_of_leds, transmissions): img = np.zeros((num_of_leds * 50 + 50, 50, 3), np.uint8) for led_id in range(num_of_leds): add_led(img, (1 + led_id) * 50, 25, transmissions[led_id]) - if background: - img = np.clip(img.astype(np.int16) + int(background), 0, 255).astype(np.uint8) return img diff --git a/ledsa/tests/UnitTests/test_analysis_quantities.py b/ledsa/tests/UnitTests/test_analysis_quantities.py index 82e785e..04e856b 100644 --- a/ledsa/tests/UnitTests/test_analysis_quantities.py +++ b/ledsa/tests/UnitTests/test_analysis_quantities.py @@ -29,17 +29,17 @@ def test_single_quantity_is_kept(self): assert parse_reference_properties('sum_col_val') == ['sum_col_val'] def test_quantities_are_split_on_whitespace(self): - assert parse_reference_properties('sum_col_val bgsub_sum_col_val\tsidebg_sum_col_val') == [ - 'sum_col_val', 'bgsub_sum_col_val', 'sidebg_sum_col_val'] + assert parse_reference_properties('sum_col_val mean_col_val\tmax_col_val') == [ + 'sum_col_val', 'mean_col_val', 'max_col_val'] def test_order_of_the_option_is_kept(self): - assert parse_reference_properties('bgsub_sum_col_val sum_col_val') == [ - 'bgsub_sum_col_val', 'sum_col_val'] + assert parse_reference_properties('mean_col_val sum_col_val') == [ + 'mean_col_val', 'sum_col_val'] def test_repeated_quantity_is_dropped(self): # both would be written into the same file - assert parse_reference_properties('sum_col_val bgsub_sum_col_val sum_col_val') == [ - 'sum_col_val', 'bgsub_sum_col_val'] + assert parse_reference_properties('sum_col_val mean_col_val sum_col_val') == [ + 'sum_col_val', 'mean_col_val'] @pytest.mark.parametrize('value', ['', ' ', 'None', 'none']) def test_option_without_a_quantity_raises(self, value): @@ -47,8 +47,8 @@ def test_option_without_a_quantity_raises(self, value): parse_reference_properties(value) def test_list_is_written_as_one_value(self): - assert format_reference_properties(['sum_col_val', 'bgsub_sum_col_val']) == \ - 'sum_col_val bgsub_sum_col_val' + assert format_reference_properties(['sum_col_val', 'mean_col_val']) == \ + 'sum_col_val mean_col_val' assert format_reference_properties('sum_col_val') == 'sum_col_val' @@ -56,8 +56,8 @@ class TestConfigDataAnalysis: def test_template_holds_the_given_quantities(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) ConfigDataAnalysis(load_config_file=False, - reference_property=['sum_col_val', 'bgsub_sum_col_val']) - assert ConfigDataAnalysis().get_reference_properties() == ['sum_col_val', 'bgsub_sum_col_val'] + reference_property=['sum_col_val', 'mean_col_val']) + assert ConfigDataAnalysis().get_reference_properties() == ['sum_col_val', 'mean_col_val'] def test_single_quantity_of_an_older_config_is_read(self, tmp_path, monkeypatch): # configs written before several quantities could be given hold a single name @@ -81,8 +81,8 @@ def write_config(self, tmp_path, options): return ConfigDataStacked(filename=str(path)) def test_several_quantities_are_read(self, tmp_path): - config = self.write_config(tmp_path, 'reference_property = sum_col_val sidebg_sum_col_val\n') - assert config.reference_properties == ['sum_col_val', 'sidebg_sum_col_val'] + config = self.write_config(tmp_path, 'reference_property = sum_col_val max_col_val\n') + assert config.reference_properties == ['sum_col_val', 'max_col_val'] assert config.reference_property == 'sum_col_val' def test_unset_option_falls_back_to_the_default_quantity(self, tmp_path): @@ -126,9 +126,8 @@ def test_file_lies_in_the_directory_of_its_solver_and_quantity(self, tmp_path): def test_quantities_of_one_solver_are_told_apart(self, tmp_path): sum_path = get_extinction_coefficients_file_path('linear', 0, 'sum_col_val', 0, root=tmp_path) - bgsub_path = get_extinction_coefficients_file_path('linear', 0, 'bgsub_sum_col_val', 0, - root=tmp_path) - assert sum_path.parent != bgsub_path.parent + mean_path = get_extinction_coefficients_file_path('linear', 0, 'mean_col_val', 0, root=tmp_path) + assert sum_path.parent != mean_path.parent class TestSave: @@ -148,7 +147,7 @@ def test_results_are_written_into_the_directory_of_their_quantity(self, tmp_path def test_quantities_of_one_run_do_not_overwrite_each_other(self, tmp_path, monkeypatch): self.setup_analysis_dir(tmp_path, monkeypatch) - quantities = ['sum_col_val', 'bgsub_sum_col_val', 'sidebg_sum_col_val'] + quantities = ['sum_col_val', 'mean_col_val', 'max_col_val'] for quantity in quantities: save_coefficients(tmp_path, quantity) for quantity in quantities: @@ -185,7 +184,7 @@ def make_sim_data(self, reference_property='sum_col_val', solver='linear'): def test_only_the_files_of_the_own_quantity_are_read(self, tmp_path): solver_dir = tmp_path / 'analysis' / 'extinction_coefficients' / 'linear' wanted = self.write_result_file(solver_dir / 'sum_col_val', 'sum_col_val') - self.write_result_file(solver_dir / 'bgsub_sum_col_val', 'bgsub_sum_col_val') + self.write_result_file(solver_dir / 'mean_col_val', 'mean_col_val') files = self.make_sim_data('sum_col_val')._extco_files(str(tmp_path)) @@ -214,10 +213,10 @@ def test_missing_results_of_the_quantity_are_reported(self, tmp_path): analysis_dir.mkdir(parents=True) (analysis_dir / 'image_infos_analysis.csv').write_text(IMAGE_INFOS, encoding='utf-8') sim_data = SimData(str(tmp_path), read_all=False, load_config_params=False, - reference_property='bgsub_sum_col_val') + reference_property='mean_col_val') sim_data.solver = 'linear' - with pytest.raises(FileNotFoundError, match='bgsub_sum_col_val'): + with pytest.raises(FileNotFoundError, match='mean_col_val'): sim_data._get_extco_df_from_path() @@ -250,7 +249,7 @@ def test_an_unknown_quantity_points_at_the_option(self, coefficients): def test_the_columns_that_are_there_are_named(self, coefficients): # the quantity may simply have been written by a newer step 3 than the one that ran - coefficients.reference_property = 'bgsub_sum_col_val' + coefficients.reference_property = 'mean_col_val' with pytest.raises(ValueError, match=r'led_array_id, sum_col_val'): coefficients.load_img_data() @@ -258,3 +257,33 @@ def test_a_quantity_that_is_there_is_loaded(self, coefficients): coefficients.reference_property = 'sum_col_val' coefficients.load_img_data() assert coefficients.analysed_img_ids == [1, 2] + + +class TestNanReferenceProperty: + """A NaN of the analysed quantity stops the analysis instead of being hidden by the solvers.""" + + @pytest.fixture + def coefficients(self, monkeypatch): + from types import SimpleNamespace + import pandas as pd + import ledsa.analysis.ExtinctionCoefficients as extinction_coefficients + from ledsa.analysis.ExtinctionCoefficientsLinear import ExtinctionCoefficientsLinear + index = pd.MultiIndex.from_product([[1, 2], [0, 1, 2]], names=['img_id', 'led_id']) + img_data = pd.DataFrame({'led_array_id': 0, 'sum_col_val': 100.0, + 'mean_col_val': [0.9, np.nan, 0.9, 0.9, np.nan, 0.9]}, index=index) + monkeypatch.setattr(extinction_coefficients, 'read_hdf', lambda channel, path: img_data) + coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) + coefficients.experiment = SimpleNamespace(channel=0, path='.', led_array=0) + coefficients.average_images = False + return coefficients + + def test_nan_values_stop_the_analysis(self, coefficients): + # the nonlinear solver would return zeros, the linear one would leave the LED out without a word + coefficients.reference_property = 'mean_col_val' + with pytest.raises(ValueError, match=r'2 values of mean_col_val of led array 0 are NaN \(led ids 1\)'): + coefficients.load_img_data() + + def test_property_without_nan_is_loaded(self, coefficients): + coefficients.reference_property = 'sum_col_val' + coefficients.load_img_data() + assert coefficients.analysed_img_ids == [1, 2] diff --git a/ledsa/tests/UnitTests/test_background_subtraction.py b/ledsa/tests/UnitTests/test_background_subtraction.py deleted file mode 100644 index 3c7b461..0000000 --- a/ledsa/tests/UnitTests/test_background_subtraction.py +++ /dev/null @@ -1,240 +0,0 @@ -import numpy as np -import pytest - -from ledsa.data_extraction.step_3_functions import _estimate_ring_backgrounds, _generate_led_analysis_data - -SEARCH_AREA_RADIUS = 10 -LED_CENTER = (30, 30) -LED_AMPLITUDE = 200.0 -LED_SIGMA = 2.0 - - -def ring_median(search_area, channel_mask=None): - """The median of the border ring of a single search area.""" - medians, _ = _estimate_ring_backgrounds(search_area[None], None if channel_mask is None else channel_mask[None]) - return medians[0] - - -def make_synthetic_image(background=0.0, bayer_mask=False): - """Create a 60x60 image with a Gaussian LED spot at LED_CENTER plus a constant background. - - With bayer_mask=True, three out of four pixels are set to 0 like in the Bayer - array of a raw image where all pixels of foreign color channels are masked with 0. - """ - x, y = np.meshgrid(np.arange(60), np.arange(60), indexing='ij') - dist_sq = (x - LED_CENTER[0]) ** 2 + (y - LED_CENTER[1]) ** 2 - led = LED_AMPLITUDE * np.exp(-dist_sq / (2 * LED_SIGMA ** 2)) - img = led + background - if bayer_mask: - mask = (x % 2 == 0) & (y % 2 == 0) - img = np.where(mask, img, 0.0) - return img - - -def pure_led_integral(img_with_zero_background): - radius = SEARCH_AREA_RADIUS - return np.sum(img_with_zero_background[LED_CENTER[0] - radius:LED_CENTER[0] + radius, - LED_CENTER[1] - radius:LED_CENTER[1] + radius]) - - -def bayer_channel_mask(shape=(60, 60)): - """The mask make_synthetic_image(bayer_mask=True) leaves the channel pixels in.""" - x, y = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), indexing='ij') - return (x % 2 == 0) & (y % 2 == 0) - - -def extract_led_data(img, channel_mask=None): - search_areas = np.array([[0, LED_CENTER[0], LED_CENTER[1]]]) - return _generate_led_analysis_data(None, 0, img, False, 0, 'test_img', 0, search_areas, - SEARCH_AREA_RADIUS, fit_leds=False, channel_mask=channel_mask) - - -class TestRingMedian: - def test_constant_image_returns_background(self): - img = np.full((20, 20), 7.5) - assert ring_median(img) == pytest.approx(7.5) - - def test_foreign_pixels_are_excluded_by_the_mask_and_not_by_their_value(self): - # the data are no longer clipped at zero, so a channel pixel may be 0 or negative and only the - # Bayer mask can say which pixels belong to the channel - img = np.full((20, 20), 7.5) - channel_mask = np.zeros(img.shape, dtype=bool) - channel_mask[::2, ::2] = True - img[~channel_mask] = 0.0 - assert ring_median(img, channel_mask) == pytest.approx(7.5) - assert ring_median(img) == pytest.approx(0.0) # without the mask they all count - - def test_channel_pixels_at_or_below_zero_are_kept(self): - img = np.full((20, 20), -1.5) - assert ring_median(img) == pytest.approx(-1.5) - - def test_all_zero_border_returns_zero(self): - img = np.zeros((20, 20)) - img[10, 10] = 100.0 - assert ring_median(img) == 0.0 - - def test_too_small_search_area_returns_zero(self): - img = np.full((4, 4), 7.5) - assert ring_median(img) == 0.0 - - -class TestBackgroundSubtractedSum: - def test_bgsub_sum_is_independent_of_background(self): - led_integral = pure_led_integral(make_synthetic_image(background=0.0)) - for background in [0.0, 5.0, 20.0, 50.0]: - led_data = extract_led_data(make_synthetic_image(background=background)) - assert led_data.bgsub_sum_color_value == pytest.approx(led_integral, rel=1e-3), \ - f'bgsub_sum_color_value deviates from the pure LED integral for background {background}' - - def test_sum_col_value_scales_with_background(self): - led_integral = pure_led_integral(make_synthetic_image(background=0.0)) - num_pixels = (2 * SEARCH_AREA_RADIUS) ** 2 - for background in [5.0, 20.0]: - led_data = extract_led_data(make_synthetic_image(background=background)) - assert led_data.sum_color_value == pytest.approx(led_integral + background * num_pixels, rel=1e-6) - - def test_bgsub_sum_with_bayer_masked_pixels(self): - channel_mask = bayer_channel_mask() - led_integral = pure_led_integral(make_synthetic_image(background=0.0, bayer_mask=True)) - for background in [5.0, 20.0]: - led_data = extract_led_data(make_synthetic_image(background=background, bayer_mask=True), - channel_mask) - assert led_data.bgsub_sum_color_value == pytest.approx(led_integral, rel=1e-3) - - def test_sum_col_value_is_unchanged_by_new_quantity(self): - img = make_synthetic_image(background=10.0) - led_data = extract_led_data(img) - radius = SEARCH_AREA_RADIUS - expected = np.sum(img[LED_CENTER[0] - radius:LED_CENTER[0] + radius, - LED_CENTER[1] - radius:LED_CENTER[1] + radius]) - assert led_data.sum_color_value == pytest.approx(expected) - - def test_csv_serialization_contains_bgsub_value(self): - led_data = extract_led_data(make_synthetic_image(background=10.0)) - main_data_fields = led_data.get_main_data_string().split(',') - assert len(main_data_fields) >= 6 # further background columns are appended after bgsub - assert float(main_data_fields[5]) == pytest.approx(led_data.bgsub_sum_color_value, rel=1e-4) - - -class TestColumnNames: - def _write_led_positions_csv(self, tmp_path, num_columns, header='# header'): - channel_dir = tmp_path / 'analysis' / 'channel0' - channel_dir.mkdir(parents=True) - row = ','.join(str(float(i)) for i in range(num_columns)) - (channel_dir / '1_led_positions.csv').write_text(f'{header}\n{row}\n{row}\n', encoding='utf-8') - - def test_csv_written_by_step_3_without_fit(self, tmp_path, monkeypatch): - from ledsa.core.file_handling import _get_column_names - from ledsa.data_extraction.step_3_functions import _create_header - img_infos = np.array([['1', 'img.jpg', '12:00:00', '0']]) - header = _create_header(0, '1', 'img.jpg', img_infos, 'root', False).strip() - num_columns = len(header.split('\n')[1].split(',')) - self._write_led_positions_csv(tmp_path, num_columns, header) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert columns[:7] == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", - "max_col_val", "bgsub_sum_col_val"] - assert "sidebg_sum_col_val" in columns and "sidebg_pedestal" in columns - assert columns[-2:] == ["width", "height"] - assert len(columns) == num_columns + 3 - - def test_new_csv_with_bgsub_column(self, tmp_path, monkeypatch): - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, 6) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert columns == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", - "max_col_val", "bgsub_sum_col_val", "width", "height"] - - def test_old_csv_without_bgsub_column(self, tmp_path, monkeypatch): - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, 5) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert columns == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", - "max_col_val", "width", "height"] - - def test_fitted_csv_written_by_step_3(self, tmp_path, monkeypatch): - # a row as written by step 3 with fits: the columns must parse and match the names - from types import SimpleNamespace - from ledsa.core.file_handling import _get_column_names - from ledsa.data_extraction.LEDAnalysisData import LEDAnalysisData - led_data = LEDAnalysisData(3, 0, True) - led_data.sum_color_value, led_data.mean_color_value, led_data.max_color_value = 41210.0, 103.0, 232 - led_data.bgsub_sum_color_value = 40000.0 - led_data.sidebg_sum_color_value, led_data.sidebg_pedestal = 39000.0, 2210.0 - led_data.bgsub2_sum_color_value = 39500.0 - led_data.led_center_x, led_data.led_center_y = 50.521, 60.25 - led_data.fit_results = SimpleNamespace(x=np.arange(1.5, 9.5), success=True, fun=0.5, nfev=120) - led_data.fit_time = 0.25 - channel_dir = tmp_path / 'analysis' / 'channel0' - channel_dir.mkdir(parents=True) - (channel_dir / '1_led_positions.csv').write_text(f'# header\n{led_data}{led_data}', encoding='utf-8') - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - values = np.loadtxt(channel_dir / '1_led_positions.csv', delimiter=',')[0] - assert len(columns) - 2 == len(values) + 1 # width/height appended, img_id prepended on load - named = dict(zip(columns[1:], values)) - assert named['max_col_val'] == 232 - assert named['bgsub_sum_col_val'] == pytest.approx(40000.0) - assert named['sidebg_pedestal'] == pytest.approx(2210.0) - assert named['bgsub2_sum_col_val'] == pytest.approx(39500.0) - assert named['led_center_x'] == pytest.approx(50.521) - assert named['A'] == pytest.approx(5.5) - assert named['fit_nfev'] == 120 - assert named['fit_time'] == pytest.approx(0.25) - - @pytest.mark.parametrize('num_columns, last_main', [(6, 'bgsub_sum_col_val'), (9, 'bgsub2_sum_col_val'), - (20, 'bgsub_sum_col_val'), (23, 'bgsub2_sum_col_val')]) - def test_layouts(self, tmp_path, monkeypatch, num_columns, last_main): - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, num_columns) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert len(columns) == num_columns + 1 + 2 - main = columns[:columns.index('led_center_x')] if 'led_center_x' in columns else columns[:-2] - assert main[-1] == last_main - if num_columns > 9: - assert columns[-3:] == ['fit_time', 'width', 'height'] - - def test_fitted_csv_without_delimiter_raises_with_hint(self, tmp_path, monkeypatch): - # fitted files of this branch written before the delimiter fix: bgsub_sum_col_value and led_center_x glued - from ledsa.core.file_handling import _get_column_names - fields = ['3', '0', '4.1210e+04', '1.0300e+02', '232', '4.1210e+04', '5.0521e+01'] + ['1.0'] * 13 - row = ','.join(fields[:5]) + ',' + fields[5] + ','.join(fields[6:]) - channel_dir = tmp_path / 'analysis' / 'channel0' - channel_dir.mkdir(parents=True) - (channel_dir / '1_led_positions.csv').write_text(f'# header\n{row}\n{row}\n', encoding='utf-8') - monkeypatch.chdir(tmp_path) - with pytest.raises(ValueError, match='glued into one field. Run step 3 again'): - _get_column_names(0) - - def test_legacy_fitted_csv(self, tmp_path, monkeypatch, capsys): - # fitted files written before the delimiter fix have 18 columns; read as before, with a warning - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, 18) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert columns[:8] == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", "max_col_val", - "led_center_x", "led_center_y"] - assert columns[-3:] == ["fit_nfev", "width", "height"] - assert len(columns) == 18 + 1 + 2 - assert 'max_col_val and all fit columns are not' in capsys.readouterr().out - - @pytest.mark.parametrize('num_columns, first_names', [ - (15, ["img_id", "led_id", "led_array_id", "x"]), (4, ["img_id", "led_id", "led_array_id", "sum_col_val"])]) - def test_oldest_layouts(self, tmp_path, monkeypatch, num_columns, first_names): - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, num_columns) - monkeypatch.chdir(tmp_path) - columns = _get_column_names(0) - assert columns[:4] == first_names - assert len(columns) == num_columns + 1 + 2 - - @pytest.mark.parametrize('num_columns', [7, 19, 21, 22]) - def test_unknown_layout_raises(self, tmp_path, monkeypatch, num_columns): - from ledsa.core.file_handling import _get_column_names - self._write_led_positions_csv(tmp_path, num_columns) - monkeypatch.chdir(tmp_path) - with pytest.raises(ValueError, match=f'has {num_columns} columns'): - _get_column_names(0) diff --git a/ledsa/tests/UnitTests/test_config_file_saving.py b/ledsa/tests/UnitTests/test_config_file_saving.py index 1bf1918..7d2d0b4 100644 --- a/ledsa/tests/UnitTests/test_config_file_saving.py +++ b/ledsa/tests/UnitTests/test_config_file_saving.py @@ -121,10 +121,10 @@ def test_multiline_value_survives_repeated_saving(self, tmp_path, monkeypatch): def test_option_added_to_the_parser_is_appended_to_its_section(self, tmp_path, monkeypatch): config = write_config(tmp_path, monkeypatch) - config['analyse_photo']['background_floor'] = '0.05' + config['analyse_photo']['first_img_analysis_id'] = '5' config.save() saved = (tmp_path / 'config.ini').read_text(encoding='utf-8') - assert saved.splitlines()[-1] == 'background_floor = 0.05' + assert saved.splitlines()[-1] == 'first_img_analysis_id = 5' assert comment_lines(saved) == comment_lines(HAND_WRITTEN) def test_option_overriding_a_default_is_written_into_its_section(self, tmp_path, monkeypatch): diff --git a/ledsa/tests/UnitTests/test_file_handling.py b/ledsa/tests/UnitTests/test_file_handling.py index 32f3cd5..13972de 100644 --- a/ledsa/tests/UnitTests/test_file_handling.py +++ b/ledsa/tests/UnitTests/test_file_handling.py @@ -1,5 +1,6 @@ import os +import numpy as np import pytest from ledsa.core.file_handling import _find_first_result_file, _get_column_names @@ -57,6 +58,103 @@ def test_names_are_read_from_the_first_existing_file(self, tmp_path, monkeypatch +class TestColumnNames: + """The names of the columns are told by their number, for the layouts every LEDSA version wrote.""" + + def _write_led_positions_csv(self, tmp_path, num_columns, header='# header'): + channel_dir = tmp_path / 'analysis' / 'channel0' + channel_dir.mkdir(parents=True) + row = ','.join(str(float(i)) for i in range(num_columns)) + (channel_dir / '1_led_positions.csv').write_text(f'{header}\n{row}\n{row}\n', encoding='utf-8') + + def test_csv_written_by_step_3_without_fit(self, tmp_path, monkeypatch): + from ledsa.data_extraction.step_3_functions import _create_header + img_infos = np.array([['1', 'img.jpg', '12:00:00', '0']]) + header = _create_header(0, '1', 'img.jpg', img_infos, 'root', False).strip() + num_columns = len(header.split('\n')[1].split(',')) + self._write_led_positions_csv(tmp_path, num_columns, header) + monkeypatch.chdir(tmp_path) + assert _get_column_names(0) == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", + "max_col_val", "width", "height"] + + def test_fitted_csv_written_by_step_3(self, tmp_path, monkeypatch): + # a row as written by step 3 with fits: the columns must parse and match the names + from types import SimpleNamespace + from ledsa.data_extraction.LEDAnalysisData import LEDAnalysisData + led_data = LEDAnalysisData(3, 0, True) + led_data.sum_color_value, led_data.mean_color_value, led_data.max_color_value = 41210.0, 103.0, 232 + led_data.led_center_x, led_data.led_center_y = 50.521, 60.25 + led_data.fit_results = SimpleNamespace(x=np.arange(1.5, 9.5), success=True, fun=0.5, nfev=120) + led_data.fit_time = 0.25 + channel_dir = tmp_path / 'analysis' / 'channel0' + channel_dir.mkdir(parents=True) + (channel_dir / '1_led_positions.csv').write_text(f'# header\n{led_data}{led_data}', encoding='utf-8') + monkeypatch.chdir(tmp_path) + columns = _get_column_names(0) + values = np.loadtxt(channel_dir / '1_led_positions.csv', delimiter=',')[0] + assert len(columns) - 2 == len(values) + 1 # width/height appended, img_id prepended on load + named = dict(zip(columns[1:], values)) + assert named['sum_col_val'] == pytest.approx(41210.0) + assert named['max_col_val'] == 232 + assert named['led_center_x'] == pytest.approx(50.521) + assert named['A'] == pytest.approx(5.5) + assert named['fit_nfev'] == 120 + assert named['fit_time'] == pytest.approx(0.25) + + @pytest.mark.parametrize('num_columns, last_main', [ + (5, 'max_col_val'), (19, 'max_col_val'), # written since v0.13.0, without and with the fit + (6, 'bgsub_sum_col_val'), (20, 'bgsub_sum_col_val'), # local background subtraction + (9, 'bgsub2_sum_col_val'), (23, 'bgsub2_sum_col_val'), # v0.11.0 + (10, 'noise_sum_col_val'), (24, 'noise_sum_col_val')]) # v0.12.0 + def test_layouts(self, tmp_path, monkeypatch, num_columns, last_main): + self._write_led_positions_csv(tmp_path, num_columns) + monkeypatch.chdir(tmp_path) + columns = _get_column_names(0) + assert len(columns) == num_columns + 1 + 2 + main = columns[:columns.index('led_center_x')] if 'led_center_x' in columns else columns[:-2] + assert main[-1] == last_main + if 'led_center_x' in columns: + assert columns[-3:] == ['fit_time', 'width', 'height'] + + def test_fitted_csv_without_delimiter_raises_with_hint(self, tmp_path, monkeypatch): + # fitted files written with bgsub_sum_col_value but before the delimiter fix glue it to led_center_x + fields = ['3', '0', '4.1210e+04', '1.0300e+02', '232', '4.1210e+04', '5.0521e+01'] + ['1.0'] * 13 + row = ','.join(fields[:5]) + ',' + fields[5] + ','.join(fields[6:]) + channel_dir = tmp_path / 'analysis' / 'channel0' + channel_dir.mkdir(parents=True) + (channel_dir / '1_led_positions.csv').write_text(f'# header\n{row}\n{row}\n', encoding='utf-8') + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError, match='glued into one field. Run step 3 again'): + _get_column_names(0) + + def test_legacy_fitted_csv(self, tmp_path, monkeypatch, capsys): + # fitted files written before the delimiter fix have 18 columns; read as before, with a warning + self._write_led_positions_csv(tmp_path, 18) + monkeypatch.chdir(tmp_path) + columns = _get_column_names(0) + assert columns[:8] == ["img_id", "led_id", "led_array_id", "sum_col_val", "mean_col_val", "max_col_val", + "led_center_x", "led_center_y"] + assert columns[-3:] == ["fit_nfev", "width", "height"] + assert len(columns) == 18 + 1 + 2 + assert 'max_col_val and all fit columns are not' in capsys.readouterr().out + + @pytest.mark.parametrize('num_columns, first_names', [ + (15, ["img_id", "led_id", "led_array_id", "x"]), (4, ["img_id", "led_id", "led_array_id", "sum_col_val"])]) + def test_oldest_layouts(self, tmp_path, monkeypatch, num_columns, first_names): + self._write_led_positions_csv(tmp_path, num_columns) + monkeypatch.chdir(tmp_path) + columns = _get_column_names(0) + assert columns[:4] == first_names + assert len(columns) == num_columns + 1 + 2 + + @pytest.mark.parametrize('num_columns', [7, 21, 22]) + def test_unknown_layout_raises(self, tmp_path, monkeypatch, num_columns): + self._write_led_positions_csv(tmp_path, num_columns) + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError, match=f'has {num_columns} columns'): + _get_column_names(0) + + class BinaryDataFiles: """Fixtures and helpers shared by the tests that build the binary data of a channel.""" diff --git a/ledsa/tests/UnitTests/test_image_channels.py b/ledsa/tests/UnitTests/test_image_channels.py index c3be4eb..8b6ac3e 100644 --- a/ledsa/tests/UnitTests/test_image_channels.py +++ b/ledsa/tests/UnitTests/test_image_channels.py @@ -13,7 +13,7 @@ from ledsa.core import image_handling, image_reading from ledsa.core.image_integrity import CorruptedImageError -from ledsa.core.image_reading import ImageChannels, RawImage +from ledsa.core.image_reading import ImageChannels, RawImage, get_bayer_channel_mask from ledsa.data_extraction import step_3_functions from ledsa.data_extraction.DataExtractor import DataExtractor @@ -182,3 +182,36 @@ def test_unreadable_image_is_skipped_for_all_channels(self, extractor, monkeypat error = extractor.process_img_file('V001_Cam01_0042.CR3') # there is no such file assert isinstance(error, CorruptedImageError) assert written == [] + + +class TestChannelMask: + # RGGB pattern as in rawpy raw_colors_visible: 0 red, 1 and 3 green, 2 blue + FILTER_ARRAY = np.tile(np.array([[0, 1], [3, 2]]), (2, 2)) + + @pytest.mark.parametrize('channel, colors', [(0, [0]), (1, [1, 3]), (2, [2])]) + def test_bayer_channel_mask(self, channel, colors): + mask = get_bayer_channel_mask(self.FILTER_ARRAY, channel) + np.testing.assert_array_equal(mask, np.isin(self.FILTER_ARRAY, colors)) + + def test_raw_mask_keeps_channel_pixels_with_value_zero(self, monkeypatch): + # a pixel of the channel may be 0 or below: the values are not clipped after the black level + data = np.where(self.FILTER_ARRAY == 1, 0, 7) + # mocked at the one pass over the RAW file, so that every reader derived from it stays consistent. + # A black level of 0 and a white level of 255 leave the stored values as they are. + raw = RawImage(data, self.FILTER_ARRAY, np.zeros(4), 255.0, image_reading.UPRIGHT_ORIENTATION) + monkeypatch.setattr(image_reading, '_read_raw_image', lambda filename: raw) + channel_array, mask = image_reading.read_channel_data_and_mask_from_img('img.CR3', 1) + np.testing.assert_array_equal(mask, np.isin(self.FILTER_ARRAY, [1, 3])) + np.testing.assert_array_equal(channel_array, np.where(mask, data, 0)) + np.testing.assert_array_equal(channel_array, image_reading._read_channel_data_from_raw_file('img.CR3', 1)) + + def test_all_pixels_of_a_png_belong_to_the_channel(self, tmp_path): + path = tmp_path / 'img.png' + plt.imsave(path, np.random.default_rng(0).random((6, 8, 3))) + channel_array, mask = image_reading.read_channel_data_and_mask_from_img(str(path), 1) + np.testing.assert_array_equal(channel_array, image_reading.read_channel_data_from_img(str(path), 1)) + assert mask.shape == channel_array.shape and mask.all() + + def test_unsupported_format_raises(self): + with pytest.raises(CorruptedImageError, match='unsupported file extension'): + image_reading.read_channel_data_and_mask_from_img('img.xyz', 0) diff --git a/ledsa/tests/UnitTests/test_ring_background_mean.py b/ledsa/tests/UnitTests/test_ring_background_mean.py deleted file mode 100644 index c2de990..0000000 --- a/ledsa/tests/UnitTests/test_ring_background_mean.py +++ /dev/null @@ -1,146 +0,0 @@ -import numpy as np -import pytest - -from ledsa.data_extraction.step_3_functions import _estimate_ring_backgrounds, _generate_led_analysis_data - -SEARCH_AREA_RADIUS = 8 -CENTER = (40, 40) - - -def ring_median(search_area, channel_mask=None): - """The median of the border ring of a single search area.""" - medians, _ = _estimate_ring_backgrounds(search_area[None], None if channel_mask is None else channel_mask[None]) - return medians[0] - - -def ring_mean(search_area, channel_mask=None): - """The mean of the border ring of a single search area.""" - _, means = _estimate_ring_backgrounds(search_area[None], None if channel_mask is None else channel_mask[None]) - return means[0] - - -def make_image(transmission=1.0, pedestal=0.0, noise_sigma=0.0, clip=False, bayer_mask=False, seed=0, - amplitude=4000.0, sigma=1.5): - """Single Gaussian LED plus constant pedestal and noise, optionally clipped at 0 and with an RGGB green mask.""" - rng = np.random.default_rng(seed) - rows, cols = np.mgrid[0:80, 0:80] - led = amplitude * transmission * np.exp(-((rows - CENTER[0]) ** 2 + (cols - CENTER[1]) ** 2) / (2 * sigma ** 2)) - img = led + pedestal + rng.normal(0.0, noise_sigma, led.shape) - if clip: - img = np.clip(img, 0.0, None) - mask = (rows + cols) % 2 == 1 if bayer_mask else np.ones(img.shape, dtype=bool) - return np.where(mask, img, 0.0), mask - - -def analyse(img, mask, floor=None): - search_areas = np.array([[0, CENTER[0], CENTER[1]]]) - return _generate_led_analysis_data(None, 1, img, False, 0, 'test_img', 0, search_areas, SEARCH_AREA_RADIUS, - fit_leds=False, channel_mask=mask, background_floor=floor) - - -def box(array): - r = SEARCH_AREA_RADIUS - return array[CENTER[0] - r:CENTER[0] + r, CENTER[1] - r:CENTER[1] + r] - - -class TestRingBackgroundMean: - def test_constant_image(self): - assert ring_mean(np.full((16, 16), 7.5)) == pytest.approx(7.5) - - def test_foreign_channel_pixels_are_excluded_by_mask(self): - img, mask = make_image(pedestal=7.5, bayer_mask=True, amplitude=0.0) - assert ring_mean(box(img), box(mask)) == pytest.approx(7.5) - - def test_channel_pixels_with_value_zero_are_included(self): - data = np.full((16, 16), 4.0) - data[::2, :] = 0.0 # channel pixels clipped at 0 - ring = np.ones((16, 16), dtype=bool) - ring[2:-2, 2:-2] = False - assert ring_mean(data) == pytest.approx(data[ring].mean()) - # both estimators keep the zeros now; only the Bayer mask decides which pixels count - assert ring_median(data) == pytest.approx(data[ring].mean()) - - def test_too_small_or_empty_gives_nan(self): - assert np.isnan(ring_mean(np.ones((4, 4)))) - assert np.isnan(ring_mean(np.ones((16, 16)), np.zeros((16, 16), dtype=bool))) - - -class TestRingBackgroundsOfAStack: - def test_search_areas_with_different_numbers_of_ring_pixels(self): - rng = np.random.default_rng(2) - boxes = rng.normal(5.0, 2.0, (6, 12, 12)).astype(np.float32) - masks = rng.random(boxes.shape) < 0.5 - masks[4] = False # no channel pixel at all - ring = np.ones((12, 12), dtype=bool) - ring[2:-2, 2:-2] = False - medians, means = _estimate_ring_backgrounds(boxes, masks) - for search_area, mask, median, mean in zip(boxes, masks, medians, means): - ring_pixels = search_area[ring & mask] - assert median == (np.median(ring_pixels) if ring_pixels.size else 0.0) - assert mean == pytest.approx(np.mean(ring_pixels, dtype=np.float64) if ring_pixels.size else np.nan, - nan_ok=True) - assert medians[4] == 0.0 and np.isnan(means[4]) - - -class TestBgsub2Sum: - @pytest.mark.parametrize('bayer_mask', [False, True]) - def test_subtraction_is_linear(self, bayer_mask): - img, mask = make_image(pedestal=3.0, noise_sigma=2.0, clip=True, bayer_mask=bayer_mask) - led_data = analyse(img, mask) - num_channel_pixels = np.count_nonzero(box(mask)) - expected = box(img).sum() - num_channel_pixels * ring_mean(box(img), box(mask)) - assert led_data.bgsub2_sum_color_value == pytest.approx(expected) - - def _box_sum_below_zero(self): - # LED shifted to the corner of the box: its light falls into the ring and N * b exceeds the box sum - img, mask = make_image(amplitude=0.0) - row, col = CENTER[0] - SEARCH_AREA_RADIUS + 1, CENTER[1] - SEARCH_AREA_RADIUS + 1 - img[row - 1:row + 2, col - 1:col + 2] = 1000.0 - num_channel_pixels = np.count_nonzero(box(mask)) - corrected = box(img).sum() - num_channel_pixels * ring_mean(box(img), box(mask)) - assert corrected < 0 - return img, mask, corrected - - def test_without_a_floor_the_corrected_sum_stays_negative(self): - # a corrected sum at or below zero says that the signal has vanished, which the solver discards - # by its weight; a limit would turn it into a positive value the solver trusts - img, mask, corrected = self._box_sum_below_zero() - led_data = analyse(img, mask) - assert led_data.bgsub2_sum_color_value == pytest.approx(corrected) - - def test_an_explicit_floor_still_limits_the_corrected_sum(self): - img, mask, _ = self._box_sum_below_zero() - led_data = analyse(img, mask, floor=0.05) - assert led_data.bgsub2_sum_color_value == pytest.approx(0.05 * box(img).sum()) - - def test_existing_values_are_unchanged(self): - img, mask = make_image(pedestal=3.0, noise_sigma=2.0, clip=True, bayer_mask=True) - led_data = analyse(img, mask) - reference = _generate_led_analysis_data(None, 1, img, False, 0, 'test_img', 0, np.array([[0, *CENTER]]), - SEARCH_AREA_RADIUS, fit_leds=False) - assert led_data.sum_color_value == reference.sum_color_value - # bgsub now takes its border ring from the channel mask, so it differs from a run without one - assert led_data.bgsub_sum_color_value != reference.bgsub_sum_color_value - fields = led_data.get_main_data_string().split(',') - # the fields up to max_col_value; bgsub follows and depends on the channel mask - assert ','.join(fields[:5]) == ','.join(reference.get_main_data_string().split(',')[:5]) - assert float(fields[8]) == pytest.approx(led_data.bgsub2_sum_color_value, rel=1e-4) - - def test_optical_depth_bias_with_clipped_noise(self): - # LED without halo in the ring: the mean estimator removes the pedestal including the clipped noise, - # the median of the pixels > 0 with pixel-wise clipping (bgsub) leaves a bias - transmission = 0.01 - args = dict(pedestal=2.0, noise_sigma=2.0, clip=True, bayer_mask=True) - true_depth = -np.log(transmission) - depth = {'sum': [], 'bgsub': [], 'bgsub2': []} - for seed in range(20): - ref = analyse(*make_image(transmission=1.0, seed=2 * seed, **args)) - img = analyse(*make_image(transmission=transmission, seed=2 * seed + 1, **args)) - depth['sum'].append(-np.log(img.sum_color_value / ref.sum_color_value)) - depth['bgsub'].append(-np.log(img.bgsub_sum_color_value / ref.bgsub_sum_color_value)) - depth['bgsub2'].append(-np.log(img.bgsub2_sum_color_value / ref.bgsub2_sum_color_value)) - bias = {key: np.mean(values) / true_depth - 1 for key, values in depth.items()} - assert abs(bias['bgsub2']) < 0.02 - # bgsub used to be biased as well, because it dropped the pixels at 0 and clipped pixel by pixel - assert abs(bias['bgsub']) < 0.02 - assert bias['sum'] < -0.1 diff --git a/ledsa/tests/UnitTests/test_sideband_background.py b/ledsa/tests/UnitTests/test_sideband_background.py deleted file mode 100644 index 296be70..0000000 --- a/ledsa/tests/UnitTests/test_sideband_background.py +++ /dev/null @@ -1,347 +0,0 @@ -import numpy as np -import pytest - -from ledsa.data_extraction.sideband_background import (build_sideband_pixels, estimate_sideband_background, - estimate_sideband_backgrounds, estimate_strip_directions, - get_sideband_pixels, subtract_pedestal) -import ledsa.core.image_reading as image_reading -from ledsa.core.image_reading import get_bayer_channel_mask -from ledsa.data_extraction.step_3_functions import (_generate_led_analysis_data, _get_unmerged_led_arrays, - get_sideband_parameters) - -SEARCH_AREA_RADIUS = 8 -LED_SPACING = 20 -LED_AMPLITUDE = 400.0 -LED_SIGMA = 1.5 -IMG_SIZE = 240 -SIDEBAND_DISTANCES = (25, 60) - - -def make_strip(angle_deg=0.0, num_leds=7, center=(120, 120)): - """LED positions (row, col) along a straight strip; angle 0 is a vertical strip (along the rows).""" - along = np.array([np.cos(np.radians(angle_deg)), np.sin(np.radians(angle_deg))]) - offsets = (np.arange(num_leds) - num_leds // 2) * LED_SPACING - positions = np.rint(np.array(center) + offsets[:, None] * along).astype(int) - return np.column_stack([np.arange(num_leds), positions]) - - -def make_image(search_areas, transmission=1.0, pedestal=0.0, noise_sigma=0.0, bayer_mask=False, clip=False, - seed=0, amplitude=LED_AMPLITUDE): - """Gaussian LEDs scaled by the transmission plus a constant pedestal and Gaussian noise. - - With clip=True the image is clipped at 0 like LEDSA does after the black level subtraction of raw files. - With bayer_mask=True only the green pixels of an RGGB pattern belong to the channel, all other pixels are 0. - Returns the channel array, the channel mask and the pure LED image (without pedestal and noise). - """ - rng = np.random.default_rng(seed) - rows, cols = np.mgrid[0:IMG_SIZE, 0:IMG_SIZE] - led_img = np.zeros((IMG_SIZE, IMG_SIZE)) - for _, row, col in search_areas: - led_img += amplitude * transmission * np.exp(-((rows - row) ** 2 + (cols - col) ** 2) / (2 * LED_SIGMA ** 2)) - img = led_img + pedestal + rng.normal(0.0, noise_sigma, led_img.shape) - if clip: - img = np.clip(img, 0.0, None) - if bayer_mask: - mask = (rows + cols) % 2 == 1 - else: - mask = np.ones(img.shape, dtype=bool) - img = np.where(mask, img, 0.0) - led_img = np.where(mask, led_img, 0.0) - return img, mask, led_img - - -def box_sum(img, row, col, radius=SEARCH_AREA_RADIUS): - return float(np.sum(img[row - radius:row + radius, col - radius:col + radius])) - - -def led_data_for(img, mask, search_areas, led_id, floor=0.05): - sideband_pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], img.shape, SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - return _generate_led_analysis_data(None, 1, img, False, led_id, 'test_img', 0, search_areas, SEARCH_AREA_RADIUS, - fit_leds=False, channel_mask=mask, sideband_pixels=sideband_pixels[led_id], - background_floor=floor) - - -class TestStripDirection: - @pytest.mark.parametrize('angle', [0.0, 90.0, 30.0, -45.0]) - def test_direction_follows_strip(self, angle): - search_areas = make_strip(angle) - directions = estimate_strip_directions(search_areas, [search_areas[:, 0]]) - expected = np.array([np.cos(np.radians(angle)), np.sin(np.radians(angle))]) - for direction in directions: - # direction is defined up to the sign; rounding of the LED positions tilts it slightly - assert abs(np.dot(direction, expected)) == pytest.approx(1.0, abs=2e-3) - - def test_led_without_array_has_no_direction(self): - search_areas = make_strip(0.0) - directions = estimate_strip_directions(search_areas, [search_areas[:3, 0]]) - assert np.all(np.isfinite(directions[:3])) - assert np.all(np.isnan(directions[3:])) - - -class TestSidebandPixels: - def test_vertical_strip_matches_axis_aligned_bands(self): - # same pixels as the axis-aligned bands of the H2 evaluation: rows r-R..r+R-1, columns c+25..c+59 and - # c-60..c-26 - search_areas = make_strip(0.0) - directions = np.tile([1.0, 0.0], (len(search_areas), 1)) - pixels = build_sideband_pixels(search_areas, directions, (IMG_SIZE, IMG_SIZE), SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - led_id, row, col = search_areas[3] - expected = np.zeros((IMG_SIZE, IMG_SIZE), dtype=bool) - expected[row - SEARCH_AREA_RADIUS:row + SEARCH_AREA_RADIUS, col + 25:col + 60] = True - expected[row - SEARCH_AREA_RADIUS:row + SEARCH_AREA_RADIUS, col - 60:col - 25] = True - selected = np.zeros_like(expected) - selected[pixels[led_id]] = True - np.testing.assert_array_equal(selected, expected) - - @pytest.mark.parametrize('angle', [0.0, 90.0, 30.0]) - def test_bands_do_not_touch_search_areas(self, angle): - search_areas = make_strip(angle) - pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], (IMG_SIZE, IMG_SIZE), SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - for _, row, col in search_areas: - box = np.zeros((IMG_SIZE, IMG_SIZE), dtype=bool) - box[row - SEARCH_AREA_RADIUS:row + SEARCH_AREA_RADIUS, col - SEARCH_AREA_RADIUS:col + SEARCH_AREA_RADIUS] = True - for led_pixels in pixels: - assert led_pixels[0].size > 0 - assert not box[led_pixels].any() - - def test_neighbouring_strip_is_excluded(self): - strip = make_strip(0.0, center=(120, 80)) - neighbour = make_strip(0.0, center=(120, 120)) # 40 px to the right, inside the right side band - neighbour[:, 0] += len(strip) - search_areas = np.vstack([strip, neighbour]) - exclusion = 2 * SEARCH_AREA_RADIUS - pixels = get_sideband_pixels(search_areas, [strip[:, 0], neighbour[:, 0]], (IMG_SIZE, IMG_SIZE), - SEARCH_AREA_RADIUS, SIDEBAND_DISTANCES) - rows, cols = pixels[3] - assert np.all((cols < 120 - exclusion) | (cols >= 120 + exclusion)) - assert np.any(cols >= 120 + exclusion) - - def test_bands_are_clipped_at_the_image_border(self): - search_areas = make_strip(0.0, center=(120, 30)) - pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], (IMG_SIZE, IMG_SIZE), SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - rows, cols = pixels[3] - assert cols.min() == 0 - assert cols.size > 0 - - -class TestSidebandBackground: - @pytest.mark.parametrize('bayer_mask', [False, True]) - def test_constant_pedestal_is_recovered(self, bayer_mask): - search_areas = make_strip(30.0) - img, mask, _ = make_image(search_areas, pedestal=7.0, bayer_mask=bayer_mask) - pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], img.shape, SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - for led_pixels in pixels: - assert estimate_sideband_background(img, mask, led_pixels) == pytest.approx(7.0) - - def test_clipped_zeros_are_included(self): - # pedestal of 1 count with noise of 2 counts: clipping at 0 raises the mean in the box and in the side - # bands alike, so the side band mean (including zeros) is the matching estimator; the mean of the - # pixels > 0 is too high - from scipy.stats import norm - search_areas = make_strip(0.0) - img, mask, _ = make_image(search_areas, pedestal=1.0, noise_sigma=2.0, clip=True, bayer_mask=True) - clipped_mean = 1.0 * norm.cdf(0.5) + 2.0 * norm.pdf(0.5) # expectation of max(0, N(1, 2)) - pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], img.shape, SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - estimate = estimate_sideband_background(img, mask, pixels[3]) - positive_mean = np.mean(img[pixels[3]][img[pixels[3]] > 0]) - assert estimate == pytest.approx(clipped_mean, rel=0.1) - assert positive_mean > 1.3 * estimate - - def test_no_channel_pixels_gives_nan(self): - img = np.zeros((10, 10)) - assert np.isnan(estimate_sideband_background(img, None, None)) - assert np.isnan(estimate_sideband_background(img, None, (np.array([], dtype=int), np.array([], dtype=int)))) - mask = np.zeros((10, 10), dtype=bool) - assert np.isnan(estimate_sideband_background(img, mask, (np.array([1]), np.array([1])))) - - -class TestSidebandBackgroundsOfManyLeds: - def test_every_led_gets_the_mean_of_its_own_channel_pixels(self): - # the second and the last LED have no side band pixels, the third none of the channel - num_pixels = [5, 0, 3, 4, 0] - rng = np.random.default_rng(1) - values = rng.normal(10.0, 3.0, sum(num_pixels)).astype(np.float32) - mask = rng.random(values.size) < 0.6 - mask[[0, 8]] = True - mask[5:8] = False - backgrounds = estimate_sideband_backgrounds(values, mask, num_pixels) - starts = np.cumsum(num_pixels) - num_pixels - for background, start, num in zip(backgrounds, starts, num_pixels): - led_values, led_mask = values[start:start + num], mask[start:start + num] - expected = np.mean(led_values[led_mask], dtype=np.float64) if led_mask.any() else np.nan - assert background == pytest.approx(expected, nan_ok=True) - assert np.isnan(backgrounds[[1, 2, 4]]).all() - - def test_no_side_band_pixels_at_all(self): - assert np.isnan(estimate_sideband_backgrounds(np.zeros(0), np.zeros(0, dtype=bool), [0, 0])).all() - - def test_one_led_is_estimate_sideband_background(self): - search_areas = make_strip(30.0) - img, mask, _ = make_image(search_areas, pedestal=3.0, noise_sigma=1.0, bayer_mask=True) - pixels = get_sideband_pixels(search_areas, [search_areas[:, 0]], img.shape, SEARCH_AREA_RADIUS, - SIDEBAND_DISTANCES) - backgrounds = estimate_sideband_backgrounds(np.concatenate([img[led_pixels] for led_pixels in pixels]), - np.concatenate([mask[led_pixels] for led_pixels in pixels]), - [led_pixels[0].size for led_pixels in pixels]) - assert backgrounds == pytest.approx([estimate_sideband_background(img, mask, led_pixels) - for led_pixels in pixels]) - - -class TestSidebandSum: - @pytest.mark.parametrize('bayer_mask', [False, True]) - @pytest.mark.parametrize('angle', [0.0, 90.0, 30.0]) - def test_sidebg_sum_is_independent_of_pedestal(self, bayer_mask, angle): - search_areas = make_strip(angle) - for pedestal in [0.0, 3.0, 20.0]: - img, mask, led_img = make_image(search_areas, pedestal=pedestal, bayer_mask=bayer_mask) - for led_id, row, col in search_areas: - led_data = led_data_for(img, mask, search_areas, led_id) - assert led_data.sidebg_sum_color_value == pytest.approx(box_sum(led_img, row, col), rel=1e-6) - num_channel_pixels = (2 * SEARCH_AREA_RADIUS) ** 2 // (2 if bayer_mask else 1) - assert led_data.sidebg_pedestal == pytest.approx(num_channel_pixels * pedestal, abs=1e-9) - - def test_optical_depth_bias_with_clipped_noise(self): - # optical depth -ln(I/I0) of a dimmed LED with pedestal, clipped noise and Bayer mask: the side band - # correction removes the bias of the plain box sum - search_areas = make_strip(0.0) - transmission = 0.01 - image_args = dict(pedestal=2.0, noise_sigma=2.0, clip=True, bayer_mask=True, amplitude=4000.0) - clean, mask, _ = make_image(search_areas, transmission=1.0, seed=1, **image_args) - smoke, _, _ = make_image(search_areas, transmission=transmission, seed=2, **image_args) - true_depth = -np.log(transmission) - depth_sum, depth_sidebg = [], [] - for led_id in search_areas[:, 0]: - ref = led_data_for(clean, mask, search_areas, led_id) - img = led_data_for(smoke, mask, search_areas, led_id) - depth_sum.append(-np.log(img.sum_color_value / ref.sum_color_value)) - depth_sidebg.append(-np.log(img.sidebg_sum_color_value / ref.sidebg_sum_color_value)) - assert abs(np.mean(depth_sidebg) - true_depth) / true_depth < 0.02 - assert (true_depth - np.mean(depth_sum)) / true_depth > 0.1 - - def test_floor_limits_the_corrected_sum(self): - assert subtract_pedestal(100.0, 99.0, 0.05) == pytest.approx(5.0) - assert subtract_pedestal(100.0, 20.0, 0.05) == pytest.approx(80.0) - assert np.isnan(subtract_pedestal(100.0, np.nan, 0.05)) - - def test_old_values_are_unchanged(self): - # the existing columns keep their values; the new ones are appended to the CSV line - search_areas = make_strip(0.0) - img, mask, _ = make_image(search_areas, pedestal=5.0, noise_sigma=1.0, bayer_mask=True) - led_data = led_data_for(img, mask, search_areas, 3) - reference = _generate_led_analysis_data(None, 1, img, False, 3, 'test_img', 0, search_areas, - SEARCH_AREA_RADIUS, fit_leds=False) - assert led_data.sum_color_value == reference.sum_color_value - assert led_data.mean_color_value == reference.mean_color_value - assert led_data.max_color_value == reference.max_color_value - # bgsub now takes its border ring from the channel mask, so it differs from a run without one - assert led_data.bgsub_sum_color_value != reference.bgsub_sum_color_value - fields = led_data.get_main_data_string().split(',') - # the fields up to max_col_value; bgsub follows and depends on the channel mask - assert ','.join(fields[:5]) == ','.join(reference.get_main_data_string().split(',')[:5]) - assert float(fields[6]) == pytest.approx(led_data.sidebg_sum_color_value, rel=1e-4) - assert float(fields[7]) == pytest.approx(led_data.sidebg_pedestal, rel=1e-4) - - -class TestSidebandParameters: - def test_defaults_without_config(self): - assert get_sideband_parameters(None) == ((25, 60), None, None) - - def test_defaults_for_old_config(self, tmp_path, monkeypatch): - from ledsa.core.ConfigData import ConfigData - monkeypatch.chdir(tmp_path) - (tmp_path / 'config.ini').write_text('[DEFAULT]\nimg_directory = x\n[analyse_photo]\nnum_skip_leds = 0\n', encoding='utf-8') - # a configuration written before the option existed gets no limit either - assert get_sideband_parameters(ConfigData()) == ((25, 60), None, None) - - def test_values_from_config(self, tmp_path, monkeypatch): - from ledsa.core.ConfigData import ConfigData - monkeypatch.chdir(tmp_path) - ConfigData(load_config_file=False, sideband_distances='12 20', background_floor=0.1, - sideband_exclusion_margin=3) - assert get_sideband_parameters(ConfigData()) == ((12.0, 20.0), 0.1, 3) - - -class TestUnmergedLedArrays: - def test_without_merge_the_analysis_arrays_are_used(self): - conf = {'analyse_positions': {'merge_led_array_indices': 'None', 'num_arrays': '2'}} - assert _get_unmerged_led_arrays(conf, [[0, 1], [2, 3]]) == [[0, 1], [2, 3]] - - def test_with_merge_the_arrays_of_step_2_are_read(self, tmp_path, monkeypatch): - # merged arrays may contain several strips, the strip direction is estimated per array of step 2 - from ledsa.data_extraction.step_2_functions import generate_led_array_indices_files - monkeypatch.chdir(tmp_path) - generate_led_array_indices_files([[0, 1, 2], [3, 4, 5]]) - conf = {'analyse_positions': {'merge_led_array_indices': '0 1', 'num_arrays': '2'}} - led_arrays = _get_unmerged_led_arrays(conf, [[0, 1, 2, 3, 4, 5]]) - assert [sorted(led_ids.tolist()) for led_ids in led_arrays] == [[0, 1, 2], [3, 4, 5]] - - -class TestNanReferenceProperty: - @pytest.fixture - def coefficients(self, monkeypatch): - from types import SimpleNamespace - import pandas as pd - import ledsa.analysis.ExtinctionCoefficients as extinction_coefficients - from ledsa.analysis.ExtinctionCoefficientsLinear import ExtinctionCoefficientsLinear - index = pd.MultiIndex.from_product([[1, 2], [0, 1, 2]], names=['img_id', 'led_id']) - img_data = pd.DataFrame({'led_array_id': 0, 'sum_col_val': 100.0, - 'sidebg_sum_col_val': [90.0, np.nan, 90.0, 90.0, np.nan, 90.0]}, index=index) - monkeypatch.setattr(extinction_coefficients, 'read_hdf', lambda channel, path: img_data) - coefficients = ExtinctionCoefficientsLinear.__new__(ExtinctionCoefficientsLinear) - coefficients.experiment = SimpleNamespace(channel=0, path='.', led_array=0) - coefficients.average_images = False - return coefficients - - def test_nan_values_stop_the_analysis(self, coefficients): - # the solvers would fail with an unspecific error (linear) or return zeros (nonlinear) - coefficients.reference_property = 'sidebg_sum_col_val' - with pytest.raises(ValueError, match=r'2 values of sidebg_sum_col_val of led array 0 are NaN \(led ids 1\)'): - coefficients.load_img_data() - - def test_property_without_nan_is_loaded(self, coefficients): - coefficients.reference_property = 'sum_col_val' - coefficients.load_img_data() - assert coefficients.analysed_img_ids == [1, 2] - - -class TestChannelMask: - # RGGB pattern as in rawpy raw_colors_visible: 0 red, 1 and 3 green, 2 blue - FILTER_ARRAY = np.tile(np.array([[0, 1], [3, 2]]), (2, 2)) - - @pytest.mark.parametrize('channel, colors', [(0, [0]), (1, [1, 3]), (2, [2])]) - def test_bayer_channel_mask(self, channel, colors): - mask = get_bayer_channel_mask(self.FILTER_ARRAY, channel) - np.testing.assert_array_equal(mask, np.isin(self.FILTER_ARRAY, colors)) - - def test_raw_mask_keeps_channel_pixels_with_value_zero(self, monkeypatch): - data = np.where(self.FILTER_ARRAY == 1, 0, 7) # the first green channel is clipped at 0 - # mocked at the one pass over the RAW file, so that every reader derived from it stays consistent. - # A black level of 0 and a white level of 255 leave the stored values as they are. - raw = image_reading.RawImage(data, self.FILTER_ARRAY, np.zeros(4), 255.0, - image_reading.UPRIGHT_ORIENTATION) - monkeypatch.setattr(image_reading, '_read_raw_image', lambda filename: raw) - channel_array, mask = \ - image_reading.read_channel_data_and_mask_from_img('img.CR3', 1) - np.testing.assert_array_equal(mask, np.isin(self.FILTER_ARRAY, [1, 3])) - np.testing.assert_array_equal(channel_array, np.where(mask, data, 0)) - np.testing.assert_array_equal(channel_array, image_reading._read_channel_data_from_raw_file('img.CR3', 1)) - - def test_all_pixels_of_a_png_belong_to_the_channel(self, tmp_path): - import matplotlib.pyplot as plt - path = tmp_path / 'img.png' - plt.imsave(path, np.random.default_rng(0).random((6, 8, 3))) - channel_array, mask = \ - image_reading.read_channel_data_and_mask_from_img(str(path), 1) - np.testing.assert_array_equal(channel_array, image_reading.read_channel_data_from_img(str(path), 1)) - assert mask.shape == channel_array.shape and mask.all() - - def test_unsupported_format_raises(self): - from ledsa.core.image_integrity import CorruptedImageError - with pytest.raises(CorruptedImageError, match='unsupported file extension'): - image_reading.read_channel_data_and_mask_from_img('img.xyz', 0) diff --git a/ledsa/tests/UnitTests/test_stacked_search_areas.py b/ledsa/tests/UnitTests/test_stacked_search_areas.py index 3abee3b..0db9ecf 100644 --- a/ledsa/tests/UnitTests/test_stacked_search_areas.py +++ b/ledsa/tests/UnitTests/test_stacked_search_areas.py @@ -1,7 +1,7 @@ """Unit tests for the analysis of all search areas of an image at once. -Step 3 reads only the pixels of the search areas and side bands from an image, calibrates them once for all -channels and takes every quantity along the axes of a stack of search areas. The results have to be those of +Step 3 reads only the pixels of the search areas from an image, calibrates them once for all channels and +takes every quantity along the axes of a stack of search areas. The results have to be those of the definitions applied LED by LED to the arrays of the whole image. """ import configparser @@ -11,39 +11,28 @@ from ledsa.core import image_reading from ledsa.core.image_reading import ImageChannels, RawImage -from ledsa.data_extraction.sideband_background import get_sideband_pixels -from ledsa.data_extraction.step_3_functions import (_get_analysis_pixels, generate_analysis_data, - get_sideband_parameters) +from ledsa.data_extraction.step_3_functions import _get_analysis_pixels, generate_analysis_data SEARCH_AREA_RADIUS = 4 IMG_SHAPE = (40, 48) # RGGB pattern as in rawpy raw_colors_visible: 0 red, 1 and 3 green, 2 blue FILTER_ARRAY = np.tile(np.array([[0, 1], [3, 2]]), (IMG_SHAPE[0] // 2, IMG_SHAPE[1] // 2)) SIDEWAYS = 6 -# A vertical strip and an LED alone in its array, whose strip direction and therefore side bands are unknown. -# The search area of LED 4 is cut short by the lower border of the upright image, that of LED 5 by the right +# A vertical strip and an LED alone in its array. The search area of LED 4 is cut short by the lower border of the upright image, that of LED 5 by the right # border of the image turned sideways. SEARCH_AREAS = np.array([[0, 8, 20], [1, 16, 20], [2, 24, 20], [3, 32, 20], [4, 38, 20], [5, 20, 38]]) LINE_INDICES = [np.array([0, 1, 2, 3, 4]), np.array([5])] -QUANTITIES = ['sum_color_value', 'mean_color_value', 'max_color_value', 'bgsub_sum_color_value', - 'sidebg_sum_color_value', 'sidebg_pedestal', 'bgsub2_sum_color_value'] +QUANTITIES = ['sum_color_value', 'mean_color_value', 'max_color_value'] def make_config(): config = configparser.ConfigParser() config['DEFAULT'] = {'img_directory': '.', 'img_rotation': 'auto'} config['find_search_areas'] = {'search_area_radius': str(SEARCH_AREA_RADIUS)} - config['analyse_photo'] = {'num_skip_leds': '0', 'sideband_distances': '6 12', 'background_floor': 'None', - 'sideband_exclusion_margin': 'None'} - config['analyse_positions'] = {'merge_led_array_indices': 'None'} + config['analyse_photo'] = {'num_skip_leds': '0'} return config -def sideband_pixels_of(img_channels, config): - distances, _, margin = get_sideband_parameters(config) - return get_sideband_pixels(SEARCH_AREAS, LINE_INDICES, img_channels.shape, SEARCH_AREA_RADIUS, distances, margin) - - @pytest.fixture(params=[image_reading.UPRIGHT_ORIENTATION, SIDEWAYS]) def img_channels(request, monkeypatch): """A RAW image with a black level of its own for every Bayer color.""" @@ -53,24 +42,13 @@ def img_channels(request, monkeypatch): return ImageChannels('img.CR3') -def quantities_by_definition(img_channels, channel, row, col, sideband_pixels): +def quantities_by_definition(img_channels, channel, row, col): """The quantities of one LED, from their definitions on the arrays of the whole image.""" - data, mask = img_channels.channel_data_and_mask(channel) + data, _ = img_channels.channel_data_and_mask(channel) box = np.s_[row - SEARCH_AREA_RADIUS:row + SEARCH_AREA_RADIUS, col - SEARCH_AREA_RADIUS:col + SEARCH_AREA_RADIUS] - values, in_channel = data[box], mask[box] - ring = np.ones(values.shape, dtype=bool) - ring[2:-2, 2:-2] = False - ring_pixels = values[ring & in_channel] - num_channel_pixels = np.count_nonzero(in_channel) - sideband_values = np.zeros(0) if sideband_pixels is None else data[sideband_pixels][mask[sideband_pixels]] - sideband_background = np.mean(sideband_values, dtype=np.float64) if sideband_values.size else np.nan - box_sum = float(np.sum(values)) + values = data[box] return {'sum_color_value': np.sum(values), 'mean_color_value': np.mean(values), - 'max_color_value': np.amax(values), - 'bgsub_sum_color_value': np.sum(values.astype(np.float64) - np.median(ring_pixels) * in_channel), - 'sidebg_sum_color_value': box_sum - num_channel_pixels * sideband_background, - 'sidebg_pedestal': num_channel_pixels * sideband_background, - 'bgsub2_sum_color_value': box_sum - num_channel_pixels * np.mean(ring_pixels, dtype=np.float64)} + 'max_color_value': np.amax(values)} class TestStackedAnalysis: @@ -79,11 +57,10 @@ def test_quantities_are_those_of_their_definitions(self, img_channels, channel): config = make_config() led_data = generate_analysis_data('img.CR3', channel, SEARCH_AREAS, LINE_INDICES, config, fit_leds=False, img_channels=img_channels) - sideband_pixels = sideband_pixels_of(img_channels, config) assert [(data.led_id, data.led_array) for data in led_data] == [(0, 0), (1, 0), (2, 0), (3, 0), (4, 0), (5, 1)] for data, (led_id, row, col) in zip(led_data, SEARCH_AREAS): - expected = quantities_by_definition(img_channels, channel, row, col, sideband_pixels[led_id]) + expected = quantities_by_definition(img_channels, channel, row, col) for quantity in QUANTITIES: assert getattr(data, quantity) == pytest.approx(expected[quantity], rel=1e-12, nan_ok=True), \ f'{quantity} of LED {led_id}' @@ -91,20 +68,14 @@ def test_quantities_are_those_of_their_definitions(self, img_channels, channel): def test_a_search_area_at_the_border_is_cut_short(self, img_channels): # as data[row - R:row + R, col - R:col + R] cuts it; the search areas of the other shape are a stack of # their own, so test_quantities_are_those_of_their_definitions covers both - analysis_pixels = _get_analysis_pixels(SEARCH_AREAS, SEARCH_AREAS[:, 0], sideband_pixels_of(img_channels, - make_config()), img_channels.shape, SEARCH_AREA_RADIUS) + analysis_pixels = _get_analysis_pixels(SEARCH_AREAS, SEARCH_AREAS[:, 0], img_channels.shape, + SEARCH_AREA_RADIUS) stacks = {shape: positions.tolist() for positions, shape, _ in analysis_pixels.box_groups} if img_channels.shape == IMG_SHAPE: assert stacks == {(8, 8): [0, 1, 2, 3, 5], (6, 8): [4]} else: assert stacks == {(8, 8): [0, 1, 2, 3, 4], (8, 6): [5]} - def test_the_led_without_side_bands_has_no_side_band_background(self, img_channels): - led_data = generate_analysis_data('img.CR3', 0, SEARCH_AREAS, LINE_INDICES, make_config(), fit_leds=False, - img_channels=img_channels) - assert np.isnan(led_data[5].sidebg_pedestal) - assert np.isfinite([data.sidebg_pedestal for data in led_data[:5]]).all() - def test_only_the_analysed_pixels_are_calibrated_once_for_all_channels(self, img_channels, monkeypatch): calibrated = [] channel_data = image_reading._channel_data From 4f83091f156959ec4621670e55ef5601643425fb Mon Sep 17 00:00:00 2001 From: Kristian Date: Mon, 9 Nov 2026 12:14:20 +0100 Subject: [PATCH 6/6] update version to 0.13.0 v0.13.0 is v0.12.0 with the clock offset, the faster step 3 and the removal of the features that are not validated yet: the background corrections, the SNR weighting with the read noise and the moving average. They stay on develop. The README says where they went, and it describes what remains in dense smoke without the weighting: the linear solver leaves out an LED at or below zero, and the coefficients come out low where the transmission reaches the read noise. Its Python requirement names the versions pyproject.toml allows (3.9 to 3.12) instead of 3.8. The commented-out study of lambda_reg goes from ExtinctionCoefficientsLinear.py; it wrote to a directory of one machine. Co-Authored-By: Claude Opus 5.5 --- README.md | 20 ++++++- .../analysis/ExtinctionCoefficientsLinear.py | 58 ------------------- pyproject.toml | 2 +- 3 files changed, 20 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 6a909ef..6e94ab4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ LEDSA (LEDSmokeAnalysis) is a Python-based software package for the computation ### Requirements -- Python 3.8 +- Python 3.9 to 3.12 - pip (Python package installer) ### Installation from PyPI @@ -162,6 +162,24 @@ The time counts from `start_time` of `config.ini`, like `Experiment_Time[s]` in factors before I0 is computed; images and LEDs outside the table stay uncorrected, with a warning. The header of each result file states the correction applied. +### Dense smoke and the read noise of the sensor + +The intensities are not clipped at zero after the black level of the sensor is subtracted, so an +LED that has vanished in dense smoke can give a relative intensity at or below zero. The linear +solver leaves such LEDs out. Where the transmission reaches the read noise of the sensor, the LEDs +that remain are those whose noise happened to be positive, and the extinction coefficients come +out too low: in fog tests by 14 to 31 % at 1.24 and 1.64 1/m, where the transmission of the red +LEDs was 0.1 to 0.2 % of I0. + +### Features in validation + +The background-corrected sums of step 3 (`bgsub_sum_col_val`, `bgsub2_sum_col_val`, +`sidebg_sum_col_val`), the weighting of the LEDs by their signal-to-noise ratio and the moving +average of the reference property, which v0.12.0 had, are left out of the releases until they are +validated. They are kept on the +[develop](https://github.com/FireDynamics/LEDSmokeAnalysis/tree/develop) branch. A config file that +still sets one of their options gets a note that the option is ignored. + ## Demo LEDSA includes a demo that demonstrates its functionality using sample data. diff --git a/ledsa/analysis/ExtinctionCoefficientsLinear.py b/ledsa/analysis/ExtinctionCoefficientsLinear.py index ed7b28e..1f7b98e 100644 --- a/ledsa/analysis/ExtinctionCoefficientsLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsLinear.py @@ -114,61 +114,3 @@ def calc_coefficients_of_img(self, rel_intensities: np.ndarray) -> np.ndarray: # sigmas, residuals, rank, s = np.linalg.lstsq(A_aug, b_aug, rcond=None) return sigmas - - # For investigation - # import os - - # def calc_coefficients_of_img(self, rel_intensities: np.ndarray) -> np.ndarray: - # """ - # Calculate the extinction coefficients for a single image using a linearized approach. - # This method solves the linear system derived from the Beer-Lambert law using - # non-negative least squares with Tikhonov regularization. - # - # :param rel_intensities: Array of relative (normalized) LED intensities (I_e/I_0). - # :type rel_intensities: np.ndarray - # :return: Array of the computed extinction coefficients (sigmas). - # :rtype: np.ndarray - # """ - # # Avoid log(0) by clipping intensities to a small positive value - # eps = 1e-10 - # target = np.clip(rel_intensities, eps, 1.0) - # - # # Compute the optical depth vector: b = -ln(I_e/I_0) - # # This linearizes the Beer-Lambert law: I_e/I_0 = exp(-sum(sigma_i * distance_i)) - # optical_depths = -np.log(target) - # - # # Get dimensions of the problem - # n_leds, n_layers = self.distances_per_led_and_layer.shape - # - # # Build finite-difference matrix L for second derivative (curvature penalty) - # # This matrix approximates the second derivative and is used for regularization - # # to enforce smoothness in the solution - # if n_layers >= 3: - # L = np.zeros((n_layers - 2, n_layers)) - # for i in range(n_layers - 2): - # # Coefficients for second-order finite difference approximation - # L[i, i] = 1 - # L[i, i + 1] = -2 - # L[i, i + 2] = 1 - # else: - # # Fallback: use an identity if there aren't enough layers - # L = np.eye(n_layers) - # - # # Interate lambda_reg from 1e-4 to 1 - # logspace = np.logspace(-4, 0, 50) - # - # os.chdir('/Users/kristian/Documents/NextVis/25_ledsa/lambda_iteration') - # for lambda_reg in logspace: - # sqrt_lambda = np.sqrt(lambda_reg) - # - # # Augment the system with regularization - # # A_aug = [A; sqrt(lambda)*L], where A is the distance matrix - # A_aug = np.vstack((self.distances_per_led_and_layer, sqrt_lambda * L)) - # - # # Augment the right-hand side: b_aug = [b; 0] - # b_aug = np.hstack((optical_depths, np.zeros(L.shape[0]))) - # - # # Solve the non-negative least squares problem: min ||A_aug*x - b_aug||^2 subject to x >= 0 - # sigmas, residuals = nnls(A_aug, b_aug) - # np.savetxt(f'lambda_{lambda_reg:.4f}.txt', sigmas) - # return sigmas \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 81af7e8..9897ce5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools", "wheel"] [project] name = "ledsa" -version = "0.12.0" +version = "0.13.0" description = "A scientific package to analyse smoke via the dimming of light sources." authors = [ {name = "Kristian Börger", email = "boerger@uni-wuppertal.de"},