From d42f4dc92f7709173cf55a0de8b59e0f8e56a312 Mon Sep 17 00:00:00 2001 From: Kristian <73885850+kriboe90@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:41:03 +0100 Subject: [PATCH 1/3] add the far-field pedestal of the search areas (farbg_sum_col_val) The sum over a search area holds light that does not dim with the LED: the dark level of the sensor where it differs from the optical black, ambient light and light that smoke scatters into the line of sight. In dense smoke this pedestal saturates the extinction coefficients. With farfield_distances = in [analyse_photo] of config.ini, step 3 estimates the pedestal of every image and LED in two bands parallel to the strip, one on each side, over the length of the search area and from the inner to the outer distance across the strip. Pixels outside the image and closer than the inner distance to any LED are left out. The level of a band is the mean of its pixels of the channel. The levels of each side are smoothed along the strip with a running median over 15 and a running mean over 5 LEDs, and the lower side is the pedestal per pixel. Step 3 writes farbg_sum_col_val, the sum minus N times the pedestal with N the number of channel pixels of the search area, and farbg_pedestal behind max_col_val, and the analysis knows both layouts, with and without the fit. reference_property = farbg_sum_col_val analyses the corrected sum. Merged LED arrays take the strips of step 2. Without the option, or with None, step 3 and its result files stay as they were. The unit test that writes a result file names the encoding on the line of write_text, where the encoding check of the CI looks for it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016cv6PgMB7utWgf5YHwHffz --- README.md | 29 ++ doc/data_extraction.rst | 6 +- ledsa/core/ConfigData.py | 11 +- ledsa/core/file_handling.py | 9 +- ledsa/data_extraction/LEDAnalysisData.py | 9 + ledsa/data_extraction/farfield_background.py | 295 ++++++++++++++ ledsa/data_extraction/step_3_functions.py | 139 ++++++- .../UnitTests/test_farfield_background.py | 365 ++++++++++++++++++ ledsa/tests/UnitTests/test_file_handling.py | 3 +- 9 files changed, 845 insertions(+), 21 deletions(-) create mode 100644 ledsa/data_extraction/farfield_background.py create mode 100644 ledsa/tests/UnitTests/test_farfield_background.py diff --git a/README.md b/README.md index b108ede..355d410 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,35 @@ that remain are those whose noise happened to be positive, and the extinction co out too low: in fog tests by 14 to 31 % at 1.24 and 1.64 1/m, where the transmission of the red LEDs was 0.1 to 0.2 % of I0. +### Pedestal from the far field of the LED strips (optional) + +The sum over a search area holds light that does not dim with the LED: the dark level of the +sensor where it differs from the optical black, ambient light and light that smoke scatters into +the line of sight. In dense smoke this pedestal saturates the extinction coefficients. With + +```ini +[analyse_photo] +farfield_distances = 150 300 +``` + +in `config.ini`, step 3 estimates the pedestal for every image in two bands parallel to each LED +strip, 150 to 300 px away from it on both sides and at least 150 px away from every LED, smooths +the levels along the strip, takes the lower side and writes two more quantities: +`farbg_sum_col_val`, the sum minus the pedestal of the search area, and `farbg_pedestal`, the +pedestal itself. `reference_property = farbg_sum_col_val` analyses the corrected sum. The bands +have to lie beyond the light of the LEDs and of their halo; `farbg_pedestal / sum_col_val` of the +reference images shows whether they do, it stays below about 1 % there. An LED whose bands hold +no pixel on either side, and no neighbour on its strip within the smoothing that has one, keeps +its plain sum with a pedestal of 0. The default `None` estimates no pedestal and leaves the +result files as they are. + +For the dark level of the sensor and for stray light that peaks at the strip, the far-field level +is a lower bound of the pedestal, so the corrected sum lies between the plain sum and the light +of the LED in the search area. Ambient light on a background that is brighter beside the strip +than behind the LEDs breaks this bound; it has been tested in dark rooms only. It does +not correct a search area that loses light of the LED, as behind a fogged window, which is an +error of the opposite sign. + ### Features in validation The background-corrected sums of step 3 (`bgsub_sum_col_val`, `bgsub2_sum_col_val`, diff --git a/doc/data_extraction.rst b/doc/data_extraction.rst index b1416d1..335298a 100644 --- a/doc/data_extraction.rst +++ b/doc/data_extraction.rst @@ -9,4 +9,8 @@ Data Extraction .. automodule:: ledsa.data_extraction.LEDAnalysisData :members: :undoc-members: - :show-inheritance: \ No newline at end of file + :show-inheritance: +.. automodule:: ledsa.data_extraction.farfield_background + :members: + :undoc-members: + :show-inheritance: diff --git a/ledsa/core/ConfigData.py b/ledsa/core/ConfigData.py index b0325bd..e05a9ad 100644 --- a/ledsa/core/ConfigData.py +++ b/ledsa/core/ConfigData.py @@ -30,7 +30,7 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius first_img_experiment_id=None, last_img_experiment_id=None, ref_img_id=None, ignore_led_indices=None, led_array_edge_indices=None, led_array_edge_coordinates=None, first_img_analysis_id=None, last_img_analysis_id=None, num_skip_imgs=0, ref_img_experiment_ids=None, num_skip_leds=0, - merge_led_array_indices=None): # TODO: merge LED arrays + merge_led_array_indices=None, farfield_distances=None): # TODO: merge LED arrays """ :param load_config_file: Determines whether to load the config file on initialization. Defaults to True. :type load_config_file: bool @@ -93,6 +93,10 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius :type num_skip_leds: int :param merge_led_array_indices: Flag to merge LED arrays for analysis. Defaults to None. TODO: not a flag but list of arrays to merge? :type merge_led_array_indices: bool or None + :param farfield_distances: Inner and outer distance in pixels across the LED strip of the far-field bands + the pedestal of farbg_sum_col_val is estimated in, e.g. '150 300'. None, the default, estimates no + pedestal: step 3 writes sum_col_val, mean_col_val and max_col_val only. + :type farfield_distances: str or None """ cp.ConfigParser.__init__(self, allow_no_value=True) if load_config_file: @@ -177,6 +181,11 @@ def __init__(self, load_config_file=True, img_directory=None, search_area_radius self['analyse_photo'][' ref_img_experiment_ids'] = str(ref_img_experiment_ids) self.set('analyse_photo', ' # In the analysis, only LEDs are considered with id dividable by skip_leds + 1.') self['analyse_photo'][' num_skip_leds'] = str(num_skip_leds) + self.set('analyse_photo', ' # Pedestal of the search areas from the far field of the LED strips (farbg_sum_col_val):') + self.set('analyse_photo', ' # inner and outer distance in pixels across the strip of the bands on both of its sides,') + self.set('analyse_photo', ' # e.g. 150 300. The bands have to lie beyond the light of the LEDs and of their halo.') + self.set('analyse_photo', ' # None: no pedestal is estimated.') + self['analyse_photo'][' farfield_distances'] = str(farfield_distances) with open('config.ini', 'w', encoding='utf-8') as configfile: self.write(configfile) diff --git a/ledsa/core/file_handling.py b/ledsa/core/file_handling.py index d784443..cde6daa 100644 --- a/ledsa/core/file_handling.py +++ b/ledsa/core/file_handling.py @@ -686,15 +686,20 @@ def create_binary_data(channel: int, path: str = '.') -> None: BGSUB_COLUMNS = ["bgsub_sum_col_val"] SIDEBG_COLUMNS = ["sidebg_sum_col_val", "sidebg_pedestal", "bgsub2_sum_col_val"] NOISE_SUM_COLUMNS = ["noise_sum_col_val"] +FARBG_COLUMNS = ["farbg_sum_col_val", "farbg_pedestal"] LED_CENTER_COLUMNS = ["led_center_x", "led_center_y"] FIT_COLUMNS = ["x", "y", "dx", "dy", "A", "alpha", "wx", "wy", "fit_success", "fit_fun", "fit_nfev", "fit_time"] -# Known layouts by the number of names (including img_id). Step 3 writes the first two; the others are those of -# earlier versions, whose background-corrected sums and read noise v0.13.0 no longer computes but still reads. +# Known layouts by the number of names (including img_id). Step 3 writes the first four, the two with the +# far-field pedestal if farfield_distances is set; the others are those of earlier versions, whose +# background-corrected sums and read noise v0.13.0 no longer computes but still reads. COLUMN_LAYOUTS = { len(BASE_COLUMNS): BASE_COLUMNS, # -s3_fast, before v0.11.0 and since v0.13.0 len(BASE_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): BASE_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit, since v0.13.0 + len(BASE_COLUMNS + FARBG_COLUMNS): BASE_COLUMNS + FARBG_COLUMNS, # -s3_fast with the far-field pedestal + len(BASE_COLUMNS + FARBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS): + BASE_COLUMNS + FARBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS, # -s3 with fit and the far-field pedestal len(BASE_COLUMNS + BGSUB_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS, # -s3_fast, before the side band pedestal len(BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS): BASE_COLUMNS + BGSUB_COLUMNS + SIDEBG_COLUMNS, # -s3_fast of v0.11.0 diff --git a/ledsa/data_extraction/LEDAnalysisData.py b/ledsa/data_extraction/LEDAnalysisData.py index 0e405ad..48b139f 100644 --- a/ledsa/data_extraction/LEDAnalysisData.py +++ b/ledsa/data_extraction/LEDAnalysisData.py @@ -18,6 +18,11 @@ class LEDAnalysisData: :vartype sum_color_value: float :ivar max_color_value: Maximum color value observed for the LED. :vartype max_color_value: float + :ivar farbg_sum_color_value: Integrated color value minus the far-field pedestal of the search area, None if + the pedestal is not estimated. + :vartype farbg_sum_color_value: float or None + :ivar farbg_pedestal: Far-field pedestal of the search area, None if the pedestal is not estimated. + :vartype farbg_pedestal: float or None :ivar fit_results: Fit results after fitting. :vartype fit_results: OptimizeResult :ivar fit_time: Time taken for fitting. @@ -41,6 +46,8 @@ def __init__(self, led_id, led_array, fit_leds): self.mean_color_value = None self.sum_color_value = None self.max_color_value = None + self.farbg_sum_color_value = None + self.farbg_pedestal = None self.fit_leds = fit_leds self.fit_results = None self.fit_time = None @@ -68,6 +75,8 @@ def get_main_data_string(self) -> str: """ out_str = f'{self.led_id:4d},{self.led_array:2d},' out_str += f'{self.sum_color_value:10.4e},{self.mean_color_value:10.4e},{self.max_color_value}' + if self.farbg_sum_color_value is not None: + out_str += f',{self.farbg_sum_color_value:10.4e},{self.farbg_pedestal:10.4e}' return out_str def get_fit_data_string(self) -> str: diff --git a/ledsa/data_extraction/farfield_background.py b/ledsa/data_extraction/farfield_background.py new file mode 100644 index 0000000..27051f2 --- /dev/null +++ b/ledsa/data_extraction/farfield_background.py @@ -0,0 +1,295 @@ +""" +Pedestal of the LED search areas, estimated in the far field of the LED strips. + +The sum over a search area (sum_col_val) holds, besides the light of the LED, light that does not dim with the +LED: the dark level of the sensor where it differs from the optical black the black level is taken from, ambient +light, and light that smoke scatters into the line of sight. In dense smoke the LED dims while this pedestal +stays, so the plain sum overestimates the transmission and the extinction coefficients saturate. + +The pedestal is estimated for every image where no light of an LED is expected, in the far field of the strips: + +1. Two bands run parallel to the strip, one on each side, over +-R along the strip (R = search_area_radius) and + from the inner to the outer far-field distance across it. Pixels closer than the inner distance to any LED of + any strip are left out, so neighbouring strips do not enter the bands. +2. The level of a band is the mean of its pixels of the color channel. +3. The levels of each side are smoothed along the strip, with a running median and a running mean over the LEDs. + The pedestal varies slowly along the strip, so this removes the noise of the estimate without changing it: + the pedestal corrected sum is as noisy as the plain sum. +4. The pedestal of an LED is the lower of the levels of its two sides. An object in one of the bands, another + light source or the brighter side of a gradient can only raise a level, so the lower side is the safe one. +5. N times this pedestal is subtracted from the sum of the search area, with N the number of pixels of the color + channel in the search area. + +Why the far field and not the neighbourhood of the LED: around an LED there is light of the LED itself, the wings +of its image and, behind a window or in hot gases, a halo that can reach a hundred pixels and more. A pedestal +measured there subtracts transmitted light and raises the extinction coefficients, the more the denser the smoke. +The far field holds none of it. Stray light that peaks at the strip is brighter at the LED than in the far field, so +the far-field level is a lower bound of the pedestal: the corrected sum lies between the plain sum and the light +of the LED in the search area. The bound holds for the dark level and for scattered light. Ambient light on a +background that is brighter beside the strip than behind the LED would raise the far-field level above the +pedestal of the search area. + +What the correction does not do: where the image of an LED widens beyond its search area, the search area loses +light of the LED. That error has the opposite sign and is no pedestal; the correction leaves it as it is. +""" +from typing import List, NamedTuple, Optional, Sequence, Tuple + +import numpy as np +from scipy.spatial import cKDTree + +# LEDs the levels are smoothed over along the strip: running median, then running mean +SMOOTHING_MEDIAN_LEDS = 15 +SMOOTHING_MEAN_LEDS = 5 +NUM_DIRECTION_NEIGHBOURS = 4 + + +class FarfieldGeometry(NamedTuple): + """The far-field bands of the LEDs of an image and the place of every LED on its strip.""" + + # per LED id the row and column indices of the pixels of the band on each of the two sides of the strip, + # None for an LED without a strip direction + pixels: List[Optional[Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]]] + strip_of_led: np.ndarray # per LED id the index of its strip, -1 for an LED on no strip + position_on_strip: np.ndarray # per LED id its coordinate along the strip in pixels + + +# geometry of the bands, cached per process because it only depends on the LED positions +_farfield_cache = {} + + +def parse_farfield_distances(value: Optional[str], search_area_radius: int) -> Optional[Tuple[float, float]]: + """ + Read the far-field distances from their config value. + + :param value: Value of farfield_distances: the inner and the outer distance of the bands across the strip in + pixels, or None. + :type value: str or None + :param search_area_radius: Radius R of the search areas. + :type search_area_radius: int + :return: The distances (inner, outer), None if the pedestal is not estimated. + :rtype: tuple[float, float] or None + :raises ValueError: If the value is neither None nor two distances R <= inner < outer. + """ + if value is None or str(value).strip().lower() in ('none', ''): + return None + try: + distances = tuple(float(distance) for distance in str(value).split()) + except ValueError: + distances = () + if len(distances) != 2 or not search_area_radius <= distances[0] < distances[1]: + raise ValueError(f"farfield_distances of config.ini holds '{value}', while it takes the inner and the " + f"outer distance of the far-field bands across the LED strip in pixels, with " + f"search_area_radius ({search_area_radius}) <= inner < outer, or None.") + return distances + + +def estimate_strip_directions(search_areas: np.ndarray, led_arrays: Sequence[Sequence[int]], + num_neighbours: int = NUM_DIRECTION_NEIGHBOURS) -> np.ndarray: + """ + Estimate the local direction of the LED strip at each LED from its nearest neighbours in the same LED array. + + The direction is the principal axis of the positions of the LED and its num_neighbours nearest neighbours of the + same array. It is oriented such that the row component is positive (or, for horizontal strips, the column + component), so the two sides of a strip are the same for all of its LEDs. + + :param search_areas: Array with rows (LED id, pixel row, pixel column), the row index equals the LED id. + :type search_areas: np.ndarray + :param led_arrays: LED ids of each LED array. + :type led_arrays: Sequence[Sequence[int]] + :param num_neighbours: Number of neighbours used to estimate the direction. + :type num_neighbours: int + :return: Array of shape (number of LEDs, 2) with the unit vector (row, column) along the strip, NaN for LEDs + that are not part of an array with at least two LEDs. + :rtype: np.ndarray + """ + directions = np.full((search_areas.shape[0], 2), np.nan) + for led_ids in led_arrays: + led_ids = np.atleast_1d(np.asarray(led_ids, dtype=int)) + if led_ids.size < 2: + continue + positions = search_areas[led_ids, 1:3].astype(np.float64) + num_points = min(num_neighbours, led_ids.size - 1) + 1 + for i, led_id in enumerate(led_ids): + distances = np.hypot(*(positions - positions[i]).T) + nearest = np.argsort(distances, kind='stable')[:num_points] + points = positions[nearest] - positions[nearest].mean(axis=0) + direction = np.linalg.svd(points, full_matrices=False)[2][0] + if direction[0] < 0 or (direction[0] == 0 and direction[1] < 0): + direction = -direction + directions[led_id] = direction + return directions + + +def build_farfield_geometry(search_areas: np.ndarray, led_arrays: Sequence[Sequence[int]], + img_shape: Tuple[int, int], search_area_radius: int, + distances: Tuple[float, float]) -> FarfieldGeometry: + """ + Determine the pixels of the two far-field bands of each LED and the place of each LED on its strip. + + With t the coordinate along the strip and s the coordinate across it, both relative to the center of the + search area, a pixel belongs to the band of one side if ``-R <= t < R`` and ``inner <= s < outer``, and to that + of the other side if ``-outer <= s < -inner``. For a vertical strip these are the rows r-R..r+R-1 and the + columns c+inner..c+outer-1 and c-outer..c-inner-1. Pixels outside the image and pixels closer than the inner + distance to the center of any search area are left out. + + :param search_areas: Array with rows (LED id, pixel row, pixel column), the row index equals the LED id. + :type search_areas: np.ndarray + :param led_arrays: LED ids of each LED strip, as found in step 2. + :type led_arrays: Sequence[Sequence[int]] + :param img_shape: Shape of the image. + :type img_shape: tuple[int, int] + :param search_area_radius: Radius R of the search areas. + :type search_area_radius: int + :param distances: Inner and outer distance of the bands across the strip in pixels. + :type distances: tuple[float, float] + :return: The band pixels and the place on its strip of every LED. + :rtype: FarfieldGeometry + """ + radius = int(search_area_radius) + inner, outer = float(distances[0]), float(distances[1]) + directions = estimate_strip_directions(search_areas, led_arrays) + centers = search_areas[:, 1:3].astype(np.float64) + tree = cKDTree(centers) + reach = int(np.ceil(np.hypot(outer, radius))) + 1 + d_row, d_col = np.mgrid[-reach:reach + 1, -reach:reach + 1] + + num_leds = search_areas.shape[0] + pixels = [None] * num_leds + strip_of_led = np.full(num_leds, -1, dtype=int) + position_on_strip = np.zeros(num_leds) + for strip, led_ids in enumerate(led_arrays): + led_ids = np.atleast_1d(np.asarray(led_ids, dtype=int)) + if led_ids.size < 2: + continue + # one direction for the coordinate along the strip, so that it orders the LEDs of a curved strip as well + mean_direction = np.nanmean(directions[led_ids], axis=0) + strip_of_led[led_ids] = strip + position_on_strip[led_ids] = centers[led_ids] @ mean_direction + for led_id in range(num_leds): + along = directions[led_id] + if not np.all(np.isfinite(along)): + continue + across = np.array([-along[1], along[0]]) + # rounding avoids that pixels exactly on a band border flip due to floating point errors + t = np.round(d_row * along[0] + d_col * along[1], 6) + s = np.round(d_row * across[0] + d_col * across[1], 6) + in_window = (t >= -radius) & (t < radius) + sides = [] + for in_band in (in_window & (s >= inner) & (s < outer), in_window & (s >= -outer) & (s < -inner)): + rows = int(centers[led_id, 0]) + d_row[in_band] + cols = int(centers[led_id, 1]) + d_col[in_band] + keep = (rows >= 0) & (rows < img_shape[0]) & (cols >= 0) & (cols < img_shape[1]) + rows, cols = rows[keep], cols[keep] + if rows.size: + distance_to_nearest_led, _ = tree.query(np.column_stack((rows, cols))) + keep = distance_to_nearest_led >= inner + rows, cols = rows[keep], cols[keep] + sides.append((rows.astype(np.intp), cols.astype(np.intp))) + pixels[led_id] = (sides[0], sides[1]) + return FarfieldGeometry(pixels, strip_of_led, position_on_strip) + + +def get_farfield_geometry(search_areas: np.ndarray, led_arrays: Sequence[Sequence[int]], img_shape: Tuple[int, int], + search_area_radius: int, distances: Tuple[float, float]) -> FarfieldGeometry: + """ + Cached build_farfield_geometry. The geometry only depends on the LED positions and is therefore computed once + per process and reused for all images. + + :return: See build_farfield_geometry. + """ + key = (search_areas.tobytes(), search_areas.shape, + tuple(tuple(int(i) for i in np.atleast_1d(led_ids)) for led_ids in led_arrays), + tuple(img_shape), int(search_area_radius), tuple(float(d) for d in distances)) + if key not in _farfield_cache: + _farfield_cache.clear() + _farfield_cache[key] = build_farfield_geometry(search_areas, led_arrays, img_shape, search_area_radius, + distances) + return _farfield_cache[key] + + +def mean_of_channel_pixels(values: np.ndarray, channel_mask: np.ndarray, num_pixels: Sequence[int]) -> np.ndarray: + """ + Mean over the pixels of the color channel in every band, from the pixels of all bands. + + :param values: Channel values of the pixels of all bands, the pixels of one band after those of the previous + one. + :type values: np.ndarray + :param channel_mask: Whether each of these pixels belongs to the channel. + :type channel_mask: np.ndarray + :param num_pixels: Number of pixels of every band. + :type num_pixels: Sequence[int] + :return: Mean per channel pixel of every band, NaN for a band without a pixel of the channel. + :rtype: np.ndarray + """ + num_pixels = np.asarray(num_pixels, dtype=np.intp) + means = np.full(num_pixels.size, np.nan) + # reduceat cannot sum an empty segment, and the segments left are contiguous without the empty ones + has_pixels = num_pixels > 0 + if not has_pixels.any(): + return means + starts = (np.cumsum(num_pixels) - num_pixels)[has_pixels] + sums = np.add.reduceat(np.where(channel_mask, values, 0), starts, dtype=np.float64) + counts = np.add.reduceat(channel_mask, starts, dtype=np.intp) + with np.errstate(invalid='ignore', divide='ignore'): + means[has_pixels] = sums / counts # 0 / 0 is NaN + return means + + +def _running(values: np.ndarray, size: int, function) -> np.ndarray: + """A centred running statistic that ignores NaN and takes fewer values at the ends, NaN where it has none.""" + half = size // 2 + padded = np.concatenate((np.full(half, np.nan), values, np.full(half, np.nan))) + windows = np.lib.stride_tricks.sliding_window_view(padded, 2 * half + 1) + result = np.full(values.size, np.nan) + has_values = np.isfinite(windows).any(axis=1) + result[has_values] = function(windows[has_values], axis=1) + return result + + +def smooth_along_strip(levels: np.ndarray, median_size: int = SMOOTHING_MEDIAN_LEDS, + mean_size: int = SMOOTHING_MEAN_LEDS) -> np.ndarray: + """ + Smooth the band levels of one side of a strip along the strip: a running median, which drops the outliers of + single LEDs, followed by a running mean. An LED without a level takes the one of its neighbours. At the ends + of the strip the windows are cut short. + + :param levels: Level of every LED of the strip, in their order along the strip, NaN for an LED without one. + :type levels: np.ndarray + :param median_size: Number of LEDs of the running median. + :type median_size: int + :param mean_size: Number of LEDs of the running mean. + :type mean_size: int + :return: The smoothed levels, NaN where no LED within reach has a level. + :rtype: np.ndarray + """ + levels = np.asarray(levels, dtype=np.float64) + if levels.size == 0: + return levels + return _running(_running(levels, median_size, np.nanmedian), mean_size, np.nanmean) + + +def farfield_pedestals(band_levels: np.ndarray, strip_of_led: np.ndarray, + position_on_strip: np.ndarray) -> np.ndarray: + """ + The pedestal per channel pixel of every LED: the levels of the two sides, smoothed along the strip, and of + them the lower one. + + :param band_levels: Level of the band on each of the two sides of every LED, shape (number of LEDs, 2), NaN + for a band without a pixel of the channel. + :type band_levels: np.ndarray + :param strip_of_led: Index of the strip of every LED, -1 for an LED on no strip. + :type strip_of_led: np.ndarray + :param position_on_strip: Coordinate of every LED along its strip. + :type position_on_strip: np.ndarray + :return: Pedestal per channel pixel of every LED, NaN for an LED without a level on either side. + :rtype: np.ndarray + """ + band_levels = np.asarray(band_levels, dtype=np.float64) + pedestals = np.full(band_levels.shape[0], np.nan) + for strip in np.unique(strip_of_led[strip_of_led >= 0]): + leds = np.flatnonzero(strip_of_led == strip) + leds = leds[np.argsort(position_on_strip[leds], kind='stable')] + smoothed = np.column_stack([smooth_along_strip(band_levels[leds, side]) for side in (0, 1)]) + # fmin takes the side that has a level where the other has none + pedestals[leds] = np.fmin(smoothed[:, 0], smoothed[:, 1]) + return pedestals diff --git a/ledsa/data_extraction/step_3_functions.py b/ledsa/data_extraction/step_3_functions.py index 0fd8052..e82f0c8 100644 --- a/ledsa/data_extraction/step_3_functions.py +++ b/ledsa/data_extraction/step_3_functions.py @@ -12,6 +12,9 @@ from ledsa.core.image_handling import get_img_name from ledsa.core.image_reading import ImageChannels from ledsa.data_extraction.LEDAnalysisData import LEDAnalysisData +from ledsa.data_extraction.farfield_background import (FarfieldGeometry, farfield_pedestals, + get_farfield_geometry, mean_of_channel_pixels, + parse_farfield_distances) from ledsa.data_extraction.model import target_function @@ -60,9 +63,18 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda led_ids = np.array([iled for iled, _ in leds], dtype=int) led_array_ids = np.array([led_array_idx for _, led_array_idx in leds], dtype=int) - # only the pixels of the search areas are read, which are a few percent of the image - analysis_pixels = _get_analysis_pixels(search_areas, led_ids, img_channels.shape, search_area_radius) - data, _ = img_channels.channel_data_and_mask(channel, analysis_pixels.pixels) + farfield = _get_farfield_geometry(conf, search_areas, line_indices, img_channels.shape, search_area_radius) + + # only the pixels of the search areas and of the far-field bands are read, which are a few percent of the image + analysis_pixels = _get_analysis_pixels(search_areas, led_ids, img_channels.shape, search_area_radius, farfield) + data, channel_mask = img_channels.channel_data_and_mask(channel, analysis_pixels.pixels) + pedestals = None + if farfield is not None: + bands = np.s_[analysis_pixels.num_box_pixels:] + band_levels = mean_of_channel_pixels(data[bands], channel_mask[bands], + analysis_pixels.num_band_pixels.ravel()).reshape(-1, 2) + pedestals = farfield_pedestals(band_levels, farfield.strip_of_led[led_ids], + farfield.position_on_strip[led_ids]) img_analysis_data = [None] * len(leds) for positions, shape, start in analysis_pixels.box_groups: @@ -70,7 +82,8 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda stack_shape = (positions.size,) + shape group_data = _analyse_search_areas( conf, channel, img_filename, led_ids[positions], led_array_ids[positions], - analysis_pixels.centers[positions], search_area_radius, fit_leds, data[boxes].reshape(stack_shape)) + analysis_pixels.centers[positions], search_area_radius, fit_leds, data[boxes].reshape(stack_shape), + channel_mask[boxes].reshape(stack_shape), None if pedestals is None else pedestals[positions]) for position, led_analysis_data in zip(positions, group_data): img_analysis_data[position] = led_analysis_data return img_analysis_data @@ -79,7 +92,8 @@ def generate_analysis_data(img_filename: str, channel: int, search_areas: np.nda class AnalysisPixels(NamedTuple): """ The pixels step 3 reads from an image, in one selection so that the image calibrates them once for all - channels: the search areas of the analysed LEDs, stacked by their shape. + channels: the search areas of the analysed LEDs, stacked by their shape, followed by their far-field bands + if the pedestal is estimated. """ pixels: Tuple[np.ndarray, np.ndarray] # row and column indices @@ -87,6 +101,8 @@ class AnalysisPixels(NamedTuple): # per shape of the search areas: the positions of their LEDs in the analysis, the shape and the index of the # first pixel of the stack in pixels box_groups: List[Tuple[np.ndarray, Tuple[int, int], int]] + num_box_pixels: int # the band pixels follow the search areas, LED after LED, side after side + num_band_pixels: np.ndarray # number of pixels of the two far-field bands of every analysed LED # pixels of the analysis, cached per process because they only depend on the LED positions @@ -94,7 +110,7 @@ class AnalysisPixels(NamedTuple): def _get_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_shape: Tuple[int, int], - search_area_radius: int) -> AnalysisPixels: + search_area_radius: int, farfield: Optional[FarfieldGeometry] = None) -> AnalysisPixels: """ Cached _build_analysis_pixels. The same object is returned for every image and channel, which lets the image keep the pixels it calibrated for the next channel. @@ -102,16 +118,19 @@ def _get_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_shap :return: See _build_analysis_pixels. """ key = (search_areas.tobytes(), search_areas.shape, led_ids.tobytes(), tuple(img_shape), search_area_radius) - if key not in _analysis_pixels_cache: + # the far-field geometry comes from a cache of its own, which builds a new one when its key changes; it is + # kept with the entry, so that its identity cannot pass to another one + if key not in _analysis_pixels_cache or _analysis_pixels_cache[key][0] is not farfield: _analysis_pixels_cache.clear() - _analysis_pixels_cache[key] = _build_analysis_pixels(search_areas, led_ids, img_shape, search_area_radius) - return _analysis_pixels_cache[key] + _analysis_pixels_cache[key] = (farfield, _build_analysis_pixels(search_areas, led_ids, img_shape, + search_area_radius, farfield)) + return _analysis_pixels_cache[key][1] def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_shape: Tuple[int, int], - search_area_radius: int) -> AnalysisPixels: + search_area_radius: int, farfield: Optional[FarfieldGeometry] = None) -> AnalysisPixels: """ - Collect the pixels of the search areas of the analysed LEDs. + Collect the pixels of the search areas of the analysed LEDs and, if given, of their far-field bands. The search area is cut from the image as data[row - R:row + R, col - R:col + R], so an area at the border of the image is cut short as the slice is. Its shape differs from that of the others, and the search areas @@ -125,7 +144,9 @@ def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_sh :type img_shape: tuple[int, int] :param search_area_radius: Radius R of the search areas. :type search_area_radius: int - :return: The pixels and where the analysis finds the search area of every LED in them. + :param farfield: The far-field bands of the LEDs, None if the pedestal is not estimated. + :type farfield: FarfieldGeometry or None + :return: The pixels and where the analysis finds the search area and the bands of every LED in them. :rtype: AnalysisPixels """ centers = search_areas[led_ids, 1:3].astype(int) @@ -147,7 +168,73 @@ def _build_analysis_pixels(search_areas: np.ndarray, led_ids: np.ndarray, img_sh cols.append(np.broadcast_to(first_cols[:, None, None] + np.arange(shape[1]), stack_shape).ravel()) box_groups.append((positions, shape, start)) start += positions.size * shape[0] * shape[1] - return AnalysisPixels((np.concatenate(rows), np.concatenate(cols)), centers, box_groups) + + num_band_pixels = np.zeros((led_ids.size, 2), dtype=np.intp) + if farfield is not None: + for position, led_id in enumerate(led_ids): + if farfield.pixels[led_id] is None: + continue + for side, (band_rows, band_cols) in enumerate(farfield.pixels[led_id]): + rows.append(band_rows) + cols.append(band_cols) + num_band_pixels[position, side] = band_rows.size + return AnalysisPixels((np.concatenate(rows), np.concatenate(cols)), centers, box_groups, start, + num_band_pixels) + + +def _get_farfield_geometry(conf: ConfigData, search_areas: np.ndarray, line_indices: List[List[int]], + img_shape: Tuple[int, int], search_area_radius: int) -> Optional[FarfieldGeometry]: + """ + The far-field bands of the LEDs if farfield_distances of the config asks for the pedestal. Config files + written before the option existed do not contain it, the pedestal is not estimated then. + + :param conf: Configuration data. + :type conf: ConfigData + :param search_areas: Array with rows (LED id, pixel row, pixel column), the row index equals the LED id. + :type search_areas: np.ndarray + :param line_indices: LED ids of the LED arrays used in the analysis. + :type line_indices: List[List[int]] + :param img_shape: Shape of the image. + :type img_shape: tuple[int, int] + :param search_area_radius: Radius R of the search areas. + :type search_area_radius: int + :return: The far-field geometry, None if the pedestal is not estimated. + :rtype: FarfieldGeometry or None + """ + distances = parse_farfield_distances(conf.get('analyse_photo', 'farfield_distances', fallback='None'), + search_area_radius) + if distances is None: + return None + return get_farfield_geometry(search_areas, _get_unmerged_led_arrays(conf, line_indices), img_shape, + search_area_radius, distances) + + +def _get_unmerged_led_arrays(conf: ConfigData, line_indices: List[List[int]]) -> List[List[int]]: + """ + Return the LED arrays as found in step 2. If LED arrays are merged for the analysis, the merged arrays may + contain several strips, which would spoil the strip direction and the smoothing along the strip. + + :param conf: Configuration data. + :type conf: ConfigData + :param line_indices: LED ids of the (possibly merged) LED arrays used in the analysis. + :type line_indices: List[List[int]] + :return: LED ids of the LED arrays as found in step 2. + :rtype: List[List[int]] + """ + if conf.get('analyse_positions', 'merge_led_array_indices', fallback='None') == 'None': + return line_indices + led_arrays = [] + for i in range(int(conf['analyse_positions']['num_arrays'])): + file_path = os.path.join('analysis', f'led_array_indices_{i:03}.csv') + # read_table stays silent, it would report every file of every image; its error message is unspecific + try: + led_arrays.append(read_table(file_path, dtype='int', silent=True)) + except OSError as error: + raise OSError(f'{file_path} could not be read ({error}). The LED arrays of step 2 are needed for ' + f'the far-field pedestal (farbg_sum_col_val), because merge_led_array_indices merges ' + f'LED arrays that may contain several strips. Run step 2 again to write the files, ' + f'or unset merge_led_array_indices or farfield_distances.') from error + return led_arrays def create_fit_result_file(img_data: List[LEDAnalysisData], img_id: int, channel: int) -> None: # TODO: rename because misleading @@ -291,7 +378,8 @@ def _generate_led_analysis_data(conf: ConfigData, channel: int, data: np.ndarray def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led_ids: Sequence[int], led_array_ids: Sequence[int], centers: np.ndarray, search_area_radius: int, fit_leds: bool, - boxes: np.ndarray) -> List[LEDAnalysisData]: + boxes: np.ndarray, box_masks: Optional[np.ndarray] = None, + pedestals: Optional[np.ndarray] = None) -> List[LEDAnalysisData]: """ Generate the analysis data of the LEDs of a stack of search areas of one shape. @@ -317,12 +405,22 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led :param boxes: Channel values of the search areas, 0 outside the channel, of shape (number of LEDs, rows, columns). :type boxes: np.ndarray + :param box_masks: Whether each pixel of the search areas belongs to the channel, of the shape of boxes. Only + needed with pedestals. + :type box_masks: np.ndarray or None + :param pedestals: Far-field pedestal per channel pixel of every LED, NaN for an LED without one, see + farfield_pedestals. None if the pedestal is not estimated. + :type pedestals: np.ndarray or None :return: Analysis data for every LED. :rtype: List[LEDAnalysisData] """ means = np.mean(boxes, axis=(1, 2)) sums = np.sum(boxes, axis=(1, 2)) maxima = np.amax(boxes, axis=(1, 2)) + if pedestals is not None: + # the pedestal of the search area is N * b with N its number of channel pixels; an LED without a far + # field keeps its plain sum, so that the corrected sum is a number for every LED + box_pedestals = np.sum(box_masks, axis=(1, 2)) * np.where(np.isfinite(pedestals), pedestals, 0.0) img_analysis_data = [] for i, led_id in enumerate(led_ids): @@ -342,6 +440,9 @@ def _analyse_search_areas(conf: ConfigData, channel: int, img_filename: str, led led_data.mean_color_value = means[i] led_data.sum_color_value = sums[i] led_data.max_color_value = maxima[i] + if pedestals is not None: + led_data.farbg_pedestal = float(box_pedestals[i]) + led_data.farbg_sum_color_value = float(sums[i]) - led_data.farbg_pedestal img_analysis_data.append(led_data) return img_analysis_data @@ -365,7 +466,8 @@ def _save_results_in_file(channel: int, img_data: LEDAnalysisData, img_filename: """ file_path = os.path.join('analysis', f'channel{channel}', f'{img_id}_led_positions.csv') out_file = open(file_path, 'w', encoding='utf-8') - header = _create_header(channel, img_id, img_filename, img_infos, basename, img_data[0].fit_leds) + header = _create_header(channel, img_id, img_filename, img_infos, basename, img_data[0].fit_leds, + img_data[0].farbg_sum_color_value is not None) out_file.write(header) for led_data in img_data: out_file.write(str(led_data)) @@ -502,7 +604,8 @@ def _log_warnings(img_filename, channel, led_data, cx, cy, size_of_search_area, logfile.close() -def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.ndarray, basename: str, fit_leds: str): +def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.ndarray, basename: str, fit_leds: str, + farfield: bool = False): """ Create a header for the analysis result file. @@ -518,6 +621,8 @@ def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.n :type basename: str :param fit_leds: If True, includes fit data in the header. :type fit_leds: bool + :param farfield: If True, includes the far-field pedestal columns in the header. + :type farfield: bool :return: String of the header. :rtype: str """ @@ -526,6 +631,8 @@ def _create_header(channel: int, img_id: str, img_filename: str, img_infos: np.n out_str += f"time[s] = {img_infos[int(img_id) - 1][3]}, " out_str += f"LEDSA {version('ledsa')}\n" out_str += "# id,line,sum_col_value,average_col_value,max_col_value" + if farfield: + out_str += ",farbg_sum_col_value,farbg_pedestal" if fit_leds: out_str += ",led_center_x, led_center_y" out_str += ",x,y,dx,dy,A,alpha,wx,wy,fit_success,fit_fun,fit_nfev,fit_time" diff --git a/ledsa/tests/UnitTests/test_farfield_background.py b/ledsa/tests/UnitTests/test_farfield_background.py new file mode 100644 index 0000000..180ed5f --- /dev/null +++ b/ledsa/tests/UnitTests/test_farfield_background.py @@ -0,0 +1,365 @@ +"""Unit tests for the pedestal of the search areas from the far field of the LED strips (farbg_sum_col_val). + +The pedestal is the level of the color channel in two bands parallel to the strip, far from every LED, smoothed +along the strip, and of the two sides the lower one. Step 3 subtracts N times it from the sum of the search area +when farfield_distances of config.ini is set, and writes the result files as before when it is not. +""" +import configparser + +import numpy as np +import pytest + +from ledsa.core import image_reading +from ledsa.core.ConfigData import ConfigData +from ledsa.core.file_handling import (BASE_COLUMNS, COLUMN_LAYOUTS, FARBG_COLUMNS, FIT_COLUMNS, + LED_CENTER_COLUMNS, _get_column_names) +from ledsa.core.image_reading import ImageChannels, RawImage +from ledsa.data_extraction import farfield_background as farfield +from ledsa.data_extraction.step_3_functions import _create_header, generate_analysis_data + +RADIUS = 4 +DISTANCES = (10.0, 20.0) +IMG_SHAPE = (120, 200) +# a vertical strip of ten LEDs at column 100 +VERTICAL = np.array([[i, 15 + 10 * i, 100] for i in range(10)]) + + +def band_set(pixels): + return set(zip(pixels[0].tolist(), pixels[1].tolist())) + + +class TestDistances: + @pytest.mark.parametrize('value', [None, 'None', 'none', '', ' ']) + def test_none_estimates_no_pedestal(self, value): + assert farfield.parse_farfield_distances(value, RADIUS) is None + + def test_two_distances(self): + assert farfield.parse_farfield_distances('150 300', 9) == (150.0, 300.0) + + @pytest.mark.parametrize('value', ['150', '300 150', '150 150', 'far away', '150 300 400', '5 300']) + def test_anything_else_names_the_option(self, value): + # the last one puts the band into the search area + with pytest.raises(ValueError, match='farfield_distances of config.ini'): + farfield.parse_farfield_distances(value, 9) + + +class TestStripDirections: + def test_vertical_horizontal_and_oblique_strips(self): + horizontal = np.array([[i, 50, 20 + 10 * i] for i in range(6)]) + oblique = np.array([[i, 20 + 7 * i, 20 + 7 * i] for i in range(6)]) + for strip, expected in ((VERTICAL, (1.0, 0.0)), (horizontal, (0.0, 1.0)), + (oblique, (np.sqrt(0.5), np.sqrt(0.5)))): + directions = farfield.estimate_strip_directions(strip, [strip[:, 0]]) + np.testing.assert_allclose(directions, np.tile(expected, (len(strip), 1)), atol=1e-12) + + def test_an_led_alone_in_its_array_has_no_direction(self): + search_areas = np.vstack([VERTICAL, [[10, 60, 30]]]) + directions = farfield.estimate_strip_directions(search_areas, [VERTICAL[:, 0], [10]]) + assert np.isnan(directions[10]).all() + assert np.isfinite(directions[:10]).all() + + +class TestBandPixels: + def test_bands_of_a_vertical_strip(self): + geometry = farfield.build_farfield_geometry(VERTICAL, [VERTICAL[:, 0]], IMG_SHAPE, RADIUS, DISTANCES) + row, col = VERTICAL[3, 1:] + rows = range(row - RADIUS, row + RADIUS) + right, left = geometry.pixels[3] + assert band_set(right) == {(r, c) for r in rows for c in range(col + 10, col + 20)} + assert band_set(left) == {(r, c) for r in rows for c in range(col - 20, col - 10)} + + def test_bands_of_a_horizontal_strip(self): + horizontal = np.array([[i, 60, 30 + 10 * i] for i in range(10)]) + geometry = farfield.build_farfield_geometry(horizontal, [horizontal[:, 0]], IMG_SHAPE, RADIUS, DISTANCES) + row, col = horizontal[4, 1:] + cols = range(col - RADIUS, col + RADIUS) + # the across coordinate of a strip along the columns points towards smaller rows + above, below = geometry.pixels[4] + assert band_set(above) == {(r, c) for c in cols for r in range(row - 19, row - 9)} + assert band_set(below) == {(r, c) for c in cols for r in range(row + 11, row + 21)} + + def test_pixels_outside_the_image_are_left_out(self): + near_border = VERTICAL.copy() + near_border[:, 2] = IMG_SHAPE[1] - 15 + geometry = farfield.build_farfield_geometry(near_border, [near_border[:, 0]], IMG_SHAPE, RADIUS, DISTANCES) + right, left = geometry.pixels[3] + assert right[1].min() == IMG_SHAPE[1] - 5 and right[1].max() == IMG_SHAPE[1] - 1 + assert left[0].size == 2 * RADIUS * 10 + + def test_pixels_close_to_another_strip_are_left_out(self): + # a second strip 15 px to the right: its LEDs lie in the right bands of the first strip + second = VERTICAL.copy() + second[:, 0] += 10 + second[:, 2] += 15 + search_areas = np.vstack([VERTICAL, second]) + geometry = farfield.build_farfield_geometry(search_areas, [VERTICAL[:, 0], second[:, 0]], IMG_SHAPE, RADIUS, + DISTANCES) + right, left = geometry.pixels[3] + centers = search_areas[:, 1:3] + for rows, cols in (right, left): + distances = np.hypot(rows[:, None] - centers[:, 0], cols[:, None] - centers[:, 1]) + assert (distances >= DISTANCES[0]).all() + assert right[0].size < 2 * RADIUS * 10 + assert left[0].size == 2 * RADIUS * 10 + + def test_place_on_the_strip(self): + search_areas = np.vstack([VERTICAL, [[10, 60, 30]]]) + geometry = farfield.build_farfield_geometry(search_areas, [VERTICAL[:, 0], [10]], IMG_SHAPE, RADIUS, + DISTANCES) + assert geometry.strip_of_led.tolist() == [0] * 10 + [-1] + assert np.all(np.diff(geometry.position_on_strip[:10]) > 0) + assert geometry.pixels[10] is None + + def test_the_geometry_is_built_once_for_all_images(self): + first = farfield.get_farfield_geometry(VERTICAL, [VERTICAL[:, 0]], IMG_SHAPE, RADIUS, DISTANCES) + assert farfield.get_farfield_geometry(VERTICAL.copy(), [VERTICAL[:, 0]], IMG_SHAPE, RADIUS, + DISTANCES) is first + assert farfield.get_farfield_geometry(VERTICAL, [VERTICAL[:, 0]], IMG_SHAPE, RADIUS, (10.0, 30.0)) \ + is not first + + +class TestBandLevels: + def test_mean_over_the_channel_pixels(self): + values = np.array([1.0, 3.0, 100.0, 5.0, 7.0, 9.0]) + mask = np.array([True, True, False, True, True, True]) + np.testing.assert_allclose(farfield.mean_of_channel_pixels(values, mask, [3, 3]), [2.0, 7.0]) + + def test_a_band_without_a_pixel_of_the_channel_has_no_level(self): + values = np.array([1.0, 3.0, 5.0, 7.0]) + mask = np.array([True, True, False, False]) + levels = farfield.mean_of_channel_pixels(values, mask, [2, 0, 2, 0]) + assert levels[0] == 2.0 + assert np.isnan(levels[1:]).all() + + def test_no_band_at_all(self): + assert np.isnan(farfield.mean_of_channel_pixels(np.zeros(0), np.zeros(0, dtype=bool), [0, 0])).all() + + +class TestSmoothing: + def test_a_constant_level_stays(self): + np.testing.assert_allclose(farfield.smooth_along_strip(np.full(40, 0.6)), 0.6) + + def test_the_outlier_of_a_single_led_is_dropped(self): + levels = np.full(40, 0.6) + levels[17] = 50.0 + np.testing.assert_allclose(farfield.smooth_along_strip(levels), 0.6) + + def test_a_slope_along_the_strip_is_kept(self): + levels = 0.5 + 0.01 * np.arange(60) + inner = slice(12, -12) + np.testing.assert_allclose(farfield.smooth_along_strip(levels)[inner], levels[inner]) + + def test_an_led_without_a_level_takes_that_of_its_neighbours(self): + levels = np.full(40, 0.6) + levels[[0, 20, 21, 39]] = np.nan + np.testing.assert_allclose(farfield.smooth_along_strip(levels), 0.6) + + def test_a_side_without_any_level_has_none(self): + assert np.isnan(farfield.smooth_along_strip(np.full(40, np.nan))).all() + + def test_short_strips(self): + # the windows are cut short at the ends of the strip, here they hold the whole strip + np.testing.assert_allclose(farfield.smooth_along_strip(np.array([0.5, 0.7, 0.9])), 0.7) + assert farfield.smooth_along_strip(np.zeros(0)).size == 0 + + def test_the_noise_of_the_estimate_is_reduced(self): + levels = 0.6 + 0.2 * np.random.default_rng(1).standard_normal(126) + assert np.std(farfield.smooth_along_strip(levels)) < 0.4 * np.std(levels) + + +class TestPedestals: + def test_the_lower_side_is_the_pedestal(self): + levels = np.column_stack((np.full(30, 0.9), np.full(30, 0.6))) + pedestals = farfield.farfield_pedestals(levels, np.zeros(30, dtype=int), np.arange(30.0)) + np.testing.assert_allclose(pedestals, 0.6) + + def test_the_lower_side_is_chosen_led_by_led(self): + levels = np.column_stack((np.linspace(0.2, 1.0, 60), np.full(60, 0.6))) + pedestals = farfield.farfield_pedestals(levels, np.zeros(60, dtype=int), np.arange(60.0)) + assert pedestals[12] == pytest.approx(levels[12, 0], abs=1e-12) + assert levels[12, 0] < 0.6 < levels[50, 0] + assert pedestals[50] == pytest.approx(0.6) + + def test_a_side_without_a_level_leaves_the_other(self): + levels = np.column_stack((np.full(30, np.nan), np.full(30, 0.6))) + pedestals = farfield.farfield_pedestals(levels, np.zeros(30, dtype=int), np.arange(30.0)) + np.testing.assert_allclose(pedestals, 0.6) + + def test_strips_are_smoothed_separately_and_in_their_order_along_the_strip(self): + # two strips with different levels, their LEDs interleaved and the second strip given backwards + strip_of_led = np.tile([0, 1], 30) + position = np.where(strip_of_led == 0, np.arange(60.0), -np.arange(60.0)) + level = np.where(strip_of_led == 0, 0.5, 2.0 + 0.01 * position) + pedestals = farfield.farfield_pedestals(np.column_stack((level, level)), strip_of_led, position) + np.testing.assert_allclose(pedestals[strip_of_led == 0], 0.5) + inner = np.flatnonzero(strip_of_led == 1)[10:-10] + np.testing.assert_allclose(pedestals[inner], level[inner]) + + def test_an_led_on_no_strip_or_without_any_level_has_no_pedestal(self): + levels = np.column_stack((np.full(5, 0.6), np.full(5, 0.6))) + assert np.isnan(farfield.farfield_pedestals(levels, np.full(5, -1), np.arange(5.0))).all() + assert np.isnan(farfield.farfield_pedestals(np.full((5, 2), np.nan), np.zeros(5, dtype=int), + np.arange(5.0))).all() + + +# ------------------------------------------------------------------------------------------------------- step 3 +STEP3_SHAPE = (100, 120) +# RGGB pattern as in rawpy raw_colors_visible: 0 red, 1 and 3 green, 2 blue +FILTER_ARRAY = np.tile(np.array([[0, 1], [3, 2]]), (STEP3_SHAPE[0] // 2, STEP3_SHAPE[1] // 2)) +BLACK_LEVELS = np.array([2040.3, 2041.7, 2042.1, 2043.9]) +STRIP = np.array([[i, 14 + 10 * i, 60] for i in range(8)]) +LINE_INDICES = [STRIP[:, 0]] + + +def make_config(farfield_distances=None): + config = configparser.ConfigParser() + config['DEFAULT'] = {'img_directory': '.', 'img_rotation': 'auto'} + config['find_search_areas'] = {'search_area_radius': str(RADIUS)} + config['analyse_photo'] = {'num_skip_leds': '0'} + if farfield_distances is not None: + config['analyse_photo']['farfield_distances'] = farfield_distances + return config + + +def make_image(monkeypatch, background): + """A RAW image of the strip: an LED in every search area on a background given per pixel in stored counts.""" + stored = np.asarray(background, dtype=np.float64) + BLACK_LEVELS[FILTER_ARRAY] + for _, row, col in STRIP: + stored[row - 2:row + 2, col - 2:col + 2] += 3000.0 + raw = RawImage(np.round(stored).astype(np.uint16), FILTER_ARRAY, BLACK_LEVELS, 16383.0, + image_reading.UPRIGHT_ORIENTATION) + monkeypatch.setattr(image_reading, '_read_raw_image', lambda filename: raw) + return ImageChannels('img.CR3') + + +def pedestal_by_definition(img_channels, channel, row, col, side): + """Level of the band on one side of an LED of the vertical strip, from the arrays of the whole image.""" + data, mask = img_channels.channel_data_and_mask(channel) + cols = slice(col + 10, col + 20) if side == 'right' else slice(col - 20, col - 10) + band = np.s_[row - RADIUS:row + RADIUS, cols] + return data[band][mask[band]].mean() + + +class TestStep3: + @pytest.mark.parametrize('channel', [0, 1, 2]) + def test_no_pedestal_without_the_option(self, monkeypatch, channel): + img_channels = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + led_data = generate_analysis_data('img.CR3', channel, STRIP, LINE_INDICES, make_config(), fit_leds=False, + img_channels=img_channels) + assert all(data.farbg_sum_color_value is None and data.farbg_pedestal is None for data in led_data) + assert len(str(led_data[0]).split(',')) == 5 + + @pytest.mark.parametrize('channel', [0, 1, 2]) + def test_a_flat_background_is_removed(self, monkeypatch, channel): + flat = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + with_pedestal = generate_analysis_data('img.CR3', channel, STRIP, LINE_INDICES, make_config('10 20'), + fit_leds=False, img_channels=flat) + dark = make_image(monkeypatch, np.zeros(STEP3_SHAPE)) + without = generate_analysis_data('img.CR3', channel, STRIP, LINE_INDICES, make_config('10 20'), + fit_leds=False, img_channels=dark) + for led, led_dark in zip(with_pedestal, without): + # the corrected sum of the image with the background is the one of the image without it + assert led.farbg_sum_color_value == pytest.approx(led_dark.farbg_sum_color_value, abs=0.2) + assert led.farbg_sum_color_value == pytest.approx(led.sum_color_value - led.farbg_pedestal, rel=1e-9) + assert led.farbg_pedestal > 0.02 * led.sum_color_value + assert led.sum_color_value > led_dark.sum_color_value + + @pytest.mark.parametrize('channel', [0, 1, 2]) + def test_the_pedestal_is_the_lower_side_times_the_channel_pixels(self, monkeypatch, channel): + # brighter towards larger columns: the left band is the lower one + background = 20.0 + 0.5 * np.arange(STEP3_SHAPE[1]) * np.ones(STEP3_SHAPE) + img_channels = make_image(monkeypatch, background) + led_data = generate_analysis_data('img.CR3', channel, STRIP, LINE_INDICES, make_config('10 20'), + fit_leds=False, img_channels=img_channels) + _, mask = img_channels.channel_data_and_mask(channel) + for data, (_, row, col) in zip(led_data, STRIP): + left = pedestal_by_definition(img_channels, channel, row, col, 'left') + assert left < pedestal_by_definition(img_channels, channel, row, col, 'right') + num_channel_pixels = mask[row - RADIUS:row + RADIUS, col - RADIUS:col + RADIUS].sum() + # the background does not change along the strip, so the smoothing leaves the level as it is + assert data.farbg_pedestal == pytest.approx(num_channel_pixels * left, rel=1e-6) + + def test_an_led_without_a_strip_keeps_its_plain_sum(self, monkeypatch): + img_channels = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + search_areas = np.vstack([STRIP, [[8, 50, 20]]]) + led_data = generate_analysis_data('img.CR3', 0, search_areas, [STRIP[:, 0], np.array([8])], + make_config('10 20'), fit_leds=False, img_channels=img_channels) + assert led_data[-1].farbg_pedestal == 0.0 + assert led_data[-1].farbg_sum_color_value == pytest.approx(float(led_data[-1].sum_color_value)) + assert led_data[0].farbg_pedestal > 0.0 + + def test_the_pixels_are_calibrated_once_for_all_channels(self, monkeypatch): + img_channels = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + calibrated = [] + channel_data = image_reading._channel_data + monkeypatch.setattr(image_reading, '_channel_data', + lambda raw: calibrated.append(raw.data.size) or channel_data(raw)) + for channel in [0, 1, 2]: + generate_analysis_data('img.CR3', channel, STRIP, LINE_INDICES, make_config('10 20'), fit_leds=False, + img_channels=img_channels) + assert len(calibrated) == 1 + + def test_the_plain_quantities_do_not_change(self, monkeypatch): + img_channels = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + plain = generate_analysis_data('img.CR3', 1, STRIP, LINE_INDICES, make_config(), fit_leds=False, + img_channels=img_channels) + corrected = generate_analysis_data('img.CR3', 1, STRIP, LINE_INDICES, make_config('10 20'), fit_leds=False, + img_channels=img_channels) + for a, b in zip(plain, corrected): + assert (a.sum_color_value, a.mean_color_value, a.max_color_value) == \ + (b.sum_color_value, b.mean_color_value, b.max_color_value) + + +class TestResultFiles: + IMG_INFOS = np.array([['1', 'img.jpg', '12:00:00', '0']]) + + def test_layouts_with_the_pedestal(self): + assert COLUMN_LAYOUTS[len(BASE_COLUMNS + FARBG_COLUMNS)] == BASE_COLUMNS + FARBG_COLUMNS + assert COLUMN_LAYOUTS[len(BASE_COLUMNS + FARBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS)] == \ + BASE_COLUMNS + FARBG_COLUMNS + LED_CENTER_COLUMNS + FIT_COLUMNS + + @pytest.mark.parametrize('fit_leds', [False, True]) + def test_header_names_the_columns_of_the_rows(self, fit_leds): + header = _create_header(0, '1', 'img.jpg', self.IMG_INFOS, 'root', fit_leds, True) + names = [name.strip() for name in header.split('\n')[1].lstrip('#').split('//')[0].split(',')] + assert names[2:7] == ['sum_col_value', 'average_col_value', 'max_col_value', 'farbg_sum_col_value', + 'farbg_pedestal'] + assert ('fit_success' in names) == fit_leds + + def test_result_file_written_by_step_3_is_read_with_its_names(self, monkeypatch, tmp_path): + img_channels = make_image(monkeypatch, np.full(STEP3_SHAPE, 40.0)) + led_data = generate_analysis_data('img.CR3', 0, STRIP, LINE_INDICES, make_config('10 20'), fit_leds=False, + img_channels=img_channels) + channel_dir = tmp_path / 'analysis' / 'channel0' + channel_dir.mkdir(parents=True) + header = _create_header(0, '1', 'img.jpg', self.IMG_INFOS, 'root', False, True) + content = header + ''.join(str(data) for data in led_data) + (channel_dir / '1_led_positions.csv').write_text(content, encoding='utf-8') + monkeypatch.chdir(tmp_path) + columns = _get_column_names(0) + assert columns == BASE_COLUMNS + FARBG_COLUMNS + ['width', 'height'] + values = np.loadtxt(channel_dir / '1_led_positions.csv', delimiter=',') + named = dict(zip(columns[1:], values[0])) + assert named['farbg_sum_col_val'] == pytest.approx(led_data[0].farbg_sum_color_value, rel=1e-4) + assert named['farbg_pedestal'] == pytest.approx(led_data[0].farbg_pedestal, rel=1e-4) + assert named['sum_col_val'] - named['farbg_pedestal'] == pytest.approx(named['farbg_sum_col_val'], rel=1e-3) + + +class TestConfig: + def test_template_holds_the_option_without_a_pedestal(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + ConfigData(load_config_file=False) + config = ConfigData() + assert config.get('analyse_photo', 'farfield_distances') == 'None' + assert farfield.parse_farfield_distances(config.get('analyse_photo', 'farfield_distances'), 10) is None + + def test_template_with_distances(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + ConfigData(load_config_file=False, farfield_distances='150 300') + assert farfield.parse_farfield_distances(ConfigData().get('analyse_photo', 'farfield_distances'), 10) == \ + (150.0, 300.0) + + def test_a_config_written_before_the_option_estimates_no_pedestal(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / 'config.ini').write_text('[analyse_photo]\nnum_skip_leds = 0\n', encoding='utf-8') + assert farfield.parse_farfield_distances(ConfigData().get('analyse_photo', 'farfield_distances'), 10) is None diff --git a/ledsa/tests/UnitTests/test_file_handling.py b/ledsa/tests/UnitTests/test_file_handling.py index 13972de..d983775 100644 --- a/ledsa/tests/UnitTests/test_file_handling.py +++ b/ledsa/tests/UnitTests/test_file_handling.py @@ -103,6 +103,7 @@ def test_fitted_csv_written_by_step_3(self, tmp_path, monkeypatch): @pytest.mark.parametrize('num_columns, last_main', [ (5, 'max_col_val'), (19, 'max_col_val'), # written since v0.13.0, without and with the fit + (7, 'farbg_pedestal'), (21, 'farbg_pedestal'), # with the far-field pedestal (farfield_distances) (6, 'bgsub_sum_col_val'), (20, 'bgsub_sum_col_val'), # local background subtraction (9, 'bgsub2_sum_col_val'), (23, 'bgsub2_sum_col_val'), # v0.11.0 (10, 'noise_sum_col_val'), (24, 'noise_sum_col_val')]) # v0.12.0 @@ -147,7 +148,7 @@ def test_oldest_layouts(self, tmp_path, monkeypatch, num_columns, first_names): assert columns[:4] == first_names assert len(columns) == num_columns + 1 + 2 - @pytest.mark.parametrize('num_columns', [7, 21, 22]) + @pytest.mark.parametrize('num_columns', [8, 22, 25]) def test_unknown_layout_raises(self, tmp_path, monkeypatch, num_columns): self._write_led_positions_csv(tmp_path, num_columns) monkeypatch.chdir(tmp_path) From 0545abbcad7e686f606bf94ec23bc4a51bc421a8 Mon Sep 17 00:00:00 2001 From: Kristian <73885850+kriboe90@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:41:04 +0100 Subject: [PATCH 2/3] test the far-field pedestal on images with a pedestal The test images can be made wider than 50 px, with the strip down their middle, and get a pixel value added everywhere, light that does not dim with the LEDs. On images 650 px wide with a pedestal of 3, step 3 with farfield_distances = 150 300 measures 3.0 per pixel of every search area, farbg_sum_col_val reproduces the input extinction coefficients (RMSE 0.019 to 0.040 1/m), and sum_col_val gives them about 30 % too low. The defaults leave the images of the other suites as they were. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016cv6PgMB7utWgf5YHwHffz --- .../13_test_farfield_pedestal.robot | 64 +++++++++++++++++++ .../AcceptanceTests/LedsaATestLibrary.py | 51 +++++++++++---- 2 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 ledsa/tests/AcceptanceTests/13_test_farfield_pedestal.robot diff --git a/ledsa/tests/AcceptanceTests/13_test_farfield_pedestal.robot b/ledsa/tests/AcceptanceTests/13_test_farfield_pedestal.robot new file mode 100644 index 0000000..9001d1d --- /dev/null +++ b/ledsa/tests/AcceptanceTests/13_test_farfield_pedestal.robot @@ -0,0 +1,64 @@ +*** Settings *** +Resource global_keywords.resource + +Suite Setup Prepare Images With A Pedestal +Suite Teardown Change Directory ${WORKDIR} + +Force Tags farfield_pedestal + + +*** Variables *** +${PEDESTAL_DIR} ${WORKDIR}${/}farfield_pedestal +# Light that does not dim with the LEDs, added to every pixel of the images. It is about a quarter +# of the sum over the search area of an undimmed LED. +${PEDESTAL} 3 +# The strip runs down the middle of the images, so that bands 150 to 300 px away from it fit on +# both sides. +${IMG_WIDTH} 650 +${FARFIELD_DISTANCES} 150 300 + + +*** Test Cases *** +Step 3 Measures The Pedestal In The Far Field + [Documentation] With farfield_distances in config.ini, step 3 writes the pedestal of every search area, + ... estimated in bands beside the strip, and the sum minus it. Per pixel of the search + ... area it is the pixel value added to the images. + Farfield Pedestal Per Pixel Should Be 1 ${PEDESTAL} 0.1 + Farfield Pedestal Per Pixel Should Be 4 ${PEDESTAL} 0.1 + +The Pedestal Corrected Sum Reproduces The Extinction Coefficients + [Documentation] The analysis of farbg_sum_col_val finds the extinction coefficients the images were + ... made with. + Create And Fill Config Analysis linear farbg_sum_col_val + Execute Ledsa --analysis + Extinction Coefficients Should Match The Input 2 + Extinction Coefficients Should Match The Input 3 + Extinction Coefficients Should Match The Input 4 + +The Plain Sum Underestimates The Extinction Coefficients + [Documentation] The pedestal does not dim with the LEDs, so the plain sum of the same images gives too + ... high a transmission and too low extinction coefficients. This is what the far-field + ... pedestal corrects. + Create And Fill Config Analysis linear sum_col_val + Execute Ledsa --analysis + ${ratio} = Mean Ratio Of Computed To Input Extinction Coefficients 2 linear + Should Be True ${ratio} < 0.85 + + +*** Keywords *** +Prepare Images With A Pedestal + Create Directory ${PEDESTAL_DIR} + Change Directory ${PEDESTAL_DIR} + Create Test Data img_width=${IMG_WIDTH} pedestal=${PEDESTAL} + Create And Fill Config farfield_distances=${FARFIELD_DISTANCES} + ${leds} = Execute Ledsa S1 ${TRUE} + Should Be Equal ${leds} 100 + Execute Ledsa -s2 + Execute Ledsa --coordinates + Execute Ledsa -s3_fast + +Extinction Coefficients Should Match The Input + [Arguments] ${image_id} + ${rmse} = Check Input Vs Computed Extinction Coefficients ${image_id} linear + ... reference_property=farbg_sum_col_val + Should Be True ${rmse} < 0.05 diff --git a/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py b/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py index 5408ff8..35e1e3c 100644 --- a/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py +++ b/ledsa/tests/AcceptanceTests/LedsaATestLibrary.py @@ -40,12 +40,14 @@ def change_dir(self, new_dir): @keyword def create_test_data(self, num_of_leds=100, num_of_layers=20, bottom_border=0, top_border=3, - pane_distance=None): + pane_distance=None, img_width=50, pedestal=0): """Create the test images of four extinction profiles, seen by a camera at (0, 0, 2). :param pane_distance: Distance [m] between the camera and a window pane it looks through into the smoke, so that the smoke dims the LEDs only behind the pane. None for smoke up to the camera. + :param img_width: Width of the images in pixels, the LED strip runs down their middle. + :param pedestal: Pixel value added to the whole image, light that does not dim with the LEDs. """ # Create test_data directory if it doesn't exist if not os.path.exists('test_data'): @@ -89,7 +91,7 @@ def extco_quad(z): for z in np.linspace(bottom_border + 0.05, top_border - 0.05, num_of_leds): ex.add_led(0, 4, z) ex.set_extinction_coefficients(extinction_coefficients) - create_test_image(image_id, ex) + create_test_image(image_id, ex, img_width=int(img_width), pedestal=int(pedestal)) # ------------------------------------------------------------------ # Stacked / multi-camera keywords @@ -406,16 +408,18 @@ def check_input_vs_computed_extinction_coefficients(self, image_id, solver, led_ return rmse @keyword - def mean_ratio_of_computed_to_input_extinction_coefficients(self, image_id, solver, led_array=0, channel=0): + def mean_ratio_of_computed_to_input_extinction_coefficients(self, image_id, solver, led_array=0, channel=0, + reference_property='sum_col_val'): """Return the mean computed extinction coefficient of an image over the mean of its input.""" - _, extinction_coefficients_computed = load_extinction_coefficients_computed(solver, channel, led_array) + _, extinction_coefficients_computed = load_extinction_coefficients_computed(solver, channel, led_array, + reference_property) extinction_coefficients_input = np.loadtxt( os.path.join('test_data', f'test_extinction_coefficients_input_{image_id}.csv'), delimiter=',') return float(np.mean(extinction_coefficients_computed[int(image_id) - 1, :]) / np.mean(extinction_coefficients_input)) @keyword - def create_and_fill_config(self, first=1, last=4): + def create_and_fill_config(self, first=1, last=4, farfield_distances=None): # Create test_data directory if it doesn't exist if not os.path.exists('test_data'): os.makedirs('test_data') @@ -427,7 +431,7 @@ def create_and_fill_config(self, first=1, last=4): last_img_experiment_id=last, ref_img_id=1, ignore_led_indices=None, led_array_edge_indices=None, led_array_edge_coordinates=None, first_img_analysis_id=first, last_img_analysis_id=last, num_skip_imgs=0, num_skip_leds=0, - merge_led_array_indices=None) + merge_led_array_indices=None, farfield_distances=farfield_distances) conf.set('analyse_positions', ' led_array_edge_indices', '49 0') conf.set('analyse_positions', ' led_array_edge_coordinates', '0 4 0.05 0 4 2.95') conf.set('DEFAULT', ' date', '2018:11:27') @@ -727,6 +731,27 @@ def count_config_help_texts(self): with open('config.ini', encoding='utf-8-sig') as config_file: return len([line for line in config_file if line.strip().startswith(('#', ';'))]) + @keyword + def farfield_pedestal_per_pixel_should_be(self, img_id, expected, tolerance, channel=0): + """Fail unless the far-field pedestal step 3 wrote for every LED of an image, divided by the number of + pixels of its search area, lies within tolerance of expected.""" + file_path = os.path.join('analysis', f'channel{channel}', f'{img_id}_led_positions.csv') + with open(file_path, encoding='utf-8') as result_file: + header = [line for line in result_file if line.startswith('#')][-1] + columns = [name.strip() for name in header.lstrip('#').split(',')] + if 'farbg_pedestal' not in columns: + raise AssertionError(f'{file_path} has no column farbg_pedestal: {columns}') + data = np.atleast_2d(np.loadtxt(file_path, delimiter=',', comments='#')) + num_pixels = (2 * int(ConfigData()['find_search_areas']['search_area_radius'])) ** 2 + pedestals = data[:, columns.index('farbg_pedestal')] / num_pixels + corrected = data[:, columns.index('sum_col_value')] - data[:, columns.index('farbg_pedestal')] + if not np.allclose(data[:, columns.index('farbg_sum_col_value')], corrected, rtol=1e-4): + raise AssertionError('farbg_sum_col_value is not sum_col_value minus farbg_pedestal') + deviation = np.abs(pedestals - float(expected)) + if np.max(deviation) > float(tolerance): + raise AssertionError(f'Pedestals per pixel range from {pedestals.min():.3f} to {pedestals.max():.3f}, ' + f'expected {expected} +- {tolerance}') + @keyword def analysed_image_names_should_be(self, *expected_img_names): """Fail unless exactly the given images are listed in 'image_infos_analysis.csv'.""" @@ -773,7 +798,7 @@ def load_extinction_coefficients_computed(solver, channel, led_array, reference_ extinction_coefficients_computed = data[:, 1:] return time, extinction_coefficients_computed -def create_test_image(image_id, experiment, img_dir='test_data'): +def create_test_image(image_id, experiment, img_dir='test_data', img_width=50, pedestal=0): """ Creates three test images with black and gray pixels representing 3 leds and sets the exif data needed The first image has 100% transmission on all LEDs, the second image has 50% transmission on all LEDs, the third has 50%, 70% and 80% transmission on the top, middle and bottom LEDs. @@ -787,7 +812,7 @@ def create_test_image(image_id, experiment, img_dir='test_data'): transmissions = experiment.calc_all_led_transmissions() # Reverse transmissions because images are created from top down - img_array = create_img_array(num_of_leds, list(reversed(transmissions))) + img_array = create_img_array(num_of_leds, list(reversed(transmissions)), img_width, pedestal) img = Image.fromarray(img_array, 'RGB') # Save image without EXIF data @@ -825,21 +850,21 @@ def turn_img_sideways(path, write_orientation=True): turned_img.writeMetadata() -def create_img_array(num_of_leds, transmissions): - img = np.zeros((num_of_leds * 50 + 50, 50, 3), np.uint8) +def create_img_array(num_of_leds, transmissions, img_width=50, pedestal=0): + img = np.full((num_of_leds * 50 + 50, img_width, 3), pedestal, np.uint8) for led_id in range(num_of_leds): - add_led(img, (1 + led_id) * 50, 25, transmissions[led_id]) + add_led(img, (1 + led_id) * 50, img_width // 2, transmissions[led_id], pedestal) return img -def add_led(img, x_pos, y_pos, transmission): +def add_led(img, x_pos, y_pos, transmission, pedestal=0): rv = norm() size = 20 led = np.zeros((size, size)) for x in range(size): for y in range(size): led[x, y] = calc_color_val(x, y, size, rv) * transmission - img[x_pos - size // 2:x_pos + size // 2, y_pos - size // 2:y_pos + size // 2, 0] = led + img[x_pos - size // 2:x_pos + size // 2, y_pos - size // 2:y_pos + size // 2, 0] = led + pedestal img[:, :, 1] = img[:, :, 0] img[:, :, 2] = img[:, :, 0] From 7f0fe9fca3f7745448b313f412541f931d09e7cf Mon Sep 17 00:00:00 2001 From: Kristian <73885850+kriboe90@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:41:04 +0100 Subject: [PATCH 3/3] name the pedestal as what limits sum_col_val in dense fog The README and the docstring of the linear solver put the extinction coefficients that came out too low in the Warsaw fog test V008 down to the read noise, which leaves the LEDs with positive noise only. For sum_col_val that does not hold: there the pedestal of the search areas keeps the relative intensities above zero, only 0.3 to 3.8 % of the red LEDs are not positive in a single image, and single images come out 21 to 27 % too low at 1.24 1/m and 36 to 42 % at 1.64 1/m. The read noise limits once the pedestal is subtracted, with farbg_sum_col_val by 5 to 6 % and 24 to 26 %, and averaging seven images lowers this to 13 to 17 % at 1.64 1/m. V009 has no extinction this high and is no longer named. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016cv6PgMB7utWgf5YHwHffz --- README.md | 9 +++++++-- ledsa/analysis/ExtinctionCoefficientsLinear.py | 6 ++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 355d410..bdd68b9 100644 --- a/README.md +++ b/README.md @@ -192,8 +192,13 @@ The intensities are not clipped at zero after the black level of the sensor is s LED that has vanished in dense smoke can give a relative intensity at or below zero. The linear solver leaves such LEDs out. Where the transmission reaches the read noise of the sensor, the LEDs that remain are those whose noise happened to be positive, and the extinction coefficients come -out too low: in fog tests by 14 to 31 % at 1.24 and 1.64 1/m, where the transmission of the red -LEDs was 0.1 to 0.2 % of I0. +out too low. With `sum_col_val` this hardly happens, because the pedestal of the search areas (see +below) keeps the relative intensities above zero and limits the extinction coefficients first: in +a fog test, single images came out 21 to 27 % too low at 1.24 1/m and 36 to 42 % at 1.64 1/m. +With the pedestal subtracted (`farbg_sum_col_val`) the read noise limits, by 5 to 6 % and 24 to +26 %, where the transmission of the red LEDs was 0.02 to 0.15 % of I0 and the noise of a single +image about 0.2 % of I0. Averaging seven images (`average_images`) lowered this to 13 to 17 % at +1.64 1/m. ### Pedestal from the far field of the LED strips (optional) diff --git a/ledsa/analysis/ExtinctionCoefficientsLinear.py b/ledsa/analysis/ExtinctionCoefficientsLinear.py index 1f7b98e..ad29fc3 100644 --- a/ledsa/analysis/ExtinctionCoefficientsLinear.py +++ b/ledsa/analysis/ExtinctionCoefficientsLinear.py @@ -58,8 +58,10 @@ def calc_coefficients_of_img(self, rel_intensities: np.ndarray) -> np.ndarray: Known limitation: where the transmission approaches the read noise of the sensor, the LEDs that are left are those whose noise happened to be positive, so the extinction coefficients come out - too low. On the Warsaw tests V008 and V009 this was 14 to 31 % at 1.24 and 1.64 1/m, where the - transmission of the red LEDs was 0.1 to 0.2 % of I0. + too low. In single images of the Warsaw fog test V008 this was 5 to 6 % at 1.24 1/m and 24 to 26 % + at 1.64 1/m for farbg_sum_col_val, where the transmission of the red LEDs was 0.02 to 0.15 % of I0. + For sum_col_val the pedestal of the search areas keeps the relative intensities above zero and + limits first, by 21 to 27 % and 36 to 42 %. :param rel_intensities: Array of relative (normalized) LED intensities (I_e/I_0). :type rel_intensities: np.ndarray