From 78bd097a8a40db0be906b392dc212195ea3fcd6d Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:10:23 +0200 Subject: [PATCH 01/18] Load pose metadata from DLC config Add internal state for keypoint names and skeleton definitions, plus a helper that extracts this metadata from `dlc.cfg` after initialization. The loader validates config shape (bodyparts and skeleton edges), rejects duplicate bodypart names and unknown edge references, and derives a stable skeleton ID from the DLC task name. --- dlclivegui/services/dlc_processor.py | 56 ++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/dlclivegui/services/dlc_processor.py b/dlclivegui/services/dlc_processor.py index 09d25525..d355dfc2 100644 --- a/dlclivegui/services/dlc_processor.py +++ b/dlclivegui/services/dlc_processor.py @@ -108,6 +108,10 @@ def __init__(self) -> None: self._processor: Any | None = None self._processor_spec: ProcessorSpec | None = None self._processor_built_from_spec = False + # Keypoint/skeleton definitions + self._keypoint_names: tuple[str, ...] | None = None + self._skeleton_id: str | None = None + self._skeleton_edges: tuple[tuple[str, str], ...] | None = None # Worker thread and queue self._queue: queue.Queue[Any] | None = None self._worker_thread: threading.Thread | None = None @@ -650,6 +654,58 @@ def _process_frame( self._worker_timing.maybe_log() self.frame_processed.emit() + def _load_pose_metadata_from_dlc_config(self) -> None: + """Extract name-based pose metadata from the initialized DLC config.""" + dlc = self._dlc + if dlc is None: + raise RuntimeError("DLCLive instance is not initialized.") + + cfg = getattr(dlc, "cfg", None) + if not isinstance(cfg, dict): + logger.info("DLCLive did not expose dictionary configuration metadata.") + return + + raw_bodyparts = cfg.get("bodyparts") + if not isinstance(raw_bodyparts, (list, tuple)): + logger.info("DLCLive configuration does not define bodyparts.") + return + + keypoint_names = tuple(str(name) for name in raw_bodyparts) + + if not keypoint_names: + return + + if len(set(keypoint_names)) != len(keypoint_names): + raise RuntimeError("DLCLive configuration contains duplicate bodypart names.") + + self._keypoint_names = keypoint_names + + raw_edges = cfg.get("skeleton") + if not isinstance(raw_edges, (list, tuple)): + return + + edges: list[tuple[str, str]] = [] + + for raw_edge in raw_edges: + if not isinstance(raw_edge, (list, tuple)) or len(raw_edge) != 2: + raise RuntimeError("DLCLive configuration contains an invalid skeleton edge.") + + start = str(raw_edge[0]) + end = str(raw_edge[1]) + + if start not in keypoint_names or end not in keypoint_names: + raise RuntimeError(f"DLCLive skeleton references an unknown bodypart: {start!r} -> {end!r}.") + + edges.append((start, end)) + + if not edges: + return + + task_name = str(cfg.get("Task", "model")).strip() or "model" + + self._skeleton_id = f"deeplabcut.{task_name}" + self._skeleton_edges = tuple(edges) + def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None: try: # -------- Initialization -------- From e4fe6056fa90f18bd996bd2fe33c8eb245d521ca Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:32:14 +0200 Subject: [PATCH 02/18] Propagate skeleton metadata for pose display DLCLiveProcessor now includes keypoint names, skeleton ID, and skeleton edges in emitted PosePacket data, and loads pose metadata from the DLC config during initialization. In the GUI, pose callbacks now refresh a cached resolved skeleton based on packet metadata, avoid repeated resolution work with a signature check, and deduplicate warning logs when skeleton resolution fails. --- dlclivegui/gui/main_window.py | 58 +++++++++++++++++++++++----- dlclivegui/services/dlc_processor.py | 6 ++- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index 6e056b74..f514ab31 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -66,6 +66,7 @@ ) from ..display import BBoxColors, compute_tile_info, create_tiled_frame, draw_bbox, draw_pose +from ..display.skeleton import ResolvedSkeleton, SkeletonResolutionError, resolve_packet_skeleton from ..processors.processor_utils import ( create_spec_from_scan, default_processors_dir, @@ -181,6 +182,17 @@ def __init__(self, config: ApplicationSettings | None = None): self._p_cutoff = 0.6 self._colormap = "hot" self._bbox_color = (0, 0, 255) # BGR: red + ## Skeleton + self._resolved_skeleton: ResolvedSkeleton | None = None + self._resolved_skeleton_signature: ( + tuple[ + str | None, + tuple[str, ...], + tuple[tuple[str, str], ...], + ] + | None + ) = None + self._last_skeleton_warning: str | None = None # Multi-camera state self._multi_camera_mode = False @@ -2300,6 +2312,42 @@ def _update_camera_controls_enabled(self) -> None: if hasattr(self, "load_config_action"): self.load_config_action.setEnabled(allow_changes) + def _refresh_resolved_skeleton( + self, + result: PoseResult, + ) -> None: + packet = result.packet + + if packet is None: + self._resolved_skeleton = None + self._resolved_skeleton_signature = None + return + + signature = ( + packet.skeleton_id, + tuple(packet.keypoint_names or ()), + tuple(packet.skeleton_edges or ()), + ) + + if signature == self._resolved_skeleton_signature: + return + + self._resolved_skeleton_signature = signature + + try: + self._resolved_skeleton = resolve_packet_skeleton(packet) + self._last_skeleton_warning = None + except SkeletonResolutionError as exc: + self._resolved_skeleton = None + + message = str(exc) + if message != self._last_skeleton_warning: + self._last_skeleton_warning = message + logger.warning( + "Skeleton could not be resolved: %s", + message, + ) + def _display_frame(self, frame: np.ndarray, *, force: bool = False) -> None: if frame is None: return @@ -2560,18 +2608,10 @@ def _on_pose_ready(self, result: PoseResult) -> None: with self._dlc_timing.measure("DLC.pose_ready_callback"): self._last_pose = result - - # try: - # latency_ms = (time.time() - float(result.timestamp)) * 1000.0 - # if logger.isEnabledFor(logging.DEBUG): - # logger.debug("DLC pose latency camera_timestamp_to_gui=%.2f ms", latency_ms) - # except Exception: - # pass + self._refresh_resolved_skeleton(result) if self._current_frame is not None: self._display_dirty = True - # with self._dlc_timing.measure("DLC.display_after_pose"): - # self._display_frame(self._current_frame, force=True) self._dlc_timing.maybe_log() diff --git a/dlclivegui/services/dlc_processor.py b/dlclivegui/services/dlc_processor.py index d355dfc2..97838313 100644 --- a/dlclivegui/services/dlc_processor.py +++ b/dlclivegui/services/dlc_processor.py @@ -614,8 +614,10 @@ def _process_frame( pose_packet = PosePacket( schema_version=0, keypoints=pose_arr, - keypoint_names=None, + keypoint_names=list(self._keypoint_names) if self._keypoint_names is not None else None, individual_ids=None, + skeleton_id=self._skeleton_id, + skeleton_edges=self._skeleton_edges, source=PoseSource(backend=PoseBackends.DLC_LIVE, model_type=self._settings.model_type), raw=pose_arr, ) @@ -806,6 +808,8 @@ def _worker_loop(self, init_frame: np.ndarray, init_timestamp: float) -> None: self._dlc.init_inference(init_frame) log_processor_context("DLCLive init_inference completed", logger) + self._load_pose_metadata_from_dlc_config() + self._debug_log_dlc_runner_device() self._worker_timing.note_frame() self._worker_timing.maybe_log() From 0e67d6761a1e43005c8ad947aa3a15a6bbf27cd7 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:32:30 +0200 Subject: [PATCH 03/18] Add overlay settings and render helper Introduce a new `display.overlays` module with typed dataclass settings for pose and bounding-box overlays, plus a `render_overlays` function that composes drawing calls on a copied frame. Also add a reusable `DisplayTransform` dataclass in `display.py` to centralize point mapping with offset and scale. --- dlclivegui/display/display.py | 20 ++++++++++ dlclivegui/display/overlays.py | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 dlclivegui/display/overlays.py diff --git a/dlclivegui/display/display.py b/dlclivegui/display/display.py index 7dcf3208..4d0fee7d 100644 --- a/dlclivegui/display/display.py +++ b/dlclivegui/display/display.py @@ -2,6 +2,7 @@ from __future__ import annotations import enum +from dataclasses import dataclass import cv2 import matplotlib.pyplot as plt @@ -23,6 +24,25 @@ def get_all_display_names() -> list[str]: return [color.name.capitalize() for color in BBoxColors] +@dataclass(frozen=True, slots=True) +class DisplayTransform: + offset: tuple[int, int] = (0, 0) + scale: tuple[float, float] = (1.0, 1.0) + + def map_point( + self, + x: float, + y: float, + ) -> tuple[int, int]: + offset_x, offset_y = self.offset + scale_x, scale_y = self.scale + + return ( + round(x * scale_x + offset_x), + round(y * scale_y + offset_y), + ) + + def color_to_rgb(color_name: str) -> tuple[int, int, int]: """Convert a color name to an RGB tuple.""" try: diff --git a/dlclivegui/display/overlays.py b/dlclivegui/display/overlays.py new file mode 100644 index 00000000..81c7c767 --- /dev/null +++ b/dlclivegui/display/overlays.py @@ -0,0 +1,71 @@ +# dlclivegui/display/overlays.py +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from dlclivegui.config import BGR + +from .display import draw_bbox, draw_pose +from .skeleton import ResolvedSkeleton, SkeletonStyle + + +@dataclass(frozen=True, slots=True) +class PoseOverlaySettings: + visible: bool + p_cutoff: float + colormap: str + + +@dataclass(frozen=True, slots=True) +class BoundingBoxOverlaySettings: + visible: bool + coordinates: tuple[int, int, int, int] + color_bgr: BGR + + +@dataclass(frozen=True, slots=True) +class SkeletonOverlaySettings: + visible: bool + resolved: ResolvedSkeleton | None + style: SkeletonStyle + + +@dataclass(frozen=True, slots=True) +class OverlaySettings: + pose: PoseOverlaySettings + bounding_box: BoundingBoxOverlaySettings + + +def render_overlays( + frame: np.ndarray, + *, + pose: np.ndarray | None, + settings: OverlaySettings, + offset: tuple[int, int] = (0, 0), + scale: tuple[float, float] = (1.0, 1.0), +) -> np.ndarray: + """Return a frame containing the requested overlays.""" + output = frame.copy() + + if settings.pose.visible and pose is not None: + output = draw_pose( + output, + pose, + p_cutoff=settings.pose.p_cutoff, + colormap=settings.pose.colormap, + offset=offset, + scale=scale, + ) + + if settings.bounding_box.visible: + output = draw_bbox( + output, + settings.bounding_box.coordinates, + color_bgr=settings.bounding_box.color_bgr, + offset=offset, + scale=scale, + ) + + return output From 2e04fff4c3d372c1b89aca1889a0da288000c48b Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:39:55 +0200 Subject: [PATCH 04/18] Refactor video overlay rendering pipeline Move main window frame rendering to shared display utilities by building `OverlaySettings`, calling `render_overlays`, and converting frames with a new `frame_to_pixmap` helper. Add `SkeletonMetadata`/`SkeletonResolver` caching support and an `OverlayRenderer` runtime-state container to prepare for stable skeleton resolution across frames. Also fix module path header comments in display utilities. --- dlclivegui/display/display.py | 2 +- dlclivegui/display/overlays.py | 18 ++++++++- dlclivegui/display/skeleton.py | 45 ++++++++++++++++++++++ dlclivegui/gui/main_window.py | 62 +++++++++++++++--------------- dlclivegui/gui/qt_display/utils.py | 40 +++++++++++++++++++ 5 files changed, 135 insertions(+), 32 deletions(-) create mode 100644 dlclivegui/gui/qt_display/utils.py diff --git a/dlclivegui/display/display.py b/dlclivegui/display/display.py index 4d0fee7d..471b67c6 100644 --- a/dlclivegui/display/display.py +++ b/dlclivegui/display/display.py @@ -1,4 +1,4 @@ -# dlclivegui/utils/display.py +# dlclivegui/display/display.py from __future__ import annotations import enum diff --git a/dlclivegui/display/overlays.py b/dlclivegui/display/overlays.py index 81c7c767..d32d31bc 100644 --- a/dlclivegui/display/overlays.py +++ b/dlclivegui/display/overlays.py @@ -8,7 +8,7 @@ from dlclivegui.config import BGR from .display import draw_bbox, draw_pose -from .skeleton import ResolvedSkeleton, SkeletonStyle +from .skeleton import ResolvedSkeleton, SkeletonResolver, SkeletonStyle @dataclass(frozen=True, slots=True) @@ -69,3 +69,19 @@ def render_overlays( ) return output + + +@dataclass(frozen=True, slots=True) +class OverlayRenderResult: + frame: np.ndarray + warning: str | None = None + + +class OverlayRenderer: + def __init__(self) -> None: + self._skeleton_resolver = SkeletonResolver() + self._last_warning: str | None = None + + def clear_runtime_state(self) -> None: + self._skeleton_resolver.clear() + self._last_warning = None diff --git a/dlclivegui/display/skeleton.py b/dlclivegui/display/skeleton.py index 8da0edcb..ba2595bb 100644 --- a/dlclivegui/display/skeleton.py +++ b/dlclivegui/display/skeleton.py @@ -86,6 +86,51 @@ def resolve_skeleton( ) +@dataclass(frozen=True, slots=True) +class SkeletonMetadata: + identifier: str + keypoint_names: tuple[str, ...] + edges: tuple[tuple[str, str], ...] + + +class SkeletonResolver: + """Caches skeleton resolution for stable pose metadata.""" + + def __init__(self) -> None: + self._metadata: SkeletonMetadata | None = None + self._resolved: ResolvedSkeleton | None = None + + def resolve( + self, + metadata: SkeletonMetadata | None, + ) -> ResolvedSkeleton | None: + if metadata is None: + self.clear() + return None + + if metadata == self._metadata: + return self._resolved + + definition = SkeletonDefinition( + identifier=metadata.identifier, + display_name=metadata.identifier, + edges=tuple(SkeletonEdge(start, end) for start, end in metadata.edges), + ) + + resolved = resolve_skeleton( + definition, + metadata.keypoint_names, + ) + + self._metadata = metadata + self._resolved = resolved + return resolved + + def clear(self) -> None: + self._metadata = None + self._resolved = None + + def skeleton_definition_from_metadata( *, identifier: str, diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index f514ab31..aaab76ad 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -10,7 +10,6 @@ import time from pathlib import Path -import cv2 import numpy as np from PySide6.QtCore import QRect, QSettings, Qt, QTimer, QUrl, Signal from PySide6.QtGui import ( @@ -21,7 +20,6 @@ QDesktopServices, QFont, QIcon, - QImage, QPainter, QPixmap, ) @@ -66,6 +64,7 @@ ) from ..display import BBoxColors, compute_tile_info, create_tiled_frame, draw_bbox, draw_pose +from ..display.overlays import BoundingBoxOverlaySettings, OverlaySettings, PoseOverlaySettings, render_overlays from ..display.skeleton import ResolvedSkeleton, SkeletonResolutionError, resolve_packet_skeleton from ..processors.processor_utils import ( create_spec_from_scan, @@ -87,6 +86,7 @@ from .misc import layouts as lyts from .misc.drag_spinbox import ScrubSpinBox from .misc.eliding_label import ElidingPathLabel +from .qt_display.utils import frame_to_pixmap from .theme import LOGO, LOGO_ALPHA, AppStyle, apply_theme logger = logging.getLogger("DLCLiveGUI") @@ -2619,40 +2619,42 @@ def _on_dlc_error(self, message: str) -> None: self._stop_inference(show_message=False) self._show_error(message) - def _update_video_display(self, frame: np.ndarray) -> None: - display_frame = frame - - if self.show_predictions_checkbox.isChecked() and self._last_pose and self._last_pose.pose is not None: - display_frame = draw_pose( - frame, - self._last_pose.pose, + def _update_video_display( + self, + frame: np.ndarray, + ) -> None: + settings = OverlaySettings( + pose=PoseOverlaySettings( + visible=self.show_predictions_checkbox.isChecked(), p_cutoff=self._p_cutoff, colormap=self._colormap, - offset=self._dlc_tile_offset, - scale=self._dlc_tile_scale, - ) - - if self._bbox_enabled: - display_frame = draw_bbox( - display_frame, - (self._bbox_x0, self._bbox_y0, self._bbox_x1, self._bbox_y1), + ), + bounding_box=BoundingBoxOverlaySettings( + visible=self._bbox_enabled, + coordinates=( + self._bbox_x0, + self._bbox_y0, + self._bbox_x1, + self._bbox_y1, + ), color_bgr=self._bbox_color, - offset=self._dlc_tile_offset, - scale=self._dlc_tile_scale, - ) - - rgb = cv2.cvtColor(display_frame, cv2.COLOR_BGR2RGB) + ), + ) - h, w, ch = rgb.shape - bytes_per_line = ch * w - image = QImage(rgb.data, w, h, bytes_per_line, QImage.Format.Format_RGB888) - pixmap = QPixmap.fromImage(image) + display_frame = render_overlays( + frame, + pose=(self._last_pose.pose if self._last_pose is not None else None), + settings=settings, + offset=self._dlc_tile_offset, + scale=self._dlc_tile_scale, + ) - # Scale pixmap to fit label while preserving aspect ratio - scaled_pixmap = pixmap.scaled( - self.video_label.size(), Qt.AspectRatioMode.KeepAspectRatio, Qt.TransformationMode.SmoothTransformation + self.video_label.setPixmap( + frame_to_pixmap( + display_frame, + self.video_label.size(), + ) ) - self.video_label.setPixmap(scaled_pixmap) def _on_show_predictions_changed(self, _state: int) -> None: if self._current_frame is not None: diff --git a/dlclivegui/gui/qt_display/utils.py b/dlclivegui/gui/qt_display/utils.py new file mode 100644 index 00000000..16e96fca --- /dev/null +++ b/dlclivegui/gui/qt_display/utils.py @@ -0,0 +1,40 @@ +# dlclivegui/gui/display/utils.py +from __future__ import annotations + +import cv2 +import numpy as np +from PySide6.QtCore import QSize, Qt +from PySide6.QtGui import QImage, QPixmap + + +def frame_to_pixmap( + frame_bgr: np.ndarray, + target_size: QSize, +) -> QPixmap: + """Convert a BGR image to a smoothly scaled Qt pixmap.""" + frame = np.asarray(frame_bgr) + + if frame.ndim != 3 or frame.shape[2] != 3: + raise ValueError(f"Display frame must have shape (H, W, 3); received {frame.shape!r}.") + + if frame.dtype != np.uint8: + raise ValueError(f"Display frame must use uint8 pixels; received {frame.dtype}.") + + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + height, width, channels = rgb.shape + + image = QImage( + rgb.data, + width, + height, + channels * width, + QImage.Format.Format_RGB888, + ) + + pixmap = QPixmap.fromImage(image) + + return pixmap.scaled( + target_size, + Qt.AspectRatioMode.KeepAspectRatio, + Qt.TransformationMode.SmoothTransformation, + ) From a5b60e5b48d2784e2a0fa692dfef1832e74ce21d Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:49:57 +0200 Subject: [PATCH 05/18] Remove unused overlay/skeleton resolver state Cleans up display-layer dead code by removing `DisplayTransform`, `OverlayRenderer`, and the cached `SkeletonResolver`/`SkeletonMetadata` path that was no longer used. `OverlaySettings` now explicitly includes skeleton settings, `render_overlays` uses a clearer `overlay_settings` argument name, and the main window stops calling the obsolete skeleton refresh hook in the pose callback. --- dlclivegui/display/display.py | 20 --------------- dlclivegui/display/overlays.py | 27 +++++++------------- dlclivegui/display/skeleton.py | 45 ---------------------------------- dlclivegui/gui/main_window.py | 3 +-- 4 files changed, 10 insertions(+), 85 deletions(-) diff --git a/dlclivegui/display/display.py b/dlclivegui/display/display.py index 471b67c6..ac2b45b0 100644 --- a/dlclivegui/display/display.py +++ b/dlclivegui/display/display.py @@ -2,7 +2,6 @@ from __future__ import annotations import enum -from dataclasses import dataclass import cv2 import matplotlib.pyplot as plt @@ -24,25 +23,6 @@ def get_all_display_names() -> list[str]: return [color.name.capitalize() for color in BBoxColors] -@dataclass(frozen=True, slots=True) -class DisplayTransform: - offset: tuple[int, int] = (0, 0) - scale: tuple[float, float] = (1.0, 1.0) - - def map_point( - self, - x: float, - y: float, - ) -> tuple[int, int]: - offset_x, offset_y = self.offset - scale_x, scale_y = self.scale - - return ( - round(x * scale_x + offset_x), - round(y * scale_y + offset_y), - ) - - def color_to_rgb(color_name: str) -> tuple[int, int, int]: """Convert a color name to an RGB tuple.""" try: diff --git a/dlclivegui/display/overlays.py b/dlclivegui/display/overlays.py index d32d31bc..a1eb0da1 100644 --- a/dlclivegui/display/overlays.py +++ b/dlclivegui/display/overlays.py @@ -8,7 +8,7 @@ from dlclivegui.config import BGR from .display import draw_bbox, draw_pose -from .skeleton import ResolvedSkeleton, SkeletonResolver, SkeletonStyle +from .skeleton import ResolvedSkeleton, SkeletonStyle @dataclass(frozen=True, slots=True) @@ -36,34 +36,35 @@ class SkeletonOverlaySettings: class OverlaySettings: pose: PoseOverlaySettings bounding_box: BoundingBoxOverlaySettings + skeleton: SkeletonOverlaySettings def render_overlays( frame: np.ndarray, *, pose: np.ndarray | None, - settings: OverlaySettings, + overlay_settings: OverlaySettings, offset: tuple[int, int] = (0, 0), scale: tuple[float, float] = (1.0, 1.0), ) -> np.ndarray: """Return a frame containing the requested overlays.""" output = frame.copy() - if settings.pose.visible and pose is not None: + if overlay_settings.pose.visible and pose is not None: output = draw_pose( output, pose, - p_cutoff=settings.pose.p_cutoff, - colormap=settings.pose.colormap, + p_cutoff=overlay_settings.pose.p_cutoff, + colormap=overlay_settings.pose.colormap, offset=offset, scale=scale, ) - if settings.bounding_box.visible: + if overlay_settings.bounding_box.visible: output = draw_bbox( output, - settings.bounding_box.coordinates, - color_bgr=settings.bounding_box.color_bgr, + overlay_settings.bounding_box.coordinates, + color_bgr=overlay_settings.bounding_box.color_bgr, offset=offset, scale=scale, ) @@ -75,13 +76,3 @@ def render_overlays( class OverlayRenderResult: frame: np.ndarray warning: str | None = None - - -class OverlayRenderer: - def __init__(self) -> None: - self._skeleton_resolver = SkeletonResolver() - self._last_warning: str | None = None - - def clear_runtime_state(self) -> None: - self._skeleton_resolver.clear() - self._last_warning = None diff --git a/dlclivegui/display/skeleton.py b/dlclivegui/display/skeleton.py index ba2595bb..8da0edcb 100644 --- a/dlclivegui/display/skeleton.py +++ b/dlclivegui/display/skeleton.py @@ -86,51 +86,6 @@ def resolve_skeleton( ) -@dataclass(frozen=True, slots=True) -class SkeletonMetadata: - identifier: str - keypoint_names: tuple[str, ...] - edges: tuple[tuple[str, str], ...] - - -class SkeletonResolver: - """Caches skeleton resolution for stable pose metadata.""" - - def __init__(self) -> None: - self._metadata: SkeletonMetadata | None = None - self._resolved: ResolvedSkeleton | None = None - - def resolve( - self, - metadata: SkeletonMetadata | None, - ) -> ResolvedSkeleton | None: - if metadata is None: - self.clear() - return None - - if metadata == self._metadata: - return self._resolved - - definition = SkeletonDefinition( - identifier=metadata.identifier, - display_name=metadata.identifier, - edges=tuple(SkeletonEdge(start, end) for start, end in metadata.edges), - ) - - resolved = resolve_skeleton( - definition, - metadata.keypoint_names, - ) - - self._metadata = metadata - self._resolved = resolved - return resolved - - def clear(self) -> None: - self._metadata = None - self._resolved = None - - def skeleton_definition_from_metadata( *, identifier: str, diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index aaab76ad..3596b559 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -2608,7 +2608,6 @@ def _on_pose_ready(self, result: PoseResult) -> None: with self._dlc_timing.measure("DLC.pose_ready_callback"): self._last_pose = result - self._refresh_resolved_skeleton(result) if self._current_frame is not None: self._display_dirty = True @@ -2644,7 +2643,7 @@ def _update_video_display( display_frame = render_overlays( frame, pose=(self._last_pose.pose if self._last_pose is not None else None), - settings=settings, + overlay_settings=settings, offset=self._dlc_tile_offset, scale=self._dlc_tile_scale, ) From 0a0cabeaf0d309e05d33c630ef0623618e9c7fa9 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 11:55:07 +0200 Subject: [PATCH 06/18] Refactor overlay rendering with skeleton cache Introduce an `OverlayRenderer` class to centralize pose, skeleton, and bounding-box composition with runtime state for skeleton resolution caching and warning deduplication. The renderer now resolves packet skeleton metadata on demand, surfaces resolution/render warnings once, and supports gradient skeleton coloring via a new `keypoint_colors_bgr` helper in `display.py`. --- dlclivegui/display/display.py | 21 ++++ dlclivegui/display/overlays.py | 207 +++++++++++++++++++++++++++------ 2 files changed, 194 insertions(+), 34 deletions(-) diff --git a/dlclivegui/display/display.py b/dlclivegui/display/display.py index ac2b45b0..d72457d5 100644 --- a/dlclivegui/display/display.py +++ b/dlclivegui/display/display.py @@ -215,6 +215,27 @@ def draw_keypoints(overlay, p_cutoff, sx, ox, sy, oy, radius, cmap, keypoints: n cv2.drawMarker(overlay, (xs, ys), bgr, marker, radius * 2, 2) +def keypoint_colors_bgr( + colormap: str, + count: int, +) -> tuple[tuple[int, int, int], ...]: + """Return the display color assigned to each keypoint.""" + if count < 0: + raise ValueError(f"Keypoint count must be non-negative; received {count}.") + + cmap = plt.get_cmap(colormap) + denominator = max(count - 1, 1) + + return tuple( + ( + round(cmap(index / denominator)[2] * 255), + round(cmap(index / denominator)[1] * 255), + round(cmap(index / denominator)[0] * 255), + ) + for index in range(count) + ) + + def draw_pose( frame: np.ndarray, pose: np.ndarray, diff --git a/dlclivegui/display/overlays.py b/dlclivegui/display/overlays.py index a1eb0da1..27f0b4a6 100644 --- a/dlclivegui/display/overlays.py +++ b/dlclivegui/display/overlays.py @@ -1,14 +1,36 @@ -# dlclivegui/display/overlays.py +"""Composition of pose, skeleton, and bounding-box overlays.""" + from __future__ import annotations from dataclasses import dataclass +from typing import Protocol import numpy as np -from dlclivegui.config import BGR +from dlclivegui.config import ( + BGR, + SkeletonColorMode, + SkeletonStyle, +) + +from .display import ( + draw_bbox, + draw_pose, + keypoint_colors_bgr, +) +from .skeleton import ( + ResolvedSkeleton, + SkeletonRenderCode, + SkeletonResolutionError, + draw_skeleton, + resolve_packet_skeleton, +) + -from .display import draw_bbox, draw_pose -from .skeleton import ResolvedSkeleton, SkeletonStyle +class PosePacketLike(Protocol): + keypoint_names: list[str] | None + skeleton_id: str | None + skeleton_edges: tuple[tuple[str, str], ...] | None @dataclass(frozen=True, slots=True) @@ -28,51 +50,168 @@ class BoundingBoxOverlaySettings: @dataclass(frozen=True, slots=True) class SkeletonOverlaySettings: visible: bool - resolved: ResolvedSkeleton | None style: SkeletonStyle @dataclass(frozen=True, slots=True) class OverlaySettings: pose: PoseOverlaySettings - bounding_box: BoundingBoxOverlaySettings skeleton: SkeletonOverlaySettings + bounding_box: BoundingBoxOverlaySettings + + +@dataclass(frozen=True, slots=True) +class OverlayRenderResult: + frame: np.ndarray + warning: str | None = None -def render_overlays( - frame: np.ndarray, - *, - pose: np.ndarray | None, - overlay_settings: OverlaySettings, - offset: tuple[int, int] = (0, 0), - scale: tuple[float, float] = (1.0, 1.0), -) -> np.ndarray: - """Return a frame containing the requested overlays.""" - output = frame.copy() - - if overlay_settings.pose.visible and pose is not None: - output = draw_pose( - output, +class OverlayRenderer: + """Compose overlays and cache packet-derived skeleton resolution.""" + + def __init__(self) -> None: + self._skeleton_signature: object | None = None + self._resolved_skeleton: ResolvedSkeleton | None = None + self._last_warning: str | None = None + + def clear_runtime_state(self) -> None: + self._skeleton_signature = None + self._resolved_skeleton = None + self._last_warning = None + + def render( + self, + frame: np.ndarray, + *, + pose: np.ndarray | None, + packet: PosePacketLike | None, + settings: OverlaySettings, + offset: tuple[int, int] = (0, 0), + scale: tuple[float, float] = (1.0, 1.0), + ) -> OverlayRenderResult: + output = frame.copy() + warning: str | None = None + + if settings.pose.visible and pose is not None: + output = draw_pose( + output, + pose, + p_cutoff=settings.pose.p_cutoff, + colormap=settings.pose.colormap, + offset=offset, + scale=scale, + ) + + if settings.skeleton.visible and pose is not None: + warning = self._draw_packet_skeleton( + output, + pose=pose, + packet=packet, + settings=settings, + offset=offset, + scale=scale, + ) + + if settings.bounding_box.visible: + output = draw_bbox( + output, + settings.bounding_box.coordinates, + color_bgr=settings.bounding_box.color_bgr, + offset=offset, + scale=scale, + ) + + return OverlayRenderResult( + frame=output, + warning=warning, + ) + + def _draw_packet_skeleton( + self, + frame: np.ndarray, + *, + pose: np.ndarray, + packet: PosePacketLike | None, + settings: OverlaySettings, + offset: tuple[int, int], + scale: tuple[float, float], + ) -> str | None: + try: + resolved = self._resolve_packet(packet) + except SkeletonResolutionError as exc: + return self._deduplicate_warning(str(exc)) + + if resolved is None: + return self._deduplicate_warning("Skeleton metadata is unavailable for this pose output.") + + style = settings.skeleton.style + keypoint_colors = None + + if style.color_mode == SkeletonColorMode.GRADIENT_KEYPOINTS: + keypoint_colors = tuple( + keypoint_colors_bgr( + settings.pose.colormap, + len(resolved.keypoint_names), + ) + ) + + result = draw_skeleton( + frame, pose, - p_cutoff=overlay_settings.pose.p_cutoff, - colormap=overlay_settings.pose.colormap, + resolved, + style, + p_cutoff=settings.pose.p_cutoff, offset=offset, scale=scale, + keypoint_colors=keypoint_colors, ) - if overlay_settings.bounding_box.visible: - output = draw_bbox( - output, - overlay_settings.bounding_box.coordinates, - color_bgr=overlay_settings.bounding_box.color_bgr, - offset=offset, - scale=scale, + if result.code in { + SkeletonRenderCode.RENDERED, + SkeletonRenderCode.NO_POSE, + }: + self._last_warning = None + return None + + return self._deduplicate_warning(result.message) + + def _resolve_packet( + self, + packet: PosePacketLike | None, + ) -> ResolvedSkeleton | None: + if packet is None: + self._clear_skeleton_cache() + return None + + signature = ( + packet.skeleton_id, + tuple(packet.keypoint_names or ()), + tuple(packet.skeleton_edges or ()), ) - return output + if signature == self._skeleton_signature: + return self._resolved_skeleton + try: + resolved = resolve_packet_skeleton(packet) + except SkeletonResolutionError: + self._clear_skeleton_cache() + raise -@dataclass(frozen=True, slots=True) -class OverlayRenderResult: - frame: np.ndarray - warning: str | None = None + self._skeleton_signature = signature + self._resolved_skeleton = resolved + return resolved + + def _clear_skeleton_cache(self) -> None: + self._skeleton_signature = None + self._resolved_skeleton = None + + def _deduplicate_warning( + self, + message: str, + ) -> str | None: + if not message or message == self._last_warning: + return None + + self._last_warning = message + return message From c058a30e5baeade44509fc1a6f4d27740b7290a4 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 12:10:03 +0200 Subject: [PATCH 07/18] Unify overlay rendering with skeleton support Refactors overlay drawing to use `OverlayRenderer` end-to-end in the main window, including pose, skeleton, and bounding-box settings from UI state. Adds skeleton controls (visibility, color mode/color, thickness), wires them into visualization/config sync, and surfaces renderer warnings in logs/status bar with deduped messaging. Also resets cached overlay runtime state on DLC/session resets and clears pose metadata fields before reloading DLC config to prevent stale skeleton/keypoint metadata from leaking between runs. --- dlclivegui/display/overlays.py | 131 +++++------- dlclivegui/gui/main_window.py | 285 +++++++++++++++++---------- dlclivegui/services/dlc_processor.py | 5 + 3 files changed, 239 insertions(+), 182 deletions(-) diff --git a/dlclivegui/display/overlays.py b/dlclivegui/display/overlays.py index 27f0b4a6..f0c5207c 100644 --- a/dlclivegui/display/overlays.py +++ b/dlclivegui/display/overlays.py @@ -56,8 +56,8 @@ class SkeletonOverlaySettings: @dataclass(frozen=True, slots=True) class OverlaySettings: pose: PoseOverlaySettings - skeleton: SkeletonOverlaySettings bounding_box: BoundingBoxOverlaySettings + skeleton: SkeletonOverlaySettings @dataclass(frozen=True, slots=True) @@ -67,7 +67,7 @@ class OverlayRenderResult: class OverlayRenderer: - """Compose overlays and cache packet-derived skeleton resolution.""" + """Render overlays and cache packet-derived skeleton resolution.""" def __init__(self) -> None: self._skeleton_signature: object | None = None @@ -85,38 +85,65 @@ def render( *, pose: np.ndarray | None, packet: PosePacketLike | None, - settings: OverlaySettings, + overlay_settings: OverlaySettings, offset: tuple[int, int] = (0, 0), scale: tuple[float, float] = (1.0, 1.0), ) -> OverlayRenderResult: output = frame.copy() warning: str | None = None - if settings.pose.visible and pose is not None: + if overlay_settings.pose.visible and pose is not None: output = draw_pose( output, pose, - p_cutoff=settings.pose.p_cutoff, - colormap=settings.pose.colormap, + p_cutoff=overlay_settings.pose.p_cutoff, + colormap=overlay_settings.pose.colormap, offset=offset, scale=scale, ) - if settings.skeleton.visible and pose is not None: - warning = self._draw_packet_skeleton( - output, - pose=pose, - packet=packet, - settings=settings, - offset=offset, - scale=scale, - ) - - if settings.bounding_box.visible: + if overlay_settings.skeleton.visible and pose is not None: + try: + resolved = self._resolve_packet_skeleton(packet) + except SkeletonResolutionError as exc: + warning = self._new_warning(str(exc)) + else: + if resolved is None: + warning = self._new_warning("Skeleton metadata is unavailable for this pose output.") + else: + style = overlay_settings.skeleton.style + colors = None + + if style.color_mode == SkeletonColorMode.GRADIENT_KEYPOINTS: + colors = keypoint_colors_bgr( + overlay_settings.pose.colormap, + len(resolved.keypoint_names), + ) + + result = draw_skeleton( + output, + pose, + resolved, + style, + p_cutoff=overlay_settings.pose.p_cutoff, + offset=offset, + scale=scale, + keypoint_colors=colors, + ) + + if result.code in { + SkeletonRenderCode.RENDERED, + SkeletonRenderCode.NO_POSE, + }: + self._last_warning = None + else: + warning = self._new_warning(result.message) + + if overlay_settings.bounding_box.visible: output = draw_bbox( output, - settings.bounding_box.coordinates, - color_bgr=settings.bounding_box.color_bgr, + overlay_settings.bounding_box.coordinates, + color_bgr=overlay_settings.bounding_box.color_bgr, offset=offset, scale=scale, ) @@ -126,61 +153,13 @@ def render( warning=warning, ) - def _draw_packet_skeleton( - self, - frame: np.ndarray, - *, - pose: np.ndarray, - packet: PosePacketLike | None, - settings: OverlaySettings, - offset: tuple[int, int], - scale: tuple[float, float], - ) -> str | None: - try: - resolved = self._resolve_packet(packet) - except SkeletonResolutionError as exc: - return self._deduplicate_warning(str(exc)) - - if resolved is None: - return self._deduplicate_warning("Skeleton metadata is unavailable for this pose output.") - - style = settings.skeleton.style - keypoint_colors = None - - if style.color_mode == SkeletonColorMode.GRADIENT_KEYPOINTS: - keypoint_colors = tuple( - keypoint_colors_bgr( - settings.pose.colormap, - len(resolved.keypoint_names), - ) - ) - - result = draw_skeleton( - frame, - pose, - resolved, - style, - p_cutoff=settings.pose.p_cutoff, - offset=offset, - scale=scale, - keypoint_colors=keypoint_colors, - ) - - if result.code in { - SkeletonRenderCode.RENDERED, - SkeletonRenderCode.NO_POSE, - }: - self._last_warning = None - return None - - return self._deduplicate_warning(result.message) - - def _resolve_packet( + def _resolve_packet_skeleton( self, packet: PosePacketLike | None, ) -> ResolvedSkeleton | None: if packet is None: - self._clear_skeleton_cache() + self._skeleton_signature = None + self._resolved_skeleton = None return None signature = ( @@ -192,21 +171,13 @@ def _resolve_packet( if signature == self._skeleton_signature: return self._resolved_skeleton - try: - resolved = resolve_packet_skeleton(packet) - except SkeletonResolutionError: - self._clear_skeleton_cache() - raise + resolved = resolve_packet_skeleton(packet) self._skeleton_signature = signature self._resolved_skeleton = resolved return resolved - def _clear_skeleton_cache(self) -> None: - self._skeleton_signature = None - self._resolved_skeleton = None - - def _deduplicate_warning( + def _new_warning( self, message: str, ) -> str | None: diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index 3596b559..12109460 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -60,12 +60,19 @@ DLCProcessorSettings, MultiCameraSettings, RecordingSettings, + SkeletonColorMode, + SkeletonStyle, VisualizationSettings, ) -from ..display import BBoxColors, compute_tile_info, create_tiled_frame, draw_bbox, draw_pose -from ..display.overlays import BoundingBoxOverlaySettings, OverlaySettings, PoseOverlaySettings, render_overlays -from ..display.skeleton import ResolvedSkeleton, SkeletonResolutionError, resolve_packet_skeleton +from ..display import BBoxColors, compute_tile_info, create_tiled_frame +from ..display.overlays import ( + BoundingBoxOverlaySettings, + OverlayRenderer, + OverlaySettings, + PoseOverlaySettings, + SkeletonOverlaySettings, +) from ..processors.processor_utils import ( create_spec_from_scan, default_processors_dir, @@ -182,17 +189,7 @@ def __init__(self, config: ApplicationSettings | None = None): self._p_cutoff = 0.6 self._colormap = "hot" self._bbox_color = (0, 0, 255) # BGR: red - ## Skeleton - self._resolved_skeleton: ResolvedSkeleton | None = None - self._resolved_skeleton_signature: ( - tuple[ - str | None, - tuple[str, ...], - tuple[tuple[str, str], ...], - ] - | None - ) = None - self._last_skeleton_warning: str | None = None + self._overlay_renderer = OverlayRenderer() # Multi-camera state self._multi_camera_mode = False @@ -812,6 +809,46 @@ def _build_viz_group(self) -> QGroupBox: ) form.addRow(bbox_settings) + # Skeleton overlay + self.show_skeleton_checkbox = QCheckBox("Display skeleton") + self.show_skeleton_checkbox.setChecked(False) + self.show_skeleton_checkbox.setToolTip( + "Display connections between keypoints when the pose output provides a compatible skeleton definition." + ) + + self.skeleton_color_combo = color_ui.make_skeleton_color_combo( # FIXME + BBoxColors, + current_mode=SkeletonColorMode.SOLID.value, + current_color=(0, 255, 255), + include_icons=True, + tooltip=("Select a solid skeleton color or a gradient between endpoint keypoint colors."), + sizing=color_ui.ComboSizing( + min_width=80, + max_width=200, + ), + ) + + skeleton_settings = lyts.make_two_field_row( + "Skeleton:", + self.skeleton_color_combo, + None, + self.show_skeleton_checkbox, + key_width=120, + left_stretch=0, + right_stretch=0, + ) + form.addRow(skeleton_settings) + + self.skeleton_thickness_spin = QSpinBox() + self.skeleton_thickness_spin.setRange(1, 20) + self.skeleton_thickness_spin.setValue(2) + self.skeleton_thickness_spin.setToolTip("Skeleton line thickness.") + + form.addRow( + "Skeleton thickness:", + self.skeleton_thickness_spin, + ) + bbox_layout = QHBoxLayout() self.bbox_x0_spin = ScrubSpinBox() self.bbox_x0_spin.setRange(0, 7680) @@ -864,6 +901,10 @@ def _connect_signals(self) -> None: self.bbox_x1_spin.valueChanged.connect(self._on_bbox_changed) self.bbox_y1_spin.valueChanged.connect(self._on_bbox_changed) self.bbox_color_combo.currentIndexChanged.connect(self._on_bbox_color_changed) + ## Skeleton settings + self.show_skeleton_checkbox.stateChanged.connect(self._on_skeleton_display_changed) + self.skeleton_color_combo.currentIndexChanged.connect(self._on_skeleton_display_changed) + self.skeleton_thickness_spin.valueChanged.connect(self._on_skeleton_display_changed) # Multi-camera controller signals (used for both single and multi-camera modes) self.multi_camera_controller.frame_ready.connect(self._on_multi_frame_processing_ready) @@ -947,11 +988,54 @@ def _apply_config(self, config: ApplicationSettings, *, restore_local_prefs: boo viz = config.visualization self._p_cutoff = viz.p_cutoff self._colormap = viz.colormap - if hasattr(self, "cmap_combo"): - color_ui.set_cmap_combo_from_name(self.cmap_combo, self._colormap, fallback="viridis") self._bbox_color = viz.get_bbox_color_bgr() - if hasattr(self, "bbox_color_combo"): - color_ui.set_bbox_combo_from_bgr(self.bbox_color_combo, self._bbox_color) + + self.show_predictions_checkbox.blockSignals(True) + try: + self.show_predictions_checkbox.setChecked(bool(viz.show_pose)) + finally: + self.show_predictions_checkbox.blockSignals(False) + + self.show_skeleton_checkbox.blockSignals(True) + try: + self.show_skeleton_checkbox.setChecked(bool(viz.show_skeleton)) + finally: + self.show_skeleton_checkbox.blockSignals(False) + + self.cmap_combo.blockSignals(True) + try: + color_ui.set_cmap_combo_from_name( + self.cmap_combo, + self._colormap, + fallback="viridis", + ) + finally: + self.cmap_combo.blockSignals(False) + + self.bbox_color_combo.blockSignals(True) + try: + color_ui.set_bbox_combo_from_bgr( + self.bbox_color_combo, + self._bbox_color, + ) + finally: + self.bbox_color_combo.blockSignals(False) + + self.skeleton_color_combo.blockSignals(True) + try: + color_ui.set_skeleton_combo_from_style( + self.skeleton_color_combo, + mode=viz.skeleton_style.color_mode.value, + color=viz.skeleton_style.color_bgr, + ) + finally: + self.skeleton_color_combo.blockSignals(False) + + self.skeleton_thickness_spin.blockSignals(True) + try: + self.skeleton_thickness_spin.setValue(int(viz.skeleton_style.thickness)) + finally: + self.skeleton_thickness_spin.blockSignals(False) # Processor ## Use custom processor checkbox state @@ -1058,11 +1142,62 @@ def _bbox_settings_from_ui(self) -> BoundingBoxSettings: y1=self.bbox_y1_spin.value(), ) - def _visualization_settings_from_ui(self) -> VisualizationSettings: + def _visualization_settings_from_ui( + self, + ) -> VisualizationSettings: return VisualizationSettings( p_cutoff=self._p_cutoff, colormap=self._colormap, bbox_color=self._bbox_color, + show_pose=self.show_predictions_checkbox.isChecked(), + show_skeleton=self.show_skeleton_checkbox.isChecked(), + skeleton_style=self._skeleton_style_from_ui(), + ) + + def _overlay_settings_from_ui( + self, + ) -> OverlaySettings: + return OverlaySettings( + pose=PoseOverlaySettings( + visible=self.show_predictions_checkbox.isChecked(), + p_cutoff=self._p_cutoff, + colormap=self._colormap, + ), + skeleton=SkeletonOverlaySettings( + visible=self.show_skeleton_checkbox.isChecked(), + style=self._skeleton_style_from_ui(), + ), + bounding_box=BoundingBoxOverlaySettings( + visible=self._bbox_enabled, + coordinates=( + self._bbox_x0, + self._bbox_y0, + self._bbox_x1, + self._bbox_y1, + ), + color_bgr=self._bbox_color, + ), + ) + + def _skeleton_style_from_ui( + self, + ) -> SkeletonStyle: + mode, color = color_ui.get_skeleton_style_from_combo( + self.skeleton_color_combo, + fallback_mode=SkeletonColorMode.SOLID.value, + fallback_color=(0, 255, 255), + ) + + color_bgr = tuple(color) if color is not None else (0, 255, 255) + + current_style = self._config.visualization.skeleton_style + + return SkeletonStyle( + color_mode=SkeletonColorMode(mode), + color_bgr=color_bgr, + thickness=self.skeleton_thickness_spin.value(), + gradient_steps=current_style.gradient_steps, + scale_with_zoom=current_style.scale_with_zoom, ) def _suggest_config_dialog_path(self) -> str: @@ -1480,6 +1615,13 @@ def _on_colormap_changed(self, _index: int) -> None: if self._current_frame is not None: self._display_frame(self._current_frame, force=True) + def _on_skeleton_display_changed( + self, + _value: int = 0, + ) -> None: + if self._current_frame is not None: + self._display_dirty = True + def _on_bbox_color_changed(self, _index: int) -> None: bgr = color_ui.get_bbox_bgr_from_combo(self.bbox_color_combo, fallback=self._bbox_color) if bgr is None: @@ -1680,31 +1822,6 @@ def _on_dlc_camera_changed(self, _index: int) -> None: # ------------------------------------------------------------------ # Multi-camera event handlers - def _render_overlays_for_recording(self, cam_id, frame): - # Copy so we don't affect GUI preview pipeline - output = frame.copy() - offset, scale = (0, 0), (1.0, 1.0) - - # If this is the inference camera, apply pose overlays - if cam_id == self._active_inference_camera_id and self._last_pose and self._last_pose.pose is not None: - output = draw_pose( - output, - self._last_pose.pose, - p_cutoff=self._p_cutoff, - colormap=self._colormap, - offset=offset, - scale=scale, - ) - if self._bbox_enabled: - output = draw_bbox( - frame=output, - bbox_xyxy=(self._bbox_x0, self._bbox_y0, self._bbox_x1, self._bbox_y1), - color_bgr=self._bbox_color, - offset=offset, - scale=scale, - ) - return output - def _on_multi_frame_processing_ready(self, frame_data: MultiFrameData) -> None: """Handle frames from multiple cameras. @@ -2138,6 +2255,7 @@ def _start_preview(self) -> None: self._current_frame = None self._raw_frame = None self._last_pose = None + self._overlay_renderer.clear_runtime_state() self._multi_camera_frames.clear() self._multi_camera_display_ids.clear() self._fps_tracker.clear() @@ -2312,42 +2430,6 @@ def _update_camera_controls_enabled(self) -> None: if hasattr(self, "load_config_action"): self.load_config_action.setEnabled(allow_changes) - def _refresh_resolved_skeleton( - self, - result: PoseResult, - ) -> None: - packet = result.packet - - if packet is None: - self._resolved_skeleton = None - self._resolved_skeleton_signature = None - return - - signature = ( - packet.skeleton_id, - tuple(packet.keypoint_names or ()), - tuple(packet.skeleton_edges or ()), - ) - - if signature == self._resolved_skeleton_signature: - return - - self._resolved_skeleton_signature = signature - - try: - self._resolved_skeleton = resolve_packet_skeleton(packet) - self._last_skeleton_warning = None - except SkeletonResolutionError as exc: - self._resolved_skeleton = None - - message = str(exc) - if message != self._last_skeleton_warning: - self._last_skeleton_warning = message - logger.warning( - "Skeleton could not be resolved: %s", - message, - ) - def _display_frame(self, frame: np.ndarray, *, force: bool = False) -> None: if frame is None: return @@ -2502,6 +2584,7 @@ def _start_inference(self) -> None: return self._dlc.reset() self._last_pose = None + self._overlay_renderer.clear_runtime_state() self._dlc_active = True self._dlc_initialized = False @@ -2602,7 +2685,10 @@ def _stop_recording(self) -> None: """Stop recording from all cameras.""" self._stop_multi_camera_recording() - def _on_pose_ready(self, result: PoseResult) -> None: + def _on_pose_ready( + self, + result: PoseResult, + ) -> None: if not self._dlc_active: return @@ -2622,35 +2708,30 @@ def _update_video_display( self, frame: np.ndarray, ) -> None: - settings = OverlaySettings( - pose=PoseOverlaySettings( - visible=self.show_predictions_checkbox.isChecked(), - p_cutoff=self._p_cutoff, - colormap=self._colormap, - ), - bounding_box=BoundingBoxOverlaySettings( - visible=self._bbox_enabled, - coordinates=( - self._bbox_x0, - self._bbox_y0, - self._bbox_x1, - self._bbox_y1, - ), - color_bgr=self._bbox_color, - ), - ) + pose_result = self._last_pose - display_frame = render_overlays( + render_result = self._overlay_renderer.render( frame, - pose=(self._last_pose.pose if self._last_pose is not None else None), - overlay_settings=settings, + pose=(pose_result.pose if pose_result is not None else None), + packet=(pose_result.packet if pose_result is not None else None), + overlay_settings=self._overlay_settings_from_ui(), offset=self._dlc_tile_offset, scale=self._dlc_tile_scale, ) + if render_result.warning is not None: + logger.warning( + "Overlay rendering warning: %s", + render_result.warning, + ) + self.statusBar().showMessage( + render_result.warning, + 5000, + ) + self.video_label.setPixmap( frame_to_pixmap( - display_frame, + render_result.frame, self.video_label.size(), ) ) diff --git a/dlclivegui/services/dlc_processor.py b/dlclivegui/services/dlc_processor.py index 97838313..696c8e27 100644 --- a/dlclivegui/services/dlc_processor.py +++ b/dlclivegui/services/dlc_processor.py @@ -658,6 +658,11 @@ def _process_frame( def _load_pose_metadata_from_dlc_config(self) -> None: """Extract name-based pose metadata from the initialized DLC config.""" + + self._keypoint_names = None + self._skeleton_id = None + self._skeleton_edges = None + dlc = self._dlc if dlc is None: raise RuntimeError("DLCLive instance is not initialized.") From 4d1599262a91b29ad0b8b2bcb8c42741253711f5 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 12:58:11 +0200 Subject: [PATCH 08/18] Add skeleton style settings and color combo Extend visualization config with skeleton-related defaults (`show_pose`, `show_skeleton`, and `skeleton_style`) and tighten bbox color typing to `BGR`. Add reusable skeleton color combo helpers that support both solid colors and a keypoint-gradient mode, including gradient swatch icon rendering, combo population, and style get/set mapping. --- dlclivegui/config.py | 22 ++- dlclivegui/gui/misc/color_dropdowns.py | 225 ++++++++++++++++++++++++- 2 files changed, 239 insertions(+), 8 deletions(-) diff --git a/dlclivegui/config.py b/dlclivegui/config.py index e8f5c349..fb7b4367 100644 --- a/dlclivegui/config.py +++ b/dlclivegui/config.py @@ -510,7 +510,7 @@ class SkeletonStyle(BaseModel): visible: bool = False color_mode: SkeletonColorMode = SkeletonColorMode.SOLID color_bgr: BGR = (0, 255, 255) # default if SOLID - thickness: int = Field(defalt=2, ge=1, le=20) # base thickness in pixels + thickness: int = Field(default=2, ge=1, le=20) # base thickness in pixels gradient_steps: int = Field(default=16, ge=2, le=128) # segments per edge when gradient scale_with_zoom: bool = True # scale thickness with (sx, sy) @@ -521,15 +521,23 @@ def effective_thickness(self, sx: float, sy: float) -> int: class VisualizationSettings(BaseModel): - p_cutoff: float = Field(default=0.6, ge=0.0, le=1.0) + p_cutoff: float = Field( + default=0.6, + ge=0.0, + le=1.0, + ) colormap: str = "hot" - bbox_color: tuple[int, int, int] = (0, 0, 255) + bbox_color: BGR = (0, 0, 255) - def get_bbox_color_bgr(self) -> tuple[int, int, int]: - """Get bounding box color in BGR format""" + show_pose: bool = True + show_skeleton: bool = False + skeleton_style: SkeletonStyle = Field(default_factory=SkeletonStyle) + + def get_bbox_color_bgr(self) -> BGR: if isinstance(self.bbox_color, (list, tuple)) and len(self.bbox_color) == 3: - return tuple(int(c) for c in self.bbox_color) - return (0, 0, 255) # default red + return tuple(int(value) for value in self.bbox_color) + + return (0, 0, 255) class RecordingSettings(BaseModel): diff --git a/dlclivegui/gui/misc/color_dropdowns.py b/dlclivegui/gui/misc/color_dropdowns.py index bb0f0ac4..577ff8e9 100644 --- a/dlclivegui/gui/misc/color_dropdowns.py +++ b/dlclivegui/gui/misc/color_dropdowns.py @@ -16,7 +16,15 @@ import numpy as np from PySide6.QtCore import Qt -from PySide6.QtGui import QColor, QIcon, QImage, QPainter, QPixmap +from PySide6.QtGui import ( + QBrush, + QColor, + QIcon, + QImage, + QLinearGradient, + QPainter, + QPixmap, +) from PySide6.QtWidgets import ( QComboBox, QSizePolicy, @@ -218,6 +226,221 @@ def get_bbox_bgr_from_combo(combo: QComboBox, *, fallback: BGR | None = None) -> return getattr(enum_item, "value", fallback) +# ----------------------------------------------------------------------------- +# Skeleton color combo helpers +# ----------------------------------------------------------------------------- + + +def make_gradient_swatch_icon( + *, + width: int = 40, + height: int = 16, + border: int = 1, +) -> QIcon: + """Create a swatch representing keypoint-gradient skeleton coloring.""" + pixmap = QPixmap(width, height) + pixmap.fill(Qt.GlobalColor.transparent) + + painter = QPainter(pixmap) + try: + painter.fillRect( + 0, + 0, + width, + height, + Qt.GlobalColor.black, + ) + painter.fillRect( + border, + border, + width - 2 * border, + height - 2 * border, + Qt.GlobalColor.white, + ) + + gradient = QLinearGradient( + border + 1, + 0, + width - border - 1, + 0, + ) + gradient.setColorAt( + 0.0, + QColor(0, 140, 255), + ) + gradient.setColorAt( + 1.0, + QColor(255, 80, 0), + ) + + painter.fillRect( + border + 1, + border + 1, + width - 2 * (border + 1), + height - 2 * (border + 1), + QBrush(gradient), + ) + finally: + painter.end() + + return QIcon(pixmap) + + +def populate_skeleton_color_combo( + combo: QComboBox, + colors_enum: Iterable[TEnum], + *, + current_mode: str = "solid", + current_color: BGR | None = None, + include_icons: bool = True, + gradient_label: str = "Gradient (from keypoints)", +) -> None: + """Populate a combo with gradient mode and solid BGR colors.""" + combo.blockSignals(True) + try: + combo.clear() + + gradient_data = { + "mode": "gradient_keypoints", + "color": None, + } + + if include_icons: + combo.addItem( + make_gradient_swatch_icon(), + gradient_label, + gradient_data, + ) + else: + combo.addItem( + gradient_label, + gradient_data, + ) + + for enum_item in colors_enum: + color: BGR = tuple(enum_item.value) + label = getattr( + enum_item, + "name", + str(enum_item), + ).title() + data = { + "mode": "solid", + "color": color, + } + + if include_icons: + combo.addItem( + make_bgr_swatch_icon(color), + label, + data, + ) + else: + combo.addItem( + label, + data, + ) + + set_skeleton_combo_from_style( + combo, + mode=current_mode, + color=current_color, + ) + finally: + combo.blockSignals(False) + + +def make_skeleton_color_combo( + colors_enum: Iterable[TEnum], + *, + current_mode: str = "solid", + current_color: BGR | None = (0, 255, 255), + include_icons: bool = True, + tooltip: str = ("Select skeleton line color or keypoint gradient"), + sizing: ComboSizing | None = None, +) -> QComboBox: + """Create and populate a skeleton-color combo box.""" + combo = ShrinkCurrentWidePopupComboBox(sizing=sizing) if sizing is not None else QComboBox() + combo.setToolTip(tooltip) + + populate_skeleton_color_combo( + combo, + colors_enum, + current_mode=current_mode, + current_color=current_color, + include_icons=include_icons, + ) + + if isinstance( + combo, + ShrinkCurrentWidePopupComboBox, + ): + combo.update_shrink_width() + + return combo + + +def get_skeleton_style_from_combo( + combo: QComboBox, + *, + fallback_mode: str = "solid", + fallback_color: BGR | None = None, +) -> tuple[str, BGR | None]: + """Return the selected skeleton mode and optional solid color.""" + data = combo.currentData() + + if not isinstance(data, dict): + return fallback_mode, fallback_color + + mode = data.get("mode") + if mode not in { + "solid", + "gradient_keypoints", + }: + mode = fallback_mode + + color = data.get( + "color", + fallback_color, + ) + if color is not None: + color = tuple(color) + + return mode, color + + +def set_skeleton_combo_from_style( + combo: QComboBox, + *, + mode: str, + color: BGR | None, +) -> None: + """Select the combo item matching a skeleton style.""" + for index in range(combo.count()): + data = combo.itemData(index) + + if not isinstance(data, dict) or data.get("mode") != mode: + continue + + if mode == "gradient_keypoints": + combo.setCurrentIndex(index) + return + + item_color = data.get("color") + + if color is not None and item_color is not None and tuple(item_color) == tuple(color): + combo.setCurrentIndex(index) + return + + if mode == "solid": + for index in range(combo.count()): + data = combo.itemData(index) + + if isinstance(data, dict) and data.get("mode") == "solid": + combo.setCurrentIndex(index) + return + + # ----------------------------------------------------------------------------- # Matplotlib colormap combo helpers # ----------------------------------------------------------------------------- From 1f3a106247bba0ac3fac647aa645584a780859ed Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:02:49 +0200 Subject: [PATCH 09/18] Reset overlay state and simplify skeleton style Remove the unused `visible` field from `SkeletonStyle` and clear the overlay renderer runtime state during live-session reset. This keeps runtime overlay data from leaking across resets and aligns skeleton styling with the current rendering flow. --- dlclivegui/config.py | 1 - dlclivegui/gui/main_window.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/dlclivegui/config.py b/dlclivegui/config.py index fb7b4367..14d00250 100644 --- a/dlclivegui/config.py +++ b/dlclivegui/config.py @@ -507,7 +507,6 @@ class SkeletonColorMode(str, Enum): class SkeletonStyle(BaseModel): - visible: bool = False color_mode: SkeletonColorMode = SkeletonColorMode.SOLID color_bgr: BGR = (0, 255, 255) # default if SOLID thickness: int = Field(default=2, ge=1, le=20) # base thickness in pixels diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index 12109460..33d3c145 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -2618,6 +2618,7 @@ def _stop_inference(self, show_message: bool = True) -> None: # Does NOT invoke the normal rec-stop/save hooks. Persistence is processor-dependent. self._dlc.reset(reset_processor_plugin=True) self._last_pose = None + self._overlay_renderer.clear_runtime_state() self._last_processor_vid_recording = False self._auto_record_session_name = None From 8e3f3fe19bf48896c7717c1b679e954ea79634aa Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:03:12 +0200 Subject: [PATCH 10/18] Add tests for skeleton visualization settings Introduce a new test module covering VisualizationSettings and SkeletonStyle behavior, including default/legacy config handling, model round-tripping, validation bounds for thickness and gradient steps, and effective thickness scaling behavior with and without zoom scaling. --- tests/test_skeleton_config.py | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 tests/test_skeleton_config.py diff --git a/tests/test_skeleton_config.py b/tests/test_skeleton_config.py new file mode 100644 index 00000000..35d2d9c9 --- /dev/null +++ b/tests/test_skeleton_config.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from dlclivegui.config import ( + SkeletonColorMode, + SkeletonStyle, + VisualizationSettings, +) + + +def test_visualization_settings_apply_skeleton_defaults() -> None: + settings = VisualizationSettings( + p_cutoff=0.6, + colormap="hot", + bbox_color=(0, 0, 255), + ) + + assert settings.show_pose is True + assert settings.show_skeleton is False + assert settings.skeleton_style == SkeletonStyle() + + +def test_visualization_settings_accept_legacy_data() -> None: + settings = VisualizationSettings.model_validate( + { + "p_cutoff": 0.6, + "colormap": "hot", + "bbox_color": [0, 0, 255], + } + ) + + assert settings.show_pose is True + assert settings.show_skeleton is False + assert settings.skeleton_style.color_mode == SkeletonColorMode.SOLID + assert settings.skeleton_style.color_bgr == (0, 255, 255) + + +def test_visualization_settings_round_trip() -> None: + original = VisualizationSettings( + p_cutoff=0.7, + colormap="viridis", + bbox_color=(255, 0, 0), + show_pose=False, + show_skeleton=True, + skeleton_style=SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + color_bgr=(10, 20, 30), + thickness=5, + gradient_steps=24, + scale_with_zoom=False, + ), + ) + + restored = VisualizationSettings.model_validate(original.model_dump()) + + assert restored == original + + +@pytest.mark.parametrize("thickness", [0, 21]) +def test_skeleton_thickness_rejects_out_of_range_values( + thickness: int, +) -> None: + with pytest.raises(ValidationError): + SkeletonStyle(thickness=thickness) + + +@pytest.mark.parametrize("steps", [0, 1, 129]) +def test_skeleton_gradient_steps_reject_out_of_range_values( + steps: int, +) -> None: + with pytest.raises(ValidationError): + SkeletonStyle(gradient_steps=steps) + + +def test_skeleton_thickness_scales_with_smallest_axis() -> None: + style = SkeletonStyle( + thickness=4, + scale_with_zoom=True, + ) + + assert style.effective_thickness(0.5, 0.25) == 1 + assert style.effective_thickness(2.0, 1.5) == 6 + + +def test_skeleton_thickness_can_ignore_scale() -> None: + style = SkeletonStyle( + thickness=4, + scale_with_zoom=False, + ) + + assert style.effective_thickness(0.1, 10.0) == 4 From 34c2afb55f9bcbb07204a0bc36c63dcfbe724c17 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:03:35 +0200 Subject: [PATCH 11/18] Move display tests into display package Renames `tests/utils/test_display.py` to `tests/display/test_display.py` to keep display-related tests grouped under the display test module. No test logic was changed. --- tests/{utils => display}/test_display.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{utils => display}/test_display.py (100%) diff --git a/tests/utils/test_display.py b/tests/display/test_display.py similarity index 100% rename from tests/utils/test_display.py rename to tests/display/test_display.py From 221f2c60cdcaa6e08fd5bc1907d29dcf82b4d7ff Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:06:33 +0200 Subject: [PATCH 12/18] Add display color and skeleton resolution tests Expand unit coverage in display-related tests by adding assertions for `keypoint_colors_bgr` behavior (requested count, zero/one cases, determinism, and negative input validation). Add a new `test_skeleton.py` suite that validates skeleton name-to-index resolution, duplicate/missing keypoint handling, and packet-based skeleton topology resolution when keypoint names or edges are absent or provided. --- tests/display/test_display.py | 34 +++++++++ tests/display/test_skeleton.py | 129 +++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 tests/display/test_skeleton.py diff --git a/tests/display/test_display.py b/tests/display/test_display.py index ec6b7eec..9aaa1f10 100644 --- a/tests/display/test_display.py +++ b/tests/display/test_display.py @@ -9,6 +9,7 @@ draw_keypoints, draw_pose, ) +from dlclivegui.display.display import keypoint_colors_bgr pytestmark = pytest.mark.unit @@ -302,3 +303,36 @@ def test_draw_pose_multi_animal_draws_distinct_markers(): out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) assert out is not frame assert np.any(out != frame) + + +def test_keypoint_colors_returns_requested_count() -> None: + colors = keypoint_colors_bgr("viridis", 5) + + assert len(colors) == 5 + assert all(len(color) == 3 for color in colors) + assert all(0 <= channel <= 255 for color in colors for channel in color) + + +def test_keypoint_colors_zero_count_returns_empty_tuple() -> None: + assert keypoint_colors_bgr("viridis", 0) == () + + +def test_keypoint_colors_one_count_returns_one_color() -> None: + colors = keypoint_colors_bgr("viridis", 1) + + assert len(colors) == 1 + + +def test_keypoint_colors_are_deterministic() -> None: + first = keypoint_colors_bgr("viridis", 4) + second = keypoint_colors_bgr("viridis", 4) + + assert first == second + + +def test_keypoint_colors_reject_negative_count() -> None: + with pytest.raises( + ValueError, + match="must be non-negative", + ): + keypoint_colors_bgr("viridis", -1) diff --git a/tests/display/test_skeleton.py b/tests/display/test_skeleton.py new file mode 100644 index 00000000..7fab55fd --- /dev/null +++ b/tests/display/test_skeleton.py @@ -0,0 +1,129 @@ +from __future__ import annotations + +import pytest + +from dlclivegui.display.skeleton import ( + SkeletonDefinition, + SkeletonEdge, + SkeletonResolutionError, + resolve_packet_skeleton, + resolve_skeleton, +) + + +class PacketStub: + def __init__( + self, + *, + keypoint_names: list[str] | None, + skeleton_id: str | None, + skeleton_edges: tuple[tuple[str, str], ...] | None, + ) -> None: + self.keypoint_names = keypoint_names + self.skeleton_id = skeleton_id + self.skeleton_edges = skeleton_edges + + +@pytest.fixture +def simple_definition() -> SkeletonDefinition: + return SkeletonDefinition( + identifier="test.simple", + display_name="Simple", + edges=( + SkeletonEdge("nose", "shoulder"), + SkeletonEdge("shoulder", "hip"), + ), + ) + + +def test_resolve_skeleton_maps_names_to_pose_order( + simple_definition: SkeletonDefinition, +) -> None: + resolved = resolve_skeleton( + simple_definition, + ["hip", "nose", "shoulder"], + ) + + assert resolved.keypoint_names == ( + "hip", + "nose", + "shoulder", + ) + assert resolved.edges == ( + (1, 2), + (2, 0), + ) + + +def test_resolve_skeleton_rejects_empty_keypoint_names( + simple_definition: SkeletonDefinition, +) -> None: + with pytest.raises( + SkeletonResolutionError, + match="without keypoint names", + ): + resolve_skeleton(simple_definition, []) + + +def test_resolve_skeleton_rejects_duplicate_names( + simple_definition: SkeletonDefinition, +) -> None: + with pytest.raises( + SkeletonResolutionError, + match="duplicate", + ): + resolve_skeleton( + simple_definition, + ["nose", "nose", "hip"], + ) + + +def test_resolve_skeleton_reports_all_missing_names( + simple_definition: SkeletonDefinition, +) -> None: + with pytest.raises(SkeletonResolutionError) as exc_info: + resolve_skeleton( + simple_definition, + ["nose"], + ) + + message = str(exc_info.value) + assert "shoulder" in message + assert "hip" in message + + +def test_resolve_packet_skeleton_returns_none_without_names() -> None: + packet = PacketStub( + keypoint_names=None, + skeleton_id="test.simple", + skeleton_edges=(("nose", "shoulder"),), + ) + + assert resolve_packet_skeleton(packet) is None + + +def test_resolve_packet_skeleton_returns_none_without_edges() -> None: + packet = PacketStub( + keypoint_names=["nose", "shoulder"], + skeleton_id="test.simple", + skeleton_edges=None, + ) + + assert resolve_packet_skeleton(packet) is None + + +def test_resolve_packet_skeleton_constructs_topology() -> None: + packet = PacketStub( + keypoint_names=["shoulder", "nose"], + skeleton_id="test.simple", + skeleton_edges=(("nose", "shoulder"),), + ) + + resolved = resolve_packet_skeleton(packet) + + assert resolved is not None + assert resolved.edges == ((1, 0),) + assert resolved.keypoint_names == ( + "shoulder", + "nose", + ) From 199ffc52c5da3730816fcb5bc6d84847b287ad90 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:13:16 +0200 Subject: [PATCH 13/18] Split display tests and add shared frame fixture Refactors display unit tests by moving bbox/keypoint/pose and most skeleton rendering coverage into a new `tests/display/test_draw.py`, while keeping tiling/color tests in `test_display.py`. Adds a shared `test_frame` fixture in `tests/display/conftest.py` to remove duplicated frame helpers and standardize test data creation. Also extends skeleton tests in `test_display.py` with gradient-mode behavior checks, including missing color handling and successful gradient rendering. --- tests/display/conftest.py | 10 ++ tests/display/test_display.py | 211 ++++++++++-------------- tests/display/test_draw.py | 297 ++++++++++++++++++++++++++++++++++ 3 files changed, 394 insertions(+), 124 deletions(-) create mode 100644 tests/display/conftest.py create mode 100644 tests/display/test_draw.py diff --git a/tests/display/conftest.py b/tests/display/conftest.py new file mode 100644 index 00000000..849abc71 --- /dev/null +++ b/tests/display/conftest.py @@ -0,0 +1,10 @@ +import numpy as np +import pytest + + +@pytest.fixture +def test_frame(h, w, c=3, value=0, dtype=np.uint8): + """Helper to create test frames with predictable content.""" + if c == 1: + return (np.ones((h, w), dtype=dtype) * value).astype(dtype) + return (np.ones((h, w, c), dtype=dtype) * value).astype(dtype) diff --git a/tests/display/test_display.py b/tests/display/test_display.py index 9aaa1f10..f3097162 100644 --- a/tests/display/test_display.py +++ b/tests/display/test_display.py @@ -1,26 +1,21 @@ import numpy as np import pytest -from dlclivegui.display import ( # noqa: E402 +from dlclivegui.config import ( + SkeletonColorMode, + SkeletonStyle, +) +from dlclivegui.display import ( compute_tile_info, compute_tiling_geometry, create_tiled_frame, - draw_bbox, - draw_keypoints, - draw_pose, ) from dlclivegui.display.display import keypoint_colors_bgr +from dlclivegui.display.skeleton import ResolvedSkeleton, SkeletonRenderCode, draw_skeleton pytestmark = pytest.mark.unit -def _frame(h, w, c=3, value=0, dtype=np.uint8): - """Helper to create test frames with predictable content.""" - if c == 1: - return (np.ones((h, w), dtype=dtype) * value).astype(dtype) - return (np.ones((h, w, c), dtype=dtype) * value).astype(dtype) - - def test_compute_tiling_geometry_empty(): cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry({}) assert cam_ids == [] @@ -28,8 +23,8 @@ def test_compute_tiling_geometry_empty(): assert (tile_w, tile_h) == (640, 480) -def test_compute_tiling_geometry_single_frame_respects_max_canvas_and_min_tile(): - frames = {"camA": _frame(480, 640, 3)} +def test_compute_tiling_geometry_single_frame_respects_max_canvas_and_min_tile(test_frame): + frames = {"camA": test_frame(480, 640, 3)} cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(1200, 800)) assert cam_ids == ["camA"] assert (rows, cols) == (1, 1) @@ -39,8 +34,8 @@ def test_compute_tiling_geometry_single_frame_respects_max_canvas_and_min_tile() assert tile_h <= 800 -def test_compute_tiling_geometry_two_frames_is_1x2(): - frames = {"camB": _frame(480, 640, 3), "camA": _frame(480, 640, 3)} +def test_compute_tiling_geometry_two_frames_is_1x2(test_frame): + frames = {"camB": test_frame(480, 640, 3), "camA": test_frame(480, 640, 3)} cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(1200, 800)) # Preserve insertion/display order, do not sort by camera ID. @@ -49,8 +44,8 @@ def test_compute_tiling_geometry_two_frames_is_1x2(): assert tile_w >= 160 and tile_h >= 120 -def test_compute_tiling_geometry_three_frames_is_2x2(): - frames = {"c3": _frame(480, 640, 3), "c1": _frame(480, 640, 3), "c2": _frame(480, 640, 3)} +def test_compute_tiling_geometry_three_frames_is_2x2(test_frame): + frames = {"c3": test_frame(480, 640, 3), "c1": test_frame(480, 640, 3), "c2": test_frame(480, 640, 3)} cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(1200, 800)) # Preserve insertion/display order. @@ -59,12 +54,12 @@ def test_compute_tiling_geometry_three_frames_is_2x2(): assert tile_w >= 160 and tile_h >= 120 -def test_compute_tiling_geometry_reference_aspect_is_first_display_order_cam(): +def test_compute_tiling_geometry_reference_aspect_is_first_display_order_cam(test_frame): # camB is first in insertion/display order and has aspect 0.5. # camA has aspect 2.0. frames = { - "camB": _frame(400, 200, 3), # aspect = 200 / 400 = 0.5 - "camA": _frame(200, 400, 3), # aspect = 400 / 200 = 2.0 + "camB": test_frame(400, 200, 3), # aspect = 200 / 400 = 0.5 + "camA": test_frame(200, 400, 3), # aspect = 400 / 200 = 2.0 } cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(1200, 800)) @@ -161,9 +156,9 @@ def test_create_tiled_frame_empty_returns_default_canvas(): assert np.all(out == 0) -def test_create_tiled_frame_grayscale_converted_and_labeled(): +def test_create_tiled_frame_grayscale_converted_and_labeled(test_frame): # Use a zero grayscale frame; any nonzero in output likely comes from putText label - frames = {"camA": _frame(120, 160, c=1, value=0)} + frames = {"camA": test_frame(120, 160, c=1, value=0)} out = create_tiled_frame(frames, max_canvas=(320, 240)) assert out.ndim == 3 and out.shape[2] == 3 @@ -171,9 +166,9 @@ def test_create_tiled_frame_grayscale_converted_and_labeled(): assert np.any(out != 0) -def test_create_tiled_frame_bgra_converted_and_labeled(): +def test_create_tiled_frame_bgra_converted_and_labeled(test_frame): # BGRA frame - bgra = _frame(120, 160, c=4, value=0) + bgra = test_frame(120, 160, c=4, value=0) frames = {"camA": bgra} out = create_tiled_frame(frames, max_canvas=(320, 240)) @@ -181,10 +176,10 @@ def test_create_tiled_frame_bgra_converted_and_labeled(): assert np.any(out != 0) -def test_create_tiled_frame_canvas_shape_matches_geometry(): +def test_create_tiled_frame_canvas_shape_matches_geometry(test_frame): frames = { - "camA": _frame(200, 400, 3, value=0), - "camB": _frame(200, 400, 3, value=0), + "camA": test_frame(200, 400, 3, value=0), + "camB": test_frame(200, 400, 3, value=0), } cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(800, 400)) out = create_tiled_frame(frames, max_canvas=(800, 400)) @@ -193,12 +188,12 @@ def test_create_tiled_frame_canvas_shape_matches_geometry(): assert np.any(out != 0) -def test_compute_tile_info_offset_and_scale_matches_tiling(): +def test_compute_tile_info_offset_and_scale_matches_tiling(test_frame): # 2 frames => 1x2 tiling, preserving insertion/display order: ["cam2", "cam1"] - frames = {"cam2": _frame(200, 400, 3), "cam1": _frame(200, 400, 3)} + frames = {"cam2": test_frame(200, 400, 3), "cam1": test_frame(200, 400, 3)} cam_ids, rows, cols, tile_w, tile_h = compute_tiling_geometry(frames, max_canvas=(1200, 800)) - original = _frame(200, 400, 3) + original = test_frame(200, 400, 3) (ox, oy), (sx, sy) = compute_tile_info("cam2", original, frames, max_canvas=(1200, 800)) assert cam_ids == ["cam2", "cam1"] @@ -211,100 +206,6 @@ def test_compute_tile_info_offset_and_scale_matches_tiling(): assert sy == pytest.approx(tile_h / 200) -def test_draw_bbox_invalid_bbox_returns_same_object(): - frame = _frame(100, 100, 3) - out = draw_bbox(frame, (10, 10, 10, 20), (0, 255, 0)) # x0 == x1 invalid - assert out is frame # passthrough for invalid bbox - - -def test_draw_bbox_draws_rectangle_and_clips(): - frame = _frame(60, 60, 3, value=0) - color = (0, 0, 255) # red in BGR - - # bbox partially outside original; with scale/offset it will be shifted/clipped - out = draw_bbox( - frame, - bbox_xyxy=(-10, -10, 50, 50), - color_bgr=color, - offset=(5, 5), - scale=(1.0, 1.0), - ) - - assert out is not frame - # Should have drawn something - assert np.any(out != frame) - # At least some red pixels should exist (allowing for thickness) - assert np.any((out[:, :, 2] > 0) & (out[:, :, 0] == 0) & (out[:, :, 1] == 0)) - - -def test_draw_keypoints_filters_by_cutoff_and_nans_and_draws(): - overlay = _frame(80, 80, 3, value=0).copy() - cmap = __import__("matplotlib.pyplot").pyplot.get_cmap("viridis") - - # keypoints: (x, y, conf) - kpts = np.array( - [ - [10.0, 10.0, 0.2], # below cutoff -> ignored - [np.nan, 15.0, 0.99], # NaN -> ignored - [20.0, np.nan, 0.99], # NaN -> ignored - [30.0, 30.0, 0.99], # should draw - ], - dtype=float, - ) - - draw_keypoints( - overlay=overlay, - p_cutoff=0.9, - sx=1.0, - ox=0, - sy=1.0, - oy=0, - radius=3, - cmap=cmap, - keypoints=kpts, - marker=None, # circle - ) - - assert np.any(overlay != 0) # something drawn - - -def test_draw_pose_single_animal_draws_when_conf_above_cutoff(): - frame = _frame(100, 100, 3, value=0) - pose = np.array( - [ - [10.0, 10.0, 0.95], - [20.0, 20.0, 0.95], - ], - dtype=float, - ) - out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) - assert out is not frame - assert np.any(out != frame) - - -def test_draw_pose_single_animal_no_draw_below_cutoff(): - frame = _frame(100, 100, 3, value=0) - pose = np.array([[10.0, 10.0, 0.1]], dtype=float) - out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) - # overlay returned, but should be identical if nothing is drawn - assert np.array_equal(out, frame) - - -def test_draw_pose_multi_animal_draws_distinct_markers(): - frame = _frame(120, 120, 3, value=0) - # A x N x 3 : 2 animals, 1 keypoint each - pose = np.array( - [ - [[30.0, 30.0, 0.99]], - [[60.0, 60.0, 0.99]], - ], - dtype=float, - ) - out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) - assert out is not frame - assert np.any(out != frame) - - def test_keypoint_colors_returns_requested_count() -> None: colors = keypoint_colors_bgr("viridis", 5) @@ -336,3 +237,65 @@ def test_keypoint_colors_reject_negative_count() -> None: match="must be non-negative", ): keypoint_colors_bgr("viridis", -1) + + +def test_draw_skeleton_requires_gradient_colors( + resolved_skeleton: ResolvedSkeleton, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 10.0, 0.9], + [40.0, 40.0, 0.9], + ], + dtype=np.float32, + ) + style = SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + style, + p_cutoff=0.5, + keypoint_colors=None, + ) + + assert result.code == (SkeletonRenderCode.COLOR_COUNT_MISMATCH) + + +def test_draw_skeleton_renders_gradient( + resolved_skeleton: ResolvedSkeleton, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 25.0, 0.9], + [40.0, 25.0, 0.9], + ], + dtype=np.float32, + ) + style = SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + thickness=2, + gradient_steps=8, + scale_with_zoom=False, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + style, + p_cutoff=0.5, + keypoint_colors=( + (255, 0, 0), + (0, 0, 255), + ), + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 1 + assert np.any(frame != 0) diff --git a/tests/display/test_draw.py b/tests/display/test_draw.py new file mode 100644 index 00000000..9d0e923d --- /dev/null +++ b/tests/display/test_draw.py @@ -0,0 +1,297 @@ +import numpy as np +import pytest + +from dlclivegui.display import ( + draw_bbox, + draw_keypoints, + draw_pose, +) +from dlclivegui.display.skeleton import ( + ResolvedSkeleton, + SkeletonColorMode, + SkeletonDefinition, + SkeletonEdge, + SkeletonRenderCode, + SkeletonStyle, + draw_skeleton, +) + + +def _frame(h, w, c=3, value=0, dtype=np.uint8): + """Helper to create test frames with predictable content.""" + if c == 1: + return (np.ones((h, w), dtype=dtype) * value).astype(dtype) + return (np.ones((h, w, c), dtype=dtype) * value).astype(dtype) + + +def test_draw_bbox_invalid_bbox_returns_same_object(): + frame = _frame(100, 100, 3) + out = draw_bbox(frame, (10, 10, 10, 20), (0, 255, 0)) # x0 == x1 invalid + assert out is frame # passthrough for invalid bbox + + +def test_draw_bbox_draws_rectangle_and_clips(): + frame = _frame(60, 60, 3, value=0) + color = (0, 0, 255) # red in BGR + + # bbox partially outside original; with scale/offset it will be shifted/clipped + out = draw_bbox( + frame, + bbox_xyxy=(-10, -10, 50, 50), + color_bgr=color, + offset=(5, 5), + scale=(1.0, 1.0), + ) + + assert out is not frame + # Should have drawn something + assert np.any(out != frame) + # At least some red pixels should exist (allowing for thickness) + assert np.any((out[:, :, 2] > 0) & (out[:, :, 0] == 0) & (out[:, :, 1] == 0)) + + +def test_draw_keypoints_filters_by_cutoff_and_nans_and_draws(): + overlay = _frame(80, 80, 3, value=0).copy() + cmap = __import__("matplotlib.pyplot").pyplot.get_cmap("viridis") + + # keypoints: (x, y, conf) + kpts = np.array( + [ + [10.0, 10.0, 0.2], # below cutoff -> ignored + [np.nan, 15.0, 0.99], # NaN -> ignored + [20.0, np.nan, 0.99], # NaN -> ignored + [30.0, 30.0, 0.99], # should draw + ], + dtype=float, + ) + + draw_keypoints( + overlay=overlay, + p_cutoff=0.9, + sx=1.0, + ox=0, + sy=1.0, + oy=0, + radius=3, + cmap=cmap, + keypoints=kpts, + marker=None, # circle + ) + + assert np.any(overlay != 0) # something drawn + + +def test_draw_pose_single_animal_draws_when_conf_above_cutoff(): + frame = _frame(100, 100, 3, value=0) + pose = np.array( + [ + [10.0, 10.0, 0.95], + [20.0, 20.0, 0.95], + ], + dtype=float, + ) + out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) + assert out is not frame + assert np.any(out != frame) + + +def test_draw_pose_single_animal_no_draw_below_cutoff(): + frame = _frame(100, 100, 3, value=0) + pose = np.array([[10.0, 10.0, 0.1]], dtype=float) + out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) + # overlay returned, but should be identical if nothing is drawn + assert np.array_equal(out, frame) + + +def test_draw_pose_multi_animal_draws_distinct_markers(): + frame = _frame(120, 120, 3, value=0) + # A x N x 3 : 2 animals, 1 keypoint each + pose = np.array( + [ + [[30.0, 30.0, 0.99]], + [[60.0, 60.0, 0.99]], + ], + dtype=float, + ) + out = draw_pose(frame, pose, p_cutoff=0.9, colormap="viridis", offset=(0, 0), scale=(1.0, 1.0)) + assert out is not frame + assert np.any(out != frame) + + +@pytest.fixture +def resolved_skeleton() -> ResolvedSkeleton: + definition = SkeletonDefinition( + identifier="test.line", + display_name="Line", + edges=(SkeletonEdge("a", "b"),), + ) + + return ResolvedSkeleton( + definition=definition, + keypoint_names=("a", "b"), + edges=((0, 1),), + ) + + +@pytest.fixture +def solid_style() -> SkeletonStyle: + return SkeletonStyle( + color_mode=SkeletonColorMode.SOLID, + color_bgr=(0, 255, 0), + thickness=2, + scale_with_zoom=False, + ) + + +def test_draw_skeleton_renders_single_pose( + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 10.0, 0.9], + [40.0, 40.0, 0.9], + ], + dtype=np.float32, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 1 + assert np.any(frame != 0) + + +def test_draw_skeleton_renders_multiple_people( + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + poses = np.array( + [ + [ + [5.0, 5.0, 0.9], + [20.0, 20.0, 0.9], + ], + [ + [25.0, 5.0, 0.9], + [40.0, 20.0, 0.9], + ], + ], + dtype=np.float32, + ) + + result = draw_skeleton( + frame, + poses, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 2 + assert np.any(frame != 0) + + +def test_draw_skeleton_skips_low_confidence_edge( + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 10.0, 0.9], + [40.0, 40.0, 0.1], + ], + dtype=np.float32, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 0 + assert not np.any(frame) + + +def test_draw_skeleton_skips_non_finite_edge( + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [np.nan, 10.0, 0.9], + [40.0, 40.0, 0.9], + ], + dtype=np.float32, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 0 + assert not np.any(frame) + + +@pytest.mark.parametrize( + "pose", + [ + np.zeros((2,), dtype=np.float32), + np.zeros((1, 2, 3, 4), dtype=np.float32), + np.zeros((2, 2), dtype=np.float32), + ], +) +def test_draw_skeleton_rejects_invalid_pose_shape( + pose: np.ndarray, + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == SkeletonRenderCode.INVALID_POSE + + +def test_draw_skeleton_reports_keypoint_count_mismatch( + resolved_skeleton: ResolvedSkeleton, + solid_style: SkeletonStyle, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.zeros((3, 3), dtype=np.float32) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + solid_style, + p_cutoff=0.5, + ) + + assert result.code == (SkeletonRenderCode.KEYPOINT_COUNT_MISMATCH) From 0e331c57da35d3d05a23339728373d5407f3ee86 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:24:39 +0200 Subject: [PATCH 14/18] Add overlay renderer tests and tidy display tests Introduces a new `tests/display/test_overlays.py` suite covering overlay rendering behavior, including frame immutability, skeleton metadata warning deduplication/reset, skeleton-only rendering, and skeleton resolution caching. Refactors the shared `test_frame` fixture into a typed frame-factory callable for clearer test setup, and moves skeleton gradient rendering assertions from `test_display.py` into `test_draw.py` to keep skeleton drawing coverage consolidated. --- tests/display/conftest.py | 26 +++- tests/display/test_display.py | 67 ----------- tests/display/test_draw.py | 62 ++++++++++ tests/display/test_overlays.py | 212 +++++++++++++++++++++++++++++++++ 4 files changed, 295 insertions(+), 72 deletions(-) create mode 100644 tests/display/test_overlays.py diff --git a/tests/display/conftest.py b/tests/display/conftest.py index 849abc71..88c22913 100644 --- a/tests/display/conftest.py +++ b/tests/display/conftest.py @@ -1,10 +1,26 @@ +# tests/display/conftest.py +from __future__ import annotations + +from collections.abc import Callable + import numpy as np import pytest @pytest.fixture -def test_frame(h, w, c=3, value=0, dtype=np.uint8): - """Helper to create test frames with predictable content.""" - if c == 1: - return (np.ones((h, w), dtype=dtype) * value).astype(dtype) - return (np.ones((h, w, c), dtype=dtype) * value).astype(dtype) +def test_frame() -> Callable[..., np.ndarray]: + """Return a factory for creating predictable test frames.""" + + def make_frame( + height: int, + width: int, + channels: int = 3, + value: int | float = 0, + dtype: np.dtype = np.uint8, + ) -> np.ndarray: + if not isinstance(dtype, np.dtype): + dtype = np.dtype(dtype) + shape = (height, width) if channels == 1 else (height, width, channels) + return np.full(shape, value, dtype=dtype) + + return make_frame diff --git a/tests/display/test_display.py b/tests/display/test_display.py index f3097162..ab6c0234 100644 --- a/tests/display/test_display.py +++ b/tests/display/test_display.py @@ -1,17 +1,12 @@ import numpy as np import pytest -from dlclivegui.config import ( - SkeletonColorMode, - SkeletonStyle, -) from dlclivegui.display import ( compute_tile_info, compute_tiling_geometry, create_tiled_frame, ) from dlclivegui.display.display import keypoint_colors_bgr -from dlclivegui.display.skeleton import ResolvedSkeleton, SkeletonRenderCode, draw_skeleton pytestmark = pytest.mark.unit @@ -237,65 +232,3 @@ def test_keypoint_colors_reject_negative_count() -> None: match="must be non-negative", ): keypoint_colors_bgr("viridis", -1) - - -def test_draw_skeleton_requires_gradient_colors( - resolved_skeleton: ResolvedSkeleton, -) -> None: - frame = np.zeros((50, 50, 3), dtype=np.uint8) - pose = np.array( - [ - [10.0, 10.0, 0.9], - [40.0, 40.0, 0.9], - ], - dtype=np.float32, - ) - style = SkeletonStyle( - color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, - ) - - result = draw_skeleton( - frame, - pose, - resolved_skeleton, - style, - p_cutoff=0.5, - keypoint_colors=None, - ) - - assert result.code == (SkeletonRenderCode.COLOR_COUNT_MISMATCH) - - -def test_draw_skeleton_renders_gradient( - resolved_skeleton: ResolvedSkeleton, -) -> None: - frame = np.zeros((50, 50, 3), dtype=np.uint8) - pose = np.array( - [ - [10.0, 25.0, 0.9], - [40.0, 25.0, 0.9], - ], - dtype=np.float32, - ) - style = SkeletonStyle( - color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, - thickness=2, - gradient_steps=8, - scale_with_zoom=False, - ) - - result = draw_skeleton( - frame, - pose, - resolved_skeleton, - style, - p_cutoff=0.5, - keypoint_colors=( - (255, 0, 0), - (0, 0, 255), - ), - ) - - assert result.code == SkeletonRenderCode.RENDERED - assert result.edges_drawn == 1 - assert np.any(frame != 0) diff --git a/tests/display/test_draw.py b/tests/display/test_draw.py index 9d0e923d..b0ddb690 100644 --- a/tests/display/test_draw.py +++ b/tests/display/test_draw.py @@ -295,3 +295,65 @@ def test_draw_skeleton_reports_keypoint_count_mismatch( ) assert result.code == (SkeletonRenderCode.KEYPOINT_COUNT_MISMATCH) + + +def test_draw_skeleton_requires_gradient_colors( + resolved_skeleton: ResolvedSkeleton, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 10.0, 0.9], + [40.0, 40.0, 0.9], + ], + dtype=np.float32, + ) + style = SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + style, + p_cutoff=0.5, + keypoint_colors=None, + ) + + assert result.code == (SkeletonRenderCode.COLOR_COUNT_MISMATCH) + + +def test_draw_skeleton_renders_gradient( + resolved_skeleton: ResolvedSkeleton, +) -> None: + frame = np.zeros((50, 50, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 25.0, 0.9], + [40.0, 25.0, 0.9], + ], + dtype=np.float32, + ) + style = SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + thickness=2, + gradient_steps=8, + scale_with_zoom=False, + ) + + result = draw_skeleton( + frame, + pose, + resolved_skeleton, + style, + p_cutoff=0.5, + keypoint_colors=( + (255, 0, 0), + (0, 0, 255), + ), + ) + + assert result.code == SkeletonRenderCode.RENDERED + assert result.edges_drawn == 1 + assert np.any(frame != 0) diff --git a/tests/display/test_overlays.py b/tests/display/test_overlays.py new file mode 100644 index 00000000..158c0d41 --- /dev/null +++ b/tests/display/test_overlays.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np + +from dlclivegui.config import ( + SkeletonStyle, +) +from dlclivegui.display.overlays import ( + BoundingBoxOverlaySettings, + OverlayRenderer, + OverlaySettings, + PoseOverlaySettings, + SkeletonOverlaySettings, +) + + +@dataclass +class PacketStub: + keypoint_names: list[str] | None + skeleton_id: str | None + skeleton_edges: tuple[tuple[str, str], ...] | None + + +def make_settings( + *, + show_pose: bool = False, + show_skeleton: bool = False, + show_bbox: bool = False, + style: SkeletonStyle | None = None, +) -> OverlaySettings: + return OverlaySettings( + pose=PoseOverlaySettings( + visible=show_pose, + p_cutoff=0.5, + colormap="viridis", + ), + skeleton=SkeletonOverlaySettings( + visible=show_skeleton, + style=style or SkeletonStyle(), + ), + bounding_box=BoundingBoxOverlaySettings( + visible=show_bbox, + coordinates=(1, 1, 8, 8), + color_bgr=(0, 0, 255), + ), + ) + + +def make_pose() -> np.ndarray: + return np.array( + [ + [2.0, 2.0, 0.9], + [8.0, 8.0, 0.9], + ], + dtype=np.float32, + ) + + +def make_packet() -> PacketStub: + return PacketStub( + keypoint_names=["a", "b"], + skeleton_id="test.line", + skeleton_edges=(("a", "b"),), + ) + + +def test_renderer_does_not_modify_input_frame() -> None: + renderer = OverlayRenderer() + frame = np.zeros((10, 10, 3), dtype=np.uint8) + original = frame.copy() + + result = renderer.render( + frame, + pose=None, + packet=None, + overlay_settings=make_settings(), + ) + + assert np.array_equal(frame, original) + assert result.frame is not frame + + +def test_renderer_warns_once_for_missing_skeleton_metadata() -> None: + renderer = OverlayRenderer() + frame = np.zeros((10, 10, 3), dtype=np.uint8) + + first = renderer.render( + frame, + pose=make_pose(), + packet=None, + overlay_settings=make_settings( + show_skeleton=True, + ), + ) + second = renderer.render( + frame, + pose=make_pose(), + packet=None, + overlay_settings=make_settings( + show_skeleton=True, + ), + ) + + assert first.warning is not None + assert second.warning is None + + +def test_renderer_does_not_warn_when_skeleton_is_hidden() -> None: + renderer = OverlayRenderer() + frame = np.zeros((10, 10, 3), dtype=np.uint8) + + result = renderer.render( + frame, + pose=make_pose(), + packet=None, + overlay_settings=make_settings( + show_skeleton=False, + ), + ) + + assert result.warning is None + + +def test_renderer_renders_skeleton_without_pose_markers() -> None: + renderer = OverlayRenderer() + frame = np.zeros((12, 12, 3), dtype=np.uint8) + + result = renderer.render( + frame, + pose=make_pose(), + packet=make_packet(), + overlay_settings=make_settings( + show_pose=False, + show_skeleton=True, + ), + ) + + assert result.warning is None + assert np.any(result.frame != 0) + + +def test_renderer_clear_runtime_state_resets_warning_deduplication() -> None: + renderer = OverlayRenderer() + frame = np.zeros((12, 12, 3), dtype=np.uint8) + settings = make_settings(show_skeleton=True) + pose = make_pose() + + first = renderer.render( + frame, + pose=pose, + packet=None, + overlay_settings=settings, + ) + second = renderer.render( + frame, + pose=pose, + packet=None, + overlay_settings=settings, + ) + + renderer.clear_runtime_state() + + third = renderer.render( + frame, + pose=pose, + packet=None, + overlay_settings=settings, + ) + + expected_warning = "Skeleton metadata is unavailable for this pose output." + + assert first.warning == expected_warning + assert second.warning is None + assert third.warning == expected_warning + + +def test_renderer_caches_skeleton_resolution( + monkeypatch, +) -> None: + renderer = OverlayRenderer() + frame = np.zeros((12, 12, 3), dtype=np.uint8) + packet = make_packet() + calls = 0 + + from dlclivegui.display import overlays + + original = overlays.resolve_packet_skeleton + + def counting_resolver(value): + nonlocal calls + calls += 1 + return original(value) + + monkeypatch.setattr( + overlays, + "resolve_packet_skeleton", + counting_resolver, + ) + + for _ in range(3): + renderer.render( + frame, + pose=make_pose(), + packet=packet, + overlay_settings=make_settings( + show_skeleton=True, + ), + ) + + assert calls == 1 From c85a6db7accecf5873eddd68ad849b0954c8c2b7 Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:24:58 +0200 Subject: [PATCH 15/18] Add main display skeleton overlay tests Introduce a new test module for main window display behavior focused on skeleton overlays. The tests cover default skeleton UI values, restoring skeleton settings from config, collecting visualization settings from UI, delegating frame rendering to the overlay renderer, clearing overlay runtime state when inference starts, and an end-to-end renderer check that verifies skeleton lines are drawn without mutating the input frame. --- tests/gui/main_window/test_main_display.py | 203 +++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 tests/gui/main_window/test_main_display.py diff --git a/tests/gui/main_window/test_main_display.py b/tests/gui/main_window/test_main_display.py new file mode 100644 index 00000000..2df01a47 --- /dev/null +++ b/tests/gui/main_window/test_main_display.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import numpy as np +from PySide6.QtGui import QPixmap + +from dlclivegui.config import ( + SkeletonColorMode, + SkeletonStyle, +) +from dlclivegui.display.overlays import ( + BoundingBoxOverlaySettings, + OverlayRenderer, + OverlayRenderResult, + OverlaySettings, + PoseOverlaySettings, + SkeletonOverlaySettings, +) +from dlclivegui.gui.misc import color_dropdowns as color_ui + + +def test_skeleton_controls_use_default_settings(window) -> None: + assert not window.show_skeleton_checkbox.isChecked() + assert window.skeleton_thickness_spin.value() == 2 + + mode, color = ( + window._skeleton_style_from_ui().color_mode, + window._skeleton_style_from_ui().color_bgr, + ) + + assert mode == SkeletonColorMode.SOLID + assert color == (0, 255, 255) + + +def test_apply_config_restores_skeleton_settings( + window, +) -> None: + config = window._config.model_copy(deep=True) + config.visualization.show_pose = False + config.visualization.show_skeleton = True + config.visualization.skeleton_style = SkeletonStyle( + color_mode=SkeletonColorMode.GRADIENT_KEYPOINTS, + thickness=6, + gradient_steps=24, + scale_with_zoom=False, + ) + + window._apply_config(config) + + assert not window.show_predictions_checkbox.isChecked() + assert window.show_skeleton_checkbox.isChecked() + assert window.skeleton_thickness_spin.value() == 6 + + style = window._skeleton_style_from_ui() + assert style.color_mode == SkeletonColorMode.GRADIENT_KEYPOINTS + assert style.gradient_steps == 24 + assert style.scale_with_zoom is False + + +def test_visualization_settings_from_ui_include_skeleton( + window, +) -> None: + window.show_predictions_checkbox.setChecked(False) + window.show_skeleton_checkbox.setChecked(True) + window.skeleton_thickness_spin.setValue(5) + + color_ui.set_skeleton_combo_from_style( + window.skeleton_color_combo, + mode="solid", + color=(0, 255, 0), + ) + + settings = window._visualization_settings_from_ui() + + assert settings.show_pose is False + assert settings.show_skeleton is True + assert settings.skeleton_style.thickness == 5 + assert settings.skeleton_style.color_bgr == (0, 255, 0) + + +def test_update_video_display_delegates_to_overlay_renderer( + window, + monkeypatch, +) -> None: + frame = np.zeros((20, 30, 3), dtype=np.uint8) + calls = [] + + def fake_render( + rendered_frame, + **kwargs, + ): + calls.append((rendered_frame, kwargs)) + return OverlayRenderResult( + frame=rendered_frame.copy(), + ) + + monkeypatch.setattr( + window._overlay_renderer, + "render", + fake_render, + ) + monkeypatch.setattr( + "dlclivegui.gui.main_window.frame_to_pixmap", + lambda _frame, _size: QPixmap(10, 10), + ) + + window._update_video_display(frame) + + assert len(calls) == 1 + assert calls[0][1]["offset"] == window._dlc_tile_offset + assert calls[0][1]["scale"] == window._dlc_tile_scale + + +def test_start_inference_clears_overlay_runtime_state( + window, + monkeypatch, +) -> None: + cleared = False + + def fake_clear() -> None: + nonlocal cleared + cleared = True + + monkeypatch.setattr( + window._overlay_renderer, + "clear_runtime_state", + fake_clear, + ) + monkeypatch.setattr( + window.multi_camera_controller, + "is_running", + lambda: True, + ) + monkeypatch.setattr( + window, + "_configure_dlc", + lambda: True, + ) + monkeypatch.setattr( + window._dlc, + "reset", + lambda: None, + ) + + window._start_inference() + + assert cleared + + +@dataclass +class PacketStub: + keypoint_names: list[str] | None + skeleton_id: str | None + skeleton_edges: tuple[tuple[str, str], ...] | None + + +def test_end_to_end_skeleton_overlay() -> None: + renderer = OverlayRenderer() + frame = np.zeros((64, 64, 3), dtype=np.uint8) + pose = np.array( + [ + [10.0, 10.0, 0.9], + [50.0, 50.0, 0.9], + ], + dtype=np.float32, + ) + packet = PacketStub( + keypoint_names=["start", "end"], + skeleton_id="test.line", + skeleton_edges=(("start", "end"),), + ) + settings = OverlaySettings( + pose=PoseOverlaySettings( + visible=False, + p_cutoff=0.5, + colormap="viridis", + ), + skeleton=SkeletonOverlaySettings( + visible=True, + style=SkeletonStyle( + color_bgr=(0, 255, 0), + thickness=2, + scale_with_zoom=False, + ), + ), + bounding_box=BoundingBoxOverlaySettings( + visible=False, + coordinates=(0, 0, 0, 0), + color_bgr=(0, 0, 255), + ), + ) + + result = renderer.render( + frame, + pose=pose, + packet=packet, + overlay_settings=settings, + ) + + assert result.warning is None + assert np.any(result.frame != 0) + assert not np.any(frame) From 01ac1fa5ac711df4396156dc33ac9053f0410a3b Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:25:22 +0200 Subject: [PATCH 16/18] Add UI block tests for skeleton color combo Adds a new `tests/gui/ui_blocks/test_ui_blocks.py` suite to validate skeleton color dropdown behavior. The tests cover presence of the gradient option, round-tripping both gradient and solid styles, fallback to the default solid color when an unknown color is provided, and icon creation when enabled. --- tests/gui/ui_blocks/test_ui_blocks.py | 99 +++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/gui/ui_blocks/test_ui_blocks.py diff --git a/tests/gui/ui_blocks/test_ui_blocks.py b/tests/gui/ui_blocks/test_ui_blocks.py new file mode 100644 index 00000000..a01bd8d4 --- /dev/null +++ b/tests/gui/ui_blocks/test_ui_blocks.py @@ -0,0 +1,99 @@ +from __future__ import annotations + +from dlclivegui.display import BBoxColors +from dlclivegui.gui.misc.color_dropdowns import ( + get_skeleton_style_from_combo, + make_skeleton_color_combo, + set_skeleton_combo_from_style, +) + + +def test_skeleton_combo_contains_gradient_and_all_colors( + qtbot, +) -> None: + combo = make_skeleton_color_combo( + BBoxColors, + include_icons=False, + ) + qtbot.addWidget(combo) + + assert combo.count() == 1 + len(BBoxColors) + + gradient = combo.itemData(0) + assert gradient == { + "mode": "gradient_keypoints", + "color": None, + } + + +def test_skeleton_combo_round_trips_gradient_style( + qtbot, +) -> None: + combo = make_skeleton_color_combo( + BBoxColors, + include_icons=False, + ) + qtbot.addWidget(combo) + + set_skeleton_combo_from_style( + combo, + mode="gradient_keypoints", + color=None, + ) + + assert get_skeleton_style_from_combo(combo) == ( + "gradient_keypoints", + None, + ) + + +def test_skeleton_combo_round_trips_solid_color( + qtbot, +) -> None: + combo = make_skeleton_color_combo( + BBoxColors, + include_icons=False, + ) + qtbot.addWidget(combo) + + set_skeleton_combo_from_style( + combo, + mode="solid", + color=(0, 255, 0), + ) + + assert get_skeleton_style_from_combo(combo) == ( + "solid", + (0, 255, 0), + ) + + +def test_skeleton_combo_falls_back_to_first_solid_color( + qtbot, +) -> None: + combo = make_skeleton_color_combo( + BBoxColors, + include_icons=False, + ) + qtbot.addWidget(combo) + + set_skeleton_combo_from_style( + combo, + mode="solid", + color=(1, 2, 3), + ) + + mode, color = get_skeleton_style_from_combo(combo) + + assert mode == "solid" + assert color == BBoxColors.RED.value + + +def test_skeleton_combo_can_create_icons(qtbot) -> None: + combo = make_skeleton_color_combo( + BBoxColors, + include_icons=True, + ) + qtbot.addWidget(combo) + + assert not combo.itemIcon(0).isNull() From 02e758bab8596ac6d0283b29272f0c2e596129aa Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:25:32 +0200 Subject: [PATCH 17/18] Add tests for DLC pose metadata loading Expand `test_dlc_processor.py` with focused unit tests for `_load_pose_metadata_from_dlc_config()`. The new cases verify successful loading of bodyparts/skeleton/task metadata, clearing of stale metadata when config is empty, support for bodyparts without skeleton data, and validation failures for duplicate bodyparts and skeleton links to unknown bodyparts. --- tests/services/test_dlc_processor.py | 99 +++++++++++++++++++++++++++- 1 file changed, 96 insertions(+), 3 deletions(-) diff --git a/tests/services/test_dlc_processor.py b/tests/services/test_dlc_processor.py index 28b192b1..401ab96b 100644 --- a/tests/services/test_dlc_processor.py +++ b/tests/services/test_dlc_processor.py @@ -2,6 +2,7 @@ import queue import threading +from types import SimpleNamespace import numpy as np import pytest @@ -10,9 +11,7 @@ # from dlclivegui.config import DLCProcessorSettings from dlclivegui.processors.processor_utils import ProcessorSpec -from dlclivegui.services.dlc_processor import ( - DLCLiveProcessor, -) +from dlclivegui.services.dlc_processor import DLCLiveProcessor from dlclivegui.services.inference.base import ( ProcessorStats, WorkerState, @@ -385,3 +384,97 @@ def test_queue_accounting_clears_after_processed_frame(qtbot, monkeypatch_dlcliv finally: proc.reset() + + +def test_load_pose_metadata_from_dlc_config() -> None: + processor = DLCLiveProcessor() + processor._dlc = SimpleNamespace( + cfg={ + "Task": "mouse", + "bodyparts": [ + "nose", + "shoulder", + "tail", + ], + "skeleton": [ + ["nose", "shoulder"], + ["shoulder", "tail"], + ], + } + ) + + processor._load_pose_metadata_from_dlc_config() + + assert processor._keypoint_names == ( + "nose", + "shoulder", + "tail", + ) + assert processor._skeleton_id == "deeplabcut.mouse" + assert processor._skeleton_edges == ( + ("nose", "shoulder"), + ("shoulder", "tail"), + ) + + +def test_metadata_loader_clears_stale_metadata() -> None: + processor = DLCLiveProcessor() + processor._keypoint_names = ("stale",) + processor._skeleton_id = "stale" + processor._skeleton_edges = (("stale", "other"),) + processor._dlc = SimpleNamespace(cfg={}) + + processor._load_pose_metadata_from_dlc_config() + + assert processor._keypoint_names is None + assert processor._skeleton_id is None + assert processor._skeleton_edges is None + + +def test_metadata_loader_allows_bodyparts_without_skeleton() -> None: + processor = DLCLiveProcessor() + processor._dlc = SimpleNamespace( + cfg={ + "bodyparts": ["nose", "tail"], + } + ) + + processor._load_pose_metadata_from_dlc_config() + + assert processor._keypoint_names == ( + "nose", + "tail", + ) + assert processor._skeleton_id is None + assert processor._skeleton_edges is None + + +def test_metadata_loader_rejects_duplicate_bodyparts() -> None: + processor = DLCLiveProcessor() + processor._dlc = SimpleNamespace( + cfg={ + "bodyparts": ["nose", "nose"], + } + ) + + with pytest.raises( + RuntimeError, + match="duplicate", + ): + processor._load_pose_metadata_from_dlc_config() + + +def test_metadata_loader_rejects_unknown_skeleton_endpoint() -> None: + processor = DLCLiveProcessor() + processor._dlc = SimpleNamespace( + cfg={ + "bodyparts": ["nose", "tail"], + "skeleton": [["nose", "shoulder"]], + } + ) + + with pytest.raises( + RuntimeError, + match="unknown bodypart", + ): + processor._load_pose_metadata_from_dlc_config() From 9fb68bf4e7c02a5e29ffcd473db0c4af906d05ef Mon Sep 17 00:00:00 2001 From: Cyril Achard Date: Thu, 27 Aug 2026 13:32:07 +0200 Subject: [PATCH 18/18] Preserve skeleton style settings from UI Keep a dedicated `SkeletonStyle` state on the main window and sync it when loading visualization settings, so non-UI-exposed fields (`gradient_steps`, `scale_with_zoom`) are preserved instead of being pulled from stale config data. The UI-to-settings path now updates and reuses this state consistently. Also renames `test_frame` factory parameters in display tests (`height/width/channels` -> `h/w/c`) to match current call patterns. --- dlclivegui/gui/main_window.py | 13 ++++++++----- tests/display/conftest.py | 8 ++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/dlclivegui/gui/main_window.py b/dlclivegui/gui/main_window.py index 33d3c145..6ecb496c 100644 --- a/dlclivegui/gui/main_window.py +++ b/dlclivegui/gui/main_window.py @@ -189,6 +189,7 @@ def __init__(self, config: ApplicationSettings | None = None): self._p_cutoff = 0.6 self._colormap = "hot" self._bbox_color = (0, 0, 255) # BGR: red + self._skeleton_style = SkeletonStyle() self._overlay_renderer = OverlayRenderer() # Multi-camera state @@ -989,6 +990,7 @@ def _apply_config(self, config: ApplicationSettings, *, restore_local_prefs: boo self._p_cutoff = viz.p_cutoff self._colormap = viz.colormap self._bbox_color = viz.get_bbox_color_bgr() + self._skeleton_style = viz.skeleton_style.model_copy(deep=True) self.show_predictions_checkbox.blockSignals(True) try: @@ -1145,13 +1147,16 @@ def _bbox_settings_from_ui(self) -> BoundingBoxSettings: def _visualization_settings_from_ui( self, ) -> VisualizationSettings: + skeleton_style = self._skeleton_style_from_ui() + self._skeleton_style = skeleton_style + return VisualizationSettings( p_cutoff=self._p_cutoff, colormap=self._colormap, bbox_color=self._bbox_color, show_pose=self.show_predictions_checkbox.isChecked(), show_skeleton=self.show_skeleton_checkbox.isChecked(), - skeleton_style=self._skeleton_style_from_ui(), + skeleton_style=skeleton_style, ) def _overlay_settings_from_ui( @@ -1190,14 +1195,12 @@ def _skeleton_style_from_ui( color_bgr = tuple(color) if color is not None else (0, 255, 255) - current_style = self._config.visualization.skeleton_style - return SkeletonStyle( color_mode=SkeletonColorMode(mode), color_bgr=color_bgr, thickness=self.skeleton_thickness_spin.value(), - gradient_steps=current_style.gradient_steps, - scale_with_zoom=current_style.scale_with_zoom, + gradient_steps=self._skeleton_style.gradient_steps, + scale_with_zoom=self._skeleton_style.scale_with_zoom, ) def _suggest_config_dialog_path(self) -> str: diff --git a/tests/display/conftest.py b/tests/display/conftest.py index 88c22913..8d0d1812 100644 --- a/tests/display/conftest.py +++ b/tests/display/conftest.py @@ -12,15 +12,15 @@ def test_frame() -> Callable[..., np.ndarray]: """Return a factory for creating predictable test frames.""" def make_frame( - height: int, - width: int, - channels: int = 3, + h: int, + w: int, + c: int = 3, value: int | float = 0, dtype: np.dtype = np.uint8, ) -> np.ndarray: if not isinstance(dtype, np.dtype): dtype = np.dtype(dtype) - shape = (height, width) if channels == 1 else (height, width, channels) + shape = (h, w) if c == 1 else (h, w, c) return np.full(shape, value, dtype=dtype) return make_frame