diff --git a/docs/configuration.rst b/docs/configuration.rst index cd006a70..a9c70387 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -567,6 +567,41 @@ subclass) that unwinds the stack and releases the lock via ``finally`` / detected. Useful when running under a process supervisor (e.g. systemd) that handles the restart externally. +Resource Monitoring +~~~~~~~~~~~~~~~~~~~~ + +Enabled by default: a background thread samples CPU, memory, disk, network, +GPU (NVML), and process-level usage and logs each value as a signal, visible +in Weights Studio like any other loss/metric curve. See +:doc:`resource_monitoring` for the full metric list, the ``resource_monitoring.yaml`` +schema, and category-level toggles. + +.. list-table:: + :header-rows: 1 + :widths: 35 15 50 + + * - Variable + - Default + - Description + * - ``WEIGHTSLAB_DISABLE_RESOURCE_MONITORING`` + - ``0`` + - If set to ``1`` / ``true`` / ``yes`` / ``on``, disables resource + monitoring entirely. + * - ``WL_RESOURCE_MONITOR_INTERVAL_SECONDS`` + - ``15`` + - How often (seconds) the monitor samples and logs a new batch of metrics. + * - ``WL_RESOURCE_MONITOR_CATEGORIES`` + - *(unset — all on)* + - Comma-separated category allowlist (``cpu``, ``memory``, ``disk``, + ``network``, ``process``, ``gpu``). Anything not listed is disabled. + * - ``WL_RESOURCE_MONITOR_DISK_PATH`` + - OS root + - Filesystem path reported by the ``disk`` category's usage metrics. + * - ``WL_RESOURCE_MONITOR_CONFIG_PATH`` + - *(empty)* + - Optional directory override for ``resource_monitoring.yaml``. + + Data and Cache ~~~~~~~~~~~~~~ diff --git a/docs/export.rst b/docs/export.rst new file mode 100644 index 00000000..5a145def --- /dev/null +++ b/docs/export.rst @@ -0,0 +1,137 @@ +Annotation Export +================== + +WeightsLab can export bounding-box/segmentation annotations to a +relabeling-tool format, so a dataset (or a slice of one) can be handed off +for an outsourced relabeling pass. Three ways to trigger it, all backed by +the same code path: + +- **Weights Studio UI** — an "Export" button next to Save/Grid settings, with + a format picker (CVAT / Label Studio / V7). Triggers a browser download. +- **CLI** — ``weightslab export`` connects over gRPC to a running experiment, + same as ``weightslab cli``. +- **Python** — :func:`wl.export_annotations`, called in-process (no gRPC + round-trip needed since it already runs alongside the registered dataframe). + +Supported formats +------------------ + +.. list-table:: + :header-rows: 1 + :widths: 20 40 40 + + * - Format + - Output shape + - Schema reference + * - ``cvat`` + - A single CVAT XML 1.1 file (one ```` element per sample, with + ````/```` children). + - `CVAT XML format `_ + * - ``label_studio`` + - A single JSON file — a list of "tasks", each with a ``result`` list of + ``rectanglelabels``/``polygonlabels`` entries. Coordinates are + percentages (0-100) of the image's width/height, per Label Studio's + convention. + - `Label Studio export format `_ + * - ``v7`` + - A zip of one Darwin JSON 2.0 file per image (V7 matches annotations to + images by filename on import). + - `Darwin JSON reference `_ + +Bounding boxes are exported for every format. Segmentation masks are +converted to polygons via OpenCV contour extraction — this needs the +optional ``export`` extra: + +.. code-block:: bash + + pip install weightslab[export] + +Bounding-box-only export needs no extra dependency; if OpenCV isn't +installed, segmentation samples still export their boxes and a warning is +logged once, rather than failing the whole export. + +Usage +------ + +**Python** + +.. code-block:: python + + import weightslab as wl + + wl.export_annotations("cvat") # everything, under root_log_dir + wl.export_annotations("label_studio", "val.json", origin="val_loader") + wl.export_annotations("v7", "out/", class_names=["bg", "cat", "dog"]) + wl.export_annotations("cvat", tags=["ToReview"]) # only samples tagged ToReview + +See :doc:`user_functions` for the full :func:`wl.export_annotations` reference. + +**CLI** + +.. code-block:: bash + + weightslab export --format cvat # everything, CVAT XML, into "." + weightslab export -f v7 out/ --origin val_loader # V7/Darwin, val split only + weightslab export -f cvat --tag ToReview # only samples tagged ToReview + +Connects over gRPC to a running experiment (``127.0.0.1:50051`` by default), +same as ``weightslab cli``. See :doc:`user_commands` for every flag. + +**Weights Studio UI** + +The "Export" button sits next to the Save and Grid settings controls in the +Details panel. Clicking it opens a small format picker (CVAT / Label +Studio / V7) with an optional tag selector; the chosen format (and tags, if +any) trigger an ``ExportAnnotations`` gRPC call and the response downloads +as a file in your browser. + +**In-app chat agent** + +Because the chat agent (see :doc:`agent`) has general tool access to the +live experiment process, you can also just ask for this in plain language -- +e.g. "export the samples tagged ToReview to CVAT format for relabeling" -- +and it calls :func:`wl.export_annotations` with the matching ``tags=`` +argument itself. No special wiring is needed beyond the API existing. + +Filtering by tag +------------------ + +All three entry points accept a tag filter (``tags=`` in Python, ``--tag`` on +the CLI, repeatable; the tag picker in the UI) that restricts the export to +samples carrying **any** of the given tags -- boolean tags set via +:func:`wl.tag_samples` or categorical values set via +:func:`wl.set_categorical_tag` both work, since they share the same +``tag:`` column. Omit it to export every sample. This is the mechanism +for a "send only what needs another look" relabeling handoff, e.g. tagging +uncertain samples as ``ToReview`` during data exploration and exporting just +that subset. + +How annotations are resolved +------------------------------ + +Every export path collects annotations from the same registered dataframe +that backs the rest of WeightsLab (`get_dataframe()`), grouping the +``(sample_id, annotation_id)`` multi-index rows by sample: + +- **Boxes** — read from the ``target`` (or ``prediction``, with + ``use_predictions=True``) column when it holds coordinate-shaped data + (``(x1, y1, x2, y2[, conf][, cls])``), whether that's a single box per + sample or several boxes exploded across annotation rows. +- **Masks -> polygons** — read from the same column when it holds a dense + ``(H, W)`` array (pixel value = class id); one polygon per connected + region per class id. + +Two real gaps in the current data model drive the "best effort" behavior +below — call these out explicitly if an export looks wrong: + +- **No dedicated class-id -> name registry.** Labels are resolved, in order: + an explicit ``class_names`` argument; else a ``class_names`` attribute on + the dataset object backing the relevant split; else ``"class_"``. +- **No per-sample stored image path or dimensions.** A real image path is + best-effort resolved from a few common dataset attribute names + (``image_paths``, ``img_files``, ``images``, ``imgs``, ``files``, + ``samples``); dimensions come from that file (via Pillow) or, for + segmentation samples, directly from the mask's own shape. When no path + resolves, the exported filename is synthetic (``sample_.jpg``) — **no + image file is copied or embedded**, so you must ensure the filenames you + upload to CVAT/Label Studio/V7 match the ones in the export. diff --git a/docs/index.rst b/docs/index.rst index 1213033a..d01c8fa5 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -109,6 +109,12 @@ Weightslab is a Python SDK to inspect, monitor, and edit training behavior for c Ask the agent for a branded HTML report: signal health plots, dataset stats, and a written analysis. + .. grid-item-card:: Annotation Export + :link: export + :link-type: doc + + Export bounding boxes and segmentation masks to CVAT, Label Studio, or V7 for relabeling. + .. grid-item-card:: gRPC Communication :link: grpc/index :link-type: doc @@ -156,6 +162,8 @@ Weightslab is a Python SDK to inspect, monitor, and edit training behavior for c agent hyperparameters logger + resource_monitoring + export checkpointing experiment_reports .. weights_studio diff --git a/docs/resource_monitoring.rst b/docs/resource_monitoring.rst new file mode 100644 index 00000000..7cb0af48 --- /dev/null +++ b/docs/resource_monitoring.rst @@ -0,0 +1,174 @@ +Resource Monitoring +==================== + +WeightsLab automatically tracks system and process resource usage — CPU, +memory, disk, network, and GPU — for the whole lifetime of a running +backend, and logs every value through the same signal pipeline used for +losses and metrics. The resulting curves appear in Weights Studio exactly +like any other signal, under graph names prefixed with ``resource/``. + +This is enabled by default and requires no setup. It runs independently of +the training loop — metrics are sampled on a wall-clock interval, not tied +to training steps, so they keep updating even while training is paused or +between experiments. + +What gets logged +----------------- + +.. list-table:: + :header-rows: 1 + :widths: 20 45 35 + + * - Category + - Metrics + - Signal names + * - ``cpu`` + - System-wide CPU utilization (%) + - ``resource/cpu/utilization_percent`` + * - ``memory`` + - System-wide memory utilization (%) + - ``resource/memory/system_utilization_percent`` + * - ``disk`` + - Disk usage (%, GB) of ``disk_path``; cumulative bytes read/written (MB) + - ``resource/disk/utilization_percent``, ``resource/disk/utilization_gb``, + ``resource/disk/read_mb``, ``resource/disk/written_mb`` + * - ``network`` + - Cumulative bytes sent/received + - ``resource/network/bytes_sent``, ``resource/network/bytes_received`` + * - ``process`` + - CPU %, thread count, RSS memory (MB/%), and system memory available (MB) + for the WeightsLab backend process itself + - ``resource/process/cpu_utilization_percent``, + ``resource/process/cpu_threads_in_use``, + ``resource/process/memory_in_use_mb``, + ``resource/process/memory_in_use_percent``, + ``resource/process/memory_available_mb`` + * - ``gpu`` + - Per-device memory/SM clock (MHz), memory used (bytes/%), temperature (°C) + - ``resource/gpu//memory_clock_mhz``, + ``resource/gpu//sm_clock_mhz``, + ``resource/gpu//memory_allocated_bytes``, + ``resource/gpu//memory_allocated_percent``, + ``resource/gpu//temperature_celsius`` + +CPU/memory/disk/network/process metrics come from `psutil +`_. GPU metrics come from NVML (the +``pynvml`` import name, shipped by the ``nvidia-ml-py`` package) and are +per-device — multi-GPU machines get one full set of ``gpu`` signals per +device index. On a machine with no NVIDIA driver, the ``gpu`` category +degrades silently to a no-op; every other category is unaffected. + +Because sampling is wall-clock driven rather than step-driven, the logged +"step" for every resource signal is **elapsed seconds since the monitor +started** — so these curves plot against time, not batch count. + +Disabling monitoring +--------------------- + +To turn everything off, either: + +.. code-block:: bash + + export WEIGHTSLAB_DISABLE_RESOURCE_MONITORING=1 + +or set ``enabled: false`` in ``resource_monitoring.yaml`` (see below) — +the YAML value wins if both are set. + +Enabling only specific categories +---------------------------------- + +Two ways to restrict which categories are sampled: + +- **Env var**, comma-separated category list (anything not listed is + disabled): + + .. code-block:: bash + + export WL_RESOURCE_MONITOR_CATEGORIES=cpu,memory,gpu + +- **YAML file** (``resource_monitoring.yaml``), per-category booleans — + lets you leave everything on and disable just one or two: + + .. code-block:: yaml + + resource_monitoring: + categories: + disk: false + network: false + +Config file +------------ + +Create ``resource_monitoring.yaml`` at your repository root (next to +``agent_config.yaml``, if you use the agent) to control monitoring without +touching env vars: + +.. code-block:: yaml + + resource_monitoring: + enabled: true # master switch + interval_seconds: 15 # how often (seconds) to sample + log a batch of metrics + disk_path: "/" # filesystem path reported by the `disk` category + categories: + cpu: true + memory: true + disk: true + network: true + process: true + gpu: true + +Config lookup order +~~~~~~~~~~~~~~~~~~~~ + +1. ``/.resource_monitoring.yaml`` / + ``/resource_monitoring.yaml`` (if + ``WL_RESOURCE_MONITOR_CONFIG_PATH`` is set) +2. Repository-level ``resource_monitoring.yaml`` +3. Package-level ``resource_monitoring.yaml`` +4. Current working directory ``resource_monitoring.yaml`` + +Any key present in the YAML file overrides the corresponding env var (or +the built-in default); keys the file omits keep whatever the env var (or +default) already resolved to. + +Environment variables +----------------------- + +.. list-table:: + :header-rows: 1 + :widths: 35 15 50 + + * - Variable + - Default + - Description + * - ``WEIGHTSLAB_DISABLE_RESOURCE_MONITORING`` + - ``0`` + - If set to ``1`` / ``true`` / ``yes`` / ``on``, disables resource + monitoring entirely (no background thread is started). + * - ``WL_RESOURCE_MONITOR_INTERVAL_SECONDS`` + - ``15`` + - How often (seconds) the monitor samples and logs a new batch of + metrics. Clamped to a 1-second floor. + * - ``WL_RESOURCE_MONITOR_CATEGORIES`` + - *(unset — all categories on)* + - Comma-separated list of categories to enable + (``cpu``, ``memory``, ``disk``, ``network``, ``process``, ``gpu``). + When set, any category not listed is disabled. + * - ``WL_RESOURCE_MONITOR_DISK_PATH`` + - OS root (``/`` or ``C:\``) + - Filesystem path reported by the ``disk`` category's usage metrics. + * - ``WL_RESOURCE_MONITOR_CONFIG_PATH`` + - *(empty)* + - Optional directory override for ``resource_monitoring.yaml``. When + set, WeightsLab first checks + ``/resource_monitoring.yaml`` before + the built-in fallback paths. + +Where it runs +--------------- + +The monitor is started once, alongside the watchdog, from +``grpc_serve()`` (``weightslab/trainer/trainer_services.py``) — so it covers +the whole backend server lifetime, not just active training. It is a single +daemon thread (``WL-ResourceMonitor``) and stops automatically when the +process exits. diff --git a/docs/user_commands.rst b/docs/user_commands.rst index 8eebbdbe..7dd9dd29 100644 --- a/docs/user_commands.rst +++ b/docs/user_commands.rst @@ -10,7 +10,7 @@ Installed as a console script via pyproject.toml: .. code-block:: text - weightslab {se,start,cli,tunnel,help} ... + weightslab {se,start,cli,tunnel,export,help} ... Run weightslab, weightslab -h, or weightslab help to print the full built-in help. @@ -29,6 +29,8 @@ Run weightslab, weightslab -h, or weightslab help to print the full built-in hel - Connect to a running experiment interactive console. * - weightslab tunnel - Forward a remote gRPC backend to a local TCP port. + * - weightslab export + - Export bounding-box/segmentation annotations to CVAT, Label Studio, or V7. * - weightslab help - Show the help/banner (same as no command, or -h). @@ -285,6 +287,49 @@ up), and runs until ``Ctrl+C``. See the classification Colab notebook (``examples/Notebooks/PyTorch/ws-classification.ipynb``) for the end-to-end setup. +weightslab export +~~~~~~~~~~~~~~~~~~ + +**Syntax** + +.. code-block:: bash + + weightslab export --format {cvat,label_studio,v7} [OUTPUT] + [--origin ORIGIN] [--predictions] [--tag TAG ...] [--host HOST] [--port PORT] + +Exports bounding-box/segmentation annotations from a **running** experiment +to a relabeling-tool format — connects over gRPC exactly like ``weightslab +cli`` does, and is the CLI counterpart to Weights Studio's "Export" button +and :func:`wl.export_annotations`. See :doc:`export` for the format +reference, class-name/image-path resolution, and caveats. + +**Arguments** + +- ``--format``, ``-f`` *(required)* — ``cvat`` (XML), ``label_studio`` + (JSON), or ``v7`` (Darwin JSON, zipped — one file per image). +- ``OUTPUT`` *(positional, optional)* — output file path or directory. + Default: the current directory, using the format's default filename + (e.g. ``annotations_cvat.xml``). +- ``--origin`` *(str)* — restrict to one registered split/loader (e.g. + ``train_loader``). Default: every registered split. +- ``--predictions`` — export model predictions instead of ground-truth targets. +- ``--tag`` *(str, repeatable)* — restrict to samples carrying this tag + (e.g. ``ToReview``); repeat for multiple tags (matches ANY of them). + Default: every sample. +- ``--host`` *(str)* — backend host to connect to. Default: **127.0.0.1**. +- ``--port`` *(int)* — backend gRPC port to connect to. Default: + ``$GRPC_BACKEND_PORT`` or **50051**. + +**Examples** + +.. code-block:: bash + + weightslab export --format cvat # everything, CVAT XML, into "." + weightslab export -f label_studio annotations.json # explicit output file + weightslab export -f v7 out/ --origin val_loader # V7/Darwin, val split only + weightslab export -f cvat --predictions # export model predictions + weightslab export -f cvat --tag ToReview # only samples tagged ToReview + .. _cli-console: Interactive CLI console diff --git a/docs/user_functions.rst b/docs/user_functions.rst index ba515ce7..d55a4559 100644 --- a/docs/user_functions.rst +++ b/docs/user_functions.rst @@ -30,6 +30,7 @@ Public API surface - ``wl.trigger_pending_evaluation_async`` *(optional, for the background gRPC/CLI worker)* - ``wl.pointcloud_thumbnail`` / ``wl.pointcloud_boxes`` *(decorators — LiDAR / point-cloud tasks)* - ``wl.ai_report_generation`` *(agent-written HTML experiment report)* +- ``wl.export_annotations`` *(export boxes/masks to CVAT, Label Studio, or V7)* - ``wl.clear_all`` - ``wl.seed_everything`` - ``wl.set_log_directory`` @@ -1380,6 +1381,65 @@ Dump the ``loss_shape`` categorical tag and signals for sample-level rows only instance_id=0, ) +export_annotations +------------------- + +**Signature** + +.. code-block:: python + + wl.export_annotations( + fmt, # "cvat" | "label_studio" | "v7" + path=None, + origin=None, + class_names=None, + use_predictions=False, + tags=None, + ) + +**Purpose** + +Export bounding-box/segmentation annotations to a relabeling-tool format — +the Python-API counterpart to Weights Studio's "Export" button and the +``weightslab export`` CLI command. See :doc:`export` for the full format +reference and known limitations (image-path/class-name resolution). + +**Arguments** + +- ``fmt`` *(str)* — ``"cvat"`` (single XML file), ``"label_studio"`` (single + JSON file), or ``"v7"`` (zip of per-image Darwin JSON files). +- ``path`` *(str, optional)* — output file path **or** directory. ``None`` + (default) uses ``root_log_dir`` from the active checkpoint manager, with + the format's default filename (e.g. ``annotations_cvat.xml``). +- ``origin`` *(str, optional)* — restrict to one registered split/loader + (e.g. ``"train_loader"``). ``None`` exports every registered split. +- ``class_names`` *(dict or list, optional)* — explicit class-id -> name + mapping, overriding any auto-detected ``dataset.class_names`` attribute. + Without either, labels fall back to ``"class_"``. +- ``use_predictions`` *(bool)* — export model predictions instead of + ground-truth targets. Default ``False``. +- ``tags`` *(list of str, optional)* — restrict to samples carrying ANY of + these tags (``tag:`` prefix optional, e.g. ``["ToReview"]``), matching a + boolean tag from :func:`tag_samples` or a categorical value from + :func:`set_categorical_tag`. ``None`` (default) exports every sample. + +**Examples** + +Export everything to CVAT, auto-named under ``root_log_dir``:: + + wl.export_annotations("cvat") + +Export only the validation split to Label Studio, with explicit class names:: + + wl.export_annotations( + "label_studio", "val_annotations.json", + origin="val_loader", class_names=["background", "cat", "dog"], + ) + +Export only the samples tagged "ToReview" to CVAT, for a relabeling pass:: + + wl.export_annotations("cvat", tags=["ToReview"]) + ai_report_generation -------------------- diff --git a/pyproject.toml b/pyproject.toml index a1cf32fb..6f7936a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,18 @@ dependencies = [ # Reporting "matplotlib>=3.7,<4", + + # Resource monitoring (CPU/memory/disk/network/process + NVIDIA GPU via NVML). + # "nvidia-ml-py" is NVIDIA's officially maintained NVML binding, published + # under the `pynvml` import name (the standalone `pynvml` PyPI package is the + # unmaintained community fork -- do not swap this for it). Core dependency so + # GPU metrics work out of the box on GPU machines; the monitor no-ops + # gracefully when no NVIDIA driver is present (CPU-only machines). Versioned + # after the CUDA driver generation it binds (already at 13.x), not semver -- + # keep the upper bound loose so a newer driver-generation release isn't + # blocked. + "psutil>=5.9,<8", + "nvidia-ml-py>=11,<14", ] [project.optional-dependencies] @@ -115,6 +127,14 @@ dev = [ "httpx>=0.27,<1", ] +# Optional: segmentation mask -> polygon extraction for annotation export +# (CVAT/Label Studio/V7). Bounding-box export needs none of this; only +# lazily imported inside weightslab.export when a segmentation mask is +# actually encountered. +export = [ + "opencv-python-headless>=4.8,<5", +] + utest = [ "torchaudio>=2.1,<2.9; python_version < '3.13'", "torchaudio>=2.9,<3; python_version >= '3.13'", diff --git a/resource_monitoring.yaml b/resource_monitoring.yaml new file mode 100644 index 00000000..8fd841f5 --- /dev/null +++ b/resource_monitoring.yaml @@ -0,0 +1,31 @@ +# Resource Monitoring Configuration +# ################################## +# Controls the automatic background monitor that samples system/process +# resource usage (CPU, memory, disk, network, GPU) while WeightsLab is +# running and logs each value as a signal (visible in Weights Studio like +# any other loss/metric curve). +# +# Variables can be defined from env. variables or directly in this file. +# Config file values here override env. variables if both are set. +# See docs/resource_monitoring.rst for the full reference. +resource_monitoring: + # Master switch. Set to false (or env WEIGHTSLAB_DISABLE_RESOURCE_MONITORING=1) + # to disable resource monitoring entirely. + enabled: true + + # How often (seconds) to sample and log a new batch of metrics. + interval_seconds: 15 + + # Filesystem path used to report disk usage (resource/disk/utilization_*). + # Defaults to the OS root ("/" on Linux/macOS, "C:\" on Windows). + # disk_path: "/" + + # Per-category toggles. Set any of these to false to stop collecting (and + # logging) that category, without touching the others. + categories: + cpu: true # system-wide CPU utilization (%) + memory: true # system-wide memory utilization (%) + disk: true # disk usage (%/GB) + cumulative read/written MB + network: true # cumulative bytes sent/received + process: true # this process's CPU/thread/memory usage + gpu: true # NVIDIA GPU clocks/memory/temperature via NVML (no-ops without a GPU) diff --git a/tests/backend/test_export_annotations_api.py b/tests/backend/test_export_annotations_api.py new file mode 100644 index 00000000..0d7b7310 --- /dev/null +++ b/tests/backend/test_export_annotations_api.py @@ -0,0 +1,62 @@ +"""Tests for wl.export_annotations — the Python API wrapper around +weightslab.export.exporter.save_export, with the same root_log_dir path +resolution convention as wl.write_dataframe. +""" + +import os +from unittest.mock import MagicMock, patch + +from weightslab.src import export_annotations + + +class TestExportAnnotationsApi: + + def test_explicit_path_and_kwargs_passed_through(self, tmp_path): + target = str(tmp_path / "out.xml") + with patch("weightslab.export.exporter.save_export", return_value=target) as mock_save: + result = export_annotations( + "cvat", target, origin="train_loader", class_names=["bg", "car"], use_predictions=True, + tags=["ToReview"], + ) + + mock_save.assert_called_once_with( + "cvat", target, + origin="train_loader", class_names=["bg", "car"], use_predictions=True, tags=["ToReview"], + ) + assert result == os.path.abspath(target) + + def test_defaults_are_none_and_false(self, tmp_path): + target = str(tmp_path / "out.json") + with patch("weightslab.export.exporter.save_export", return_value=target) as mock_save: + export_annotations("label_studio", target) + + mock_save.assert_called_once_with( + "label_studio", target, + origin=None, class_names=None, use_predictions=False, tags=None, + ) + + def test_path_none_falls_back_to_root_log_dir(self, tmp_path): + mock_logger = MagicMock() + mock_logger.chkpt_manager.root_log_dir = tmp_path + written = str(tmp_path / "annotations_v7_darwin.zip") + + with patch("weightslab.src.get_logger", return_value=mock_logger), \ + patch("weightslab.export.exporter.save_export", return_value=written) as mock_save: + result = export_annotations("v7") + + mock_save.assert_called_once_with( + "v7", str(tmp_path), + origin=None, class_names=None, use_predictions=False, tags=None, + ) + assert result == os.path.abspath(written) + + def test_path_none_no_logger_falls_back_to_cwd(self): + written = os.path.join(".", "annotations_cvat.xml") + with patch("weightslab.src.get_logger", return_value=None), \ + patch("weightslab.export.exporter.save_export", return_value=written) as mock_save: + export_annotations("cvat") + + mock_save.assert_called_once_with( + "cvat", ".", + origin=None, class_names=None, use_predictions=False, tags=None, + ) diff --git a/tests/export/__init__.py b/tests/export/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/export/test_collect.py b/tests/export/test_collect.py new file mode 100644 index 00000000..cacb2f05 --- /dev/null +++ b/tests/export/test_collect.py @@ -0,0 +1,247 @@ +"""Unit tests for weightslab.export.collect. + +Covers: + - Box-cell flattening (single box, multi-box cell, non-box cell) + - The bbox-vs-mask shape heuristic + - Class-id -> label resolution (dict / list / missing / no override) + - Image dimension resolution priority (mask shape > default) + - End-to-end collect_image_annotations() against a mocked dataframe manager +""" + +from unittest.mock import MagicMock, patch + +import numpy as np +import pandas as pd +import pytest + +from weightslab.export.collect import ( + _boxes_from_cell, + _filter_by_tags, + _is_bbox_array, + _label_for_class, + _resolve_image_dims, + _tag_column, + collect_image_annotations, +) + + +class TestIsBboxArray: + + def test_single_box_1d_is_not_bbox_array(self): + # _is_bbox_array only recognizes the 2-D (stacked) shape; a lone 1-D + # box is handled separately by _boxes_from_cell. + assert _is_bbox_array(np.array([1.0, 2.0, 3.0, 4.0, 5.0])) is False + + def test_stacked_boxes_2d_is_bbox_array(self): + assert _is_bbox_array(np.array([[1, 2, 3, 4, 5], [6, 7, 8, 9, 1]])) is True + + def test_dense_mask_is_not_bbox_array(self): + assert _is_bbox_array(np.zeros((32, 32), dtype=int)) is False + + def test_none_is_not_bbox_array(self): + assert _is_bbox_array(None) is False + + +class TestBoxesFromCell: + + def test_none_returns_empty(self): + assert _boxes_from_cell(None) == [] + + def test_single_box_5_wide_extracts_class(self): + boxes = _boxes_from_cell(np.array([10.0, 20.0, 100.0, 200.0, 2.0])) + assert boxes == [(10.0, 20.0, 100.0, 200.0, 2)] + + def test_single_box_4_wide_has_no_class(self): + boxes = _boxes_from_cell(np.array([10.0, 20.0, 100.0, 200.0])) + assert boxes == [(10.0, 20.0, 100.0, 200.0, None)] + + def test_multi_box_cell(self): + cell = np.array([[1.0, 2.0, 3.0, 4.0, 1.0], [5.0, 6.0, 7.0, 8.0, 2.0]]) + boxes = _boxes_from_cell(cell) + assert boxes == [(1.0, 2.0, 3.0, 4.0, 1), (5.0, 6.0, 7.0, 8.0, 2)] + + def test_mask_shaped_cell_returns_empty(self): + assert _boxes_from_cell(np.zeros((32, 40), dtype=int)) == [] + + def test_empty_array_returns_empty(self): + assert _boxes_from_cell(np.array([])) == [] + + +class TestLabelForClass: + + def test_none_id_is_generic_object(self): + assert _label_for_class(None, None) == "object" + + def test_no_class_names_falls_back_to_class_id(self): + assert _label_for_class(3, None) == "class_3" + + def test_list_class_names(self): + assert _label_for_class(1, ["bg", "cat", "dog"]) == "cat" + + def test_dict_class_names_int_key(self): + assert _label_for_class(2, {1: "cat", 2: "dog"}) == "dog" + + def test_dict_class_names_string_key_fallback(self): + assert _label_for_class(2, {"1": "cat", "2": "dog"}) == "dog" + + def test_out_of_range_falls_back(self): + assert _label_for_class(9, ["bg", "cat"]) == "class_9" + + +class TestResolveImageDims: + + def test_mask_shape_wins_over_default(self): + mask = np.zeros((10, 12), dtype=int) + assert _resolve_image_dims(None, mask, (640, 480)) == (12, 10) + + def test_default_used_when_no_mask_and_no_path(self): + assert _resolve_image_dims(None, None, (640, 480)) == (640, 480) + + +class TestTagColumn: + + def test_bare_name_gets_prefixed(self): + assert _tag_column("ToReview") == "tag:ToReview" + + def test_already_prefixed_name_is_untouched(self): + assert _tag_column("tag:ToReview") == "tag:ToReview" + + def test_strips_whitespace(self): + assert _tag_column(" ToReview ") == "tag:ToReview" + + +class TestFilterByTags: + + def _df(self): + return pd.DataFrame({ + "tag:ToReview": [True, False, None], + "tag:weather": ["rainy", "", None], + }) + + def test_no_tags_returns_df_unchanged(self): + df = self._df() + assert _filter_by_tags(df, None) is df + assert _filter_by_tags(df, []) is df + + def test_boolean_tag_keeps_only_true_rows(self): + result = _filter_by_tags(self._df(), ["ToReview"]) + assert list(result.index) == [0] + + def test_categorical_tag_keeps_non_empty_string_rows(self): + result = _filter_by_tags(self._df(), ["weather"]) + assert list(result.index) == [0] + + def test_multiple_tags_are_ored(self): + df = pd.DataFrame({"tag:a": [True, False], "tag:b": [False, True]}) + result = _filter_by_tags(df, ["a", "b"]) + assert list(result.index) == [0, 1] + + def test_unknown_tag_column_returns_empty(self): + result = _filter_by_tags(self._df(), ["doesnotexist"]) + assert result.empty + + def test_tag_prefix_optional(self): + result = _filter_by_tags(self._df(), ["tag:ToReview"]) + assert list(result.index) == [0] + + +class TestCollectImageAnnotations: + + def _mock_manager(self, df): + manager = MagicMock() + manager.get_combined_df.return_value = df + return manager + + def _fixture_df(self): + index = pd.MultiIndex.from_tuples( + [ + ("s1", 0), # single box, on the sample row itself + ("s2", 0), # placeholder sample row (no target) + ("s2", 1), # box instance 1 + ("s2", 2), # box instance 2 + ("s3", 0), # segmentation mask + ], + names=["sample_id", "annotation_id"], + ) + mask = np.zeros((10, 12), dtype=int) + mask[2:5, 2:5] = 1 # "cat" + mask[6:9, 6:9] = 2 # "dog" + return pd.DataFrame( + { + "origin": ["train", "train", "train", "train", "val"], + "target": [ + np.array([10.0, 20.0, 100.0, 200.0, 4.0]), # person + None, + np.array([1.0, 1.0, 50.0, 50.0, 1.0]), # cat + np.array([60.0, 60.0, 120.0, 120.0, 2.0]), # dog + mask, + ], + }, + index=index, + ) + + def test_end_to_end_boxes_and_polygons(self): + class_names = ["bg", "cat", "dog", "car", "person"] + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(self._fixture_df())): + images = collect_image_annotations(class_names=class_names, default_image_size=(640, 480)) + + by_id = {img.sample_id: img for img in images} + assert set(by_id) == {"s1", "s2", "s3"} + + s1 = by_id["s1"] + assert s1.filename == "sample_s1.jpg" + assert s1.width, s1.height == (640, 480) + assert len(s1.boxes) == 1 + assert s1.boxes[0].label == "person" + + s2 = by_id["s2"] + assert len(s2.boxes) == 2 + assert {b.label for b in s2.boxes} == {"cat", "dog"} + + s3 = by_id["s3"] + assert (s3.width, s3.height) == (12, 10) # from the mask shape, not the default + assert len(s3.boxes) == 0 + + def test_segmentation_mask_extracts_polygons_per_class(self): + pytest.importorskip("cv2", reason="opencv not installed") + class_names = {1: "cat", 2: "dog"} + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(self._fixture_df())): + images = collect_image_annotations(class_names=class_names) + + s3 = next(img for img in images if img.sample_id == "s3") + labels = sorted(p.label for p in s3.polygons) + assert labels == ["cat", "dog"] + for poly in s3.polygons: + assert len(poly.points) >= 3 + + def test_origin_filter(self): + class_names = ["bg", "cat", "dog", "car", "person"] + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(self._fixture_df())): + images = collect_image_annotations(origin="val", class_names=class_names) + assert [img.sample_id for img in images] == ["s3"] + + def test_empty_dataframe_returns_empty_list(self): + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(pd.DataFrame())): + assert collect_image_annotations() == [] + + def test_no_target_column_returns_empty_list(self): + index = pd.MultiIndex.from_tuples([("s1", 0)], names=["sample_id", "annotation_id"]) + df = pd.DataFrame({"origin": ["train"]}, index=index) + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(df)): + assert collect_image_annotations() == [] + + def test_tag_filter_restricts_to_tagged_samples(self): + df = self._fixture_df() + df["tag:ToReview"] = [True, False, False, False, False] + class_names = ["bg", "cat", "dog", "car", "person"] + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(df)): + images = collect_image_annotations(tags=["ToReview"], class_names=class_names) + assert [img.sample_id for img in images] == ["s1"] + + def test_tag_filter_no_match_returns_empty_list(self): + with patch("weightslab.backend.ledgers.get_dataframe", return_value=self._mock_manager(self._fixture_df())): + assert collect_image_annotations(tags=["doesnotexist"]) == [] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/export/test_exporter.py b/tests/export/test_exporter.py new file mode 100644 index 00000000..0f46aeaf --- /dev/null +++ b/tests/export/test_exporter.py @@ -0,0 +1,86 @@ +"""Unit tests for weightslab.export.exporter (the shared dispatcher used by +the Python API, the gRPC handler, and the CLI).""" + +import os +import tempfile +from unittest.mock import patch + +import pytest + +from weightslab.export.exporter import SUPPORTED_FORMATS, export_annotations, save_export +from weightslab.export.models import BoxAnnotation, ImageAnnotations + + +@pytest.fixture +def fake_images(): + return [ + ImageAnnotations( + sample_id="s1", filename="img1.jpg", width=100, height=100, + boxes=[BoxAnnotation(0.0, 0.0, 10.0, 10.0, "car")], + ) + ] + + +class TestExportAnnotations: + + def test_unknown_format_raises(self): + with pytest.raises(ValueError): + export_annotations("not-a-real-format") + + @pytest.mark.parametrize("fmt", SUPPORTED_FORMATS) + def test_each_supported_format_dispatches(self, fmt, fake_images): + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images): + payload, filename, mime_type, image_count = export_annotations(fmt) + assert isinstance(payload, bytes) and len(payload) > 0 + assert filename + assert mime_type + assert image_count == 1 + + def test_format_is_case_and_whitespace_insensitive(self, fake_images): + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images): + export_annotations(" CVAT ") + + def test_passes_kwargs_through_to_collect(self, fake_images): + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images) as mock_collect: + export_annotations( + "cvat", origin="train_loader", class_names=["bg", "car"], use_predictions=True, + tags=["ToReview"], + ) + mock_collect.assert_called_once_with( + origin="train_loader", class_names=["bg", "car"], use_predictions=True, tags=["ToReview"], + ) + + def test_tags_default_to_none(self, fake_images): + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images) as mock_collect: + export_annotations("cvat") + mock_collect.assert_called_once_with( + origin=None, class_names=None, use_predictions=False, tags=None, + ) + + +class TestSaveExport: + + def test_writes_bytes_to_explicit_file_path(self, fake_images, tmp_path): + output_path = str(tmp_path / "out.xml") + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images): + written = save_export("cvat", output_path) + assert written == output_path + assert os.path.exists(output_path) + assert os.path.getsize(output_path) > 0 + + def test_appends_default_filename_when_given_a_directory(self, fake_images, tmp_path): + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images): + written = save_export("label_studio", str(tmp_path)) + assert os.path.dirname(written) == str(tmp_path) + assert os.path.basename(written) == "annotations_label_studio.json" + assert os.path.exists(written) + + def test_creates_missing_parent_directories(self, fake_images, tmp_path): + nested = tmp_path / "a" / "b" / "out.zip" + with patch("weightslab.export.exporter.collect_image_annotations", return_value=fake_images): + save_export("v7", str(nested)) + assert nested.exists() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/export/test_formats.py b/tests/export/test_formats.py new file mode 100644 index 00000000..81fe439a --- /dev/null +++ b/tests/export/test_formats.py @@ -0,0 +1,142 @@ +"""Unit tests for the CVAT / Label Studio / V7 Darwin format encoders. + +Each encoder is tested against a manually built ImageAnnotations fixture -- +no dependency on the dataframe/ledger layer here, just IR -> bytes. +""" + +import json +import xml.etree.ElementTree as ET +import zipfile +from io import BytesIO + +import pytest + +from weightslab.export.formats.cvat import to_cvat_xml +from weightslab.export.formats.label_studio import to_label_studio_json +from weightslab.export.formats.v7_darwin import to_v7_darwin_zip +from weightslab.export.models import BoxAnnotation, ImageAnnotations, PolygonAnnotation + + +@pytest.fixture +def images(): + return [ + ImageAnnotations( + sample_id="s1", + filename="img1.jpg", + width=200, + height=100, + boxes=[BoxAnnotation(10.0, 20.0, 110.0, 70.0, "car")], + polygons=[PolygonAnnotation(points=[(0.0, 0.0), (50.0, 0.0), (25.0, 50.0)], label="person")], + ), + ImageAnnotations( + sample_id="s2", + filename="img2.jpg", + width=50, + height=50, + boxes=[], + polygons=[], + ), + ] + + +class TestCvatEncoder: + + def test_produces_well_formed_xml_with_expected_structure(self, images): + payload = to_cvat_xml(images) + root = ET.fromstring(payload) + assert root.tag == "annotations" + + image_els = root.findall("image") + assert len(image_els) == 2 + assert image_els[0].get("name") == "img1.jpg" + assert image_els[0].get("width") == "200" + assert image_els[0].get("height") == "100" + + box_els = image_els[0].findall("box") + assert len(box_els) == 1 + assert box_els[0].get("label") == "car" + assert box_els[0].get("xtl") == "10.00" + assert box_els[0].get("ybr") == "70.00" + + poly_els = image_els[0].findall("polygon") + assert len(poly_els) == 1 + assert poly_els[0].get("label") == "person" + assert poly_els[0].get("points") == "0.00,0.00;50.00,0.00;25.00,50.00" + + def test_labels_declared_once_in_meta(self, images): + payload = to_cvat_xml(images) + root = ET.fromstring(payload) + label_names = [el.find("name").text for el in root.findall("./meta/task/labels/label")] + assert label_names == ["car", "person"] + + def test_empty_images_list_still_produces_valid_xml(self): + payload = to_cvat_xml([]) + root = ET.fromstring(payload) + assert root.findall("image") == [] + + +class TestLabelStudioEncoder: + + def test_coordinates_converted_to_percent(self, images): + payload = to_label_studio_json(images) + tasks = json.loads(payload) + assert len(tasks) == 2 + + result = tasks[0]["annotations"][0]["result"] + box_result = next(r for r in result if r["type"] == "rectanglelabels") + # x=10/200*100=5.0, y=20/100*100=20.0, width=100/200*100=50.0, height=50/100*100=50.0 + assert box_result["value"]["x"] == pytest.approx(5.0) + assert box_result["value"]["y"] == pytest.approx(20.0) + assert box_result["value"]["width"] == pytest.approx(50.0) + assert box_result["value"]["height"] == pytest.approx(50.0) + assert box_result["value"]["rectanglelabels"] == ["car"] + + poly_result = next(r for r in result if r["type"] == "polygonlabels") + assert poly_result["value"]["polygonlabels"] == ["person"] + assert poly_result["value"]["points"][1] == pytest.approx([25.0, 0.0]) + + def test_task_without_annotations_has_empty_result(self, images): + payload = to_label_studio_json(images) + tasks = json.loads(payload) + assert tasks[1]["annotations"][0]["result"] == [] + + def test_data_image_field_is_filename(self, images): + payload = to_label_studio_json(images) + tasks = json.loads(payload) + assert tasks[0]["data"]["image"] == "img1.jpg" + + +class TestV7DarwinEncoder: + + def test_zip_contains_one_json_per_image(self, images): + payload = to_v7_darwin_zip(images) + with zipfile.ZipFile(BytesIO(payload)) as zf: + names = sorted(zf.namelist()) + assert names == ["img1.json", "img2.json"] + + doc = json.loads(zf.read("img1.json")) + assert doc["version"] == "2.0" + assert doc["item"]["name"] == "img1.jpg" + + box_ann = next(a for a in doc["annotations"] if "bounding_box" in a) + assert box_ann["name"] == "car" + assert box_ann["bounding_box"] == {"x": 10.0, "y": 20.0, "w": 100.0, "h": 50.0} + + poly_ann = next(a for a in doc["annotations"] if "polygon" in a) + assert poly_ann["name"] == "person" + assert poly_ann["polygon"]["paths"][0][0] == {"x": 0.0, "y": 0.0} + + def test_filename_collisions_are_disambiguated(self): + images = [ + ImageAnnotations(sample_id="a", filename="dup.jpg", width=10, height=10), + ImageAnnotations(sample_id="b", filename="dup.jpg", width=10, height=10), + ] + payload = to_v7_darwin_zip(images) + with zipfile.ZipFile(BytesIO(payload)) as zf: + names = sorted(zf.namelist()) + assert len(names) == 2 + assert len(set(names)) == 2 + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/general/test_cli_export.py b/tests/general/test_cli_export.py new file mode 100644 index 00000000..d60dc897 --- /dev/null +++ b/tests/general/test_cli_export.py @@ -0,0 +1,141 @@ +"""Tests for `weightslab export` (weightslab.cli.export_annotations_cli) -- +the CLI's gRPC client for the annotation-export feature. +""" + +import argparse +import os +import unittest +from unittest.mock import MagicMock, patch + +import grpc + +import weightslab.cli as wl_cli + + +def _args(**overrides): + base = dict(format="cvat", output=None, origin=None, predictions=False, tags=None, host=None, port=None) + base.update(overrides) + return argparse.Namespace(**base) + + +class TestExportAnnotationsCli(unittest.TestCase): + + def test_connection_failure_exits_1(self): + with patch("grpc.insecure_channel", side_effect=RuntimeError("no route to host")): + with self.assertRaises(SystemExit) as cm: + wl_cli.export_annotations_cli(_args()) + self.assertEqual(cm.exception.code, 1) + + def test_rpc_error_exits_1(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.side_effect = grpc.RpcError("boom") + + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + with self.assertRaises(SystemExit) as cm: + wl_cli.export_annotations_cli(_args()) + self.assertEqual(cm.exception.code, 1) + + def test_unsuccessful_response_exits_1(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock(success=False, message="no dataframe registered") + + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + with self.assertRaises(SystemExit) as cm: + wl_cli.export_annotations_cli(_args()) + self.assertEqual(cm.exception.code, 1) + + def test_success_writes_payload_to_explicit_output_path(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock( + success=True, payload=b"", filename="annotations_cvat.xml", + mime_type="application/xml", image_count=2, + ) + + import tempfile + with tempfile.TemporaryDirectory() as tmp_dir: + output = os.path.join(tmp_dir, "out.xml") + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + wl_cli.export_annotations_cli(_args(output=output)) + + self.assertTrue(os.path.isfile(output)) + with open(output, "rb") as f: + self.assertEqual(f.read(), b"") + + def test_success_appends_default_filename_to_directory_output(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock( + success=True, payload=b"[]", filename="annotations_label_studio.json", + mime_type="application/json", image_count=0, + ) + + import tempfile + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + wl_cli.export_annotations_cli(_args(format="label_studio", output=tmp_dir)) + + expected = os.path.join(tmp_dir, "annotations_label_studio.json") + self.assertTrue(os.path.isfile(expected)) + + def test_request_uses_correct_format_enum_and_options(self): + from weightslab.proto import experiment_service_pb2 as pb2 + + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock( + success=True, payload=b"", filename="f", mime_type="application/zip", image_count=0, + ) + + import tempfile + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + wl_cli.export_annotations_cli(_args(format="v7", output=tmp_dir, origin="val_loader", predictions=True)) + + sent_request = mock_stub.ExportAnnotations.call_args[0][0] + self.assertEqual(sent_request.format, pb2.EXPORT_FORMAT_V7_DARWIN) + self.assertEqual(sent_request.origin, "val_loader") + self.assertTrue(sent_request.include_predictions) + + def test_request_carries_repeated_tags(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock( + success=True, payload=b"", filename="f", mime_type="application/xml", image_count=0, + ) + + import tempfile + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + wl_cli.export_annotations_cli(_args(output=tmp_dir, tags=["ToReview", "outlier"])) + + sent_request = mock_stub.ExportAnnotations.call_args[0][0] + self.assertEqual(list(sent_request.tags), ["ToReview", "outlier"]) + + def test_request_tags_default_empty(self): + mock_stub = MagicMock() + mock_stub.ExportAnnotations.return_value = MagicMock( + success=True, payload=b"", filename="f", mime_type="application/xml", image_count=0, + ) + + import tempfile + with tempfile.TemporaryDirectory() as tmp_dir: + with patch("grpc.insecure_channel", return_value=MagicMock()), \ + patch("grpc.channel_ready_future", return_value=MagicMock()), \ + patch("weightslab.proto.experiment_service_pb2_grpc.ExperimentServiceStub", return_value=mock_stub): + wl_cli.export_annotations_cli(_args(output=tmp_dir)) + + sent_request = mock_stub.ExportAnnotations.call_args[0][0] + self.assertEqual(list(sent_request.tags), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/monitoring/__init__.py b/tests/monitoring/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/monitoring/test_resource_monitor.py b/tests/monitoring/test_resource_monitor.py new file mode 100644 index 00000000..da367b24 --- /dev/null +++ b/tests/monitoring/test_resource_monitor.py @@ -0,0 +1,269 @@ +"""Unit tests for weightslab.monitoring.resource_monitor. + +Covers: + - Config resolution: hardcoded defaults, env var overrides, YAML overriding env vars + - ResourceMonitor category gating (only enabled categories get sampled/logged) + - Individual psutil-backed samplers return the expected metric keys + - GPU sampling degrades gracefully with no NVIDIA driver present + - start()/stop() lifecycle of the background thread + - Process-wide singleton helpers +""" + +import os +import tempfile +import threading +import unittest +from pathlib import Path +from unittest import mock + +from weightslab.monitoring.resource_monitor import ( + DEFAULT_CATEGORIES, + ResourceMonitor, + load_resource_monitoring_config, + start_resource_monitor_from_config, + get_resource_monitor, + stop_resource_monitor, +) + +_ENV_KEYS = ( + "WEIGHTSLAB_DISABLE_RESOURCE_MONITORING", + "WL_RESOURCE_MONITOR_INTERVAL_SECONDS", + "WL_RESOURCE_MONITOR_CATEGORIES", + "WL_RESOURCE_MONITOR_DISK_PATH", + "WL_RESOURCE_MONITOR_CONFIG_PATH", +) + + +class TestLoadResourceMonitoringConfig(unittest.TestCase): + + def setUp(self): + # Point config resolution at an empty directory so no real + # resource_monitoring.yaml (repo root, cwd, ...) leaks into these tests. + self._empty_dir = tempfile.TemporaryDirectory() + self.addCleanup(self._empty_dir.cleanup) + + def _patched_env(self, **overrides): + # The loader falls back to Path.cwd()/resource_monitoring.yaml (same + # design as agent_config.yaml's lookup chain) -- patch cwd too so the + # real repo-root config (pytest's cwd) can't leak into these tests. + env = {"WL_RESOURCE_MONITOR_CONFIG_PATH": self._empty_dir.name} + env.update(overrides) + return self._enter_isolated(env) + + def _enter_isolated(self, env): + from contextlib import ExitStack + + stack = ExitStack() + stack.enter_context(mock.patch.dict(os.environ, env, clear=False)) + stack.enter_context(mock.patch("pathlib.Path.cwd", return_value=Path(self._empty_dir.name))) + return stack + + def test_defaults_when_no_overrides(self): + with self._patched_env(): + for key in _ENV_KEYS: + os.environ.pop(key, None) + os.environ["WL_RESOURCE_MONITOR_CONFIG_PATH"] = self._empty_dir.name + config = load_resource_monitoring_config() + + self.assertTrue(config["enabled"]) + self.assertEqual(config["interval_seconds"], 15.0) + self.assertEqual(set(config["categories"]), set(DEFAULT_CATEGORIES)) + self.assertTrue(all(config["categories"].values())) + + def test_env_var_disables_monitoring(self): + with self._patched_env(WEIGHTSLAB_DISABLE_RESOURCE_MONITORING="1"): + config = load_resource_monitoring_config() + self.assertFalse(config["enabled"]) + + def test_env_var_restricts_categories(self): + with self._patched_env(WL_RESOURCE_MONITOR_CATEGORIES="cpu,gpu"): + config = load_resource_monitoring_config() + self.assertTrue(config["categories"]["cpu"]) + self.assertTrue(config["categories"]["gpu"]) + self.assertFalse(config["categories"]["memory"]) + self.assertFalse(config["categories"]["disk"]) + self.assertFalse(config["categories"]["network"]) + self.assertFalse(config["categories"]["process"]) + + def test_yaml_overrides_env_vars(self): + yaml_dir = tempfile.TemporaryDirectory() + self.addCleanup(yaml_dir.cleanup) + yaml_path = Path(yaml_dir.name) / "resource_monitoring.yaml" + yaml_path.write_text( + "resource_monitoring:\n" + " enabled: false\n" + " interval_seconds: 5\n" + " disk_path: /data\n" + " categories:\n" + " cpu: false\n" + ) + + # Env says "enabled" but YAML (which takes precedence) says disabled. + with self._patched_env( + WEIGHTSLAB_DISABLE_RESOURCE_MONITORING="0", + WL_RESOURCE_MONITOR_CONFIG_PATH=yaml_dir.name, + ): + config = load_resource_monitoring_config() + + self.assertFalse(config["enabled"]) + self.assertEqual(config["interval_seconds"], 5.0) + self.assertEqual(config["disk_path"], "/data") + self.assertFalse(config["categories"]["cpu"]) + # Categories not mentioned in the YAML keep their (default) value. + self.assertTrue(config["categories"]["memory"]) + + +class TestResourceMonitorTick(unittest.TestCase): + + def _monitor(self, categories): + return ResourceMonitor(interval_seconds=1.0, categories=categories) + + def test_tick_only_logs_enabled_categories(self): + categories = {name: False for name in DEFAULT_CATEGORIES} + categories["cpu"] = True + monitor = self._monitor(categories) + monitor._start_monotonic = 0.0 + + logged = [] + monitor._log_metric = lambda reg_name, value, step: logged.append(reg_name) + + monitor._tick() + + self.assertTrue(logged, "expected at least one metric to be logged") + self.assertTrue(all(name.startswith("resource/cpu/") for name in logged)) + + def test_tick_skips_gpu_when_unavailable(self): + categories = {name: True for name in DEFAULT_CATEGORIES} + monitor = self._monitor(categories) + monitor._start_monotonic = 0.0 + monitor._gpu_available = False # simulate "no NVIDIA driver" + + logged = [] + monitor._log_metric = lambda reg_name, value, step: logged.append(reg_name) + + monitor._tick() + + self.assertFalse(any(name.startswith("resource/gpu/") for name in logged)) + + def test_log_metric_delegates_to_signal_logger(self): + monitor = self._monitor({name: True for name in DEFAULT_CATEGORIES}) + with mock.patch("weightslab.src._log_signal") as mock_log_signal: + monitor._log_metric("resource/cpu/utilization_percent", 42.0, 7) + mock_log_signal.assert_called_once_with( + 42.0, None, "resource/cpu/utilization_percent", step=7 + ) + + def test_log_metric_swallows_exceptions(self): + monitor = self._monitor({name: True for name in DEFAULT_CATEGORIES}) + with mock.patch("weightslab.src._log_signal", side_effect=RuntimeError("boom")): + monitor._log_metric("resource/cpu/utilization_percent", 42.0, 7) # must not raise + + +class TestResourceMonitorSamplers(unittest.TestCase): + + def setUp(self): + self.monitor = ResourceMonitor(interval_seconds=1.0) + + def test_sample_cpu_keys(self): + metrics = self.monitor._sample_cpu() + self.assertIn("resource/cpu/utilization_percent", metrics) + self.assertIsInstance(metrics["resource/cpu/utilization_percent"], float) + + def test_sample_memory_keys(self): + metrics = self.monitor._sample_memory() + self.assertIn("resource/memory/system_utilization_percent", metrics) + + def test_sample_process_keys(self): + metrics = self.monitor._sample_process() + for key in ( + "resource/process/cpu_utilization_percent", + "resource/process/cpu_threads_in_use", + "resource/process/memory_in_use_mb", + "resource/process/memory_in_use_percent", + "resource/process/memory_available_mb", + ): + self.assertIn(key, metrics) + self.assertIsInstance(metrics[key], float) + self.assertGreater(metrics["resource/process/cpu_threads_in_use"], 0) + + def test_sample_disk_keys(self): + metrics = self.monitor._sample_disk() + # At least the usage percent/GB pair should always resolve on any OS. + self.assertIn("resource/disk/utilization_percent", metrics) + self.assertIn("resource/disk/utilization_gb", metrics) + + def test_sample_network_keys(self): + metrics = self.monitor._sample_network() + if metrics: # some sandboxed/CI environments expose no network counters + self.assertIn("resource/network/bytes_sent", metrics) + self.assertIn("resource/network/bytes_received", metrics) + + def test_init_gpu_never_raises_without_driver(self): + # No assertion on the outcome (depends on the test machine's hardware) -- + # only that a missing/absent NVIDIA driver degrades gracefully. + self.monitor._init_gpu() + self.assertIsInstance(self.monitor._gpu_available, bool) + if not self.monitor._gpu_available: + self.assertEqual(self.monitor._gpu_handles, []) + self.monitor._shutdown_gpu() + self.assertFalse(self.monitor._gpu_available) + + +class TestResourceMonitorLifecycle(unittest.TestCase): + + def test_start_stop_clean_thread_teardown(self): + monitor = ResourceMonitor( + interval_seconds=1.0, + categories={name: False for name in DEFAULT_CATEGORIES}, + ) + monitor.start() + try: + self.assertIsInstance(monitor._thread, threading.Thread) + self.assertTrue(monitor._thread.is_alive()) + finally: + monitor.stop() + self.assertFalse(monitor._thread.is_alive()) + + +class TestResourceMonitorSingleton(unittest.TestCase): + + def tearDown(self): + stop_resource_monitor() + + def test_start_from_config_returns_none_when_disabled(self): + with mock.patch( + "weightslab.monitoring.resource_monitor.load_resource_monitoring_config", + return_value={ + "enabled": False, + "interval_seconds": 15.0, + "categories": {name: True for name in DEFAULT_CATEGORIES}, + "disk_path": os.sep, + }, + ): + monitor = start_resource_monitor_from_config() + self.assertIsNone(monitor) + self.assertIsNone(get_resource_monitor()) + + def test_start_from_config_is_idempotent(self): + with mock.patch( + "weightslab.monitoring.resource_monitor.load_resource_monitoring_config", + return_value={ + "enabled": True, + "interval_seconds": 1.0, + "categories": {name: False for name in DEFAULT_CATEGORIES}, + "disk_path": os.sep, + }, + ): + first = start_resource_monitor_from_config() + second = start_resource_monitor_from_config() + + self.assertIsNotNone(first) + self.assertIs(first, second) + self.assertIs(get_resource_monitor(), first) + + stop_resource_monitor() + self.assertIsNone(get_resource_monitor()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/trainer/services/test_trainer_services_server.py b/tests/trainer/services/test_trainer_services_server.py index 61e38134..8ec50f06 100644 --- a/tests/trainer/services/test_trainer_services_server.py +++ b/tests/trainer/services/test_trainer_services_server.py @@ -43,6 +43,7 @@ def test_servicer_delegates_to_subservices(self): servicer.GetDataSamples(req, ctx) servicer.EditDataSample(req, ctx) servicer.GetDataSplits(req, ctx) + servicer.ExportAnnotations(req, ctx) servicer.CheckAgentHealth(req, ctx) servicer.GetLatestLoggerData(req, ctx) servicer.ExperimentCommand(req, ctx) @@ -56,6 +57,7 @@ def test_servicer_delegates_to_subservices(self): exp_service.data_service.GetDataSamples.assert_called_once_with(req, ctx) exp_service.data_service.EditDataSample.assert_called_once_with(req, ctx) exp_service.data_service.GetDataSplits.assert_called_once_with(req, ctx) + exp_service.data_service.ExportAnnotations.assert_called_once_with(req, ctx) exp_service.data_service.CheckAgentHealth.assert_called_once_with(req, ctx) exp_service.GetLatestLoggerData.assert_called_once_with(req, ctx) exp_service.ExperimentCommand.assert_called_once_with(req, ctx) diff --git a/tests/trainer/services/test_trainer_services_unit.py b/tests/trainer/services/test_trainer_services_unit.py index ef7ecbb6..28d5ac0f 100644 --- a/tests/trainer/services/test_trainer_services_unit.py +++ b/tests/trainer/services/test_trainer_services_unit.py @@ -510,5 +510,87 @@ def test_manual_save_data_state_force_enables_h5_and_flushes(self): self.assertTrue(service._df_manager.flush.called) self.assertTrue(checkpoint_manager.save_data_snapshot.called) + +class TestDataServiceExportAnnotationsUnit(unittest.TestCase): + + def test_export_annotations_success(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=pb2.EXPORT_FORMAT_CVAT, origin="train_loader") + + with patch( + "weightslab.export.exporter.export_annotations", + return_value=(b"", "annotations_cvat.xml", "application/xml", 3), + ) as mock_export: + response = service.ExportAnnotations(req, None) + + mock_export.assert_called_once_with("cvat", origin="train_loader", use_predictions=False, tags=None) + self.assertTrue(response.success) + self.assertEqual(response.payload, b"") + self.assertEqual(response.filename, "annotations_cvat.xml") + self.assertEqual(response.mime_type, "application/xml") + self.assertEqual(response.image_count, 3) + + def test_export_annotations_passes_predictions_flag(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=pb2.EXPORT_FORMAT_LABEL_STUDIO, include_predictions=True) + + with patch( + "weightslab.export.exporter.export_annotations", + return_value=(b"[]", "annotations_label_studio.json", "application/json", 0), + ) as mock_export: + service.ExportAnnotations(req, None) + + mock_export.assert_called_once_with("label_studio", origin=None, use_predictions=True, tags=None) + + def test_export_annotations_passes_tags(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=pb2.EXPORT_FORMAT_CVAT, tags=["ToReview", "outlier"]) + + with patch( + "weightslab.export.exporter.export_annotations", + return_value=(b"", "annotations_cvat.xml", "application/xml", 1), + ) as mock_export: + service.ExportAnnotations(req, None) + + mock_export.assert_called_once_with( + "cvat", origin=None, use_predictions=False, tags=["ToReview", "outlier"], + ) + + def test_export_annotations_unknown_format_value(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=99) + + response = service.ExportAnnotations(req, None) + + self.assertFalse(response.success) + self.assertIn("99", response.message) + + def test_export_annotations_missing_optional_dependency(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=pb2.EXPORT_FORMAT_V7_DARWIN) + + with patch( + "weightslab.export.exporter.export_annotations", + side_effect=ImportError("Segmentation polygon export requires OpenCV"), + ): + response = service.ExportAnnotations(req, None) + + self.assertFalse(response.success) + self.assertIn("OpenCV", response.message) + + def test_export_annotations_generic_failure_does_not_raise(self): + service = DataService.__new__(DataService) + req = pb2.ExportAnnotationsRequest(format=pb2.EXPORT_FORMAT_CVAT) + + with patch( + "weightslab.export.exporter.export_annotations", + side_effect=RuntimeError("boom"), + ): + response = service.ExportAnnotations(req, None) # must not raise + + self.assertFalse(response.success) + self.assertIn("boom", response.message) + + if __name__ == "__main__": unittest.main() diff --git a/weightslab/__init__.py b/weightslab/__init__.py index 8802753e..48d8f3e8 100644 --- a/weightslab/__init__.py +++ b/weightslab/__init__.py @@ -43,7 +43,7 @@ "drain_signals", "clear_all", "run_pending_evaluation", "trigger_pending_evaluation_async", "query_signal_history", "query_sample_history", "query_instance_history", "write_history", - "write_dataframe", "classify_loss_shape", "trajectory_stats", + "write_dataframe", "export_annotations", "classify_loss_shape", "trajectory_stats", "write_loss_shapes", "write_signal_shapes", "enable_loss_shape_signal", "enable_loss_shape_autotag", "disable_loss_shape_autotag", "auto_loss_shape_signal_names", @@ -220,6 +220,7 @@ def _clean(v: str) -> str: "write_history", "write_dataframe", + "export_annotations", "ai_report_generation", "classify_loss_shape", "write_loss_shapes", diff --git a/weightslab/cli.py b/weightslab/cli.py index b1dde92d..e236d5c9 100644 --- a/weightslab/cli.py +++ b/weightslab/cli.py @@ -228,6 +228,19 @@ def _banner() -> str: --listen-host H interface to bind (default: auto) --remote-port N remote port, if not in ENDPOINT + export Export bounding-box/segmentation annotations from + a running experiment (connects over gRPC, like + `cli`) to a relabeling-tool format, ready to hand + off to an outsourced relabeling pass. + --format, -f cvat | label_studio | v7 (required) + OUTPUT file path or directory (default: ".") + --origin O restrict to one split/loader (default: all) + --predictions export model predictions, not ground truth + --tag TAG restrict to samples with this tag; repeat for + multiple (matches ANY), default: all samples + --host H backend host (default: 127.0.0.1) + --port N backend gRPC port (default: 50051) + examples: weightslab se # one-time secure setup (then export WEIGHTSLAB_CERTS_DIR) weightslab se --force-certs # regenerate the certs @@ -244,6 +257,9 @@ def _banner() -> str: weightslab cli # connect a terminal to the running experiment weightslab cli --port 60000 # connect to a specific CLI port weightslab tunnel bore.pub:12345 # expose a remote (Colab) backend at localhost:50051 + weightslab export --format cvat # export all annotations to CVAT XML in "." + weightslab export -f v7 out/ --origin val_loader # V7/Darwin, val split only, into out/ + weightslab export -f cvat --tag ToReview # only samples tagged ToReview, for relabeling """ @@ -615,6 +631,76 @@ def cli_connect(args): sys.exit(exit_code) +_EXPORT_FORMAT_VALUES = { + "cvat": "EXPORT_FORMAT_CVAT", + "label_studio": "EXPORT_FORMAT_LABEL_STUDIO", + "v7": "EXPORT_FORMAT_V7_DARWIN", +} + + +def export_annotations_cli(args): + """`weightslab export --format FMT [OUTPUT]`: export bounding-box/ + segmentation annotations from a running experiment to a relabeling-tool + format (CVAT, Label Studio, or V7/Darwin), over the same gRPC connection + the Weights Studio "Export" button uses. + """ + try: + import grpc + from weightslab.proto import experiment_service_pb2 as pb2 + from weightslab.proto import experiment_service_pb2_grpc as pb2_grpc + except Exception as exc: + logger.error(f"Could not load the WeightsLab gRPC client: {exc}") + sys.exit(1) + + host = args.host or "127.0.0.1" + port = args.port or int(os.getenv("GRPC_BACKEND_PORT", "50051")) + + try: + channel = grpc.insecure_channel(f"{host}:{port}") + grpc.channel_ready_future(channel).result(timeout=10) + except Exception as exc: + logger.error( + f"Could not connect to a running WeightsLab backend at {host}:{port}: {exc}\n" + f"Is training running with wl.serve(serving_grpc=True)?" + ) + sys.exit(1) + + stub = pb2_grpc.ExperimentServiceStub(channel) + request = pb2.ExportAnnotationsRequest( + format=getattr(pb2, _EXPORT_FORMAT_VALUES[args.format]), + origin=args.origin or "", + include_predictions=bool(args.predictions), + tags=args.tags or [], + ) + try: + response = stub.ExportAnnotations(request, timeout=120) + except grpc.RpcError as exc: + logger.error(f"Export RPC failed: {exc}") + sys.exit(1) + + if not response.success: + logger.error(f"Export failed: {response.message}") + sys.exit(1) + + output = args.output + if not output or os.path.isdir(output) or str(output).endswith(("/", "\\")): + directory = output or "." + os.makedirs(directory, exist_ok=True) + output = os.path.join(directory, response.filename) + else: + parent = os.path.dirname(output) + if parent: + os.makedirs(parent, exist_ok=True) + + with open(output, "wb") as f: + f.write(response.payload) + + logger.info( + f"Exported {response.image_count} image(s) to {os.path.abspath(output)} " + f"({args.format} format)." + ) + + # Friendly two-word experiment names (adjective-noun), e.g. "wl-brisk-otter". # Readable and low-collision — nicer than a raw UUID for a directory the user # will `cd` into and share. @@ -840,6 +926,7 @@ def _build_parser() -> argparse.ArgumentParser: weightslab start example [--cls|--seg|--det|--clus|--gen|--3d_det|--2d_det] weightslab cli [--port PORT] [--host HOST] weightslab tunnel ENDPOINT + weightslab export --format {cvat,label_studio,v7} [OUTPUT] """ parser = argparse.ArgumentParser( prog="weightslab", @@ -847,7 +934,7 @@ def _build_parser() -> argparse.ArgumentParser: epilog=_EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter, ) - sub = parser.add_subparsers(dest="command", metavar="{se,start,cli,tunnel,help}") + sub = parser.add_subparsers(dest="command", metavar="{se,start,cli,tunnel,export,help}") # weightslab se [--force-certs] [certs_dir] se_parser = sub.add_parser("se", help="Set up the secure environment (TLS certs + gRPC auth token)") @@ -882,6 +969,35 @@ def _build_parser() -> argparse.ArgumentParser: '--remote-port', type=int, default=None, help="Remote port, if not included in ENDPOINT") + # weightslab export --format FMT [OUTPUT] [--origin O] [--predictions] [--host H] [--port N] + export_parser = sub.add_parser( + "export", + help="Export bounding-box/segmentation annotations from a running " + "experiment to a relabeling-tool format (CVAT, Label Studio, or V7/Darwin)") + export_parser.add_argument( + '--format', '-f', required=True, choices=['cvat', 'label_studio', 'v7'], + help="Target format: 'cvat' (XML), 'label_studio' (JSON), or 'v7' (Darwin JSON, zipped)") + export_parser.add_argument( + 'output', nargs='?', default=None, + help="Output file path or directory (default: current directory, using " + "the format's default filename)") + export_parser.add_argument( + '--origin', default=None, + help="Restrict to one registered split/loader (e.g. train_loader). Default: all splits.") + export_parser.add_argument( + '--predictions', action='store_true', + help="Export model predictions instead of ground-truth targets.") + export_parser.add_argument( + '--tag', dest='tags', action='append', default=None, + help="Restrict to samples carrying this tag (e.g. ToReview). Repeat " + "for multiple tags -- a sample matching ANY of them is included. " + "Default: all samples.") + export_parser.add_argument( + '--host', default=None, help="Backend host to connect to (default: 127.0.0.1)") + export_parser.add_argument( + '--port', type=int, default=None, + help="Backend gRPC port to connect to (default: $GRPC_BACKEND_PORT or 50051)") + # weightslab start [DIR] [--port N ...] -> native UI server for an experiment dir # weightslab start example [--cls|...] -> bundled example (rewritten in main() # to the `example` alias below, because a @@ -932,6 +1048,8 @@ def main(): elif args.command == "tunnel": from weightslab.tunnel import tunnel_connect tunnel_connect(args) + elif args.command == "export": + export_annotations_cli(args) elif args.command == "se": ui_secure_environment(args) elif args.command == "start": diff --git a/weightslab/examples/PyTorch/wl-detection-relabeling/README.md b/weightslab/examples/PyTorch/wl-detection-relabeling/README.md new file mode 100644 index 00000000..e709e52a --- /dev/null +++ b/weightslab/examples/PyTorch/wl-detection-relabeling/README.md @@ -0,0 +1,95 @@ +# WeightsLab — Detection Data Exploration & Relabeling (dataloader only) + +A WeightsLab example with **no model, optimizer, loss, or training loop** — +only a `Dataset`/`DataLoader` registered with `wl.watch_or_edit(..., flag="data")`. +It exists purely to browse, inspect, and tag a detection dataset, then export a +tagged subset to a relabeling tool (CVAT / Label Studio / V7). If you're +looking for the trainable version of this same dataset, see +[`../wl-detection`](../wl-detection). + +## Quick start + +```bash +cd weightslab/examples/PyTorch/wl-detection-relabeling +pip install -r ../wl-detection/requirements.txt # same deps, no extra training-only packages needed +python main.py +``` + +The **first run downloads** the Penn-Fudan Pedestrian dataset (~50 MB, ~170 +real photos, one class: `person`) into `./data/`. Then open Weights Studio +(e.g. `http://localhost:5173`) — every sample and its ground-truth bounding +box is already there; nothing needs to run first. + +## The workflow this demonstrates + +1. **Explore** — browse the grid or List view; ground-truth boxes render as + overlays (`task_type = "detection"`). +2. **Inspect** — open a sample's modal view for the full-size image + boxes. +3. **Tag** — flag samples that need another look. The script seeds 10 samples + with a `tag:ToReview` boolean tag on startup (`seed_review_tag_count` in + `config.yaml`; set to `0` to start clean) so there's something to export + right away, but real usage is tagging whatever you actually flag while + browsing — right-click a sample in the grid, or just ask the in-app chat + agent (it has full tool access to this running process) something like + *"tag the 5 blurriest samples as ToReview"* or *"mark sample 12 as + ToReview"* (`wl.tag_samples([...], "ToReview")` under the hood). +4. **Export for relabeling** — hand off just the tagged subset: + + ```bash + # CLI, from a second terminal (connects over gRPC to the running process) + weightslab export -f cvat --tag ToReview + + # Or ask the chat agent directly: + # "export the samples tagged ToReview to CVAT format for relabeling" + + # Or the Weights Studio UI's Export button (format picker + tag picker) + ``` + + Only samples carrying the `ToReview` tag end up in the exported file — see + [`../../../../docs/export.rst`](../../../../docs/export.rst) for the full + tag-filter reference (also works with categorical tags, and with + `wl.export_annotations(..., tags=[...])` from Python). + +## Why no training loop + +Every other PyTorch example in this repo (`wl-detection`, `wl-classification`, +...) wires a model + optimizer + loss and calls `guard_training_context`/ +`guard_testing_context` around each step, plus `wl.start_training(timeout=...)` +to let the UI attach before stepping starts. None of that applies here: + +- `watch_or_edit(..., flag="data", ...)` preloads every sample's ground-truth + boxes and metadata into the dataframe **at registration time** + (`preload_labels`/`preload_metadata` default to `True`) — the grid is fully + populated before a single batch is ever pulled from the loader. There is no + "step" to gate, so `guard_training_context`/`guard_testing_context` are + skipped entirely. +- `wl.start_training()` only sleeps for a timeout then resumes the pause + controller — meaningless without a stepping loop, so it's skipped too. +- `wl.serve(...)` + `wl.keep_serving()` are still called, so the process stays + up and inspectable/taggable/exportable for as long as you need it. + +If you want to iterate the loader anyway (e.g. to sanity-check `det_collate` +output shapes), a plain `for batch in loader: ...` works — it's a real +`DataLoader`, just never wrapped in a guard context here. + +## Files + +``` +utils/data.py PennFudanDetectionDataset + det_collate, copied unchanged from + ../wl-detection/utils/data.py (same dataset, no model/loss + utilities needed here). +main.py Registers hyperparameters + the dataset, seeds a demo + "ToReview" tag, serves, and keeps serving. No model/optimizer/ + loss/training loop. +config.yaml Dataset + serving config only (no model/optimizer/training + hyperparameters, since none exist in this usecase). +``` + +## Using your own dataset + +Same as [`../wl-detection`](../wl-detection#using-your-own-dataset-eg-traffic-lights): +write a `Dataset` whose `get_items(idx, ...)` returns +`(image_tensor, uid, target, metadata)` with `target` an `[N, 6]` +`[x1, y1, x2, y2, class_id, confidence]` array normalized to `[0, 1]`, set +`self.task_type = "detection"` / `self.class_names`, and swap it in for +`PennFudanDetectionDataset` in `main.py`. diff --git a/weightslab/examples/PyTorch/wl-detection-relabeling/config.yaml b/weightslab/examples/PyTorch/wl-detection-relabeling/config.yaml new file mode 100644 index 00000000..f85eaa90 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-detection-relabeling/config.yaml @@ -0,0 +1,29 @@ +# Global configuration +experiment_name: pennfudan_detection_relabeling +# root_log_dir: # Empty to write in tmp directory, or specify a path to store logs + +# Serving +serving_grpc: true +serving_cli: true + +# Configure global dataframe storage +ledger_enable_h5_persistence: true +ledger_enable_flushing_threads: true +ledger_flush_max_rows: 100 +ledger_flush_interval: 60.0 + +# Dataset +num_classes: 1 # Penn-Fudan: single class (person) +image_size: 256 + +# How many samples get auto-tagged "ToReview" on first launch, purely to give +# the export-filtered-by-tag workflow something to demo immediately. Set to 0 +# to start with a clean, untagged dataset. +seed_review_tag_count: 10 + +# Data (Penn-Fudan pedestrians; ~170 images downloaded on first run under data_root) +data_root: .\data +data: + data_loader: + batch_size: 8 + max_samples: null # null = use the full train split diff --git a/weightslab/examples/PyTorch/wl-detection-relabeling/main.py b/weightslab/examples/PyTorch/wl-detection-relabeling/main.py new file mode 100644 index 00000000..9b6116ef --- /dev/null +++ b/weightslab/examples/PyTorch/wl-detection-relabeling/main.py @@ -0,0 +1,107 @@ +import os +import tempfile + +import yaml + +import weightslab as wl + +from utils.data import PennFudanDetectionDataset, det_collate + + +# ============================================================================= +# Main -- data exploration / inspection / tagging / export-for-relabeling. +# +# No model, optimizer, loss, or training loop: `watch_or_edit(..., flag="data")` +# already preloads every sample's ground-truth boxes and metadata into the +# dataframe at registration time (`preload_labels`/`preload_metadata` default +# to True), so the grid is fully browsable the moment this script starts +# serving -- there is nothing to train towards, only data to look at. +# ============================================================================= +if __name__ == "__main__": + config_path = os.path.join(os.path.dirname(__file__), "config.yaml") + if os.path.exists(config_path): + with open(config_path, "r") as fh: + parameters = yaml.safe_load(fh) or {} + else: + parameters = {} + + parameters.setdefault("experiment_name", "pennfudan_detection_relabeling") + parameters.setdefault("num_classes", 1) # Penn-Fudan: single class (person) + parameters.setdefault("image_size", 256) + parameters.setdefault("seed_review_tag_count", 10) + + exp_name = parameters["experiment_name"] + + wl.watch_or_edit( + parameters, + flag="hyperparameters", + name=exp_name, + defaults=parameters, + poll_interval=1.0, + ) + + num_classes = int(parameters["num_classes"]) + image_size = int(parameters["image_size"]) + + if not parameters.get("root_log_dir"): + parameters["root_log_dir"] = tempfile.mkdtemp() + print(f"No root_log_dir specified, using temporary directory: {parameters['root_log_dir']}") + os.makedirs(parameters["root_log_dir"], exist_ok=True) + + # --- Data (Penn-Fudan pedestrians, downloaded on first run) --- + default_data_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "data")) + data_root = parameters.get("data_root", default_data_root) + data_cfg = parameters.get("data", {}).get("data_loader", {}) + + dataset = PennFudanDetectionDataset( + root=data_root, + split="train", # only the train split is loaded here, for simplicity + num_classes=num_classes, + image_size=image_size, + max_samples=data_cfg.get("max_samples", None), + ) + + loader = wl.watch_or_edit( + dataset, + flag="data", + loader_name="data_loader", + batch_size=data_cfg.get("batch_size", 8), + shuffle=False, + is_training=False, + compute_hash=False, + array_autoload_arrays=False, + array_return_proxies=True, + array_use_cache=True, + collate_fn=det_collate, + ) + + # Seed a few samples with a "ToReview" tag purely so the tag-filtered + # export workflow has something to demo on first launch. Real usage: tag + # whatever you actually flag while browsing (grid right-click, or just + # ask the in-app chat agent), then export that subset -- see README.md. + # Sample ids are filename-derived (e.g. "FudanPed00001"), not 0..N indices. + seed_count = min(int(parameters.get("seed_review_tag_count", 10)), len(dataset)) + seed_sample_ids = [ + os.path.splitext(os.path.basename(p))[0] for p in dataset.images[:seed_count] + ] + if seed_sample_ids: + wl.tag_samples(seed_sample_ids, "ToReview") + + wl.serve( + serving_grpc=parameters.get("serving_grpc", True), + serving_cli=parameters.get("serving_cli", True), + ) + + print("=" * 60) + print(" PENN-FUDAN DETECTION -- DATA EXPLORATION / TAGGING / RELABELING") + print(f" {len(dataset)} samples registered ({seed_count} seeded with tag:ToReview)") + print(f" Data root: {data_root}") + print(" Open Weights Studio, browse/tag samples, then export a tagged") + print(" subset for relabeling, e.g.:") + print(" weightslab export -f cvat --tag ToReview") + print("=" * 60 + "\n") + + # No wl.start_training()/guard_training_context/guard_testing_context: this + # usecase never runs a training step, so there is nothing to gate. See + # README.md ("Why no training loop") for the reasoning. + wl.keep_serving() diff --git a/weightslab/examples/PyTorch/wl-detection-relabeling/utils/data.py b/weightslab/examples/PyTorch/wl-detection-relabeling/utils/data.py new file mode 100644 index 00000000..dbff4a02 --- /dev/null +++ b/weightslab/examples/PyTorch/wl-detection-relabeling/utils/data.py @@ -0,0 +1,226 @@ +import os +import ssl +import zipfile +import urllib.request + +import numpy as np +import torch + +from torchvision import transforms + +from torch.utils.data import Dataset + +from PIL import Image + + +# ============================================================================= +# Penn-Fudan Pedestrian detection dataset +# ============================================================================= +# A small, real object-detection dataset (~170 photos, one class: "person"). +# It ships per-instance segmentation masks; we derive an axis-aligned bounding +# box per pedestrian from each mask. Downloaded + extracted on first use. +# +# On-disk layout after extraction: +# /PennFudanPed/ +# PNGImages/FudanPed00001.png ... +# PedMasks/FudanPed00001_mask.png ... # pixel value k = k-th pedestrian, 0 = bg +# +# WL renders detection targets/predictions from a per-sample [N, 6] array +# ``[x1, y1, x2, y2, class_id, confidence]`` normalized to [0, 1] (GT conf = 1.0) +# — see ``get_items`` below and ``data_service.py`` (task_type == "detection"). + +CLASS_NAMES = ["person"] + +_URL = "https://www.cis.upenn.edu/~jshi/ped_html/PennFudanPed.zip" + +# ImageNet statistics — the MobileNetV3 backbone is pretrained with these, so we +# normalize model inputs the same way. (The UI still shows the original PNG.) +IMAGENET_MEAN = [0.485, 0.456, 0.406] +IMAGENET_STD = [0.229, 0.224, 0.225] + + +def download_penn_fudan(root): + """Download + extract Penn-Fudan into /PennFudanPed (idempotent).""" + base = os.path.join(root, "PennFudanPed") + if os.path.isdir(os.path.join(base, "PNGImages")): + return base + + os.makedirs(root, exist_ok=True) + zip_path = os.path.join(root, "PennFudanPed.zip") + + if not os.path.exists(zip_path): + print(f"[data] Downloading Penn-Fudan dataset to {zip_path} ...", flush=True) + try: + urllib.request.urlretrieve(_URL, zip_path) + except Exception as e: + # Some corporate environments break TLS verification - retry unverified. + print(f"[data] TLS verification failed ({e}); retrying without verification.", flush=True) + ctx = ssl._create_unverified_context() + with urllib.request.urlopen(_URL, context=ctx) as resp, open(zip_path, "wb") as fh: + fh.write(resp.read()) + + print("[data] Extracting Penn-Fudan ...", flush=True) + with zipfile.ZipFile(zip_path) as zf: + zf.extractall(root) + return base + + +def _boxes_from_mask(mask_path): + """Derive one bbox per pedestrian from a Penn-Fudan instance mask. + + Returns (boxes_px [N, 4] int xyxy, height, width). Background (0) skipped. + """ + mask = np.array(Image.open(mask_path)) + h, w = mask.shape[:2] + obj_ids = np.unique(mask) + obj_ids = obj_ids[obj_ids != 0] # drop background + + boxes = [] + for oid in obj_ids: + ys, xs = np.where(mask == oid) + if xs.size == 0: + continue + x1, x2 = int(xs.min()), int(xs.max()) + y1, y2 = int(ys.min()), int(ys.max()) + if x2 > x1 and y2 > y1: + boxes.append([x1, y1, x2, y2]) + return np.asarray(boxes, dtype=np.float32).reshape(-1, 4), h, w + + +class PennFudanDetectionDataset(Dataset): + """Pedestrian bounding-box detection over the Penn-Fudan images. + + Args: + root: directory to download/extract the dataset into. + split: "train" or "val" (deterministic split of the 170 images). + image_size: square resize fed to the model. + val_fraction: fraction of images held out for validation. + max_samples: optional cap on the split size (for quick runs). + """ + + def __init__( + self, + root, + split="train", + num_classes=1, + image_size=256, + val_fraction=0.2, + max_samples=None, + ): + super().__init__() + self.root = root + self.split = split + self.num_classes = num_classes + self.image_size = image_size + # Explicit task type; bypasses WL's label-shape heuristic so bboxes are + # rendered as detection overlays (not mistaken for classification). + self.task_type = "detection" + self.class_names = CLASS_NAMES[:num_classes] + + base = download_penn_fudan(root) + img_dir = os.path.join(base, "PNGImages") + mask_dir = os.path.join(base, "PedMasks") + + all_imgs = sorted(f for f in os.listdir(img_dir) if f.lower().endswith(".png")) + + # Deterministic train/val split: every k-th image goes to val. + k = max(2, int(round(1.0 / max(val_fraction, 1e-6)))) + if split == "val": + selected = all_imgs[::k] + else: + val_set = set(all_imgs[::k]) + selected = [f for f in all_imgs if f not in val_set] + + selected = selected[:max_samples] if max_samples != None else selected + + self.images = [] + self.masks = [] + for fname in selected: + base_name, _ = os.path.splitext(fname) + mask_path = os.path.join(mask_dir, base_name + "_mask.png") + if os.path.exists(mask_path): + self.images.append(os.path.join(img_dir, fname)) + self.masks.append(mask_path) + + if len(self.images) == 0: + raise RuntimeError(f"No image/mask pairs found under {base}") + + self.image_transform = transforms.Compose( + [ + transforms.Resize((image_size, image_size), interpolation=Image.BILINEAR), + transforms.ToTensor(), + transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD), + ] + ) + + def __len__(self): + return len(self.images) + + def _load_boxes(self, mask_path): + """Read mask → [N, 6] float32 = [x1, y1, x2, y2, cls=0, conf=1.0] (normalized).""" + boxes_px, h, w = _boxes_from_mask(mask_path) + if boxes_px.shape[0] == 0: + return np.zeros((0, 6), dtype=np.float32) + norm = boxes_px.copy() + norm[:, [0, 2]] /= float(w) + norm[:, [1, 3]] /= float(h) + n = norm.shape[0] + cls = np.zeros((n, 1), dtype=np.float32) # single class: person + conf = np.ones((n, 1), dtype=np.float32) + return np.concatenate([norm, cls, conf], axis=1).astype(np.float32) + + def __getitem__(self, idx): + """Returns (item, uid, target, metadata). + + - item: normalized image tensor [C, H, W] + - uid: unique sample id (string) + - target: [N, 6] float32 = [x1, y1, x2, y2, class_id, confidence] + - metadata: dict with source paths + """ + return self.get_items(idx, include_metadata=True, include_labels=True, include_images=True) + + def get_items(self, idx, include_metadata=False, include_labels=False, include_images=False): + img_path = self.images[idx] + mask_path = self.masks[idx] + uid = os.path.splitext(os.path.basename(img_path))[0] + + metadata = { + "img_path": img_path, + "mask_path": mask_path, + } if include_metadata else None + + img_t = None + if include_images: + img = Image.open(img_path).convert("RGB") + img_t = self.image_transform(img) + + target = None + if include_labels: + target = self._load_boxes(mask_path) + + return img_t, uid, target, metadata + + +def det_collate(batch): + """Collate WL per-sample tuples for object detection. + + A sample owns a variable number of boxes, so targets cannot be stacked. We + keep them as a Python list (one [N_i, 6] tensor per sample), exactly the + layout WL's per-instance helpers expect (``targets[s]`` iterates that + sample's boxes in annotation order). + + Returns: + images: FloatTensor [B, C, H, W] + ids: list[str] of length B + targets: list[B] of [N_i, 6] float tensors ([x1, y1, x2, y2, cls, conf]) + metas: list[B] of metadata dicts + """ + images = torch.stack([b[0] for b in batch], dim=0) + ids = [b[1] for b in batch] + targets = [ + torch.as_tensor(b[2], dtype=torch.float32) + if not isinstance(b[2], torch.Tensor) else b[2].float() + for b in batch + ] + metas = [b[3] if len(b) > 3 else None for b in batch] + return images, ids, targets, metas diff --git a/weightslab/export/__init__.py b/weightslab/export/__init__.py new file mode 100644 index 00000000..62bf3930 --- /dev/null +++ b/weightslab/export/__init__.py @@ -0,0 +1,16 @@ +"""weightslab.export -- annotation export to relabeling-tool formats. + +Public API +---------- +SUPPORTED_FORMATS : tuple -- ("cvat", "label_studio", "v7") +export_annotations : func -- collect + encode, returns (bytes, filename, mime_type, image_count) +save_export : func -- export_annotations() + write to disk, returns the path written +collect_image_annotations : func -- IR extraction only (no format encoding) +""" + +from weightslab.export.collect import collect_image_annotations # noqa: F401 +from weightslab.export.exporter import ( # noqa: F401 + SUPPORTED_FORMATS, + export_annotations, + save_export, +) diff --git a/weightslab/export/collect.py b/weightslab/export/collect.py new file mode 100644 index 00000000..07eea006 --- /dev/null +++ b/weightslab/export/collect.py @@ -0,0 +1,330 @@ +"""Collect bounding-box/segmentation annotations from the registered dataframe +into the format-agnostic ``ImageAnnotations`` intermediate representation. + +Two real architectural gaps drive most of the "best effort" logic here (see +docs/export.rst for the user-facing caveat): + +- There is no dedicated class-id -> class-name registry. We fall back to a + ``dataset.class_names`` attribute (the same fallback ``data_service.py`` + already uses for the studio UI), or an explicit ``class_names`` override. +- There is no per-sample stored image path/dimensions. We best-effort resolve + a path via a few common dataset attribute names, and fall back to a + segmentation mask's own shape, or a caller-supplied default size. +""" + +import logging +import os +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np + +from weightslab.data.sample_stats import SampleStats +from weightslab.export.models import BoxAnnotation, ImageAnnotations, PolygonAnnotation + +logger = logging.getLogger(__name__) + +SID = SampleStats.Ex.SAMPLE_ID.value +ORIGIN = SampleStats.Ex.ORIGIN.value +TARGET = SampleStats.Ex.TARGET.value +PREDICTION = SampleStats.Ex.PREDICTION.value + +_PATH_ATTR_CANDIDATES = ("image_paths", "img_files", "images", "imgs", "files", "samples") + +_warned_no_cv2 = False + + +def _is_bbox_array(value: Any) -> bool: + """Mirrors ``LedgeredDataFrameManager._is_bbox_array``: a 2-D array whose + last dim is 4..6 is bbox coordinates; dense (H, W) masks have a much + larger trailing dim and never collide with this shape. + """ + try: + arr = np.asanyarray(value) + except Exception: + return False + return arr.ndim == 2 and arr.shape[-1] in range(3, 10) + + +def _boxes_from_cell(value: Any) -> List[Tuple[float, float, float, float, Optional[int]]]: + """Flatten one target/prediction cell into (x1, y1, x2, y2, cls_id) tuples. + + Handles a single box (1-D, len 4..6), or multiple boxes stacked in one + cell (2-D, last dim 4..6). Row layout is ``(x1, y1, x2, y2[, conf][, cls])`` + -- class id is the last column when the row is 5 or 6 wide. + """ + if value is None: + return [] + try: + arr = np.asanyarray(value) + except Exception: + return [] + if arr.size == 0: + return [] + + if arr.ndim == 1 and arr.shape[0] in (4, 5, 6): + rows = [arr] + elif arr.ndim == 2 and arr.shape[-1] in range(4, 7): + rows = list(arr) + else: + return [] + + boxes = [] + for row in rows: + row = np.asanyarray(row, dtype=float) + cls_id = int(round(row[-1])) if row.shape[0] >= 5 else None + boxes.append((float(row[0]), float(row[1]), float(row[2]), float(row[3]), cls_id)) + return boxes + + +def _mask_to_polygons(mask: np.ndarray, class_id: int) -> List[List[Tuple[float, float]]]: + """Extract polygon contours for one class id from a dense (H, W) mask. + + Requires OpenCV (lazily imported) -- install with ``pip install + weightslab[export]``. Callers should catch ``ImportError`` and degrade to + bbox-only export rather than let this abort the whole run. + """ + import cv2 # raises ImportError if the [export] extra isn't installed + + binary = (mask == class_id).astype(np.uint8) + contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + polygons = [] + for contour in contours: + if len(contour) < 3: + continue + polygons.append([(float(pt[0][0]), float(pt[0][1])) for pt in contour]) + return polygons + + +def _polygons_from_mask_cell(value: Any, class_names) -> List[PolygonAnnotation]: + global _warned_no_cv2 + try: + arr = np.asanyarray(value) + except Exception: + return [] + if arr.ndim != 2 or arr.size == 0 or _is_bbox_array(arr): + return [] + + class_ids = sorted(int(c) for c in np.unique(arr) if c != 0) + if not class_ids: + return [] + + polygons = [] + for cid in class_ids: + label = _label_for_class(cid, class_names) + try: + for points in _mask_to_polygons(arr, cid): + polygons.append(PolygonAnnotation(points=points, label=label)) + except ImportError: + if not _warned_no_cv2: + logger.warning( + "[export] OpenCV not installed -- skipping segmentation polygon " + "export (bboxes are unaffected). Install with `pip install " + "weightslab[export]` to enable polygon export." + ) + _warned_no_cv2 = True + return [] + return polygons + + +def _label_for_class(cls_id: Optional[int], class_names: Union[dict, list, tuple, None]) -> str: + if cls_id is None: + return "object" + if class_names is not None: + try: + if isinstance(class_names, dict): + if cls_id in class_names: + return str(class_names[cls_id]) + if str(cls_id) in class_names: + return str(class_names[str(cls_id)]) + elif isinstance(class_names, (list, tuple)) and 0 <= cls_id < len(class_names): + return str(class_names[cls_id]) + except Exception: + pass + return f"class_{cls_id}" + + +def _resolve_dataset_for_origin(origin: Optional[str]): + try: + from weightslab.backend.ledgers import get_dataloader, list_dataloaders + names = [origin] if origin else list_dataloaders() + for name in names: + loader = get_dataloader(name) + dataset = getattr(loader, "dataset", None) + if dataset is not None: + return dataset + except Exception: + logger.debug("[export] Could not resolve dataset for origin=%r", origin, exc_info=True) + return None + + +def _resolve_class_names(explicit, origin: Optional[str]): + if explicit is not None: + return explicit + dataset = _resolve_dataset_for_origin(origin) + class_names = getattr(dataset, "class_names", None) if dataset is not None else None + if not class_names: + logger.debug( + "[export] No class_names resolved (no override, no dataset.class_names " + "attribute) -- labels will fall back to 'class_'." + ) + return class_names + + +def _positional_index(dataset, sample_id) -> int: + if dataset is not None and hasattr(dataset, "get_index_from_sample_id"): + try: + return dataset.get_index_from_sample_id(sample_id) + except Exception: + pass + try: + return int(sample_id) + except (TypeError, ValueError): + return 0 + + +def _resolve_image_path(dataset, sample_id) -> Optional[str]: + if dataset is None: + return None + idx = _positional_index(dataset, sample_id) + for attr in _PATH_ATTR_CANDIDATES: + seq = getattr(dataset, attr, None) + if seq is None: + continue + try: + item = seq[idx] + except Exception: + continue + if isinstance(item, (tuple, list)) and item: + item = item[0] + if isinstance(item, (str, os.PathLike)): + return str(item) + return None + + +def _resolve_image_dims(path: Optional[str], mask_arr: Optional[np.ndarray], default_size: Tuple[int, int]) -> Tuple[int, int]: + if mask_arr is not None: + h, w = mask_arr.shape[0], mask_arr.shape[1] + return int(w), int(h) + if path: + try: + from PIL import Image + with Image.open(path) as im: + return int(im.width), int(im.height) + except Exception: + logger.debug("[export] Could not open %s to read dimensions", path, exc_info=True) + return default_size + + +def _tag_column(tag: str) -> str: + tag = tag.strip() + prefix = f"{SampleStats.Ex.TAG.value}:" + return tag if tag.startswith(prefix) else f"{prefix}{tag}" + + +def _filter_by_tags(df, tags: Optional[List[str]]): + """Keep only rows carrying ANY of `tags` (boolean True, or a non-empty + categorical value -- both kinds of tag reuse the same ``tag:`` + column, see ``wl.tag_samples``/``wl.set_categorical_tag``). + """ + if not tags: + return df + + tag_cols = [col for col in (_tag_column(t) for t in tags) if col in df.columns] + if not tag_cols: + return df.iloc[0:0] + + mask = False + for col in tag_cols: + col_mask = df[col].notna() & (df[col] != False) & (df[col] != "") + mask = col_mask if mask is False else (mask | col_mask) + return df[mask] + + +def collect_image_annotations( + origin: Optional[str] = None, + class_names: Optional[Union[dict, list, tuple]] = None, + use_predictions: bool = False, + default_image_size: Tuple[int, int] = (0, 0), + image_extension: str = ".jpg", + tags: Optional[List[str]] = None, +) -> List[ImageAnnotations]: + """Build the per-image annotation list from the registered dataframe. + + Args: + origin: restrict to one split/loader name (e.g. "train_loader"); + ``None`` exports every registered split. + class_names: explicit class-id -> name mapping (dict or list), wins + over any auto-detected ``dataset.class_names``. + use_predictions: export model predictions instead of ground-truth targets. + default_image_size: (width, height) fallback when no dimensions can + be resolved from a mask or the source image file. + image_extension: extension used for the synthetic filename + (``sample_``) when no real image path can be resolved. + tags: restrict to samples carrying ANY of these tags (``tag:`` prefix + optional, e.g. ``["ToReview"]``); ``None``/empty exports every sample. + """ + from weightslab.backend.ledgers import get_dataframe + + dfm = get_dataframe() + if dfm is None or not hasattr(dfm, "get_combined_df"): + return [] + df = dfm.get_combined_df() + if df is None or df.empty: + return [] + + df = df.reset_index() + if origin: + if ORIGIN not in df.columns: + return [] + df = df[df[ORIGIN] == origin] + if df.empty: + return [] + + df = _filter_by_tags(df, tags) + if df.empty: + return [] + + value_col = PREDICTION if use_predictions else TARGET + if value_col not in df.columns: + return [] + + resolved_class_names = _resolve_class_names(class_names, origin) + dataset_cache: Dict[str, Any] = {} + + images: List[ImageAnnotations] = [] + for sample_id, group in df.groupby(SID, sort=False): + sample_origin = str(group[ORIGIN].iloc[0]) if ORIGIN in group.columns else (origin or "") + dataset = dataset_cache.setdefault(sample_origin, _resolve_dataset_for_origin(sample_origin or origin)) + path = _resolve_image_path(dataset, sample_id) + filename = os.path.basename(path) if path else f"sample_{sample_id}{image_extension}" + + boxes: List[BoxAnnotation] = [] + polygons: List[PolygonAnnotation] = [] + mask_arr: Optional[np.ndarray] = None + + for cell in group[value_col]: + if cell is None: + continue + for x1, y1, x2, y2, cls_id in _boxes_from_cell(cell): + boxes.append(BoxAnnotation(x1, y1, x2, y2, _label_for_class(cls_id, resolved_class_names))) + + try: + arr = np.asanyarray(cell) + except Exception: + continue + if arr.ndim == 2 and arr.size > 0 and not _is_bbox_array(arr): + mask_arr = arr + polygons.extend(_polygons_from_mask_cell(arr, resolved_class_names)) + + width, height = _resolve_image_dims(path, mask_arr, default_image_size) + images.append(ImageAnnotations( + sample_id=str(sample_id), + filename=filename, + width=width, + height=height, + origin=sample_origin, + boxes=boxes, + polygons=polygons, + )) + + return images diff --git a/weightslab/export/exporter.py b/weightslab/export/exporter.py new file mode 100644 index 00000000..9e05e14d --- /dev/null +++ b/weightslab/export/exporter.py @@ -0,0 +1,81 @@ +"""Top-level dispatcher for annotation export -- the single entry point shared +by the Python API (``wl.export_annotations``), the gRPC handler backing the +Weights Studio "Export" button, and the ``weightslab export`` CLI command. +""" + +import logging +import os +from typing import List, Optional, Tuple, Union + +from weightslab.export.collect import collect_image_annotations +from weightslab.export.formats.cvat import to_cvat_xml +from weightslab.export.formats.label_studio import to_label_studio_json +from weightslab.export.formats.v7_darwin import to_v7_darwin_zip + +logger = logging.getLogger(__name__) + +SUPPORTED_FORMATS = ("cvat", "label_studio", "v7") + +# format -> (encoder, default filename, mime type) +_ENCODERS = { + "cvat": (to_cvat_xml, "annotations_cvat.xml", "application/xml"), + "label_studio": (to_label_studio_json, "annotations_label_studio.json", "application/json"), + "v7": (to_v7_darwin_zip, "annotations_v7_darwin.zip", "application/zip"), +} + + +def export_annotations( + fmt: str, + origin: Optional[str] = None, + class_names: Optional[Union[dict, list, tuple]] = None, + use_predictions: bool = False, + tags: Optional[List[str]] = None, +) -> Tuple[bytes, str, str, int]: + """Collect annotations from the registered dataframe and encode them as `fmt`. + + Args: + fmt: one of ``SUPPORTED_FORMATS`` (``"cvat"``, ``"label_studio"``, ``"v7"``). + origin: restrict to one split/loader name; ``None`` exports every split. + class_names: explicit class-id -> name mapping, overriding any + auto-detected ``dataset.class_names``. + use_predictions: export model predictions instead of ground-truth targets. + tags: restrict to samples carrying ANY of these tags (e.g. ``["ToReview"]``); + ``None``/empty exports every sample. + + Returns: + ``(payload_bytes, filename, mime_type, image_count)``. + """ + fmt = (fmt or "").strip().lower() + if fmt not in _ENCODERS: + raise ValueError(f"Unknown export format {fmt!r}. Supported: {', '.join(SUPPORTED_FORMATS)}") + + images = collect_image_annotations( + origin=origin, class_names=class_names, use_predictions=use_predictions, tags=tags, + ) + encoder, filename, mime_type = _ENCODERS[fmt] + payload = encoder(images) + logger.info("[export] Encoded %d image(s) to %s format (%d bytes)", len(images), fmt, len(payload)) + return payload, filename, mime_type, len(images) + + +def save_export(fmt: str, output_path: str, **kwargs) -> str: + """Export and write the result to `output_path`. + + If `output_path` is an existing directory (or ends in a path separator), + the format's default filename is appended. Returns the path written. + """ + payload, default_filename, _mime_type, image_count = export_annotations(fmt, **kwargs) + + if os.path.isdir(output_path) or output_path.endswith(("/", "\\")): + os.makedirs(output_path, exist_ok=True) + output_path = os.path.join(output_path, default_filename) + else: + parent = os.path.dirname(output_path) + if parent: + os.makedirs(parent, exist_ok=True) + + with open(output_path, "wb") as f: + f.write(payload) + + logger.info("[export] Wrote %d image(s) to %s (%s format)", image_count, output_path, fmt) + return output_path diff --git a/weightslab/export/formats/__init__.py b/weightslab/export/formats/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/weightslab/export/formats/cvat.py b/weightslab/export/formats/cvat.py new file mode 100644 index 00000000..a9059133 --- /dev/null +++ b/weightslab/export/formats/cvat.py @@ -0,0 +1,56 @@ +"""CVAT XML 1.1 (images task) encoder. + +Schema reference: https://opencv.github.io/cvat/docs/manual/advanced/xml_format/ +""" + +import xml.etree.ElementTree as ET +from typing import List +from xml.dom import minidom + +from weightslab.export.models import ImageAnnotations + + +def to_cvat_xml(images: List[ImageAnnotations]) -> bytes: + root = ET.Element("annotations") + ET.SubElement(root, "version").text = "1.1" + + meta = ET.SubElement(root, "meta") + task = ET.SubElement(meta, "task") + ET.SubElement(task, "name").text = "weightslab-export" + ET.SubElement(task, "size").text = str(len(images)) + ET.SubElement(task, "mode").text = "annotation" + + labels_el = ET.SubElement(task, "labels") + seen_labels: List[str] = [] + for img in images: + for ann in (*img.boxes, *img.polygons): + if ann.label not in seen_labels: + seen_labels.append(ann.label) + for label in seen_labels: + label_el = ET.SubElement(labels_el, "label") + ET.SubElement(label_el, "name").text = label + + for image_id, img in enumerate(images): + image_el = ET.SubElement( + root, "image", + id=str(image_id), name=img.filename, + width=str(img.width), height=str(img.height), + ) + for box in img.boxes: + ET.SubElement( + image_el, "box", + label=box.label, + xtl=f"{box.x1:.2f}", ytl=f"{box.y1:.2f}", + xbr=f"{box.x2:.2f}", ybr=f"{box.y2:.2f}", + occluded="0", z_order="0", + ) + for poly in img.polygons: + points_str = ";".join(f"{x:.2f},{y:.2f}" for x, y in poly.points) + ET.SubElement( + image_el, "polygon", + label=poly.label, points=points_str, + occluded="0", z_order="0", + ) + + raw = ET.tostring(root, encoding="utf-8") + return minidom.parseString(raw).toprettyxml(indent=" ", encoding="utf-8") diff --git a/weightslab/export/formats/label_studio.py b/weightslab/export/formats/label_studio.py new file mode 100644 index 00000000..17d77cfc --- /dev/null +++ b/weightslab/export/formats/label_studio.py @@ -0,0 +1,61 @@ +"""Label Studio JSON export encoder. + +Schema reference: https://labelstud.io/guide/export.html#JSON +Rectangle/polygon coordinates are stored as percentages (0-100) of the +image's original width/height, per Label Studio's ``rectanglelabels`` / +``polygonlabels`` result types. +""" + +import json +import uuid +from typing import List + +from weightslab.export.models import ImageAnnotations + + +def to_label_studio_json(images: List[ImageAnnotations]) -> bytes: + tasks = [] + for task_id, img in enumerate(images, start=1): + width = img.width or 1 + height = img.height or 1 + results = [] + + for box in img.boxes: + results.append({ + "id": uuid.uuid4().hex[:10], + "type": "rectanglelabels", + "from_name": "label", + "to_name": "image", + "original_width": img.width, + "original_height": img.height, + "value": { + "x": box.x1 / width * 100.0, + "y": box.y1 / height * 100.0, + "width": (box.x2 - box.x1) / width * 100.0, + "height": (box.y2 - box.y1) / height * 100.0, + "rotation": 0, + "rectanglelabels": [box.label], + }, + }) + + for poly in img.polygons: + results.append({ + "id": uuid.uuid4().hex[:10], + "type": "polygonlabels", + "from_name": "label", + "to_name": "image", + "original_width": img.width, + "original_height": img.height, + "value": { + "points": [[x / width * 100.0, y / height * 100.0] for x, y in poly.points], + "polygonlabels": [poly.label], + }, + }) + + tasks.append({ + "id": task_id, + "data": {"image": img.filename}, + "annotations": [{"id": task_id, "result": results}], + }) + + return json.dumps(tasks, indent=2).encode("utf-8") diff --git a/weightslab/export/formats/v7_darwin.py b/weightslab/export/formats/v7_darwin.py new file mode 100644 index 00000000..ed391898 --- /dev/null +++ b/weightslab/export/formats/v7_darwin.py @@ -0,0 +1,70 @@ +"""V7 (Darwin JSON 2.0) export encoder. + +Schema reference: https://docs.v7labs.com/reference/darwin-json +Darwin expects one JSON annotation file per image (matched by filename on +import), so this bundles all per-image documents into a single zip. +""" + +import io +import json +import os +import uuid +import zipfile +from typing import List + +from weightslab.export.models import ImageAnnotations + + +def _image_document(img: ImageAnnotations) -> dict: + annotations = [] + for box in img.boxes: + annotations.append({ + "id": str(uuid.uuid4()), + "name": box.label, + "bounding_box": { + "x": box.x1, + "y": box.y1, + "w": box.x2 - box.x1, + "h": box.y2 - box.y1, + }, + }) + for poly in img.polygons: + annotations.append({ + "id": str(uuid.uuid4()), + "name": poly.label, + "polygon": {"paths": [[{"x": x, "y": y} for x, y in poly.points]]}, + }) + + return { + "version": "2.0", + "item": { + "name": img.filename, + "path": "/", + "source_info": {"item_id": img.sample_id}, + "slots": [{ + "type": "image", + "slot_name": "0", + "width": img.width, + "height": img.height, + }], + }, + "annotations": annotations, + } + + +def to_v7_darwin_zip(images: List[ImageAnnotations]) -> bytes: + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w", zipfile.ZIP_DEFLATED) as zf: + used_names = set() + for img in images: + json_name = os.path.splitext(img.filename)[0] + ".json" + # Guard against filename collisions (e.g. synthetic "sample_" + # names sharing a stem after extension-stripping). + candidate = json_name + suffix = 1 + while candidate in used_names: + candidate = f"{os.path.splitext(json_name)[0]}_{suffix}.json" + suffix += 1 + used_names.add(candidate) + zf.writestr(candidate, json.dumps(_image_document(img), indent=2)) + return buf.getvalue() diff --git a/weightslab/export/models.py b/weightslab/export/models.py new file mode 100644 index 00000000..eb68095e --- /dev/null +++ b/weightslab/export/models.py @@ -0,0 +1,35 @@ +"""Plain intermediate-representation types for annotation export. + +Every format encoder (CVAT / Label Studio / V7 Darwin) consumes a list of +``ImageAnnotations`` — this is the single point where WeightsLab's internal +dataframe representation gets translated into something format-agnostic. +""" + +from dataclasses import dataclass, field +from typing import List, Tuple + + +@dataclass +class BoxAnnotation: + x1: float + y1: float + x2: float + y2: float + label: str + + +@dataclass +class PolygonAnnotation: + points: List[Tuple[float, float]] + label: str + + +@dataclass +class ImageAnnotations: + sample_id: str + filename: str + width: int + height: int + origin: str = "" + boxes: List[BoxAnnotation] = field(default_factory=list) + polygons: List[PolygonAnnotation] = field(default_factory=list) diff --git a/weightslab/monitoring/__init__.py b/weightslab/monitoring/__init__.py new file mode 100644 index 00000000..1c584119 --- /dev/null +++ b/weightslab/monitoring/__init__.py @@ -0,0 +1,18 @@ +"""weightslab.monitoring — automatic system resource monitoring. + +Public API +---------- +ResourceMonitor : class — background thread sampling CPU/memory/disk/network/GPU/process +load_resource_monitoring_config : func — resolve config from env vars + YAML +start_resource_monitor_from_config : func — start the process-wide singleton if enabled +get_resource_monitor : func — return the running singleton, if any +stop_resource_monitor : func — stop and clear the singleton +""" + +from weightslab.monitoring.resource_monitor import ( # noqa: F401 + ResourceMonitor, + load_resource_monitoring_config, + start_resource_monitor_from_config, + get_resource_monitor, + stop_resource_monitor, +) diff --git a/weightslab/monitoring/resource_monitor.py b/weightslab/monitoring/resource_monitor.py new file mode 100644 index 00000000..5029fba3 --- /dev/null +++ b/weightslab/monitoring/resource_monitor.py @@ -0,0 +1,384 @@ +"""System resource monitoring — CPU / memory / disk / network / GPU / process. + +Runs a single background daemon thread (started alongside the gRPC server in +``weightslab.trainer.trainer_services.grpc_serve``) that periodically samples +machine- and process-level resource usage and logs each value through the +same signal pipeline used for losses/metrics +(``weightslab.src._log_signal`` -> registered ``LoggerQueue.add_scalars``), +so the resulting curves show up in Weights Studio exactly like any other +signal. Since sampling is on a wall-clock cadence rather than tied to +training steps, the logged "step" for these signals is the number of +elapsed seconds since the monitor started. + +GPU metrics use NVML (via the ``pynvml`` import name, shipped by the +``nvidia-ml-py`` package) and are skipped gracefully when no NVIDIA driver +is present. Every other category is provided by ``psutil``. + +See ``docs/resource_monitoring.rst`` for the user-facing config reference. +""" + +import logging +import os +import threading +import time +from pathlib import Path +from typing import Dict, Optional + +import psutil +import yaml + +logger = logging.getLogger(__name__) + +DEFAULT_INTERVAL_SECONDS = 15.0 +DEFAULT_CATEGORIES = ("cpu", "memory", "gpu", "disk", "network", "process") + + +def _bool_env(name: str, default: bool) -> bool: + val = os.environ.get(name) + if val is None: + return default + return val.strip().lower() in {"1", "true", "yes", "on"} + + +def load_resource_monitoring_config() -> dict: + """Resolve resource-monitoring config with env-vars-then-YAML precedence + (YAML overrides env vars if both set), mirroring the agent config loader + in ``weightslab.trainer.services.agent.agent``. + """ + enabled = not _bool_env("WEIGHTSLAB_DISABLE_RESOURCE_MONITORING", False) + interval_seconds = float(os.environ.get("WL_RESOURCE_MONITOR_INTERVAL_SECONDS", DEFAULT_INTERVAL_SECONDS)) + disk_path = os.environ.get("WL_RESOURCE_MONITOR_DISK_PATH", os.sep) + categories = {name: True for name in DEFAULT_CATEGORIES} + + env_categories = os.environ.get("WL_RESOURCE_MONITOR_CATEGORIES") + if env_categories is not None: + selected = {c.strip().lower() for c in env_categories.split(",") if c.strip()} + categories = {name: (name in selected) for name in DEFAULT_CATEGORIES} + + repo_root = Path(__file__).resolve().parents[2] # weightslab/ root + inner_pkg = Path(__file__).resolve().parents[1] # weightslab package dir + + config_dir = Path(os.environ.get("WL_RESOURCE_MONITOR_CONFIG_PATH", repo_root)) + config_paths = [ + config_dir / ".resource_monitoring.yaml", + config_dir / "resource_monitoring.yaml", + inner_pkg / "resource_monitoring.yaml", + Path.cwd() / "resource_monitoring.yaml", + ] + for path in config_paths: + if not path.exists(): + continue + try: + with open(path, "r") as f: + cfg = yaml.safe_load(f) + if not cfg or "resource_monitoring" not in cfg: + continue + rm_cfg = cfg["resource_monitoring"] or {} + + enabled = bool(rm_cfg.get("enabled", enabled)) + interval_seconds = float(rm_cfg.get("interval_seconds", interval_seconds)) + disk_path = rm_cfg.get("disk_path", disk_path) + + cats_cfg = rm_cfg.get("categories") + if isinstance(cats_cfg, dict): + categories = { + name: bool(cats_cfg.get(name, categories[name])) + for name in DEFAULT_CATEGORIES + } + + logger.debug("[ResourceMonitor] Loaded config from %s", path) + except Exception as e: + logger.warning("[ResourceMonitor] Failed to load config %s: %s", path, e) + break + + return { + "enabled": enabled, + "interval_seconds": interval_seconds, + "categories": categories, + "disk_path": disk_path, + } + + +class ResourceMonitor: + """Background daemon thread sampling system resources on a fixed interval.""" + + def __init__( + self, + interval_seconds: float = DEFAULT_INTERVAL_SECONDS, + categories: Optional[Dict[str, bool]] = None, + disk_path: str = os.sep, + ) -> None: + self._interval_seconds = max(float(interval_seconds), 1.0) + self._categories = dict(categories) if categories else {name: True for name in DEFAULT_CATEGORIES} + self._disk_path = disk_path or os.sep + + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + self._start_monotonic: Optional[float] = None + + self._process = psutil.Process(os.getpid()) + self._gpu_available = False + self._gpu_handles: list = [] + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def start(self) -> None: + """Start the resource-monitoring background thread.""" + self._start_monotonic = time.monotonic() + + # First call to cpu_percent() always returns a meaningless baseline + # (0.0) since there's no prior sample to compare against — prime it + # now so the first real tick already reports a useful value. + try: + self._process.cpu_percent(interval=None) + psutil.cpu_percent(interval=None) + except Exception: + pass + + if self._categories.get("gpu", True): + self._init_gpu() + + self._stop.clear() + self._thread = threading.Thread(target=self._loop, name="WL-ResourceMonitor", daemon=True) + self._thread.start() + logger.info( + "[ResourceMonitor] Started (interval=%.1fs categories=%s)", + self._interval_seconds, + sorted(name for name, on in self._categories.items() if on), + ) + + def stop(self) -> None: + """Stop the resource-monitoring background thread.""" + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=self._interval_seconds + 1.0) + self._shutdown_gpu() + + # ------------------------------------------------------------------ + # Main loop + # ------------------------------------------------------------------ + + def _loop(self) -> None: + while not self._stop.wait(self._interval_seconds): + try: + self._tick() + except Exception: + logger.exception("[ResourceMonitor] Unexpected error while sampling resources") + + def _tick(self) -> None: + step = int(round(time.monotonic() - self._start_monotonic)) + metrics: Dict[str, float] = {} + + if self._categories.get("cpu", True): + metrics.update(self._sample_cpu()) + if self._categories.get("memory", True): + metrics.update(self._sample_memory()) + if self._categories.get("process", True): + metrics.update(self._sample_process()) + if self._categories.get("disk", True): + metrics.update(self._sample_disk()) + if self._categories.get("network", True): + metrics.update(self._sample_network()) + if self._categories.get("gpu", True) and self._gpu_available: + metrics.update(self._sample_gpu()) + + for reg_name, value in metrics.items(): + self._log_metric(reg_name, value, step) + + # ------------------------------------------------------------------ + # Samplers (psutil) — one dict per category, graph name -> value + # ------------------------------------------------------------------ + + def _sample_cpu(self) -> Dict[str, float]: + try: + return {"resource/cpu/utilization_percent": float(psutil.cpu_percent(interval=None))} + except Exception: + return {} + + def _sample_memory(self) -> Dict[str, float]: + try: + vm = psutil.virtual_memory() + return {"resource/memory/system_utilization_percent": float(vm.percent)} + except Exception: + return {} + + def _sample_process(self) -> Dict[str, float]: + try: + available_mb = float(psutil.virtual_memory().available) / (1024 ** 2) + with self._process.oneshot(): + cpu_percent = self._process.cpu_percent(interval=None) + num_threads = self._process.num_threads() + mem_info = self._process.memory_info() + mem_percent = self._process.memory_percent() + return { + "resource/process/cpu_utilization_percent": float(cpu_percent), + "resource/process/cpu_threads_in_use": float(num_threads), + "resource/process/memory_in_use_mb": float(mem_info.rss) / (1024 ** 2), + "resource/process/memory_in_use_percent": float(mem_percent), + "resource/process/memory_available_mb": available_mb, + } + except Exception: + return {} + + def _sample_disk(self) -> Dict[str, float]: + metrics: Dict[str, float] = {} + try: + usage = psutil.disk_usage(self._disk_path) + metrics["resource/disk/utilization_percent"] = float(usage.percent) + metrics["resource/disk/utilization_gb"] = float(usage.used) / (1024 ** 3) + except Exception: + pass + try: + io_counters = psutil.disk_io_counters() + if io_counters is not None: + metrics["resource/disk/read_mb"] = float(io_counters.read_bytes) / (1024 ** 2) + metrics["resource/disk/written_mb"] = float(io_counters.write_bytes) / (1024 ** 2) + except Exception: + pass + return metrics + + def _sample_network(self) -> Dict[str, float]: + try: + counters = psutil.net_io_counters() + if counters is None: + return {} + return { + "resource/network/bytes_sent": float(counters.bytes_sent), + "resource/network/bytes_received": float(counters.bytes_recv), + } + except Exception: + return {} + + # ------------------------------------------------------------------ + # GPU / NVML + # ------------------------------------------------------------------ + + def _init_gpu(self) -> None: + try: + import pynvml + except ImportError: + logger.debug("[ResourceMonitor] pynvml not available; GPU metrics disabled.") + return + try: + pynvml.nvmlInit() + count = pynvml.nvmlDeviceGetCount() + self._gpu_handles = [pynvml.nvmlDeviceGetHandleByIndex(i) for i in range(count)] + self._gpu_available = bool(self._gpu_handles) + if self._gpu_available: + logger.info("[ResourceMonitor] NVML initialized (%d GPU device(s)).", count) + else: + pynvml.nvmlShutdown() + except Exception as e: + logger.debug("[ResourceMonitor] NVML unavailable, GPU metrics disabled: %s", e) + self._gpu_available = False + self._gpu_handles = [] + + def _shutdown_gpu(self) -> None: + if not self._gpu_available: + return + try: + import pynvml + pynvml.nvmlShutdown() + except Exception: + pass + finally: + self._gpu_available = False + self._gpu_handles = [] + + def _sample_gpu(self) -> Dict[str, float]: + import pynvml + + metrics: Dict[str, float] = {} + for index, handle in enumerate(self._gpu_handles): + prefix = f"resource/gpu/{index}" + try: + metrics[f"{prefix}/memory_clock_mhz"] = float( + pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_MEM) + ) + except Exception: + pass + try: + metrics[f"{prefix}/sm_clock_mhz"] = float( + pynvml.nvmlDeviceGetClockInfo(handle, pynvml.NVML_CLOCK_SM) + ) + except Exception: + pass + try: + mem_info = pynvml.nvmlDeviceGetMemoryInfo(handle) + metrics[f"{prefix}/memory_allocated_bytes"] = float(mem_info.used) + if mem_info.total: + metrics[f"{prefix}/memory_allocated_percent"] = ( + float(mem_info.used) / float(mem_info.total) * 100.0 + ) + except Exception: + pass + try: + metrics[f"{prefix}/temperature_celsius"] = float( + pynvml.nvmlDeviceGetTemperature(handle, pynvml.NVML_TEMPERATURE_GPU) + ) + except Exception: + pass + return metrics + + # ------------------------------------------------------------------ + # Signal logging + # ------------------------------------------------------------------ + + def _log_metric(self, reg_name: str, value: float, step: int) -> None: + try: + import weightslab.src as _src + except Exception: + return + try: + _src._log_signal(value, None, reg_name, step=step) + except Exception: + logger.debug("[ResourceMonitor] Failed to log metric %s", reg_name, exc_info=True) + + +# ---------------------------------------------------------------------- +# Process-wide singleton — started once from grpc_serve() +# ---------------------------------------------------------------------- + +_singleton_lock = threading.Lock() +_singleton_instance: Optional[ResourceMonitor] = None + + +def start_resource_monitor_from_config() -> Optional[ResourceMonitor]: + """Load config (env vars + optional YAML) and start the resource monitor + singleton if enabled. Safe to call more than once — returns the existing + instance (or None if disabled) on subsequent calls. + """ + global _singleton_instance + with _singleton_lock: + if _singleton_instance is not None: + return _singleton_instance + + config = load_resource_monitoring_config() + if not config["enabled"]: + logger.info("[ResourceMonitor] Disabled via WEIGHTSLAB_DISABLE_RESOURCE_MONITORING/config — not starting.") + return None + + monitor = ResourceMonitor( + interval_seconds=config["interval_seconds"], + categories=config["categories"], + disk_path=config["disk_path"], + ) + monitor.start() + _singleton_instance = monitor + return monitor + + +def get_resource_monitor() -> Optional[ResourceMonitor]: + """Return the running resource monitor singleton, if any.""" + return _singleton_instance + + +def stop_resource_monitor() -> None: + """Stop and clear the resource monitor singleton, if running.""" + global _singleton_instance + with _singleton_lock: + if _singleton_instance is not None: + _singleton_instance.stop() + _singleton_instance = None diff --git a/weightslab/proto/experiment_service.proto b/weightslab/proto/experiment_service.proto index e926ab55..d48ddfb9 100644 --- a/weightslab/proto/experiment_service.proto +++ b/weightslab/proto/experiment_service.proto @@ -62,6 +62,12 @@ service ExperimentService { rpc TriggerEvaluation (TriggerEvaluationRequest) returns (TriggerEvaluationResponse); rpc GetEvaluationStatus (GetEvaluationStatusRequest) returns (GetEvaluationStatusResponse); rpc CancelEvaluation (CancelEvaluationRequest) returns (CancelEvaluationResponse); + + // Export bounding-box/segmentation annotations to a relabeling-tool format + // (CVAT XML, Label Studio JSON, or V7/Darwin JSON). Unary: the whole file + // (or a zip, for formats that need one file per image) comes back as bytes + // in a single response. + rpc ExportAnnotations (ExportAnnotationsRequest) returns (ExportAnnotationsResponse); } // --- Logger Data Sync --- @@ -717,3 +723,26 @@ message GenerateNotebookCodeResponse { bool ok = 3; string error = 4; } + +// --- Annotation export (relabeling-tool formats) --- +enum AnnotationExportFormat { + EXPORT_FORMAT_CVAT = 0; + EXPORT_FORMAT_LABEL_STUDIO = 1; + EXPORT_FORMAT_V7_DARWIN = 2; +} + +message ExportAnnotationsRequest { + AnnotationExportFormat format = 1; + string origin = 2; // restrict to one split/loader; "" = all loaders + bool include_predictions = 3; // export model predictions instead of ground-truth targets + repeated string tags = 4; // restrict to samples carrying ANY of these tags (tag: prefix optional); empty = all samples +} + +message ExportAnnotationsResponse { + bool success = 1; + string message = 2; + bytes payload = 3; // export file bytes (a zip, for formats that need one file per image) + string filename = 4; // suggested filename (with extension), e.g. "annotations_cvat.xml" + string mime_type = 5; // e.g. "application/xml", "application/json", "application/zip" + int32 image_count = 6; // number of images included, for UI feedback +} diff --git a/weightslab/proto/experiment_service_pb2.py b/weightslab/proto/experiment_service_pb2.py index d299a943..dd2af12c 100644 --- a/weightslab/proto/experiment_service_pb2.py +++ b/weightslab/proto/experiment_service_pb2.py @@ -2,7 +2,7 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # NO CHECKED-IN PROTOBUF GENCODE # source: weightslab/proto/experiment_service.proto -# Protobuf Python Version: 6.31.1 +# Protobuf Python Version: 5.28.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -11,8 +11,8 @@ from google.protobuf.internal import builder as _builder _runtime_version.ValidateProtobufRuntimeVersion( _runtime_version.Domain.PUBLIC, - 6, - 31, + 5, + 28, 1, '', 'weightslab/proto/experiment_service.proto' @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)weightslab/proto/experiment_service.proto\"\x89\x01\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\"\x81\x02\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\"[\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\x12\x1a\n\x12weightslab_version\x18\x02 \x01(\t\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\x1a\n\x18RestartInstanceOperation\"\x8d\x08\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x12\x39\n\x11restart_operation\x18\x0f \x01(\x0b\x32\x19.RestartInstanceOperationH\n\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operationB\x14\n\x12_restart_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\x87\x03\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x12\x19\n\x11sample_label_text\x18\x0b \x03(\t\x12\x1e\n\x16sample_prediction_text\x18\x0c \x03(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"Y\n\x1aGetSignalTrajectoryRequest\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12\x12\n\nsample_ids\x18\x02 \x03(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"4\n\x10SignalTrajectory\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\x02\"}\n\x1bGetSignalTrajectoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12\'\n\x0ctrajectories\x18\x04 \x03(\x0b\x32\x11.SignalTrajectory\"J\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"\xdc\x01\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"7\n\x16RunNotebookCellRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x65ll_id\x18\x02 \x01(\t\"2\n\x10NotebookCellDone\x12\x12\n\nexec_count\x18\x01 \x01(\x05\x12\n\n\x02ok\x18\x02 \x01(\x08\"\x1e\n\x1cInterruptNotebookCellRequest\":\n\x1dInterruptNotebookCellResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\"\xbd\x01\n\x11NotebookCellChunk\x12\x0f\n\x07\x63\x65ll_id\x18\x01 \x01(\t\x12\x10\n\x06stdout\x18\x02 \x01(\tH\x00\x12\x10\n\x06stderr\x18\x03 \x01(\tH\x00\x12\x15\n\x0bresult_text\x18\x04 \x01(\tH\x00\x12\x13\n\timage_png\x18\x05 \x01(\x0cH\x00\x12\x19\n\x0f\x65rror_traceback\x18\x06 \x01(\tH\x00\x12!\n\x04\x64one\x18\x07 \x01(\x0b\x32\x11.NotebookCellDoneH\x00\x42\t\n\x07payload\"S\n\x10NotebookResponse\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0f\n\x07\x65xisted\x18\x02 \x01(\x08\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"7\n\x13SaveNotebookRequest\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"M\n\x14SaveNotebookResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"C\n\x1bGenerateNotebookCodeRequest\x12\x0e\n\x06prompt\x18\x01 \x01(\t\x12\x14\n\x0c\x63ontext_code\x18\x02 \x01(\t\"\\\n\x1cGenerateNotebookCodeResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x13\n\x0b\x65xplanation\x18\x02 \x01(\t\x12\n\n\x02ok\x18\x03 \x01(\x08\x12\r\n\x05\x65rror\x18\x04 \x01(\t*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*,\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00\x32\x9d\x0e\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12P\n\x13GetSignalTrajectory\x12\x1b.GetSignalTrajectoryRequest\x1a\x1c.GetSignalTrajectoryResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12@\n\x0fRunNotebookCell\x12\x17.RunNotebookCellRequest\x1a\x12.NotebookCellChunk0\x01\x12V\n\x15InterruptNotebookCell\x12\x1d.InterruptNotebookCellRequest\x1a\x1e.InterruptNotebookCellResponse\x12(\n\x0bGetNotebook\x12\x06.Empty\x1a\x11.NotebookResponse\x12;\n\x0cSaveNotebook\x12\x14.SaveNotebookRequest\x1a\x15.SaveNotebookResponse\x12S\n\x14GenerateNotebookCode\x12\x1c.GenerateNotebookCodeRequest\x1a\x1d.GenerateNotebookCodeResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n)weightslab/proto/experiment_service.proto\"\x89\x01\n\x1aGetLatestLoggerDataRequest\x12\x1c\n\x14request_full_history\x18\x01 \x01(\x08\x12\x12\n\nmax_points\x18\x02 \x01(\x05\x12\x17\n\x0f\x62reak_by_slices\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\x12\x12\n\ngraph_name\x18\x05 \x01(\t\"\x81\x02\n\x0fLoggerDataPoint\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x11\n\tmodel_age\x18\x02 \x01(\x05\x12\x14\n\x0cmetric_value\x18\x03 \x01(\x02\x12\x17\n\x0f\x65xperiment_hash\x18\x04 \x01(\t\x12\x11\n\ttimestamp\x18\x05 \x01(\x03\x12\x11\n\tsample_id\x18\x06 \x01(\t\x12\x1c\n\x14is_evaluation_marker\x18\x07 \x01(\x08\x12\x12\n\nsplit_name\x18\x08 \x01(\t\x12\x17\n\x0f\x65valuation_tags\x18\t \x03(\t\x12\x12\n\npoint_note\x18\n \x01(\t\x12\x12\n\naudit_mode\x18\x0b \x01(\x08\"[\n\x1bGetLatestLoggerDataResponse\x12 \n\x06points\x18\x01 \x03(\x0b\x32\x10.LoggerDataPoint\x12\x1a\n\x12weightslab_version\x18\x02 \x01(\t\"\x07\n\x05\x45mpty\"/\n\x08NeuronId\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tneuron_id\x18\x02 \x01(\x05\"\x91\x02\n\x0fWeightOperation\x12*\n\x07op_type\x18\x01 \x01(\x0e\x32\x14.WeightOperationTypeH\x00\x88\x01\x01\x12\x15\n\x08layer_id\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1d\n\nneuron_ids\x18\x03 \x03(\x0b\x32\t.NeuronId\x12\x16\n\x0eneurons_to_add\x18\t \x01(\x05\x12 \n\x18zerofy_from_incoming_ids\x18\x0b \x03(\x05\x12\x1c\n\x14zerofy_to_neuron_ids\x18\x0c \x03(\x05\x12+\n\x11zerofy_predicates\x18\r \x03(\x0e\x32\x10.ZerofyPredicateB\n\n\x08_op_typeB\x0b\n\t_layer_id\"_\n\x17WeightsOperationRequest\x12/\n\x10weight_operation\x18\x01 \x01(\x0b\x32\x10.WeightOperationH\x00\x88\x01\x01\x42\x13\n\x11_weight_operation\"<\n\x18WeightsOperationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\xc1\x05\n\x0fHyperParameters\x12\x1c\n\x0f\x65xperiment_name\x18\x01 \x01(\tH\x00\x88\x01\x01\x12!\n\x14training_steps_to_do\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x17\n\nbatch_size\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12 \n\x13\x66ull_eval_frequency\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12 \n\x13\x63heckpont_frequency\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x18\n\x0bis_training\x18\x07 \x01(\x08H\x06\x88\x01\x01\x12\x15\n\x08nb_steps\x18\x08 \x01(\x05H\x07\x88\x01\x01\x12\x19\n\x0c\x61uditor_mode\x18\t \x01(\x08H\x08\x88\x01\x01\x12\x1d\n\x10train_batch_size\x18\n \x01(\x05H\t\x88\x01\x01\x12\x1b\n\x0eval_batch_size\x18\x0b \x01(\x05H\n\x88\x01\x01\x12\x1c\n\x0ftest_batch_size\x18\x0c \x01(\x05H\x0b\x88\x01\x01\x12\x1c\n\x0f\x65valuation_mode\x18\r \x01(\x08H\x0c\x88\x01\x01\x12\x1e\n\x11\x65valuation_config\x18\x0e \x01(\tH\r\x88\x01\x01\x42\x12\n\x10_experiment_nameB\x17\n\x15_training_steps_to_doB\x10\n\x0e_learning_rateB\r\n\x0b_batch_sizeB\x16\n\x14_full_eval_frequencyB\x16\n\x14_checkpont_frequencyB\x0e\n\x0c_is_trainingB\x0b\n\t_nb_stepsB\x0f\n\r_auditor_modeB\x13\n\x11_train_batch_sizeB\x11\n\x0f_val_batch_sizeB\x12\n\x10_test_batch_sizeB\x12\n\x10_evaluation_modeB\x14\n\x12_evaluation_config\",\n\rMetricsStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02\"~\n\rAnnotatStatus\x12\x0c\n\x04name\x18\x01 \x01(\t\x12.\n\x08metadata\x18\x02 \x03(\x0b\x32\x1c.AnnotatStatus.MetadataEntry\x1a/\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x90\x02\n\x10TrainingStatusEx\x12\x16\n\ttimestamp\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x1c\n\x0f\x65xperiment_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x16\n\tmodel_age\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12+\n\x0emetrics_status\x18\x04 \x01(\x0b\x32\x0e.MetricsStatusH\x03\x88\x01\x01\x12+\n\x0e\x61nnotat_status\x18\x05 \x01(\x0b\x32\x0e.AnnotatStatusH\x04\x88\x01\x01\x42\x0c\n\n_timestampB\x12\n\x10_experiment_nameB\x0c\n\n_model_ageB\x11\n\x0f_metrics_statusB\x11\n\x0f_annotat_status\"]\n\x15HyperParameterCommand\x12/\n\x10hyper_parameters\x18\x01 \x01(\x0b\x32\x10.HyperParametersH\x00\x88\x01\x01\x42\x13\n\x11_hyper_parameters\">\n\x14\x44\x65nySamplesOperation\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\"0\n\x17LoadCheckpointOperation\x12\x15\n\rcheckpoint_id\x18\x01 \x01(\x05\"b\n\x11PlotNoteOperation\x12\x13\n\x0bmetric_name\x18\x01 \x01(\t\x12\x17\n\x0f\x65xperiment_hash\x18\x02 \x01(\t\x12\x11\n\tmodel_age\x18\x03 \x01(\x05\x12\x0c\n\x04note\x18\x04 \x01(\t\"L\n\x17SaveCheckpointOperation\x12\x19\n\x11save_architecture\x18\x01 \x01(\x08\x12\x16\n\x0esave_optimizer\x18\x02 \x01(\x08\"\x1a\n\x18RestartInstanceOperation\"\x8d\x08\n\x0eTrainerCommand\x12\x1c\n\x14get_hyper_parameters\x18\x04 \x01(\x08\x12\x1e\n\x16get_interactive_layers\x18\x05 \x01(\x08\x12\x1d\n\x10get_data_records\x18\x06 \x01(\tH\x00\x88\x01\x01\x12%\n\x18get_single_layer_info_id\x18\x08 \x01(\x05H\x01\x88\x01\x01\x12;\n\x16hyper_parameter_change\x18\x01 \x01(\x0b\x32\x16.HyperParameterCommandH\x02\x88\x01\x01\x12:\n\x16\x64\x65ny_samples_operation\x18\x07 \x01(\x0b\x32\x15.DenySamplesOperationH\x03\x88\x01\x01\x12?\n\x1b\x64\x65ny_eval_samples_operation\x18\n \x01(\x0b\x32\x15.DenySamplesOperationH\x04\x88\x01\x01\x12@\n\x19load_checkpoint_operation\x18\t \x01(\x0b\x32\x18.LoadCheckpointOperationH\x05\x88\x01\x01\x12\x42\n\x1eremove_from_denylist_operation\x18\x0b \x01(\x0b\x32\x15.DenySamplesOperationH\x06\x88\x01\x01\x12G\n#remove_eval_from_denylist_operation\x18\x0c \x01(\x0b\x32\x15.DenySamplesOperationH\x07\x88\x01\x01\x12\x34\n\x13plot_note_operation\x18\r \x01(\x0b\x32\x12.PlotNoteOperationH\x08\x88\x01\x01\x12@\n\x19save_checkpoint_operation\x18\x0e \x01(\x0b\x32\x18.SaveCheckpointOperationH\t\x88\x01\x01\x12\x39\n\x11restart_operation\x18\x0f \x01(\x0b\x32\x19.RestartInstanceOperationH\n\x88\x01\x01\x42\x13\n\x11_get_data_recordsB\x1b\n\x19_get_single_layer_info_idB\x19\n\x17_hyper_parameter_changeB\x19\n\x17_deny_samples_operationB\x1e\n\x1c_deny_eval_samples_operationB\x1c\n\x1a_load_checkpoint_operationB!\n\x1f_remove_from_denylist_operationB&\n$_remove_eval_from_denylist_operationB\x16\n\x14_plot_note_operationB\x1c\n\x1a_save_checkpoint_operationB\x14\n\x12_restart_operation\"\x9d\x01\n\x12HyperParameterDesc\x12\r\n\x05label\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0c\n\x04type\x18\x03 \x01(\t\x12\x1c\n\x0fnumerical_value\x18\x04 \x01(\x02H\x00\x88\x01\x01\x12\x19\n\x0cstring_value\x18\x05 \x01(\tH\x01\x88\x01\x01\x42\x12\n\x10_numerical_valueB\x0f\n\r_string_value\"\xf2\x02\n\x10NeuronStatistics\x12!\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronIdH\x00\x88\x01\x01\x12\x17\n\nneuron_age\x18\x02 \x01(\x05H\x01\x88\x01\x01\x12\x1f\n\x12train_trigger_rate\x18\x03 \x01(\x02H\x02\x88\x01\x01\x12\x1e\n\x11\x65val_trigger_rate\x18\x04 \x01(\x02H\x03\x88\x01\x01\x12\x1a\n\rlearning_rate\x18\x07 \x01(\x02H\x04\x88\x01\x01\x12\x36\n\x0bincoming_lr\x18\x08 \x03(\x0b\x32!.NeuronStatistics.IncomingLrEntry\x1a\x31\n\x0fIncomingLrEntry\x12\x0b\n\x03key\x18\x01 \x01(\x05\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\x42\x0c\n\n_neuron_idB\r\n\x0b_neuron_ageB\x15\n\x13_train_trigger_rateB\x14\n\x12_eval_trigger_rateB\x10\n\x0e_learning_rate\"\xf0\x02\n\x13LayerRepresentation\x12\x15\n\x08layer_id\x18\x01 \x01(\x05H\x00\x88\x01\x01\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x1a\n\rneurons_count\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12#\n\x16incoming_neurons_count\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x05\x88\x01\x01\x12\x13\n\x06stride\x18\x07 \x01(\x05H\x06\x88\x01\x01\x12-\n\x12neurons_statistics\x18\n \x03(\x0b\x32\x11.NeuronStatisticsB\x0b\n\t_layer_idB\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x10\n\x0e_neurons_countB\x19\n\x17_incoming_neurons_countB\x0e\n\x0c_kernel_sizeB\t\n\x07_stride\"H\n\x11\x41\x63tivationRequest\x12\x10\n\x08layer_id\x18\x01 \x01(\x05\x12\x11\n\tsample_id\x18\x02 \x01(\t\x12\x0e\n\x06origin\x18\x03 \x01(\t\"H\n\rActivationMap\x12\x11\n\tneuron_id\x18\x01 \x01(\x05\x12\x0e\n\x06values\x18\x02 \x03(\x02\x12\t\n\x01H\x18\x03 \x01(\x05\x12\t\n\x01W\x18\x04 \x01(\x05\"d\n\x12\x41\x63tivationResponse\x12\x12\n\nlayer_type\x18\x01 \x01(\t\x12\x15\n\rneurons_count\x18\x02 \x01(\x05\x12#\n\x0b\x61\x63tivations\x18\x03 \x03(\x0b\x32\x0e.ActivationMap\"\x93\x01\n\tTaskField\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x15\n\x0b\x66loat_value\x18\x02 \x01(\x02H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x05H\x00\x12\x16\n\x0cstring_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x05 \x01(\x0cH\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x42\x07\n\x05value\"\x87\x03\n\x0eRecordMetadata\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x14\n\x0csample_label\x18\x02 \x03(\x05\x12\x19\n\x11sample_prediction\x18\x03 \x03(\x05\x12=\n\x10sample_last_loss\x18\x04 \x03(\x0b\x32#.RecordMetadata.SampleLastLossEntry\x12\x19\n\x11sample_encounters\x18\x05 \x01(\x05\x12\x18\n\x10sample_discarded\x18\x06 \x01(\x08\x12 \n\x0c\x65xtra_fields\x18\x07 \x03(\x0b\x32\n.TaskField\x12\x16\n\x0eprediction_raw\x18\t \x01(\x0c\x12\x11\n\ttask_type\x18\n \x01(\t\x12\x19\n\x11sample_label_text\x18\x0b \x03(\t\x12\x1e\n\x16sample_prediction_text\x18\x0c \x03(\t\x1a\x35\n\x13SampleLastLossEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x02:\x02\x38\x01\"\x93\x01\n\x10SampleStatistics\x12\x13\n\x06origin\x18\x06 \x01(\tH\x00\x88\x01\x01\x12\x19\n\x0csample_count\x18\x07 \x01(\x05H\x01\x88\x01\x01\x12\x11\n\ttask_type\x18\t \x01(\t\x12 \n\x07records\x18\x08 \x03(\x0b\x32\x0f.RecordMetadataB\t\n\x07_originB\x0f\n\r_sample_count\"\xe6\x01\n\x0f\x43ommandResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x33\n\x16hyper_parameters_descs\x18\x03 \x03(\x0b\x32\x13.HyperParameterDesc\x12\x33\n\x15layer_representations\x18\x04 \x03(\x0b\x32\x14.LayerRepresentation\x12\x31\n\x11sample_statistics\x18\x05 \x01(\x0b\x32\x11.SampleStatisticsH\x00\x88\x01\x01\x42\x14\n\x12_sample_statistics\"U\n\rSampleRequest\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_origin\"\xad\x02\n\x15SampleRequestResponse\x12\x16\n\tsample_id\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x06origin\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x12\n\x05label\x18\x03 \x01(\x05H\x02\x88\x01\x01\x12\x11\n\x04\x64\x61ta\x18\x04 \x01(\x0cH\x03\x88\x01\x01\x12\x1a\n\rerror_message\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x15\n\x08raw_data\x18\x06 \x01(\x0cH\x05\x88\x01\x01\x12\x11\n\x04mask\x18\x07 \x01(\x0cH\x06\x88\x01\x01\x12\x17\n\nprediction\x18\x08 \x01(\x0cH\x07\x88\x01\x01\x42\x0c\n\n_sample_idB\t\n\x07_originB\x08\n\x06_labelB\x07\n\x05_dataB\x10\n\x0e_error_messageB\x0b\n\t_raw_dataB\x07\n\x05_maskB\r\n\x0b_prediction\"\x92\x01\n\x12\x42\x61tchSampleRequest\x12\x12\n\nsample_ids\x18\x01 \x03(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x19\n\x0cresize_width\x18\x03 \x01(\x05H\x00\x88\x01\x01\x12\x1a\n\rresize_height\x18\x04 \x01(\x05H\x01\x88\x01\x01\x42\x0f\n\r_resize_widthB\x10\n\x0e_resize_height\">\n\x13\x42\x61tchSampleResponse\x12\'\n\x07samples\x18\x01 \x03(\x0b\x32\x16.SampleRequestResponse\".\n\x0eWeightsRequest\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\"\x9d\x02\n\x0fWeightsResponse\x12\x1c\n\tneuron_id\x18\x01 \x01(\x0b\x32\t.NeuronId\x12\x17\n\nlayer_name\x18\x02 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nlayer_type\x18\x03 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x08incoming\x18\x04 \x01(\x05\x12\x10\n\x08outgoing\x18\x05 \x01(\x05\x12\x18\n\x0bkernel_size\x18\x06 \x01(\x05H\x02\x88\x01\x01\x12\x0f\n\x07weights\x18\x07 \x03(\x02\x12\x0f\n\x07success\x18\x0b \x01(\x08\x12\x1a\n\rerror_message\x18\x0c \x01(\tH\x03\x88\x01\x01\x42\r\n\x0b_layer_nameB\r\n\x0b_layer_typeB\x0e\n\x0c_kernel_sizeB\x10\n\x0e_error_message\"R\n\x10\x44\x61taQueryRequest\x12\r\n\x05query\x18\x01 \x01(\t\x12\x12\n\naccumulate\x18\x02 \x01(\x08\x12\x1b\n\x13is_natural_language\x18\x03 \x01(\x08\"5\n\x11\x43\x61tegoricalTagDef\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ncategories\x18\x02 \x03(\t\"\xa9\x02\n\x11\x44\x61taQueryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1d\n\x15number_of_all_samples\x18\x03 \x01(\x05\x12%\n\x1dnumber_of_samples_in_the_loop\x18\x04 \x01(\x05\x12#\n\x1bnumber_of_discarded_samples\x18\x05 \x01(\x05\x12\x13\n\x0bunique_tags\x18\x06 \x03(\t\x12+\n\x11\x61gent_intent_type\x18\x07 \x01(\x0e\x32\x10.AgentIntentType\x12\x17\n\x0f\x61nalysis_result\x18\x08 \x01(\t\x12,\n\x10\x63\x61tegorical_tags\x18\t \x03(\x0b\x32\x12.CategoricalTagDef\"\xc2\x01\n\x12\x44\x61taSamplesRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12 \n\x18include_transformed_data\x18\x03 \x01(\x08\x12\x18\n\x10include_raw_data\x18\x04 \x01(\x08\x12\x19\n\x11stats_to_retrieve\x18\x05 \x03(\t\x12\x14\n\x0cresize_width\x18\x06 \x01(\x05\x12\x15\n\rresize_height\x18\x07 \x01(\x05\"m\n\x08\x44\x61taStat\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04type\x18\x02 \x01(\t\x12\r\n\x05shape\x18\x03 \x03(\x05\x12\r\n\x05value\x18\x04 \x03(\x02\x12\x14\n\x0cvalue_string\x18\x05 \x01(\t\x12\x11\n\tthumbnail\x18\x06 \x01(\x0c\">\n\nDataRecord\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x1d\n\ndata_stats\x18\x02 \x03(\x0b\x32\t.DataStat\"Z\n\x13\x44\x61taSamplesResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12!\n\x0c\x64\x61ta_records\x18\x03 \x03(\x0b\x32\x0b.DataRecord\"C\n\x0fHistogramSubBar\x12\x0e\n\x06origin\x18\x01 \x01(\t\x12\x11\n\tdiscarded\x18\x02 \x01(\x08\x12\r\n\x05\x63ount\x18\x03 \x01(\x03\"h\n\x0cHistogramBin\x12\x0b\n\x03min\x18\x01 \x01(\x01\x12\x0b\n\x03max\x18\x02 \x01(\x01\x12\x0b\n\x03\x61vg\x18\x03 \x01(\x01\x12\r\n\x05\x63ount\x18\x04 \x01(\x03\x12\"\n\x08sub_bars\x18\x05 \x03(\x0b\x32\x10.HistogramSubBar\"[\n\x17\x43\x61tegoricalHistogramBar\x12\r\n\x05label\x18\x01 \x01(\t\x12\r\n\x05\x63ount\x18\x02 \x01(\x03\x12\"\n\x08sub_bars\x18\x03 \x03(\x0b\x32\x10.HistogramSubBar\"4\n\x10HistogramRequest\x12\x0e\n\x06\x63olumn\x18\x01 \x01(\t\x12\x10\n\x08max_bins\x18\x02 \x01(\x05\"\xb2\x01\n\x11HistogramResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\ntotal_rows\x18\x03 \x01(\x03\x12\x1b\n\x04\x62ins\x18\x04 \x03(\x0b\x32\r.HistogramBin\x12\x16\n\x0eis_categorical\x18\x05 \x01(\x08\x12\x32\n\x10\x63\x61tegorical_bars\x18\x06 \x03(\x0b\x32\x18.CategoricalHistogramBar\"W\n\x12GetMetaDataRequest\x12\x13\n\x0bstart_index\x18\x01 \x01(\x05\x12\x13\n\x0brecords_cnt\x18\x02 \x01(\x05\x12\x17\n\x0fmodal_sample_id\x18\x03 \x01(\t\"\x99\x01\n\x13GetMetaDataResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x1a\n\x12\x61ll_metadata_names\x18\x03 \x03(\t\x12!\n\x0cgrid_records\x18\x04 \x03(\x0b\x32\x0b.DataRecord\x12!\n\x0cmodal_record\x18\x05 \x01(\x0b\x32\x0b.DataRecord\"Y\n\x1aGetSignalTrajectoryRequest\x12\x13\n\x0bsignal_name\x18\x01 \x01(\t\x12\x12\n\nsample_ids\x18\x02 \x03(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"4\n\x10SignalTrajectory\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\x02\"}\n\x1bGetSignalTrajectoryResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x13\n\x0bsignal_name\x18\x03 \x01(\t\x12\'\n\x0ctrajectories\x18\x04 \x03(\x0b\x32\x11.SignalTrajectory\"J\n\x11PointCloudRequest\x12\x11\n\tsample_id\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x12\n\nmax_points\x18\x03 \x01(\x05\"\xbf\x01\n\x0fPointCloudChunk\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x12\n\nnum_points\x18\x03 \x01(\x05\x12\x14\n\x0cnum_features\x18\x04 \x01(\x05\x12\x10\n\x08pc_range\x18\x05 \x03(\x02\x12\x0c\n\x04\x64\x61ta\x18\x06 \x01(\x0c\x12\x13\n\x0b\x63hunk_index\x18\x07 \x01(\x05\x12\x14\n\x0ctotal_chunks\x18\x08 \x01(\x05\x12\x15\n\rfeature_names\x18\t \x03(\t\"\xdc\x01\n\x10\x44\x61taEditsRequest\x12\x11\n\tstat_name\x18\x01 \x01(\t\x12\x13\n\x0b\x66loat_value\x18\x02 \x01(\x02\x12\x14\n\x0cstring_value\x18\x03 \x01(\t\x12\x12\n\nbool_value\x18\x04 \x01(\x08\x12\x1d\n\x04type\x18\x05 \x01(\x0e\x32\x0f.SampleEditType\x12\x13\n\x0bsamples_ids\x18\x06 \x03(\t\x12\x16\n\x0esample_origins\x18\x07 \x03(\t\x12\x16\n\x0eis_categorical\x18\x08 \x01(\x08\x12\x12\n\ncategories\x18\t \x03(\t\"5\n\x11\x44\x61taEditsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\":\n\x12\x44\x61taSplitsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x13\n\x0bsplit_names\x18\x02 \x03(\t\"9\n\x13\x41gentHealthResponse\x12\x11\n\tavailable\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"^\n\x16InitializeAgentRequest\x12\x0f\n\x07\x61pi_key\x18\x01 \x01(\t\x12$\n\x08provider\x18\x02 \x01(\x0e\x32\x12.AgentProviderType\x12\r\n\x05model\x18\x03 \x01(\t\";\n\x17InitializeAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"(\n\x17\x43hangeAgentModelRequest\x12\r\n\x05model\x18\x01 \x01(\t\"<\n\x18\x43hangeAgentModelResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x17\n\x15GetAgentModelsRequest\"J\n\x16GetAgentModelsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0e\n\x06models\x18\x02 \x03(\t\x12\x0f\n\x07message\x18\x03 \x01(\t\"6\n\x12ResetAgentResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"3\n\x18RestoreCheckpointRequest\x12\x17\n\x0f\x65xperiment_hash\x18\x01 \x01(\t\"=\n\x19RestoreCheckpointResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"R\n\x18TriggerEvaluationRequest\x12\x12\n\nsplit_name\x18\x01 \x01(\t\x12\x0c\n\x04tags\x18\x02 \x03(\t\x12\x14\n\x0cuse_full_set\x18\x03 \x01(\x08\"=\n\x19TriggerEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"\x1c\n\x1aGetEvaluationStatusRequest\"\x81\x01\n\x1bGetEvaluationStatusResponse\x12\x0e\n\x06status\x18\x01 \x01(\t\x12\x0f\n\x07\x63urrent\x18\x02 \x01(\x05\x12\r\n\x05total\x18\x03 \x01(\x05\x12\x0f\n\x07message\x18\x04 \x01(\t\x12\r\n\x05\x65rror\x18\x05 \x01(\t\x12\x12\n\nsplit_name\x18\x06 \x01(\t\")\n\x17\x43\x61ncelEvaluationRequest\x12\x0e\n\x06reason\x18\x01 \x01(\t\"<\n\x18\x43\x61ncelEvaluationResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\"7\n\x16RunNotebookCellRequest\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x0f\n\x07\x63\x65ll_id\x18\x02 \x01(\t\"2\n\x10NotebookCellDone\x12\x12\n\nexec_count\x18\x01 \x01(\x05\x12\n\n\x02ok\x18\x02 \x01(\x08\"\x1e\n\x1cInterruptNotebookCellRequest\":\n\x1dInterruptNotebookCellResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\r\n\x05\x65rror\x18\x02 \x01(\t\"\xbd\x01\n\x11NotebookCellChunk\x12\x0f\n\x07\x63\x65ll_id\x18\x01 \x01(\t\x12\x10\n\x06stdout\x18\x02 \x01(\tH\x00\x12\x10\n\x06stderr\x18\x03 \x01(\tH\x00\x12\x15\n\x0bresult_text\x18\x04 \x01(\tH\x00\x12\x13\n\timage_png\x18\x05 \x01(\x0cH\x00\x12\x19\n\x0f\x65rror_traceback\x18\x06 \x01(\tH\x00\x12!\n\x04\x64one\x18\x07 \x01(\x0b\x32\x11.NotebookCellDoneH\x00\x42\t\n\x07payload\"S\n\x10NotebookResponse\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0f\n\x07\x65xisted\x18\x02 \x01(\x08\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"7\n\x13SaveNotebookRequest\x12\x12\n\nipynb_json\x18\x01 \x01(\t\x12\x0c\n\x04name\x18\x02 \x01(\t\"M\n\x14SaveNotebookResponse\x12\n\n\x02ok\x18\x01 \x01(\x08\x12\x0c\n\x04path\x18\x02 \x01(\t\x12\r\n\x05\x65rror\x18\x03 \x01(\t\x12\x0c\n\x04name\x18\x04 \x01(\t\"C\n\x1bGenerateNotebookCodeRequest\x12\x0e\n\x06prompt\x18\x01 \x01(\t\x12\x14\n\x0c\x63ontext_code\x18\x02 \x01(\t\"\\\n\x1cGenerateNotebookCodeResponse\x12\x0c\n\x04\x63ode\x18\x01 \x01(\t\x12\x13\n\x0b\x65xplanation\x18\x02 \x01(\t\x12\n\n\x02ok\x18\x03 \x01(\x08\x12\r\n\x05\x65rror\x18\x04 \x01(\t\"~\n\x18\x45xportAnnotationsRequest\x12\'\n\x06\x66ormat\x18\x01 \x01(\x0e\x32\x17.AnnotationExportFormat\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x1b\n\x13include_predictions\x18\x03 \x01(\x08\x12\x0c\n\x04tags\x18\x04 \x03(\t\"\x88\x01\n\x19\x45xportAnnotationsResponse\x12\x0f\n\x07success\x18\x01 \x01(\x08\x12\x0f\n\x07message\x18\x02 \x01(\t\x12\x0f\n\x07payload\x18\x03 \x01(\x0c\x12\x10\n\x08\x66ilename\x18\x04 \x01(\t\x12\x11\n\tmime_type\x18\x05 \x01(\t\x12\x13\n\x0bimage_count\x18\x06 \x01(\x05*d\n\x13WeightOperationType\x12\n\n\x06ZEROFY\x10\x00\x12\x10\n\x0cREINITIALIZE\x10\x01\x12\n\n\x06\x46REEZE\x10\x02\x12\x12\n\x0eREMOVE_NEURONS\x10\t\x12\x0f\n\x0b\x41\x44\x44_NEURONS\x10\n*o\n\x0fZerofyPredicate\x12\x19\n\x15ZEROFY_PREDICATE_NONE\x10\x00\x12 \n\x1cZEROFY_PREDICATE_WITH_FROZEN\x10\x01\x12\x1f\n\x1bZEROFY_PREDICATE_WITH_OLDER\x10\x02*M\n\x0f\x41gentIntentType\x12\x12\n\x0eINTENT_UNKNOWN\x10\x00\x12\x11\n\rINTENT_FILTER\x10\x01\x12\x13\n\x0fINTENT_ANALYSIS\x10\x02*I\n\x0eSampleEditType\x12\x11\n\rEDIT_OVERRIDE\x10\x00\x12\x13\n\x0f\x45\x44IT_ACCUMULATE\x10\x01\x12\x0f\n\x0b\x45\x44IT_REMOVE\x10\x02*,\n\x11\x41gentProviderType\x12\x17\n\x13PROVIDER_OPENROUTER\x10\x00*m\n\x16\x41nnotationExportFormat\x12\x16\n\x12\x45XPORT_FORMAT_CVAT\x10\x00\x12\x1e\n\x1a\x45XPORT_FORMAT_LABEL_STUDIO\x10\x01\x12\x1b\n\x17\x45XPORT_FORMAT_V7_DARWIN\x10\x02\x32\xe9\x0e\n\x11\x45xperimentService\x12P\n\x13GetLatestLoggerData\x12\x1b.GetLatestLoggerDataRequest\x1a\x1c.GetLatestLoggerDataResponse\x12\x36\n\x11\x45xperimentCommand\x12\x0f.TrainerCommand\x1a\x10.CommandResponse\x12H\n\x11ManipulateWeights\x12\x18.WeightsOperationRequest\x1a\x19.WeightsOperationResponse\x12/\n\nGetWeights\x12\x0f.WeightsRequest\x1a\x10.WeightsResponse\x12\x39\n\x0eGetActivations\x12\x12.ActivationRequest\x1a\x13.ActivationResponse\x12\x37\n\nGetSamples\x12\x13.BatchSampleRequest\x1a\x14.BatchSampleResponse\x12\x37\n\x0e\x41pplyDataQuery\x12\x11.DataQueryRequest\x1a\x12.DataQueryResponse\x12;\n\x0eGetDataSamples\x12\x13.DataSamplesRequest\x1a\x14.DataSamplesResponse\x12\x35\n\x0cGetHistogram\x12\x11.HistogramRequest\x1a\x12.HistogramResponse\x12\x38\n\x0bGetMetaData\x12\x13.GetMetaDataRequest\x1a\x14.GetMetaDataResponse\x12P\n\x13GetSignalTrajectory\x12\x1b.GetSignalTrajectoryRequest\x1a\x1c.GetSignalTrajectoryResponse\x12\x37\n\rGetPointCloud\x12\x12.PointCloudRequest\x1a\x10.PointCloudChunk0\x01\x12\x37\n\x0e\x45\x64itDataSample\x12\x11.DataEditsRequest\x1a\x12.DataEditsResponse\x12,\n\rGetDataSplits\x12\x06.Empty\x1a\x13.DataSplitsResponse\x12\x30\n\x10\x43heckAgentHealth\x12\x06.Empty\x1a\x14.AgentHealthResponse\x12\x44\n\x0fInitializeAgent\x12\x17.InitializeAgentRequest\x1a\x18.InitializeAgentResponse\x12G\n\x10\x43hangeAgentModel\x12\x18.ChangeAgentModelRequest\x1a\x19.ChangeAgentModelResponse\x12\x41\n\x0eGetAgentModels\x12\x16.GetAgentModelsRequest\x1a\x17.GetAgentModelsResponse\x12)\n\nResetAgent\x12\x06.Empty\x1a\x13.ResetAgentResponse\x12@\n\x0fRunNotebookCell\x12\x17.RunNotebookCellRequest\x1a\x12.NotebookCellChunk0\x01\x12V\n\x15InterruptNotebookCell\x12\x1d.InterruptNotebookCellRequest\x1a\x1e.InterruptNotebookCellResponse\x12(\n\x0bGetNotebook\x12\x06.Empty\x1a\x11.NotebookResponse\x12;\n\x0cSaveNotebook\x12\x14.SaveNotebookRequest\x1a\x15.SaveNotebookResponse\x12S\n\x14GenerateNotebookCode\x12\x1c.GenerateNotebookCodeRequest\x1a\x1d.GenerateNotebookCodeResponse\x12J\n\x11RestoreCheckpoint\x12\x19.RestoreCheckpointRequest\x1a\x1a.RestoreCheckpointResponse\x12J\n\x11TriggerEvaluation\x12\x19.TriggerEvaluationRequest\x1a\x1a.TriggerEvaluationResponse\x12P\n\x13GetEvaluationStatus\x12\x1b.GetEvaluationStatusRequest\x1a\x1c.GetEvaluationStatusResponse\x12G\n\x10\x43\x61ncelEvaluation\x12\x18.CancelEvaluationRequest\x1a\x19.CancelEvaluationResponse\x12J\n\x11\x45xportAnnotations\x12\x19.ExportAnnotationsRequest\x1a\x1a.ExportAnnotationsResponseb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -37,16 +37,18 @@ _globals['_NEURONSTATISTICS_INCOMINGLRENTRY']._serialized_options = b'8\001' _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._loaded_options = None _globals['_RECORDMETADATA_SAMPLELASTLOSSENTRY']._serialized_options = b'8\001' - _globals['_WEIGHTOPERATIONTYPE']._serialized_start=10979 - _globals['_WEIGHTOPERATIONTYPE']._serialized_end=11079 - _globals['_ZEROFYPREDICATE']._serialized_start=11081 - _globals['_ZEROFYPREDICATE']._serialized_end=11192 - _globals['_AGENTINTENTTYPE']._serialized_start=11194 - _globals['_AGENTINTENTTYPE']._serialized_end=11271 - _globals['_SAMPLEEDITTYPE']._serialized_start=11273 - _globals['_SAMPLEEDITTYPE']._serialized_end=11346 - _globals['_AGENTPROVIDERTYPE']._serialized_start=11348 - _globals['_AGENTPROVIDERTYPE']._serialized_end=11392 + _globals['_WEIGHTOPERATIONTYPE']._serialized_start=11246 + _globals['_WEIGHTOPERATIONTYPE']._serialized_end=11346 + _globals['_ZEROFYPREDICATE']._serialized_start=11348 + _globals['_ZEROFYPREDICATE']._serialized_end=11459 + _globals['_AGENTINTENTTYPE']._serialized_start=11461 + _globals['_AGENTINTENTTYPE']._serialized_end=11538 + _globals['_SAMPLEEDITTYPE']._serialized_start=11540 + _globals['_SAMPLEEDITTYPE']._serialized_end=11613 + _globals['_AGENTPROVIDERTYPE']._serialized_start=11615 + _globals['_AGENTPROVIDERTYPE']._serialized_end=11659 + _globals['_ANNOTATIONEXPORTFORMAT']._serialized_start=11661 + _globals['_ANNOTATIONEXPORTFORMAT']._serialized_end=11770 _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_start=46 _globals['_GETLATESTLOGGERDATAREQUEST']._serialized_end=183 _globals['_LOGGERDATAPOINT']._serialized_start=186 @@ -219,6 +221,10 @@ _globals['_GENERATENOTEBOOKCODEREQUEST']._serialized_end=10883 _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_start=10885 _globals['_GENERATENOTEBOOKCODERESPONSE']._serialized_end=10977 - _globals['_EXPERIMENTSERVICE']._serialized_start=11395 - _globals['_EXPERIMENTSERVICE']._serialized_end=13216 + _globals['_EXPORTANNOTATIONSREQUEST']._serialized_start=10979 + _globals['_EXPORTANNOTATIONSREQUEST']._serialized_end=11105 + _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_start=11108 + _globals['_EXPORTANNOTATIONSRESPONSE']._serialized_end=11244 + _globals['_EXPERIMENTSERVICE']._serialized_start=11773 + _globals['_EXPERIMENTSERVICE']._serialized_end=13670 # @@protoc_insertion_point(module_scope) diff --git a/weightslab/proto/experiment_service_pb2_grpc.py b/weightslab/proto/experiment_service_pb2_grpc.py index 2f9127b2..2967171a 100644 --- a/weightslab/proto/experiment_service_pb2_grpc.py +++ b/weightslab/proto/experiment_service_pb2_grpc.py @@ -5,7 +5,7 @@ from weightslab.proto import experiment_service_pb2 as weightslab_dot_proto_dot_experiment__service__pb2 -GRPC_GENERATED_VERSION = '1.76.0' +GRPC_GENERATED_VERSION = '1.68.1' GRPC_VERSION = grpc.__version__ _version_not_supported = False @@ -18,7 +18,7 @@ if _version_not_supported: raise RuntimeError( f'The grpc package installed is at version {GRPC_VERSION},' - + ' but the generated code in weightslab/proto/experiment_service_pb2_grpc.py depends on' + + f' but the generated code in weightslab/proto/experiment_service_pb2_grpc.py depends on' + f' grpcio>={GRPC_GENERATED_VERSION}.' + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' @@ -174,6 +174,11 @@ def __init__(self, channel): request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationRequest.SerializeToString, response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationResponse.FromString, _registered_method=True) + self.ExportAnnotations = channel.unary_unary( + '/ExperimentService/ExportAnnotations', + request_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, + response_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.FromString, + _registered_method=True) class ExperimentServiceServicer(object): @@ -370,6 +375,16 @@ def CancelEvaluation(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def ExportAnnotations(self, request, context): + """Export bounding-box/segmentation annotations to a relabeling-tool format + (CVAT XML, Label Studio JSON, or V7/Darwin JSON). Unary: the whole file + (or a zip, for formats that need one file per image) comes back as bytes + in a single response. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_ExperimentServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -513,6 +528,11 @@ def add_ExperimentServiceServicer_to_server(servicer, server): request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationRequest.FromString, response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.CancelEvaluationResponse.SerializeToString, ), + 'ExportAnnotations': grpc.unary_unary_rpc_method_handler( + servicer.ExportAnnotations, + request_deserializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.FromString, + response_serializer=weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'ExperimentService', rpc_method_handlers) @@ -1279,3 +1299,30 @@ def CancelEvaluation(request, timeout, metadata, _registered_method=True) + + @staticmethod + def ExportAnnotations(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary( + request, + target, + '/ExperimentService/ExportAnnotations', + weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsRequest.SerializeToString, + weightslab_dot_proto_dot_experiment__service__pb2.ExportAnnotationsResponse.FromString, + options, + channel_credentials, + insecure, + call_credentials, + compression, + wait_for_ready, + timeout, + metadata, + _registered_method=True) diff --git a/weightslab/src.py b/weightslab/src.py index 62dd5290..f2849818 100644 --- a/weightslab/src.py +++ b/weightslab/src.py @@ -4866,6 +4866,106 @@ def write_dataframe( return path +def export_annotations( + fmt: str, + path: str | None = None, + origin: str | None = None, + class_names: dict | list | None = None, + use_predictions: bool = False, + tags: list[str] | None = None, +) -> str: + """Export bounding-box/segmentation annotations to a relabeling-tool format. + + Reads ground-truth (or, with *use_predictions*, model-predicted) boxes and + segmentation masks from the registered dataframe and writes them to + *path* in *fmt*, ready to hand off to an outsourced relabeling pass. + + Parameters + ---------- + fmt : {"cvat", "label_studio", "v7"} + Target format: + + - ``"cvat"`` — a single CVAT XML 1.1 file. + - ``"label_studio"`` — a single Label Studio JSON file. + - ``"v7"`` — a zip of per-image V7/Darwin JSON 2.0 files (Darwin + matches annotations to images by filename on import). + path : str, optional + Output file path **or** directory. When omitted (``None``), the + ``root_log_dir`` from the active checkpoint manager is used, with the + format's default filename (e.g. ``annotations_cvat.xml``). If *path* + is a directory (or has no extension), the default filename is + appended inside it. + origin : str, optional + Restrict export to one registered split/loader (e.g. ``"train_loader"``). + ``None`` exports every registered split. + class_names : dict or list, optional + Explicit class-id -> name mapping, overriding any auto-detected + ``dataset.class_names`` attribute. Without either, labels fall back + to ``"class_"``. + use_predictions : bool, optional + Export model predictions instead of ground-truth targets. + tags : list of str, optional + Restrict export to samples carrying ANY of these tags (``tag:`` + prefix optional, e.g. ``["ToReview"]``), matching either a boolean + tag set via :func:`tag_samples` or a categorical value set via + :func:`set_categorical_tag`. ``None`` (default) exports every sample. + + Returns + ------- + str + Absolute path of the file (or zip) that was written. + + Notes + ----- + Only annotation coordinates + a filename are exported — image files + themselves are neither copied nor embedded. Ensure the filenames used + when uploading images to CVAT/Label Studio/V7 match the ones in the + export (real image paths are resolved best-effort from the dataset + object; unresolved samples get a synthetic ``sample_.jpg`` name). + + Segmentation polygon export requires OpenCV (``pip install + weightslab[export]``); bounding-box export needs no extra dependency. + + Examples + -------- + Export everything to CVAT, auto-named under ``root_log_dir``:: + + wl.export_annotations("cvat") + + Export only the validation split to Label Studio, with explicit class names:: + + wl.export_annotations( + "label_studio", "val_annotations.json", + origin="val_loader", class_names=["background", "cat", "dog"], + ) + + Export only the samples tagged "ToReview" to CVAT, for a relabeling pass:: + + wl.export_annotations("cvat", tags=["ToReview"]) + """ + import os as _os + + from weightslab.export.exporter import save_export + + if path is None: + _lg = get_logger() + try: + _cm = _lg.chkpt_manager if _lg is not None else None + _rld = _cm.root_log_dir if _cm is not None else None + path = str(_rld) if _rld is not None else "." + except Exception as _e: + logger.debug("export_annotations: failed to resolve root_log_dir (%s); " + "falling back to current directory.", _e) + path = "." + logger.debug("export_annotations: no path given, using output directory %r.", path) + + written_path = save_export( + fmt, path, + origin=origin, class_names=class_names, use_predictions=use_predictions, tags=tags, + ) + return _os.path.abspath(written_path) + + def _live_data_service(): """The DataService of the experiment running in THIS process, if any. diff --git a/weightslab/trainer/services/data_service.py b/weightslab/trainer/services/data_service.py index 64192cfd..4d94955e 100755 --- a/weightslab/trainer/services/data_service.py +++ b/weightslab/trainer/services/data_service.py @@ -5404,3 +5404,47 @@ def GetDataSplits(self, request, context): success=False, split_names=[] ) + + # Format enum value -> weightslab.export's string key. + _EXPORT_FORMAT_NAMES = { + pb2.EXPORT_FORMAT_CVAT: "cvat", + pb2.EXPORT_FORMAT_LABEL_STUDIO: "label_studio", + pb2.EXPORT_FORMAT_V7_DARWIN: "v7", + } + + def ExportAnnotations(self, request, context): + """Export bounding-box/segmentation annotations to a relabeling-tool + format (CVAT XML, Label Studio JSON, or V7/Darwin JSON) and return the + encoded file as bytes for the caller (Weights Studio's Export button, + or the `weightslab export` CLI) to write/download. + """ + fmt = self._EXPORT_FORMAT_NAMES.get(request.format) + if fmt is None: + return pb2.ExportAnnotationsResponse( + success=False, + message=f"Unknown export format value: {request.format}", + ) + + try: + from weightslab.export.exporter import export_annotations + + payload, filename, mime_type, image_count = export_annotations( + fmt, + origin=request.origin or None, + use_predictions=request.include_predictions, + tags=list(request.tags) or None, + ) + return pb2.ExportAnnotationsResponse( + success=True, + message=f"Exported {image_count} image(s) to {fmt} format.", + payload=payload, + filename=filename, + mime_type=mime_type, + image_count=image_count, + ) + except ImportError as e: + logger.warning(f"ExportAnnotations missing optional dependency: {e}") + return pb2.ExportAnnotationsResponse(success=False, message=str(e)) + except Exception as e: + logger.error(f"ExportAnnotations failed: {e}", exc_info=True) + return pb2.ExportAnnotationsResponse(success=False, message=f"Export failed: {e}") diff --git a/weightslab/trainer/trainer_services.py b/weightslab/trainer/trainer_services.py index 6ea86831..a37ba409 100644 --- a/weightslab/trainer/trainer_services.py +++ b/weightslab/trainer/trainer_services.py @@ -377,6 +377,10 @@ def GetDataSplits(self, request, context): logger.debug(f"\nExperimentServiceServicer.GetDataSplits({request})") return self._exp_service.data_service.GetDataSplits(request, context) + def ExportAnnotations(self, request, context): + logger.debug(f"\nExperimentServiceServicer.ExportAnnotations({request})") + return self._exp_service.data_service.ExportAnnotations(request, context) + def CheckAgentHealth(self, request, context): logger.debug(f"\nExperimentServiceServicer.CheckAgentHealth({request})") # Prefer explicit AgentService when present (new wiring). @@ -731,5 +735,15 @@ def serving_thread_callback(): grpc_host, grpc_port, n_workers_grpc, ) + # Resource monitoring (CPU/memory/disk/network/GPU/process) runs for the + # whole server lifetime, independent of training steps. Enabled by + # default; see docs/resource_monitoring.rst for the config file and + # env var overrides (WEIGHTSLAB_DISABLE_RESOURCE_MONITORING, etc.). + try: + from weightslab.monitoring.resource_monitor import start_resource_monitor_from_config + start_resource_monitor_from_config() + except Exception: + logger.exception("[gRPC] Failed to start resource monitor") + if __name__ == "__main__": grpc_serve()