Refactor camera backends and optimize the real-time frame pipeline

.claude-plugin/marketplace.json

@@ -0,0 +1,16 @@
+{
+  "name": "scopeone",
+  "owner": {
+    "name": "ScopeOne Project"
+  },
+  "description": "Claude Code plugins for ScopeOne microscopy automation",
+  "plugins": [
+    {
+      "name": "scopeone",
+      "displayName": "ScopeOne",
+      "source": "./scopeone-skill",
+      "description": "Control ScopeOne through its bundled MCP server",
+      "homepage": "https://github.com/Experimental-Microscopy-Lab/ScopeOne"
+    }
+  ]
+}

.github/workflows/compile-check.yml

@@ -44,6 +44,6 @@ jobs:
             libtiff-dev \
             zlib1g-dev
 
-      - name: Compile MMCore and ScopeOne
+      - name: Compile mmCoreAndDevices and ScopeOne
         shell: bash
         run: ./scripts/build-linux.sh

.gitignore

@@ -13,6 +13,12 @@ ScopeOneCore/python/**
 !.github/
 !.github/**
 
+!.claude-plugin/
+!.claude-plugin/**
+
+!scopeone-skill/
+!scopeone-skill/**
+
 !resources/
 !resources/**
 

README.md

@@ -7,7 +7,7 @@
   <a href="https://doi.org/10.48550/arXiv.2606.19384"><img src="https://img.shields.io/badge/arXiv-2606.19384-b31b1b.svg" alt="Preprint DOI"></a>
 </p>
 
-ScopeOne is an open-source microscopy control software for multi-camera imaging, originally developed for in-house lab use. Built with C++ and Qt, it uses a multi-process architecture where each camera runs in its own [MMCore](https://github.com/micro-manager/mmCoreAndDevices) instance, enabling simultaneous preview and acquisition across multiple cameras.
+ScopeOne is an open-source microscopy control software for multi-camera imaging, originally developed for in-house lab use. Built with C++ and Qt, it uses a native [MMCore](https://github.com/micro-manager/mmCoreAndDevices) backend for single-camera operation and one isolated camera agent per device for simultaneous multi-camera preview and acquisition.
 It retains full compatibility with the [Micro-Manager](https://micro-manager.org/) device ecosystem and adds a modular real-time image processing pipeline with support for background calibration, temporal filtering, FFT analysis, and more.
 
 <p align="center">
@@ -154,6 +154,20 @@ The API reports which operations mutate hardware, write files, remove state, or
 
 `ScopeOneMcpServer` is a standalone C++ MCP server included with ScopeOne. It uses MCP protocol version `2025-06-18` over standard input and output and forwards validated tool calls to the running desktop app through the Local API. It does not embed a model, open a network port, or depend on Python.
 
+#### Claude Code plugin
+
+Install ScopeOne first, then run these commands inside Claude Code:
+
+```text
+/plugin marketplace add Experimental-Microscopy-Lab/ScopeOne
+/plugin install scopeone@scopeone
+/reload-plugins
+```
+
+When prompted, select `ScopeOneMcpServer.exe` from the same directory as `ScopeOne.exe`. Start ScopeOne before asking Claude to inspect or control it. The plugin supplies the MCP configuration and agent operating guidance, so no Python installation or manual MCP server launch is required.
+
+#### Other MCP hosts
+
 To use it:
 
 1. Start ScopeOne. Load the device configuration manually or ask the agent to load it after confirmation.
@@ -173,7 +187,8 @@ The MCP tool set mirrors the Local API operation catalog, including system state
 The current validation list is still short, but the codebase has been cleaned to remove early hard-coded device assumptions. In principle, ScopeOne should follow Micro-Manager device compatibility.
 
 ## 🖥️ Tested System Configurations
-- Windows 10, Dual Intel(R) Xeon(R) E5-2637 v3, 64 GB RAM, NVIDIA Quadro K620  
-- Windows 11, Intel(R) Core(TM) Ultra 5 125U, 64 GB RAM
+- Windows 10 Version 21H2, Dual Intel(R) Xeon(R) E5-2637 v3, 64 GB RAM, NVIDIA Quadro K620
+- Windows 11 Version 25H2, Intel(R) Core(TM) Ultra 5 125U, 64 GB RAM
+- Fedora Linux 44 (Workstation Edition), Intel(R) Core(TM) i7-7700, 32 GB RAM
 
-Although these machines are older and weaker than many typical lab computers, ScopeOne still provides smooth real-time preview and processing on them.
+We build and test ScopeOne on the above machines, which are comparatively older and weaker than many typical optical lab computers. However, ScopeOne still provides smooth real-time preview and processing on them.

ScopeOneCore/CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(ScopeOneCore VERSION 2.0.719 LANGUAGES CXX)
+project(ScopeOneCore VERSION 2.0.722 LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_AUTOMOC ON)
@@ -108,7 +108,10 @@ set(CORE_SOURCES
         src/DifferentialRollingModule.cpp
         src/MDAManager.cpp
         src/RecordingManager.cpp
-        src/MultiProcessCameraManager.cpp
+        src/CameraBackend.cpp
+        src/CameraManager.cpp
+        src/NativeCameraBackend.cpp
+        src/AgentCameraBackend.cpp
         src/ScopeOneCore.cpp
 )
 
@@ -129,7 +132,8 @@ set(CORE_HEADERS
         internal/DifferentialRollingModule.h
         internal/MDAManager.h
         internal/RecordingManager.h
-        internal/MultiProcessCameraManager.h
+        internal/CameraBackend.h
+        internal/CameraManager.h
         include/scopeone/ImageFrame.h
         include/scopeone/SharedFrame.h
         include/scopeone/scopeone_core_export.h

ScopeOneCore/README.md

@@ -43,7 +43,7 @@ Use this placement rule:
 | Namespace | Purpose | Examples |
 |---|---|---|
 | `scopeone::core` | Stable Core-facing types and public facades | `ScopeOneCore`, `ImageFrame`, `ExperimentDocument`, `ImageSceneModel` |
-| `scopeone::core::internal` | Core-only managers and processing implementations | `MMCoreManager`, `RecordingManager`, processing modules |
+| `scopeone::core::internal` | Core-only managers and processing implementations | `CameraManager`, `MMCoreManager`, `RecordingManager`, processing modules |
 | `scopeone::core::internal::agent` | Private camera-agent protocol details | Agent request, response and frame transport types |
 | `scopeone::ui` | Desktop application widgets and UI coordination outside this library | `MainWindow`, `PreviewWidget`, `InspectWidget` |
 

ScopeOneCore/include/scopeone/ScopeOneCore.h

@@ -3,6 +3,7 @@
 #include <QObject>
 #include <QStringList>
 #include <QByteArray>
+#include <QElapsedTimer>
 #include <QHash>
 #include <QList>
 #include <QMetaType>
@@ -21,6 +22,8 @@
 #include "scopeone/scopeone_core_export.h"
 
 class CMMCore;
+class QThreadPool;
+class QTimer;
 
 namespace scopeone::core
 {
@@ -620,6 +623,7 @@ namespace scopeone::core
         void clearProcessedFrames();
         bool getRawImageStatistics(const QString& cameraId, HistogramStats& stats) const;
         bool getLayerHistogram(const QString& layerKey, HistogramStats& stats) const;
+        void setActiveHistogramLayer(const QString& layerKey);
         bool autoLayerLevels(const QString& layerKey);
         bool fullLayerLevels(const QString& layerKey);
         bool setLayerAutoStretchEnabled(const QString& layerKey, bool enabled);
@@ -726,10 +730,12 @@ namespace scopeone::core
         void deviceStateChanged();
         void stagePositionChanged();
         void newRawFrameReady(const ImageFrame& frame);
+        void rawFramesAcquired(const QString& cameraId, quint64 frameCount);
         void previewRawFrameReady(const ImageFrame& frame);
         void previewStateChanged(bool running);
         void agentControlServerListening(const QString& cameraId, const QString& serverName);
         void processedFrameReady(const ImageFrame& frame);
+        void processedFramesCompleted(const QString& cameraId, quint64 frameCount);
         void previewProcessedFrameReady(const ImageFrame& frame);
         void staticFramePublished(const QString& sourceId,
                                   const QString& displayName,
@@ -825,6 +831,7 @@ namespace scopeone::core
         struct HistogramJobState
         {
             bool inFlight{false};
+            bool retryScheduled{false};
             qint64 lastScheduledMs{0};
             quint64 activeSequence{0};
             ImageFrame queuedFrame;
@@ -834,7 +841,7 @@ namespace scopeone::core
                                        LoadConfigResult* result,
                                        QString* errorMessage);
         std::shared_ptr<CMMCore> core() const;
-        bool isAgentCamera(const QString& deviceLabel) const;
+        bool isConfiguredCamera(const QString& deviceLabel) const;
         bool isNativeCamera(const QString& deviceLabel) const;
         bool isPropertyPreInit(const QString& deviceLabel, const QString& name) const;
         void ensureSceneLayer(const QString& layerKey,
@@ -845,13 +852,12 @@ namespace scopeone::core
         void processGraphRawFrameAsync(const ImageFrame& frame);
         void queuePreviewRawFrame(const ImageFrame& frame);
         void queuePreviewProcessedFrame(const ImageFrame& frame);
-        void flushPreviewRawFrames();
-        void flushPreviewProcessedFrames();
+        void schedulePreviewFlush();
+        void flushPreviewFrames();
         QString sessionFrameSourceId(const std::shared_ptr<RecordingSessionData>& session) const;
         bool publishSessionFrameSource(const std::shared_ptr<RecordingSessionData>& session);
-        void scheduleHistogramStats(const QString& cameraId,
-                                    bool processed,
-                                    const ImageFrame& frame);
+        bool histogramUpdatesEnabled(const QString& layerKey) const;
+        void scheduleHistogramStats(const QString& layerKey, const ImageFrame& frame);
         void clearLayerAnalysis(const QString& layerKey);
         void clearLayerAnalysisByPrefix(const QString& prefix);
         void updateLineProfile(const QString& cameraId,
@@ -891,9 +897,12 @@ namespace scopeone::core
         QHash<QString, ImageFrame> m_pendingPreviewProcessedFrames;
         QHash<QString, HistogramJobState> m_histogramJobStates;
         QHash<QString, HistogramStats> m_latestHistogramStats;
+        std::unique_ptr<QThreadPool> m_histogramThreadPool;
+        QString m_activeHistogramLayerKey;
         quint64 m_nextHistogramSequence{0};
-        bool m_previewRawFlushQueued{false};
-        bool m_previewProcessedFlushQueued{false};
+        QElapsedTimer m_lineProfileUpdateTimer;
+        QElapsedTimer m_previewPublishTimer;
+        QTimer* m_previewFlushTimer{nullptr};
         QSet<const RecordingSessionData*> m_sessionsSaving;
     };
 }

ScopeOneCore/include/scopeone/SharedFrame.h

@@ -76,8 +76,8 @@ namespace scopeone::core
         }
     }
 
-    inline constexpr int kSharedFrameNumSlots = 12;
-    inline constexpr int kSharedFrameMaxBytes = 16 * 2048 * 2048;
+    inline constexpr int kSharedFrameNumSlots = 32;
+    inline constexpr int kSharedFrameMaxBytes = 4 * 2048 * 2048;
     inline constexpr int kSharedMemoryControlSize = 64;
     inline constexpr int kSharedFrameHeaderSize = static_cast<int>(sizeof(SharedFrameHeader));
     inline constexpr int kSharedFrameSlotStride = kSharedFrameHeaderSize + kSharedFrameMaxBytes;

ScopeOneCore/internal/AgentProtocol.h

@@ -8,7 +8,7 @@
 
 namespace scopeone::core::internal::agent
 {
-    inline constexpr quint32 kProtocolVersion = 1;
+    inline constexpr quint32 kProtocolVersion = 3;
     inline constexpr quint32 kMaxControlMessageBytes = 256 * 1024;
 
     inline const QString kEnvelopeKindField = QStringLiteral("kind");
@@ -23,6 +23,7 @@ namespace scopeone::core::internal::agent
     inline const QString kCommandShutdown = QStringLiteral("Shutdown");
     inline const QString kCommandStartPreview = QStringLiteral("StartPreview");
     inline const QString kCommandStopPreview = QStringLiteral("StopPreview");
+    inline const QString kCommandSetFrameDeliveryMode = QStringLiteral("SetFrameDeliveryMode");
     inline const QString kCommandSetExposure = QStringLiteral("SetExposure");
     inline const QString kCommandListProperties = QStringLiteral("ListProperties");
     inline const QString kCommandGetProperty = QStringLiteral("GetProperty");
@@ -32,11 +33,14 @@ namespace scopeone::core::internal::agent
     inline const QString kCommandClearRoi = QStringLiteral("ClearROI");
     inline const QString kCommandGetRoi = QStringLiteral("GetROI");
 
+    inline const QString kFrameDeliveryModePreviewLatest = QStringLiteral("PreviewLatest");
+    inline const QString kFrameDeliveryModeLatestOnly = QStringLiteral("LatestOnly");
+    inline const QString kFrameDeliveryModeAllFrames = QStringLiteral("AllFrames");
+
     inline const QString kEventHello = QStringLiteral("Hello");
     inline const QString kEventFrameAvailable = QStringLiteral("FrameAvailable");
     inline const QString kEventPreviewState = QStringLiteral("PreviewState");
     inline const QString kEventAgentError = QStringLiteral("AgentError");
-    inline const QString kEventBufferOverflow = QStringLiteral("BufferOverflow");
 
     inline const QString kExecutableFileName = QStringLiteral("ScopeOne_Agent.exe");
 

ScopeOneCore/internal/BackgroundCalibrationModule.h

@@ -2,6 +2,7 @@
 
 #include "internal/ProcessingModule.h"
 #include <deque>
+#include <vector>
 
 namespace scopeone::core::internal
 {
@@ -41,9 +42,12 @@ namespace scopeone::core::internal
     private:
         void resetCalibration();
         void computeBackground();
+        ProcessingResult processRunningMean(const ImageFrame& sourceFrame,
+                                            const ImageFrame& workingFrame);
 
         int m_calibrationFrames{101};
         std::deque<ImageFrame> m_buffer;
+        std::vector<qint64> m_runningSum;
         ImageFrame m_background;
         bool m_calibrated{false};
         BackgroundOperation m_operation{BackgroundOperation::Subtract};