From 497f6a58c38165d0ac62dc9b8e628855e7d3fbe8 Mon Sep 17 00:00:00 2001 From: AlexTemirov Date: Fri, 17 Jul 2026 21:26:34 -0700 Subject: [PATCH] Improve onboarding templates and dataset replay --- README.md | 5 +- docs/packages.md | 9 +- docs/walkthrough.md | 6 +- editor-server/server.py | 173 ++++++++- editor/src/api.ts | 14 + editor/src/components/BlackNode.tsx | 156 +++++++- editor/src/components/DatasetBrowserPanel.tsx | 348 ++++++++++++++++++ editor/src/components/NodePalette.tsx | 6 +- editor/src/components/OutputNode.tsx | 16 +- editor/src/components/TemplateGallery.tsx | 113 +++++- editor/src/portColors.ts | 2 + editor/src/store.ts | 95 ++++- python/blacknode/node.py | 3 +- start.ps1 | 10 +- start.sh | 8 +- tests/test_editor_media_output.py | 35 ++ tests/test_editor_package_welcome.py | 4 +- tests/test_editor_runtime.py | 126 +++++++ tests/test_editor_template_groups.py | 27 ++ tests/test_editor_template_packages.py | 33 ++ tests/test_launchers.py | 10 + 21 files changed, 1141 insertions(+), 58 deletions(-) create mode 100644 editor/src/components/DatasetBrowserPanel.tsx create mode 100644 tests/test_editor_template_groups.py diff --git a/README.md b/README.md index 28ad4e5..29a6a1c 100644 --- a/README.md +++ b/README.md @@ -80,9 +80,10 @@ takes less than a minute; first-run time depends on network and package caches. On the first launch of a Blacknode workspace, the editor opens **Packages** with a one-time welcome message. Install the official packages needed for robotics, ROS 2, vision, CUDA, datasets, and training workflows, or continue directly -with the core graph. The acknowledgement is stored locally in +with the core templates. The acknowledgement is stored locally in `.blacknode/onboarding.json`, and the Packages tab remains available in the -left sidebar. +left sidebar. After onboarding, normal editor sessions start in **Templates**; +the full Nodes palette remains available when building a graph manually. Continue with the [Beginner Walkthrough](docs/walkthrough.md). diff --git a/docs/packages.md b/docs/packages.md index 96282a8..6b1b3e7 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -35,7 +35,9 @@ blacknode packages setup blacknode-ros2 ``` `start.ps1` and `start.sh` automatically install missing declared Python -dependencies for installed packages before starting the server. Set +dependencies for installed packages before starting the server. The launchers +stream package names, dependency resolution, downloads, and pip installation +output while this step runs. Set `BLACKNODE_PACKAGE_AUTO_SETUP=0` to disable that behavior. Automatic startup setup does not pull Docker images or run package setup scripts; use the command above when those additional prerequisites are required. @@ -97,6 +99,11 @@ Official packages are listed in the editor's Packages tab even when they are not installed. Press **Install** on an available package to clone it from the built-in Git URL without pasting a repository URL manually. +The editor's **Templates** tab groups starter workflows by Core or their source +package category. Every group starts collapsed, its templates inherit the +category color, and the search field filters across category names, template +names, slugs, and descriptions. + On a Blacknode workspace's first editor session, the editor opens this tab behind a one-time welcome message. The message directs robotics users to install the official packages their workflows require and lets core-graph users diff --git a/docs/walkthrough.md b/docs/walkthrough.md index 6608492..bcb78da 100644 --- a/docs/walkthrough.md +++ b/docs/walkthrough.md @@ -168,7 +168,7 @@ http://localhost:3000 In the browser: 1. Look at the left sidebar. -2. Click the **Templates** tab. +2. Use the **Templates** tab, which is open by default after onboarding. 3. Click **Text Pipeline**. 4. Click the **Output** node on the canvas. 5. In the right **Properties** panel, click **Cook**. @@ -179,6 +179,10 @@ Expected result: - A small run status appears while the graph cooks once. - The run is saved in the **Runs** tab. +An Output node connected to a typed image or video port opens at the same large +dashboard size used by robot monitoring nodes. Live camera frames and video +controls stay inside the node. + ## 7. Build the Same Workflow by Hand In the editor: diff --git a/editor-server/server.py b/editor-server/server.py index 894e58b..21465d4 100644 --- a/editor-server/server.py +++ b/editor-server/server.py @@ -7,7 +7,7 @@ from typing import Any from fastapi import FastAPI, HTTPException, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import StreamingResponse +from fastapi.responses import FileResponse, StreamingResponse from pydantic import BaseModel sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "python")) @@ -168,6 +168,22 @@ class UpdateParamReq(BaseModel): key: str value: Any +class NodeControlReq(BaseModel): + action: str + +class PickDirectoryReq(BaseModel): + initial_path: str = "" + +class DatasetTrimReq(BaseModel): + token: str + frame_index: int + side: str + +class DatasetReplayEventReq(BaseModel): + token: str + frame_index: int + event: str + class UpdatePortsReq(BaseModel): inputs: list[str] | None = None outputs: list[str] | None = None @@ -1234,6 +1250,78 @@ def update_param(node_id: str, req: UpdateParamReq): return meta +@app.post("/nodes/{node_id}/control") +def control_node(node_id: str, req: NodeControlReq): + meta = _session.node_meta.get(node_id) + if meta is None: + raise HTTPException(404, "Node not found") + if meta.get("type") == "TrajectorySmoother": + if req.action != "apply": + raise HTTPException(400, "TrajectorySmoother supports the apply control") + control_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "apply_configured_smoother") + if control_fn is None: + raise HTTPException(503, "blacknode-dataset smoother runtime is not loaded") + params = dict(meta.get("params") or {}) + try: + outputs = dict(control_fn( + node_id, + str(params.get("method") or "spline"), + float(params.get("strength") if params.get("strength") is not None else 1.0), + preview_source=str(params.get("preview_source") or "action"), + preview_joint=str(params.get("preview_joint") or ""), + )) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + for port, value in outputs.items(): + _session.graph._cache[(node_id, port)] = value + _session.graph._dirty.discard(node_id) + return {"ok": True, "node_id": node_id, "outputs": outputs} + if meta.get("type") != "EpisodeRecorder": + raise HTTPException(400, "This node does not expose direct controls") + control_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "control_configured_recorder") + if control_fn is None: + raise HTTPException(503, "blacknode-dataset recorder runtime is not loaded") + run_id = str(meta.get("params", {}).get("run_id") or "episode_recorder").strip() or "episode_recorder" + try: + outputs = dict(control_fn(run_id, req.action)) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"ok": True, "node_id": node_id, "outputs": outputs} + + +def _pick_directory(initial_path: str = "") -> str: + try: + import tkinter as tk + from tkinter import filedialog + except Exception as exc: # pragma: no cover - depends on the local Python GUI build + raise RuntimeError(f"native folder picker is unavailable: {exc}") from exc + initial = Path(str(initial_path or "")).expanduser() + if not initial.is_dir(): + initial = Path.home() + root = tk.Tk() + try: + root.withdraw() + root.attributes("-topmost", True) + root.update() + return str(filedialog.askdirectory( + parent=root, + title="Choose a folder that will contain Blacknode datasets", + initialdir=str(initial), + mustexist=True, + ) or "") + finally: + root.destroy() + + +@app.post("/filesystem/pick-directory") +def pick_directory(req: PickDirectoryReq): + try: + selected = _pick_directory(req.initial_path) + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + return {"selected": selected, "cancelled": not bool(selected)} + + @app.patch("/nodes/{node_id}/ports") def update_ports(node_id: str, req: UpdatePortsReq): if node_id not in _session.node_meta: @@ -2313,6 +2401,55 @@ def runtime_status(): return _runtime_status() +@app.get("/api/dataset/media/{token}") +@app.get("/dataset/media/{token}") +def dataset_media(token: str): + resolve_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "replay_media_path") + path = resolve_fn(token) if resolve_fn is not None else None + if path is None: + raise HTTPException(404, "Replay media not found") + return FileResponse(path, media_type="video/mp4") + + +@app.get("/api/dataset/frame/{token}") +@app.get("/dataset/frame/{token}") +def dataset_frame(token: str, index: int = 0): + frame_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "replay_frame") + try: + frame = frame_fn(token, index) if frame_fn is not None else None + except (RuntimeError, ValueError) as exc: + raise HTTPException(409, str(exc)) from exc + if frame is None: + raise HTTPException(404, "Replay frame not found") + return frame + + +@app.post("/api/dataset/trim") +@app.post("/dataset/trim") +def dataset_trim(req: DatasetTrimReq): + trim_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "trim_replay_episode") + if trim_fn is None: + raise HTTPException(503, "blacknode-dataset trim runtime is not loaded") + try: + return dict(trim_fn(req.token, req.frame_index, req.side)) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(500, str(exc)) from exc + + +@app.post("/api/dataset/replay-event") +@app.post("/dataset/replay-event") +def dataset_replay_event(req: DatasetReplayEventReq): + publish_fn = _runtime_callable("dataset", _RUNTIME_MODULES["dataset"], "publish_replay_event") + if publish_fn is None: + raise HTTPException(503, "blacknode-dataset replay stream runtime is not loaded") + try: + return dict(publish_fn(req.token, req.frame_index, req.event)) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + + @app.post("/runtime/stop") def stop_runtime(): _stop_active_cook() @@ -3530,6 +3667,21 @@ def _template_dirs() -> list[str]: return [_TEMPLATES_DIR, *package_template_dirs()] +def _template_sources() -> list[tuple[str, str, str]]: + """Return template directories with stable editor grouping metadata.""" + sources = [(_TEMPLATES_DIR, "Core", "#6366f1")] + for info in installed_packages(): + if not info.ok or not info.templates_dir: + continue + if info.categories: + group, color = next(iter(info.categories.items())) + else: + group = info.name.removeprefix("blacknode-").replace("-", " ").title() + color = "#6366f1" + sources.append((info.templates_dir, group, color)) + return sources + + def _template_path(slug: str) -> str: if not re.fullmatch(r"[a-zA-Z0-9_-]{1,60}", slug): raise HTTPException(400, "Invalid template slug") @@ -3549,7 +3701,13 @@ def _read_workflow_file(path: str) -> dict[str, Any]: return data -def _workflow_summary(slug: str, data: dict[str, Any]) -> dict[str, Any]: +def _workflow_summary( + slug: str, + data: dict[str, Any], + *, + group: str = "Core", + group_color: str = "#6366f1", +) -> dict[str, Any]: metadata = data.get("metadata") if isinstance(data.get("metadata"), dict) else {} return { "slug": slug, @@ -3558,6 +3716,8 @@ def _workflow_summary(slug: str, data: dict[str, Any]) -> dict[str, Any]: "description": metadata.get("description", ""), "color": metadata.get("color", "#6366f1"), "node_count": len(data.get("node_meta", {}) or {}), + "group": group, + "group_color": group_color, } @@ -3722,7 +3882,7 @@ def list_workflows(): def list_templates(): result = [] seen: set[str] = set() - for templates_dir in _template_dirs(): + for templates_dir, group, group_color in _template_sources(): if not os.path.isdir(templates_dir): continue for fname in sorted(os.listdir(templates_dir)): @@ -3733,7 +3893,12 @@ def list_templates(): continue try: data = _read_workflow_file(os.path.join(templates_dir, fname)) - result.append(_workflow_summary(slug, data)) + result.append(_workflow_summary( + slug, + data, + group=group, + group_color=group_color, + )) seen.add(slug) except Exception: pass diff --git a/editor/src/api.ts b/editor/src/api.ts index 1da00ff..e76defe 100644 --- a/editor/src/api.ts +++ b/editor/src/api.ts @@ -17,6 +17,8 @@ export interface TemplateMeta { name: string description: string color: string + group: string + group_color: string saved_at: string node_count: number } @@ -342,6 +344,18 @@ export const api = { removeNode: (id: string) => req('DELETE', `/nodes/${id}`), updateParam:(id: string, key: string, value: unknown) => req('PATCH', `/nodes/${id}/params`, { key, value }), + controlNode:(id: string, action: string) => + req<{ ok: boolean; node_id: string; outputs: Record }>('POST', `/nodes/${id}/control`, { action }), + pickDirectory:(initialPath = '') => + req<{ selected: string; cancelled: boolean }>('POST', '/filesystem/pick-directory', { initial_path: initialPath }), + datasetFrame:(token: string, index: number) => + req>('GET', `/dataset/frame/${encodeURIComponent(token)}?index=${Math.max(0, Math.floor(index))}`), + trimDatasetEpisode:(token: string, frameIndex: number, side: 'before' | 'after') => + req>('POST', '/dataset/trim', { token, frame_index: Math.max(0, Math.floor(frameIndex)), side }, 120000), + publishDatasetReplayFrame:(token: string, frameIndex: number, event: 'play' | 'seek') => + req>('POST', '/dataset/replay-event', { + token, frame_index: Math.max(0, Math.floor(frameIndex)), event, + }), updatePorts:(id: string, patch: Partial>) => req('PATCH', `/nodes/${id}/ports`, patch), updatePortVisibility:(id: string, patch: Pick) => diff --git a/editor/src/components/BlackNode.tsx b/editor/src/components/BlackNode.tsx index ca234e4..1d8cfaa 100644 --- a/editor/src/components/BlackNode.tsx +++ b/editor/src/components/BlackNode.tsx @@ -9,6 +9,7 @@ import { isWireOnlyInput } from '../inputControls' import { copyTextToClipboard } from '../clipboard' import { portDisplayHint, portDisplayName } from '../portLabels' import NodeFrame from './NodeFrame' +import DatasetBrowserPanel from './DatasetBrowserPanel' import type { NodeCookState } from '../types' const TOOLBOX_NEW_HANDLE_COLOR = '#ef444488' @@ -26,6 +27,7 @@ const LIVE_STREAM_NODE_TYPES = new Set([ 'CV2ColorObjectStream', 'VisionReasoningStream', 'CUDAImageFilterStream', + 'StreamPublisher', ]) function driverBtn(color: string, disabled = false): React.CSSProperties { @@ -424,6 +426,8 @@ function NodeImageInput({ function BlackNode({ id, data, selected }: NodeProps) { const cookNode = useStore(s => s.cookNode) const updateParam = useStore(s => s.updateParam) + const controlNode = useStore(s => s.controlNode) + const pickDirectory = useStore(s => s.pickDirectory) const resizeNode = useStore(s => s.resizeNode) const disconnectEdge = useStore(s => s.disconnectEdge) const edges = useStore(s => s.edges) @@ -438,10 +442,12 @@ function BlackNode({ id, data, selected }: NodeProps) { const driverNotInstalled = driverName ? drivers[driverName]?.packages_installed === false : false const [driverPending, setDriverPending] = useState(null) const [streamStopPending, setStreamStopPending] = useState(false) + const [streamStartPending, setStreamStartPending] = useState(false) const [rosRunStopPending, setRosRunStopPending] = useState(false) const [manualMovePending, setManualMovePending] = useState(null) const [calibrationPending, setCalibrationPending] = useState(null) const [episodePending, setEpisodePending] = useState(null) + const [datasetFolderPending, setDatasetFolderPending] = useState(false) const dashboardAutoFitDone = useRef(false) const updateNodeInternals = useUpdateNodeInternals() const color = headerColor(data.type) @@ -452,6 +458,8 @@ function BlackNode({ id, data, selected }: NodeProps) { const isManualMove = data.type === 'ROS2ManualMove' || data.type === 'ROS2TeachMode' const isRobotCalibration = data.type === 'RobotCalibrationRecorder' const isEpisodeRecorder = data.type === 'EpisodeRecorder' + const isDatasetCreate = data.type === 'DatasetCreate' + const isDatasetBrowser = data.type === 'DatasetBrowser' const availableInputs = isRobotJointList ? (data.inputs ?? []).filter(port => edges.some(edge => edge.target === id && edge.targetHandle === port)) : isVariadic @@ -552,10 +560,29 @@ function BlackNode({ id, data, selected }: NodeProps) { const episodeLastError = isEpisodeRecorder && data.portResults?.status && typeof data.portResults.status === 'object' ? String((data.portResults.status as Record).last_error ?? '') : '' + const episodeRecoverable = isEpisodeRecorder && data.portResults?.status && typeof data.portResults.status === 'object' + ? (data.portResults.status as Record).recoverable === true + : false + const episodeStoragePath = isEpisodeRecorder && data.portResults?.status && typeof data.portResults.status === 'object' + ? String( + (data.portResults.status as Record).saved_path + ?? (data.portResults.status as Record).work_path + ?? (data.portResults.status as Record).dataset_path + ?? (data.portResults.dataset && typeof data.portResults.dataset === 'object' + ? (data.portResults.dataset as Record).path + : '') + ?? '' + ) + : '' const episodeInputsReady = isEpisodeRecorder && edges.some(edge => edge.target === id && edge.targetHandle === 'dataset') && edges.some(edge => edge.target === id && edge.targetHandle === 'robot_stream') && edges.some(edge => edge.target === id && (edge.targetHandle === 'camera_stream' || edge.targetHandle === 'camera_streams')) + const datasetRoot = isDatasetCreate ? String(data.params?.root ?? '').trim() : '' + const datasetId = isDatasetCreate ? String(data.params?.dataset_id ?? 'dataset').trim() || 'dataset' : '' + const datasetResolvedPath = isDatasetCreate && typeof data.portResults?.path === 'string' + ? data.portResults.path + : '' const hasLiveOutput = data.live_capable === true && (data.outputs ?? []).includes('live') const liveStateReport = hasLiveOutput ? String(data.portResults?.report ?? '').trim() : '' const liveServiceRunning = hasLiveOutput && data.portResults?.running === true @@ -567,8 +594,12 @@ function BlackNode({ id, data, selected }: NodeProps) { .replace(/^(blocked|failed|error)\s*:\s*/i, '') .trim() const genericNodeLive = data.live_capable === true && data.portResults?.live === true && !manualMoveLive && !streamActive + // StreamPublisher gets its own Go live / Stop controls, so it should never fall + // back to the generic "snapshot" badge — that badge is what read as "broken". + const streamStartable = data.type === 'StreamPublisher' && !streamActive const snapshotResult = data.live_capable === true && !streamActive + && !streamStartable && !manualMoveLive && !genericNodeLive && !liveBlocked @@ -652,6 +683,16 @@ function BlackNode({ id, data, selected }: NodeProps) { } } + const onStartStream = async () => { + setStreamStartPending(true) + try { + await updateParam(id, 'action', 'start') + await cookNode(id, 'dashboard') + } finally { + setStreamStartPending(false) + } + } + const onStopImageStream = async () => { setStreamStopPending(true) try { @@ -726,14 +767,32 @@ function BlackNode({ id, data, selected }: NodeProps) { )) return setEpisodePending(action) try { - await updateParam(id, 'action', action) - await cookNode(id, 'dashboard', undefined, 'live') - } finally { try { - await updateParam(id, 'action', 'status') - } finally { - setEpisodePending(null) + await controlNode(id, action) + } catch (error) { + // A fresh graph has no resolved recorder handles yet. Record may cook + // once to configure them; every subsequent control is runtime-only. + if (action !== 'start') throw error + await updateParam(id, 'action', action) + try { + await cookNode(id, 'dashboard', undefined, 'live') + } finally { + await updateParam(id, 'action', 'status') + } } + } finally { + setEpisodePending(null) + } + } + + const chooseDatasetFolder = async () => { + if (datasetFolderPending) return + setDatasetFolderPending(true) + try { + const selected = await pickDirectory(String(data.params?.root ?? '')) + if (selected) await updateParam(id, 'root', selected) + } finally { + setDatasetFolderPending(false) } } @@ -934,6 +993,28 @@ function BlackNode({ id, data, selected }: NodeProps) { )} + {streamStartable && ( +
e.stopPropagation()} + style={{ display: 'flex', alignItems: 'center', gap: 6, padding: '6px 10px 2px' }}> + + + starts the WebSocket stream (action=start) + +
+ )} + {snapshotResult && (
) {
)} + {isDatasetBrowser && } + + {isDatasetCreate && ( +
+
+ DATASET STORAGE +
+
+ Root: {datasetRoot || '~/.blacknode/datasets (default)'} +
+
+ Blacknode stores this dataset in a “{datasetId}” subfolder. +
+ {datasetResolvedPath && ( +
+ Current: {datasetResolvedPath} +
+ )} +
+ + +
+
+ )} + {isEpisodeRecorder && (
) { ? `● RECORDING · ${episodeFrameCount} FRAMES · ${episodeDuration.toFixed(1)}s` : episodePaused ? `Ⅱ PAUSED · ${episodeFrameCount} FRAMES · ${episodeDuration.toFixed(1)}s` - : '○ READY FOR A NEW EPISODE'} + : episodeRecoverable + ? `↻ RECOVERABLE · ${episodeFrameCount} FRAMES · ${episodeDuration.toFixed(1)}s` + : '○ READY FOR A NEW EPISODE'} {episodeDroppedFrames > 0 ? ` · ${episodeDroppedFrames} DROPPED` : ''}
{episodeLastError && ( @@ -1289,12 +1412,17 @@ function BlackNode({ id, data, selected }: NodeProps) { {episodeLastError} )} + {episodeStoragePath && ( +
+ Saving to: {episodeStoragePath} +
+ )}
@@ -1315,10 +1443,10 @@ function BlackNode({ id, data, selected }: NodeProps) { {episodePending === 'resume' ? 'Resuming…' : '▶ Resume'} @@ -1331,10 +1459,10 @@ function BlackNode({ id, data, selected }: NodeProps) { {episodePending === 'stop' ? 'Stopping…' : '■ Stop'} diff --git a/editor/src/components/DatasetBrowserPanel.tsx b/editor/src/components/DatasetBrowserPanel.tsx new file mode 100644 index 0000000..10298d8 --- /dev/null +++ b/editor/src/components/DatasetBrowserPanel.tsx @@ -0,0 +1,348 @@ +import { useEffect, useRef, useState } from 'react' +import { api } from '../api' +import { useStore } from '../store' + +type AnyRecord = Record +type FrameCallbackVideo = HTMLVideoElement & { + requestVideoFrameCallback?: (callback: (now: number, metadata: { mediaTime: number }) => void) => number + cancelVideoFrameCallback?: (id: number) => void +} + +const panel: React.CSSProperties = { + margin: '7px 9px 3px', padding: 10, borderRadius: 8, + border: '1px solid var(--line)', background: 'rgba(255,255,255,.02)', + fontFamily: 'var(--font-ui)', +} + +const button = (disabled = false): React.CSSProperties => ({ + border: '1px solid var(--line)', borderRadius: 5, padding: '4px 9px', + background: disabled ? 'rgba(255,255,255,.03)' : 'rgba(139,92,246,.18)', + color: disabled ? 'var(--tx3)' : 'var(--tx1)', cursor: disabled ? 'default' : 'pointer', + fontSize: 10, fontWeight: 700, +}) + +const selectStyle: React.CSSProperties = { + minWidth: 150, maxWidth: 260, padding: '4px 6px', borderRadius: 5, + border: '1px solid var(--line)', background: 'var(--bg2)', color: 'var(--tx1)', fontSize: 10, + colorScheme: 'dark', +} + +const optionStyle: React.CSSProperties = { background: '#0f172a', color: '#e2e8f0' } + +export default function DatasetBrowserPanel({ id, data }: { + id: string + data: { params?: Record; portResults?: Record } +}) { + const updateParam = useStore(s => s.updateParam) + const cookNode = useStore(s => s.cookNode) + const pickDirectory = useStore(s => s.pickDirectory) + const [pending, setPending] = useState(false) + const [frame, setFrame] = useState(null) + const [playing, setPlaying] = useState(false) + const [playbackRate, setPlaybackRate] = useState(1) + const [loop, setLoop] = useState(false) + const [angleUnit, setAngleUnit] = useState<'radians' | 'degrees'>('radians') + const [trimPending, setTrimPending] = useState<'before' | 'after' | null>(null) + const [trimMessage, setTrimMessage] = useState('') + const lastFrame = useRef(-1) + const lastPublishedFrame = useRef(-1) + const videoRef = useRef(null) + + const catalog = data.portResults?.catalog && typeof data.portResults.catalog === 'object' + ? data.portResults.catalog as AnyRecord : {} + const datasets = Array.isArray(catalog.datasets) ? catalog.datasets as AnyRecord[] : [] + const selectedDataset = catalog.selected_dataset && typeof catalog.selected_dataset === 'object' + ? catalog.selected_dataset as AnyRecord : {} + const episodes = Array.isArray(selectedDataset.episodes) ? selectedDataset.episodes as AnyRecord[] : [] + const episode = catalog.selected_episode && typeof catalog.selected_episode === 'object' + ? catalog.selected_episode as AnyRecord : {} + const cameras = Array.isArray(episode.cameras) ? episode.cameras.map(String) : [] + const rawVideo = typeof catalog.video === 'string' ? catalog.video : '' + const video = rawVideo.startsWith('/dataset/') ? `/api${rawVideo}` : rawVideo + const token = typeof catalog.replay_token === 'string' ? catalog.replay_token : '' + const fps = Number(episode.fps ?? 0) + const totalFrames = Number(episode.frames ?? 0) + + useEffect(() => { + setFrame(null) + setPlaying(false) + lastFrame.current = -1 + lastPublishedFrame.current = -1 + }, [token]) + + const refresh = async (patch: Record = {}) => { + if (pending) return + setPending(true) + try { + const effectivePatch = Object.keys(patch).length > 0 ? patch : { refresh_key: Date.now() } + for (const [key, value] of Object.entries(effectivePatch)) await updateParam(id, key, value) + await cookNode(id, 'catalog') + } finally { + setPending(false) + } + } + + const chooseRoot = async () => { + if (pending) return + const selected = await pickDirectory(String(data.params?.root ?? '')) + if (selected) await refresh({ root: selected, dataset_id: '', episode_index: 0, camera: '' }) + } + + const updateReplayFrame = async (time: number) => { + if (!token || !fps) return + const index = Math.min(Math.max(0, totalFrames - 1), Math.max(0, Math.floor(time * fps))) + if (index === lastFrame.current) return + lastFrame.current = index + try { + setFrame(await api.datasetFrame(token, index)) + } catch { + // Playback remains usable even if one metadata request races a selection change. + } + } + + const publishReplayPosition = (time: number, event: 'play' | 'seek', force = false) => { + if (!token || !fps || totalFrames <= 0) return + const index = Math.min(totalFrames - 1, Math.max(0, Math.floor(time * fps))) + if (!force && index === lastPublishedFrame.current) return + lastPublishedFrame.current = index + void api.publishDatasetReplayFrame(token, index, event).catch(() => { + // A publisher is optional; local dataset replay remains usable by itself. + }) + } + + useEffect(() => { + const player = videoRef.current as FrameCallbackVideo | null + if (!playing || !player || !token || !fps || totalFrames <= 0) return + let cancelled = false + let callbackId: number | null = null + let timerId: number | null = null + const publishFrame = (_now?: number, metadata?: { mediaTime: number }) => { + if (cancelled || player.paused) return + publishReplayPosition(metadata?.mediaTime ?? player.currentTime, 'play') + if (player.requestVideoFrameCallback) callbackId = player.requestVideoFrameCallback(publishFrame) + } + if (player.requestVideoFrameCallback) { + callbackId = player.requestVideoFrameCallback(publishFrame) + } else { + timerId = window.setInterval(() => publishReplayPosition(player.currentTime, 'play'), + Math.max(16, 1000 / fps)) + } + return () => { + cancelled = true + if (callbackId !== null && player.cancelVideoFrameCallback) player.cancelVideoFrameCallback(callbackId) + if (timerId !== null) window.clearInterval(timerId) + } + }, [playing, token, fps, totalFrames]) + + const jointNames = frame && Array.isArray(frame.joint_names) ? frame.joint_names.map(String) : [] + const leader = (frame?.leader ?? {}) as AnyRecord + const observation = (frame?.observation ?? {}) as AnyRecord + const action = (frame?.action ?? {}) as AnyRecord + const storedUnits = String(episode.units ?? 'radians').toLowerCase() + const displayAngle = (value: unknown) => { + const numeric = Number(value ?? 0) + if (angleUnit === 'degrees' && storedUnits.startsWith('rad')) return numeric * 180 / Math.PI + if (angleUnit === 'radians' && storedUnits.startsWith('deg')) return numeric * Math.PI / 180 + return numeric + } + + const toggleReplay = async () => { + const player = videoRef.current + if (!player) return + if (player.paused) await player.play() + else player.pause() + } + + const restartReplay = async () => { + const player = videoRef.current + if (!player) return + player.currentTime = 0 + await updateReplayFrame(0) + await player.play() + } + + const stepFrame = async (direction: -1 | 1) => { + const player = videoRef.current + if (!player || !fps) return + player.pause() + player.currentTime = Math.max(0, Math.min(player.duration || Number.POSITIVE_INFINITY, player.currentTime + direction / fps)) + await updateReplayFrame(player.currentTime) + } + + const trimEpisode = async (side: 'before' | 'after') => { + const player = videoRef.current + if (!token || !player || trimPending) return + player.pause() + const index = Math.min( + Math.max(0, totalFrames - 1), + Math.max(0, Math.floor(player.currentTime * fps)), + ) + const removeCount = side === 'before' ? index : Math.max(0, totalFrames - index - 1) + if (removeCount <= 0) return + const label = side === 'before' ? `before frame ${index}` : `after frame ${index}` + if (!window.confirm( + `Permanently remove ${removeCount} frame(s) ${label} from episode ${episode.episode_index}?\n\n` + + 'The selected frame is kept. Every camera video and the synchronized robot data will be trimmed together.' + )) return + setTrimPending(side) + setTrimMessage('') + try { + const result = await api.trimDatasetEpisode(token, index, side) + setTrimMessage(`Trimmed ${Number(result.removed_frames ?? removeCount)} frame(s); ${Number(result.frames ?? 0)} remain.`) + await refresh({ refresh_key: Date.now() }) + } catch (error) { + setTrimMessage(error instanceof Error ? error.message : String(error)) + } finally { + setTrimPending(null) + } + } + + return ( +
+
+ DATASET BROWSER + + + + {String(catalog.root ?? data.params?.root ?? '~/.blacknode/datasets')} + +
+ +
+ + + +
+ + {video ? ( + <> +
+ + + + + + + + + + + + + + + Read-only replay · robot commands disabled +
+ {trimMessage &&
+ {trimMessage} +
} +
+
+ + ) : ( +
+ {datasets.length ? 'Select a dataset containing a saved episode.' : 'Choose a dataset root, then press Refresh.'} +
+ )} + + {episode.episode_path &&
+
Episode: {episode.episode_path}
+
Video: {episode.video_path}
+
Robot data: {episode.data_path}
+
Task: {episode.task || '—'} · saved {episode.saved_at || '—'}
+
} + {episode.episode_path &&
+ All episode and current-frame metadata + 
+          {JSON.stringify({ episode, frame }, null, 2)}
+
+
} +
+ ) +} diff --git a/editor/src/components/NodePalette.tsx b/editor/src/components/NodePalette.tsx index e27b00c..3bd64d2 100644 --- a/editor/src/components/NodePalette.tsx +++ b/editor/src/components/NodePalette.tsx @@ -100,7 +100,7 @@ const TABS: { id: Tab; label: string; icon: React.ReactNode }[] = [ export default function NodePalette() { const { nodeTypes, nodeDefs, addNode, loadNodeTypes, learnedNodeHighlight } = useStore() - const [activeTab, setActiveTab] = useState('nodes') + const [activeTab, setActiveTab] = useState('templates') const [showPackageWelcome, setShowPackageWelcome] = useState(false) const [panelWidth, setPanelWidth] = useState(PANEL_DEFAULT_W) const [openGroups, setOpenGroups] = useState>(() => new Set()) @@ -292,10 +292,10 @@ export default function NodePalette() {
+ {isExpanded && group.templates.map(template => { const isLoading = loading === template.slug const wasLoaded = loaded === template.slug const dependencyError = missing[template.slug] @@ -137,25 +233,25 @@ export default function TemplateGallery() { key={template.slug} style={{ background: 'var(--lift)', - border: `1px solid ${dependencyError ? 'var(--warn)' : wasLoaded ? template.color : 'var(--line2)'}`, + border: `1px solid ${dependencyError ? 'var(--warn)' : group.color}`, borderRadius: 8, padding: '10px 12px', cursor: isBusy ? 'default' : 'pointer', transition: 'border-color 0.2s', }} onMouseEnter={e => { - if (!isBusy && !dependencyError) (e.currentTarget as HTMLElement).style.borderColor = template.color + if (!isBusy && !dependencyError) (e.currentTarget as HTMLElement).style.borderColor = group.color }} onMouseLeave={e => { if (!wasLoaded) { - (e.currentTarget as HTMLElement).style.borderColor = dependencyError ? 'var(--warn)' : 'var(--line2)' + (e.currentTarget as HTMLElement).style.borderColor = dependencyError ? 'var(--warn)' : group.color } }} onClick={() => !isBusy && loadTemplate(template)} >
{isLoading @@ -250,6 +346,9 @@ export default function TemplateGallery() {
)}
+ ) + })} + ) })}
diff --git a/editor/src/portColors.ts b/editor/src/portColors.ts index a828235..90bfebd 100644 --- a/editor/src/portColors.ts +++ b/editor/src/portColors.ts @@ -10,6 +10,7 @@ export const PORT_COLORS: Record = { Fn: '#ef4444', // red Model: '#76b900', // nvidia green Image: '#fb7185', // rose + Video: '#f43f5e', // deep rose Color: '#e11d48', // color picker value HSV: '#0ea5e9', // hue/saturation/value triplet Any: '#6b7280', // grey @@ -28,6 +29,7 @@ const COMPAT: Record> = { Fn: new Set(['Fn', 'Any']), Model: new Set(['Model', 'Text', 'Any']), Image: new Set(['Image', 'Any']), + Video: new Set(['Video', 'Any']), Color: new Set(['Color', 'Text', 'Any']), HSV: new Set(['HSV', 'Text', 'Any']), } diff --git a/editor/src/store.ts b/editor/src/store.ts index 02578dd..a34c25b 100644 --- a/editor/src/store.ts +++ b/editor/src/store.ts @@ -13,6 +13,8 @@ import type { GraphRunTarget } from './graphRun' const MODEL_NODE_TYPES = new Set(['Model']) const OUTPUT_NODE_TYPES = new Set(['Output']) +const MEDIA_OUTPUT_NODE_SIZE = { width: 860, height: 720 } as const +const MEDIA_OUTPUT_TYPES = new Set(['Image', 'Video']) const SUBGRAPH_NODE_TYPES = new Set(['Subnet', 'SubnetAsTool', 'VisualAgentLoop']) // In-flight cook stream, so a Stop button can abort a running/stuck cook. @@ -180,6 +182,8 @@ interface Store { copyIdMap: Record | null, ) => Promise updateParam: (id: string, key: string, value: unknown) => Promise + controlNode: (id: string, action: string) => Promise + pickDirectory: (initialPath?: string) => Promise updatePortVisibility: (id: string, promotedInputs?: string[], promotedOutputs?: string[]) => Promise cookNode: (id: string, port?: string, graphTargets?: GraphRunTarget[], runMode?: 'once' | 'live') => Promise stopCook: () => void @@ -213,6 +217,7 @@ function makeReactNode(meta: BnNodeMeta): Node { ...(meta.type === 'Dict' ? { style: { width: 260, height: 150 } } : {}), ...(meta.type === 'Output' ? { style: { width: 320, height: 200 } } : {}), ...(meta.type === 'OutputImage' ? { style: { width: 760, height: 620 } } : {}), + ...(meta.type === 'DatasetBrowser' ? { style: { width: 980, height: 860 } } : {}), ...(hasDashboardImage ? { style: { width: 860, height: 720 } } : {}), ...(meta.type === 'ROS2VisualDashboard' ? { style: { width: 840, height: 760 } } : {}), ...(meta.type === 'ROS2CompressedImageSnapshot' ? { style: { width: 700, height: 600 } } : {}), @@ -223,6 +228,38 @@ function makeReactNode(meta: BnNodeMeta): Node { } } +function ensureMediaOutputNodeSizes( + nodes: Node[], + edges: Edge[], +): Node[] { + const byId = new Map(nodes.map(node => [node.id, node])) + const mediaOutputIds = new Set() + edges.forEach(edge => { + const target = byId.get(edge.target) + const source = byId.get(edge.source) + const sourceType = edge.sourceHandle + ? source?.data.output_types?.[edge.sourceHandle] + : undefined + if (target?.data.type === 'Output' && sourceType && MEDIA_OUTPUT_TYPES.has(sourceType)) { + mediaOutputIds.add(target.id) + } + }) + + return nodes.map(node => { + if (!mediaOutputIds.has(node.id)) return node + const styleWidth = typeof node.style?.width === 'number' ? node.style.width : 0 + const styleHeight = typeof node.style?.height === 'number' ? node.style.height : 0 + const width = Math.max(node.width ?? 0, styleWidth, MEDIA_OUTPUT_NODE_SIZE.width) + const height = Math.max(node.height ?? 0, styleHeight, MEDIA_OUTPUT_NODE_SIZE.height) + return { + ...node, + width, + height, + style: { ...(node.style ?? {}), width, height }, + } + }) +} + function parseGraph(bnNodes: BnNodeMeta[], bnEdges: any[]): { nodes: Node[]; edges: Edge[] } { const nodes: Node[] = bnNodes.map(n => makeReactNode(n)) const edges: Edge[] = bnEdges.map((e: any, i: number) => { @@ -236,7 +273,7 @@ function parseGraph(bnNodes: BnNodeMeta[], bnEdges: any[]): { nodes: Node[], edges: Edge[]): Node[] { @@ -1968,15 +2005,19 @@ export const useStore = create((set, get) => ({ await api.connect(nextConn.source, nextConn.sourceHandle, nextConn.target, nextConn.targetHandle) } - set(s => ({ - nodes: [...s.nodes, node], - edges: nextConn?.source && nextConn.target ? addEdge({ + set(s => { + const nextEdges = nextConn?.source && nextConn.target ? addEdge({ ...nextConn, id: nextEdgeId(), style: { stroke: portColor(edgeType), strokeWidth: 1.5 }, - }, s.edges) : s.edges, - ...markActiveTabDirty(s), - })) + }, s.edges) : s.edges + const nextNodes = ensureMediaOutputNodeSizes([...s.nodes, node], nextEdges) + return { + nodes: nextNodes, + edges: nextEdges, + ...markActiveTabDirty(s), + } + }) }, removeNode: async (id) => { @@ -2279,7 +2320,10 @@ export const useStore = create((set, get) => ({ await api.connect(conn.source!, conn.sourceHandle!, conn.target!, conn.targetHandle!) } - const nextNodes = ensureConnectedToolBoxSlots(nodes, updatedEdges) + const nextNodes = ensureMediaOutputNodeSizes( + ensureConnectedToolBoxSlots(nodes, updatedEdges), + updatedEdges, + ) set(s => ({ nodes: nextNodes, edges: updatedEdges, ...markActiveTabDirty(s) })) }, @@ -2377,7 +2421,8 @@ export const useStore = create((set, get) => ({ style: { stroke: portColor(fromType), strokeWidth: 1.5 }, }]) const removedForPrune = [oldEdge, conflictingEdge].filter(Boolean) as Edge[] - const { nodes: nextNodes, changedIds: prunedDynamicNodes } = pruneDisconnectedDynamicPorts(nodes, nextEdges, removedForPrune) + const { nodes: prunedNodes, changedIds: prunedDynamicNodes } = pruneDisconnectedDynamicPorts(nodes, nextEdges, removedForPrune) + const nextNodes = ensureMediaOutputNodeSizes(prunedNodes, nextEdges) if (subnetStack.length > 0) { const frame = subnetStack[subnetStack.length - 1] @@ -2732,6 +2777,20 @@ export const useStore = create((set, get) => ({ ...markActiveTabDirty(s), })) await api.updateParam(id, key, value) + if (node.data.type === 'TrajectorySmoother' + && ['method', 'strength', 'preview_source', 'preview_joint'].includes(key)) { + try { + await get().controlNode(id, 'apply') + } catch (error) { + window.dispatchEvent(new CustomEvent('blacknode:notice', { + detail: { + kind: 'error', + title: 'Smoother update needs an input', + message: error instanceof Error ? error.message : String(error), + }, + })) + } + } if (profileChanged) { window.dispatchEvent(new CustomEvent('blacknode:notice', { detail: { @@ -2743,6 +2802,24 @@ export const useStore = create((set, get) => ({ } }, + controlNode: async (id, action) => { + const result = await api.controlNode(id, action) + set(s => ({ + nodes: propagateLiveTerminalValues(s.nodes.map(node => node.id === id ? { + ...node, + data: { + ...node.data, + portResults: { ...(node.data.portResults ?? {}), ...result.outputs }, + }, + } : node), s.edges), + })) + }, + + pickDirectory: async (initialPath = '') => { + const result = await api.pickDirectory(initialPath) + return result.cancelled || !result.selected ? null : result.selected + }, + updatePortVisibility: async (id, promotedInputs, promotedOutputs) => { const node = get().nodes.find(n => n.id === id) if (!node) return diff --git a/python/blacknode/node.py b/python/blacknode/node.py index 10ee0b8..2ae76be 100644 --- a/python/blacknode/node.py +++ b/python/blacknode/node.py @@ -45,6 +45,7 @@ def __str__(self) -> str: Model = PortType("Model") Number = PortType("Number") Image = PortType("Image") +Video = PortType("Video") Any = PortType("Any") @@ -173,7 +174,7 @@ def _parse_default(raw: str) -> object: _COMPACT_PORT_THRESHOLD = 8 -_DATA_INPUT_TYPES = {"Any", "Dict", "Embedding", "Fn", "Image", "List"} +_DATA_INPUT_TYPES = {"Any", "Dict", "Embedding", "Fn", "Image", "Video", "List"} _STATUS_OUTPUTS = { "active", "armed", "command_ok", "connected", "count", "data_ready", "device", "driver_running", "dropped_frames", "error", "found", "joint_count", diff --git a/start.ps1 b/start.ps1 index a4865ce..d1ab117 100644 --- a/start.ps1 +++ b/start.ps1 @@ -355,12 +355,14 @@ function Invoke-PackageHealthCheck { if ($env:BLACKNODE_PACKAGE_AUTO_SETUP -ne "0") { Write-Step "Installing missing extension package dependencies..." - $SetupResult = Invoke-PythonCapture -Arguments @("-m", "blacknode.cli", "packages", "setup", "--missing") - $SetupOutput = $SetupResult.Output - $SetupExitCode = $SetupResult.ExitCode - if ($SetupOutput) { $SetupOutput | ForEach-Object { Write-Host " $_" } } + Write-Host " Dependency download and installation output will appear below." + $SetupArguments = @("-m", "blacknode.cli", "packages", "setup", "--missing") + & $Python @SetupArguments + $SetupExitCode = $LASTEXITCODE if ($SetupExitCode -ne 0) { Write-Host " Warning: automatic package dependency setup failed; startup will continue." -ForegroundColor Yellow + } else { + Write-Step "Extension package dependency setup complete." } } diff --git a/start.sh b/start.sh index cba08f5..620a198 100755 --- a/start.sh +++ b/start.sh @@ -311,11 +311,11 @@ check_package_dependencies() { if [[ "${BLACKNODE_PACKAGE_AUTO_SETUP:-1}" == "1" ]]; then echo " Installing missing extension package dependencies..." - if ! output="$(PYTHONPATH="$ROOT_DIR/python" "$PYTHON_BIN" -m blacknode.cli packages setup --missing 2>&1)"; then - printf '%s\n' "$output" | sed 's/^/ /' + echo " Dependency download and installation output will appear below." + if ! PYTHONPATH="$ROOT_DIR/python" "$PYTHON_BIN" -m blacknode.cli packages setup --missing; then echo " Warning: automatic package dependency setup failed; startup will continue." - elif [[ -n "$output" ]]; then - printf '%s\n' "$output" | sed 's/^/ /' + else + echo " Extension package dependency setup complete." fi fi diff --git a/tests/test_editor_media_output.py b/tests/test_editor_media_output.py index 786c42f..2ae515d 100644 --- a/tests/test_editor_media_output.py +++ b/tests/test_editor_media_output.py @@ -6,6 +6,7 @@ def test_generic_output_renders_typed_images_and_video_instead_of_urls() -> None: source = (ROOT / "editor" / "src" / "components" / "OutputNode.tsx").read_text(encoding="utf-8") + store = (ROOT / "editor" / "src" / "store.ts").read_text(encoding="utf-8") assert "sourceType === 'Image'" in source assert "sourceType === 'Video'" in source @@ -13,6 +14,12 @@ def test_generic_output_renders_typed_images_and_video_instead_of_urls() -> None assert " None: @@ -64,3 +71,31 @@ def test_live_nodes_distinguish_blocked_waiting_and_snapshot_states() -> None: assert "&& !liveWaiting" in black_node assert "blockedControllerCount" in app assert "waitingControllerCount" in app + + +def test_dataset_replay_switches_units_and_keeps_canvas_wheel_zoom() -> None: + browser = (ROOT / "editor" / "src" / "components" / "DatasetBrowserPanel.tsx").read_text(encoding="utf-8") + output = (ROOT / "editor" / "src" / "components" / "OutputNode.tsx").read_text(encoding="utf-8") + api = (ROOT / "editor" / "src" / "api.ts").read_text(encoding="utf-8") + + assert "setAngleUnit('radians')" in browser + assert "setAngleUnit('degrees')" in browser + assert "numeric * 180 / Math.PI" in browser + assert "numeric * Math.PI / 180" in browser + assert 'className="nodrag"' in browser + assert 'className="nodrag nowheel"' not in browser + assert 'className="nodrag nowheel bn-output-scroll"' not in output + assert "✂ Cut before" in browser + assert "✂ Cut after" in browser + assert "The selected frame is kept" in browser + assert "trimDatasetEpisode" in api + assert "publishDatasetReplayFrame" in api + assert "requestVideoFrameCallback" in browser + assert "onSeeking=" in browser + + +def test_smoother_parameter_updates_use_direct_control_instead_of_graph_cook() -> None: + store = (ROOT / "editor" / "src" / "store.ts").read_text(encoding="utf-8") + + assert "node.data.type === 'TrajectorySmoother'" in store + assert "await get().controlNode(id, 'apply')" in store diff --git a/tests/test_editor_package_welcome.py b/tests/test_editor_package_welcome.py index 534f584..1b55cdf 100644 --- a/tests/test_editor_package_welcome.py +++ b/tests/test_editor_package_welcome.py @@ -14,4 +14,6 @@ def test_first_editor_visit_opens_packages_with_one_time_welcome(): assert "localStorage" not in source assert "Prepare your robotics workspace" in source assert "Explore essential packages" in source - assert "Continue with core graph" in source + assert "Explore core templates" in source + assert "useState('templates')" in source + assert "finishPackageWelcome('templates')" in source diff --git a/tests/test_editor_runtime.py b/tests/test_editor_runtime.py index da9ebbf..a1025e8 100644 --- a/tests/test_editor_runtime.py +++ b/tests/test_editor_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations import sys +import tempfile import unittest from pathlib import Path from types import SimpleNamespace @@ -174,6 +175,131 @@ def test_runtime_stop_endpoint_stops_cook_and_runtime_helpers(self): stop_cook.assert_called_once() fresh_cook.assert_called_once() + def test_episode_recorder_control_does_not_cook_graph(self): + server._session.node_meta["recorder-control-test"] = { + "id": "recorder-control-test", "type": "EpisodeRecorder", + "params": {"run_id": "episode-test"}, + } + control = lambda run_id, action: {"running": False, "frame_count": 12, "report": f"{run_id}:{action}"} + try: + with ( + patch.object(server, "_runtime_callable", return_value=control), + patch.object(server, "_prepare_cook") as prepare_cook, + ): + response = TestClient(server.app).post( + "/nodes/recorder-control-test/control", json={"action": "save"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["outputs"]["frame_count"], 12) + prepare_cook.assert_not_called() + finally: + server._session.node_meta.pop("recorder-control-test", None) + + def test_trajectory_smoother_control_recomputes_only_smoother(self): + node_id = "smoother-control-test" + server._session.node_meta[node_id] = { + "id": node_id, "type": "TrajectorySmoother", + "params": {"method": "gaussian", "strength": 2.5, + "preview_source": "leader", "preview_joint": "elbow"}, + } + server._session.graph._dirty.add(node_id) + apply = lambda node_id, method, strength, **preview: { + "stream": {"token": "smoothed"}, "preview": "image", + "report": f"{node_id}:{method}:{strength}:{preview['preview_source']}:{preview['preview_joint']}", + } + try: + with ( + patch.object(server, "_runtime_callable", return_value=apply), + patch.object(server, "_prepare_cook") as prepare_cook, + ): + response = TestClient(server.app).post( + f"/nodes/{node_id}/control", json={"action": "apply"}, + ) + self.assertEqual(response.status_code, 200) + self.assertIn("gaussian:2.5:leader:elbow", response.json()["outputs"]["report"]) + self.assertEqual(server._session.graph._cache[(node_id, "stream")], {"token": "smoothed"}) + self.assertNotIn(node_id, server._session.graph._dirty) + prepare_cook.assert_not_called() + finally: + server._session.node_meta.pop(node_id, None) + server._session.graph._dirty.discard(node_id) + for key in [key for key in server._session.graph._cache if key[0] == node_id]: + server._session.graph._cache.pop(key, None) + + def test_dataset_media_endpoint_serves_only_runtime_registered_video(self): + with tempfile.TemporaryDirectory() as tmp: + video = Path(tmp) / "episode.mp4" + video.write_bytes(b"synthetic-mp4") + with patch.object(server, "_runtime_callable", return_value=lambda token: video if token == "known" else None): + client = TestClient(server.app) + response = client.get("/dataset/media/known") + api_response = client.get("/api/dataset/media/known") + missing = client.get("/dataset/media/unknown") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.content, b"synthetic-mp4") + self.assertEqual(api_response.content, b"synthetic-mp4") + self.assertEqual(missing.status_code, 404) + + def test_directory_picker_endpoint_returns_native_selection(self): + with patch.object(server, "_pick_directory", return_value=r"E:\RobotData") as picker: + response = TestClient(server.app).post( + "/filesystem/pick-directory", json={"initial_path": r"C:\Users\robot"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), {"selected": r"E:\RobotData", "cancelled": False}) + picker.assert_called_once_with(r"C:\Users\robot") + + def test_dataset_frame_endpoint_returns_synchronized_robot_values(self): + frame = { + "frame_index": 12, + "timestamp": 0.4, + "leader": {"joint": 0.1}, + "observation": {"joint": 0.09}, + "action": {"joint": 0.1}, + } + with patch.object(server, "_runtime_callable", return_value=lambda token, index: frame if (token, index) == ("known", 12) else None): + client = TestClient(server.app) + response = client.get("/dataset/frame/known?index=12") + missing = client.get("/dataset/frame/unknown?index=12") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json(), frame) + self.assertEqual(missing.status_code, 404) + + def test_dataset_trim_endpoint_forwards_confirmed_frame_and_side(self): + def trim(token, frame_index, side): + if token != "known": + raise ValueError("replay selection expired") + return {"ok": True, "frames": 8, "removed_frames": 4, + "frame_index": frame_index, "side": side} + + with patch.object(server, "_runtime_callable", return_value=trim): + client = TestClient(server.app) + response = client.post( + "/dataset/trim", json={"token": "known", "frame_index": 4, "side": "before"}, + ) + expired = client.post( + "/dataset/trim", json={"token": "expired", "frame_index": 4, "side": "after"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["removed_frames"], 4) + self.assertEqual(response.json()["frame_index"], 4) + self.assertEqual(response.json()["side"], "before") + self.assertEqual(expired.status_code, 409) + + def test_dataset_replay_event_endpoint_forwards_browser_playback(self): + publish = lambda token, index, event: { + "ok": True, "token": token, "frame_index": index, "event": event, + "publishers": 1, "subscribers": 1, + } + with patch.object(server, "_runtime_callable", return_value=publish): + response = TestClient(server.app).post( + "/dataset/replay-event", + json={"token": "episode", "frame_index": 12, "event": "seek"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["frame_index"], 12) + self.assertEqual(response.json()["event"], "seek") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_editor_template_groups.py b/tests/test_editor_template_groups.py new file mode 100644 index 0000000..e0d51e4 --- /dev/null +++ b/tests/test_editor_template_groups.py @@ -0,0 +1,27 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_template_gallery_groups_are_collapsed_by_default(): + source = ( + ROOT / "editor" / "src" / "components" / "TemplateGallery.tsx" + ).read_text(encoding="utf-8") + + assert "useState>(() => new Set())" in source + assert "template.group || 'Core'" in source + assert "aria-expanded={isExpanded}" in source + assert "isExpanded && group.templates.map" in source + + +def test_template_gallery_searches_and_uses_group_colors(): + source = ( + ROOT / "editor" / "src" / "components" / "TemplateGallery.tsx" + ).read_text(encoding="utf-8") + + assert 'placeholder="Search templates or categories..."' in source + assert "const filteredTemplateGroups = useMemo" in source + assert "Boolean(query.trim()) || expandedGroups.has(group.name)" in source + assert "border: `1px solid ${dependencyError ? 'var(--warn)' : group.color}`" in source + assert "color: group.color" in source diff --git a/tests/test_editor_template_packages.py b/tests/test_editor_template_packages.py index e824c4b..ddb7360 100644 --- a/tests/test_editor_template_packages.py +++ b/tests/test_editor_template_packages.py @@ -3,6 +3,7 @@ import json import sys from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch from fastapi.testclient import TestClient @@ -73,3 +74,35 @@ def test_template_load_returns_installable_missing_package(tmp_path): "installed": False, "load_error": "", }] + + +def test_template_list_groups_core_and_package_templates(tmp_path: Path): + core_dir = tmp_path / "core" + robot_dir = tmp_path / "robot" / "templates" + core_dir.mkdir() + robot_dir.mkdir(parents=True) + (core_dir / "text-pipeline.json").write_text( + json.dumps(_template("TextInput")), encoding="utf-8", + ) + (robot_dir / "motion-test.json").write_text( + json.dumps(_template("Robot")), encoding="utf-8", + ) + robot_package = SimpleNamespace( + name="blacknode-robot", + ok=True, + templates_dir=str(robot_dir), + categories={"Robot": "#14b8a6"}, + ) + + with ( + patch.object(server, "_TEMPLATES_DIR", str(core_dir)), + patch.object(server, "installed_packages", return_value=[robot_package]), + ): + response = TestClient(server.app).get("/templates") + + assert response.status_code == 200 + templates = {template["slug"]: template for template in response.json()} + assert templates["text-pipeline"]["group"] == "Core" + assert templates["text-pipeline"]["group_color"] == "#6366f1" + assert templates["motion-test"]["group"] == "Robot" + assert templates["motion-test"]["group_color"] == "#14b8a6" diff --git a/tests/test_launchers.py b/tests/test_launchers.py index edcac3f..f53d08b 100644 --- a/tests/test_launchers.py +++ b/tests/test_launchers.py @@ -26,6 +26,16 @@ def test_core_launchers_do_not_install_optional_cuda_dependencies(): assert "pip_install cupy" not in shell.lower() +def test_launchers_stream_extension_dependency_setup_output(): + powershell = (ROOT / "start.ps1").read_text(encoding="utf-8") + shell = (ROOT / "start.sh").read_text(encoding="utf-8") + + assert "& $Python @SetupArguments" in powershell + assert "Dependency download and installation output will appear below." in powershell + assert 'output="$(PYTHONPATH="$ROOT_DIR/python" "$PYTHON_BIN" -m blacknode.cli packages setup --missing' not in shell + assert 'PYTHONPATH="$ROOT_DIR/python" "$PYTHON_BIN" -m blacknode.cli packages setup --missing' in shell + + def test_windows_markdown_launch_commands_are_powershell_explicit(): markdown_files = [ROOT / "README.md", *ROOT.joinpath("docs").rglob("*.md")] markdown_files.extend([