Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ Run the Python tests with `pytest` (config in [pyproject.toml](pyproject.toml);
- [teleoperate.py](lelab/teleoperate.py) — leader→follower teleoperation (wraps `lerobot.teleoperate`).
- [calibrate.py](lelab/calibrate.py) — step-by-step web calibration with a `CalibrationManager` singleton and `_step_complete` threading.Event.
- [train.py](lelab/train.py) — wraps the LeRobot training CLI as a subprocess (psutil for lifecycle, queue for log streaming).
- [datasets.py](lelab/datasets.py) — dataset listing (local cache + Hub, merged under a `source` field) plus the episode-browsing handlers behind `/dataset-episodes`, `/dataset-episode`, `/dataset-frame`, `/dataset-thumbnails`, `/dataset-motion` and `/dataset-video`.
- [datasets.py](lelab/datasets.py) — dataset listing (local cache + Hub, merged under a `source` field), merging local datasets into a new one (`/merge-datasets`, wraps `lerobot.datasets.dataset_tools.merge_datasets`) plus the episode-browsing handlers behind `/dataset-episodes`, `/dataset-episode`, `/dataset-frame`, `/dataset-thumbnails`, `/dataset-motion` and `/dataset-video`.
- [episode_media.py](lelab/episode_media.py) — reads the LeRobot v3.0 on-disk layout (`meta/episodes/*.parquet` + `videos/`) and decodes frames with PyAV. Deliberately does **not** go through `LeRobotDataset`, so browsing a dataset never builds a robot config or imports torch. Several episodes share one mp4, so locating an episode resolves both the file and the `from_/to_timestamp` window inside it — which is why `/dataset-video` is addressed by chunk/file rather than by episode.
- [dataset_repair.py](lelab/dataset_repair.py) — rebuilds `meta/episodes/` for a recording interrupted before `LeRobotDataset.finalize()` ran. Without an episode index LeRobot reads a local dataset as "not downloaded yet" and fetches it from the Hub, which 404s for a dataset that was never pushed. Call `repair_local_dataset(repo_id)` before opening a recorded dataset with `LeRobotDataset`.
- [utils/config.py](lelab/utils/config.py) — shared paths and persistence: calibration JSON, saved ports, saved config selections. **Import shared constants from here, do not hardcode paths in feature modules.**
Expand Down
3,958 changes: 0 additions & 3,958 deletions frontend/dist/assets/index-BX9JKcmw.js

This file was deleted.

1 change: 1 addition & 0 deletions frontend/dist/assets/index-CLxhDAmf.css

Large diffs are not rendered by default.

1 change: 0 additions & 1 deletion frontend/dist/assets/index-DJuBJMLM.css

This file was deleted.

3,956 changes: 3,956 additions & 0 deletions frontend/dist/assets/index-DLgsIoRd.js

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions frontend/dist/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@
<meta property="og:title" content="LeLab" />
<meta property="og:description" content="LeRobot but on your browser." />
<meta property="og:type" content="website" />
<script type="module" crossorigin src="/assets/index-BX9JKcmw.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DJuBJMLM.css">
<script type="module" crossorigin src="/assets/index-DLgsIoRd.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-CLxhDAmf.css">
</head>

<body>
Expand Down
174 changes: 174 additions & 0 deletions frontend/src/components/dataset/MergeDatasetsDialog.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
import React, { useEffect, useState } from "react";
import { GitMerge, Loader2 } from "lucide-react";
import { Button } from "@/components/ui/button";
import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog";
import { useApi } from "@/contexts/ApiContext";
import { useToast } from "@/hooks/use-toast";
import { mergeDatasets } from "@/lib/datasetApi";

interface MergeDatasetsDialogProps {
open: boolean;
onOpenChange: (open: boolean) => void;
/** Local datasets available to pick from. */
datasets: { repo_id: string }[];
/** Pre-checked when the dialog opens (the dataset currently being browsed). */
initialRepoId?: string | null;
/** Called with the newly merged dataset's repo_id once it's on disk. */
onMerged: (outputRepoId: string) => void;
}

/**
* Merge several local datasets into one new local dataset.
*
* Wraps `lerobot.datasets.dataset_tools.merge_datasets` via `/merge-datasets`,
* which concatenates the selected datasets' videos and data files. The result
* is written to disk only — merging never touches the Hub.
*/
const MergeDatasetsDialog: React.FC<MergeDatasetsDialogProps> = ({
open,
onOpenChange,
datasets,
initialRepoId,
onMerged,
}) => {
const { baseUrl, fetchWithHeaders } = useApi();
const { toast } = useToast();

const [selected, setSelected] = useState<Set<string>>(new Set());
const [outputName, setOutputName] = useState("");
const [submitting, setSubmitting] = useState(false);

useEffect(() => {
if (open) {
setSelected(new Set(initialRepoId ? [initialRepoId] : []));
setOutputName("");
setSubmitting(false);
}
}, [open, initialRepoId]);

const toggle = (repoId: string) => {
setSelected((prev) => {
const next = new Set(prev);
if (next.has(repoId)) next.delete(repoId);
else next.add(repoId);
return next;
});
};

const canSubmit = selected.size >= 2 && outputName.trim().length > 0 && !submitting;

const handleSubmit = async () => {
if (!canSubmit) return;
setSubmitting(true);
try {
const result = await mergeDatasets(
baseUrl,
fetchWithHeaders,
Array.from(selected),
outputName.trim(),
);
if (!result.success || !result.output_repo_id) {
toast({
title: "Merge failed",
description: result.message ?? "Could not merge the selected datasets.",
variant: "destructive",
});
return;
}
toast({
title: "Datasets merged",
description: `${selected.size} datasets merged into ${result.output_repo_id}.`,
});
onOpenChange(false);
onMerged(result.output_repo_id);
} catch (e) {
toast({
title: "Merge failed",
description: e instanceof Error ? e.message : "Could not merge the selected datasets.",
variant: "destructive",
});
} finally {
setSubmitting(false);
}
};

return (
<Dialog open={open} onOpenChange={onOpenChange}>
<DialogContent className="border-gray-800 bg-gray-950 text-white">
<DialogHeader>
<DialogTitle className="flex items-center gap-2">
<GitMerge className="h-4 w-4 text-orange-500" />
Merge datasets
</DialogTitle>
<DialogDescription className="text-gray-400">
Pick at least two local datasets. Their episodes are concatenated into a new
dataset — the originals are left untouched.
</DialogDescription>
</DialogHeader>

<div className="max-h-64 space-y-1.5 overflow-y-auto rounded-md border border-gray-800 p-2">
{datasets.length === 0 && (
<p className="p-2 text-xs text-gray-500">No local datasets available.</p>
)}
{datasets.map((d) => (
<label
key={d.repo_id}
className="flex cursor-pointer items-center gap-2 rounded px-2 py-1.5 text-sm hover:bg-gray-900"
>
<Checkbox
checked={selected.has(d.repo_id)}
onCheckedChange={() => toggle(d.repo_id)}
/>
<span className="truncate">{d.repo_id}</span>
</label>
))}
</div>

<div className="space-y-1.5">
<Label htmlFor="merge-output-name" className="text-xs text-gray-400">
New dataset name
</Label>
<Input
id="merge-output-name"
value={outputName}
onChange={(e) => setOutputName(e.target.value)}
placeholder="merged-dataset"
className="border-gray-800 bg-gray-900 text-sm text-white"
/>
</div>

<DialogFooter>
<Button
variant="outline"
size="sm"
onClick={() => onOpenChange(false)}
className="border-gray-800 bg-gray-900 text-xs text-gray-300 hover:bg-gray-800"
>
Cancel
</Button>
<Button
size="sm"
onClick={handleSubmit}
disabled={!canSubmit}
className="gap-1.5 bg-orange-600 text-xs hover:bg-orange-700"
>
{submitting && <Loader2 className="h-3.5 w-3.5 animate-spin" />}
Merge {selected.size >= 2 ? `${selected.size} datasets` : ""}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
);
};

export default MergeDatasetsDialog;
20 changes: 20 additions & 0 deletions frontend/src/lib/datasetApi.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,26 @@ export interface ThumbnailsResponse {
thumbnails: Thumbnail[];
}

export interface MergeDatasetsResponse {
success: boolean;
message?: string;
output_repo_id?: string;
num_episodes?: number;
}

export async function mergeDatasets(
baseUrl: string,
fetcher: Fetcher,
datasetRepoIds: string[],
outputRepoId: string,
): Promise<MergeDatasetsResponse> {
return apiRequest<MergeDatasetsResponse>(baseUrl, fetcher, "/merge-datasets", {
method: "POST",
body: { dataset_repo_ids: datasetRepoIds, output_repo_id: outputRepoId },
action: "Merge datasets",
});
}

export async function listEpisodes(
baseUrl: string,
fetcher: Fetcher,
Expand Down
31 changes: 28 additions & 3 deletions frontend/src/pages/EditDataset.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import React, { useEffect, useMemo } from "react";
import React, { useEffect, useMemo, useState } from "react";
import { useNavigate, useSearchParams } from "react-router-dom";
import { AlertCircle, ArrowLeft, Database, Loader2, Upload } from "lucide-react";
import { AlertCircle, ArrowLeft, Database, GitMerge, Loader2, Upload } from "lucide-react";
import {
Select,
SelectContent,
Expand All @@ -11,6 +11,7 @@ import {
import { Button } from "@/components/ui/button";
import EpisodeList from "@/components/dataset/EpisodeList";
import EpisodeViewer from "@/components/dataset/EpisodeViewer";
import MergeDatasetsDialog from "@/components/dataset/MergeDatasetsDialog";
import { useDatasets } from "@/hooks/useDatasets";
import { useEpisodeDetail, useEpisodes } from "@/hooks/useEpisodes";
import { formatDuration } from "@/lib/datasetApi";
Expand All @@ -33,13 +34,15 @@ const EditDataset = () => {
const navigate = useNavigate();
const [searchParams, setSearchParams] = useSearchParams();

const { datasets, loading: datasetsLoading } = useDatasets();
const { datasets, loading: datasetsLoading, refresh: refreshDatasets } = useDatasets();
// A Hub-only dataset has no videos on disk to decode.
const localDatasets = useMemo(
() => datasets.filter((d) => d.source === "local" || d.source === "both"),
[datasets],
);

const [mergeDialogOpen, setMergeDialogOpen] = useState(false);

const repoId = searchParams.get("dataset");
const episodeParam = searchParams.get("episode");
const episodeIndex = episodeParam === null ? null : Number(episodeParam);
Expand Down Expand Up @@ -113,6 +116,17 @@ const EditDataset = () => {
</SelectContent>
</Select>
</div>
{localDatasets.length >= 2 && (
<Button
variant="outline"
size="sm"
onClick={() => setMergeDialogOpen(true)}
className="h-9 gap-1.5 border-gray-800 bg-gray-950 text-xs text-gray-300 hover:bg-gray-900"
>
<GitMerge className="h-3.5 w-3.5" />
Merge
</Button>
)}
{/* Picking a dataset now lands here rather than on /upload, so the
upload + delete flow hangs off the page you browse from. */}
{repoId && (
Expand All @@ -138,6 +152,17 @@ const EditDataset = () => {
</div>
</div>

<MergeDatasetsDialog
open={mergeDialogOpen}
onOpenChange={setMergeDialogOpen}
datasets={localDatasets}
initialRepoId={repoId}
onMerged={(outputRepoId) => {
refreshDatasets();
setDataset(outputRepoId);
}}
/>

{!repoId && (
<div className="rounded-lg border border-gray-800 bg-gray-950 p-10 text-center">
<Database className="mx-auto mb-3 h-8 w-8 text-gray-700" />
Expand Down
75 changes: 75 additions & 0 deletions lelab/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,20 @@
from typing import Any

from huggingface_hub.errors import HfHubHTTPError
from pydantic import BaseModel

from . import episode_media
from .dataset_repair import DatasetRepairError, repair_local_dataset
from .utils.hf_auth import cached_whoami, shared_hf_api

logger = logging.getLogger(__name__)


class MergeDatasetsRequest(BaseModel):
dataset_repo_ids: list[str]
output_repo_id: str


def _lerobot_cache_root() -> Path:
return episode_media.lerobot_cache_root()

Expand Down Expand Up @@ -173,6 +180,74 @@ def list_all_datasets() -> list[dict[str, Any]]:
return out


def _resolve_new_dataset_dir(repo_id: str) -> Path:
"""Resolve `repo_id` to a not-yet-existing directory inside the cache root.

Same traversal guard as `handle_delete_dataset` (lelab/record.py): the
resolved target must stay strictly inside the cache root.
"""
root = _lerobot_cache_root()
try:
target = (root / repo_id).resolve()
except (ValueError, OSError) as e:
raise ValueError(f"Invalid dataset name: {repo_id}") from e
if target == root or root not in target.parents:
raise ValueError(f"Invalid dataset name: {repo_id}")
return target


def handle_merge_datasets(dataset_repo_ids: list[str], output_repo_id: str) -> dict[str, Any]:
"""Merge several local datasets into one new local dataset.

Wraps lerobot's `dataset_tools.merge_datasets`, which concatenates the
source datasets' videos and data files under a new repo_id. Each source
dataset is loaded (repairing an unfinalized recording first) and the
result is written to disk; nothing is pushed to the Hub.
"""
repo_ids = [r.strip() for r in dataset_repo_ids if r.strip()]
if len(repo_ids) < 2:
return {"success": False, "message": "Select at least two datasets to merge"}

output_repo_id = (output_repo_id or "").strip()
if not output_repo_id:
return {"success": False, "message": "Enter a name for the merged dataset"}

try:
output_dir = _resolve_new_dataset_dir(output_repo_id)
except ValueError as e:
return {"success": False, "message": str(e)}
if output_dir.exists():
return {"success": False, "message": f"{output_repo_id} already exists locally"}

from lerobot.datasets import LeRobotDataset
from lerobot.datasets.dataset_tools import merge_datasets as lerobot_merge_datasets

loaded = []
for repo_id in repo_ids:
try:
repair_local_dataset(repo_id)
loaded.append(LeRobotDataset(repo_id))
except DatasetRepairError as e:
return {"success": False, "message": f"Could not repair {repo_id}: {e}"}
except Exception as e:
logger.warning(f"Could not load {repo_id} for merge: {e}")
return {"success": False, "message": f"Could not load {repo_id}: {e}"}

try:
merged = lerobot_merge_datasets(loaded, output_repo_id=output_repo_id, output_dir=output_dir)
except Exception as e:
logger.error(f"Failed to merge {repo_ids} into {output_repo_id}: {e}")
return {"success": False, "message": f"Failed to merge datasets: {e}"}

logger.info(f"Merged {repo_ids} into {output_repo_id}")
return {
"success": True,
"message": f"Merged {len(loaded)} datasets into {output_repo_id}",
"output_repo_id": output_repo_id,
"num_episodes": merged.num_episodes,
}


# ── episode browsing ────────────────────────────────────────────────────────
#
# Everything below reads a *local* dataset only. Browsing a Hub-only dataset
Expand Down
Loading