diff --git a/AGENTS.md b/AGENTS.md index bbc08b5..9c1d9e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -11,14 +11,14 @@ This is `ytx`, a .NET Global Tool that extracts YouTube video metadata and trans **Main Components:** - `src/Ytx/Program.cs` - Single-file console application with async main method - `Input` record - Simple DTO for JSON input parsing -- `Output` class - JSON output contract (1.1.0): canonical url, videoId, metadata, caption status and track, transcriptRaw, transcript, segments. Property order is the field order; README documents each field. Changing a field name or order is a breaking change +- `Output` class - JSON output contract (1.1.0): canonical url, videoId, metadata, caption status and track, transcriptRaw, transcript, segments, chapters. Property order is the field order; new fields go at the end; README documents each field. Changing a field name or order is a breaking change - Uses YoutubeExplode library for YouTube API interactions and caption extraction **Data Flow:** 1. Input validation (command-line args or JSON via stdin) 2. YouTube video data extraction via YoutubeExplode 3. Caption track discovery and selection (`SelectTrack`: best `--language` match, then English, then manual over auto-generated) -4. Transcript formatting (raw text + markdown with timestamped links) and caption status +4. Chapter parsing from the description (`ParseChapters`, YouTube's rules) and transcript formatting (raw text + markdown with timestamped links and a `##` heading per chapter) with caption status 5. JSON serialization to stdout, or a transcript-only format (`--format md|txt|srt|vtt`) built from the normalized segments ## Development Commands diff --git a/CHANGELOG.md b/CHANGELOG.md index 5752440..c106f9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ All notable changes to ytx are listed here. The format follows ## [Unreleased] +## [1.2.0] - 2026-10-08 + +### Added + +- `chapters`: chapters read from the video description as `{start, end, title}`, using YouTube's rules (starts at `0:00`, at least three, increasing, each at least 10 seconds, one block). Filled with `--metadata-only` too ([#16](https://github.com/solrevdev/solrevdev.ytx/issues/16)). +- The Markdown `transcript` and `--format md` have a `## Title` heading at each chapter. + ## [1.1.0] - 2026-10-08 This release fixes the caption bugs from the v1.0.7 review ([#3](https://github.com/solrevdev/solrevdev.ytx/issues/3)) and settles the JSON contract. @@ -92,7 +99,8 @@ This release fixes the caption bugs from the v1.0.7 review ([#3](https://github. - First release: title, description, raw transcript and Markdown transcript as JSON. -[Unreleased]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.2.0...HEAD +[1.2.0]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.1.0...v1.2.0 [1.1.0]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.0.8...v1.1.0 [1.0.8]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.0.7...v1.0.8 [1.0.7]: https://github.com/solrevdev/solrevdev.ytx/compare/v1.0.6...v1.0.7 diff --git a/README.md b/README.md index 3ec3398..c4d36f2 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,12 @@ A .NET global tool that extracts YouTube video metadata and transcripts as JSON. "captionIsAutoGenerated": false, "captionError": null, "transcriptRaw": "Full transcript as one string...", - "transcript": "- [00:03](https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=3s) First line\n- [00:07](...) Next line" + "transcript": "## Intro\n\n- [00:03](https://www.youtube.com/watch?v=dQw4w9WgXcQ&t=3s) First line\n- [00:07](...) Next line", + "segments": null, + "chapters": [ + { "start": 0, "end": 43, "title": "Intro" }, + { "start": 43, "end": 212, "title": "Chorus" } + ] } ``` @@ -28,7 +33,8 @@ A .NET global tool that extracts YouTube video metadata and transcripts as JSON. - 🎯 **Single JSON output** — Clean, structured data for easy parsing - 🌍 **Captions-aware** — Prefers human-written captions in your language, reports which track it used -- 📝 **Markdown transcript** — Human-readable format with timestamped links +- 📝 **Markdown transcript** — Human-readable format with timestamped links, split into sections by chapter +- 📑 **Chapters** — Read from the video description, using YouTube's own rules - 🚀 **Cross-platform** — Works on macOS, Windows, Linux (.NET 8/9/10) - ⚡ **Fast & lightweight** — No dependencies beyond .NET and YoutubeExplode @@ -141,6 +147,9 @@ Every document has the same keys in the same order. Fields that YouTube does not | `transcriptRaw` | All caption text as a single normalized string | | `transcript` | Markdown list with `[HH:MM:SS](link) text` format | | `segments` | With `--segments`, an array of `{start, end, text}` with times in seconds to the millisecond; otherwise `null` | +| `chapters` | Chapters from the description as `{start, end, title}` in seconds; `end` is `null` for the last chapter of a live stream. `[]` when the video has none. Also filled with `--metadata-only` | + +**Chapters.** ytx reads the timestamped lines that creators put in the description, such as `0:00 Intro` or `(03:12) - Setup`, and applies YouTube's rules: the list starts at `0:00`, has at least three entries in increasing order, each chapter lasts at least 10 seconds, and the lines form one block. A list that breaks a rule gives `[]`, so tracklists and stray timestamps are not mistaken for chapters. YouTube's auto-generated "key moments" are not included. When a video has chapters, `transcript` and `--format md` start each chapter with a `## Title` heading. With `--format md|txt|srt|vtt`, only the transcript is written, with no JSON. If there is no transcript, stdout is empty and stderr says why. The exit codes are the same as for JSON. diff --git a/docs/investigations/2026-10-ytx-review-to-1.1.0.md b/docs/investigations/2026-10-ytx-review-to-1.1.0.md index 8475d4b..cec4836 100644 --- a/docs/investigations/2026-10-ytx-review-to-1.1.0.md +++ b/docs/investigations/2026-10-ytx-review-to-1.1.0.md @@ -305,6 +305,24 @@ Each of these is worth a paragraph in the post. For startup, CPU and memory comparisons between build types, see #4 and the AOT write-up. +## 6a. Follow-up: chapters in 1.2.0 + +After 1.1.0, the user asked why chapters were left out. That had been the session's own call, because YoutubeExplode has no chapters API, and it should have been raised as a question. The design was agreed in [#16](https://github.com/solrevdev/solrevdev.ytx/issues/16) and shipped the same evening as **1.2.0**. + +- **Source.** Creator chapters are the timestamped lines in the description, which ytx already fetches. No extra request is needed, and the code stays AOT-safe because both regexes are source-generated. +- **YouTube's rules, applied as written:** + - the list starts at `0:00` + - it has at least three entries + - times strictly increase + - every chapter lasts at least 10 seconds, including the last one up to the video's duration + + A list that breaks any rule gives `[]`. +- **Trap: prose after the list.** A line such as "Follow me at 99:00 on stream" after the list has an increasing time, so the first version would have added it as a chapter. The list is now one block: blank lines are allowed, but any other line ends it. +- **Trap: empty chapters.** A chapter with no captions of its own still gets its heading, so the Markdown outline always matches `chapters`. An extra blank line between two headings was caught in a unit test and removed. +- **Output.** `chapters` is appended as the last field, so existing field order is unchanged. Live on `NYFGCESmikA`, it returns 23 chapters, matching the description, and the Markdown starts each one with `## Title`. The TED talk returns `[]`. AOT and JIT output were byte-identical on all three checks. +- **Tests:** 145. They cover 10 line layouts, 6 rejections, 7 rule violations, a null end for live streams, and heading placement. +- **Left out:** auto-generated "key moments" (needs page scraping), a chapter index on each segment, and WebVTT chapter tracks. + ## 7. Decisions not taken - **No `ytx mcp` server mode.** It was prototyped as far as checking the API of `ModelContextProtocol.Core` 2.2.0, then dropped at the user's suggestion. Reasons: @@ -313,7 +331,7 @@ For startup, CPU and memory comparisons between build types, see #4 and the AOT - Agents can already run `ytx` and parse its JSON, which now has `captionStatus`, segments and clear exit codes. - **No deduplication of caption cues.** See section 5. - **No branch protection.** The publish job pushes the version bump straight to `master`, and a protection rule would break that. The user chose to leave protection off. -- **No chapters, thumbnails or other rich metadata.** YoutubeExplode doesn't expose chapters. The 1.1.0 schema adds channel, date, duration, views and keywords, which covers most uses. +- **No chapters in 1.1.0, thumbnails or other rich metadata.** YoutubeExplode doesn't expose chapters, so they were deferred, then added in 1.2.0 by parsing the description (section 6a). Thumbnails are still left out. - **No translation, cookies or batch input.** #3 lists these in "What users expect". They weren't in the decided order (#7), so they stay as possible future work. - **Action majors not bumped.** The pins stay on v5 and v1. Dependabot will propose v6 and v7 as separate PRs. - **No CHANGELOG entries until release.** Adding them in each stacked PR would have caused conflicts, so they were written at release time. diff --git a/src/Ytx/Program.cs b/src/Ytx/Program.cs index c258b95..0aeac34 100644 --- a/src/Ytx/Program.cs +++ b/src/Ytx/Program.cs @@ -59,6 +59,15 @@ class Output public string transcriptRaw { get; set; } = ""; public string transcript { get; set; } = ""; public IReadOnlyList? segments { get; set; } + public IReadOnlyList chapters { get; set; } = []; +} + +// start and end are in seconds. end is null for the last chapter of a live stream. +class Chapter +{ + public double start { get; set; } + public double? end { get; set; } + public string title { get; set; } = ""; } // Times are in seconds, rounded to the millisecond. @@ -292,8 +301,72 @@ static async Task Main(string[] args) viewCount = video.Engagement.ViewCount, keywords = video.Keywords, description = video.Description ?? "", + chapters = ParseChapters(video.Description ?? "", video.Duration), }; + // YouTube only shows description timestamps as chapters when the list starts at 0:00, has + // at least three entries in increasing order, and every chapter lasts at least 10 seconds. + // Applying the same rules keeps tracklists and stray timestamps out. + internal static IReadOnlyList ParseChapters(string description, TimeSpan? duration) + { + const int minChapters = 3; + const int minSeconds = 10; + + var run = new List<(int Start, string Title)>(); + foreach (var line in description.Split('\n')) + { + var parsed = TryParseChapterLine(line, out var start, out var title); + if (run.Count == 0) + { + if (parsed && start == 0) run.Add((start, title)); + } + // The list is one block: blank lines are allowed, but any other line or a time that + // does not increase ends it, so later timestamps in prose are not swallowed. + else if (parsed && start > run[^1].Start) run.Add((start, title)); + else if (parsed || !string.IsNullOrWhiteSpace(line)) break; + } + + var total = duration is { } d ? (int)d.TotalSeconds : (int?)null; + if (run.Count < minChapters) return []; + for (var i = 0; i < run.Count; i++) + { + var end = i + 1 < run.Count ? run[i + 1].Start : total; + if (end - run[i].Start < minSeconds) return []; + } + + return run.Select((c, i) => new Chapter + { + start = c.Start, + end = i + 1 < run.Count ? run[i + 1].Start : total, + title = c.Title, + }).ToList(); + } + + // Accepts the common layouts: "0:00 Intro", "(00:00) - Intro", "[1:02:03] Intro", "• 3:12 | Intro" + // and "Intro - 3:12". + internal static bool TryParseChapterLine(string line, out int start, out string title) + { + start = 0; + title = ""; + var match = ChapterAtStart().Match(line); + if (!match.Success) match = ChapterAtEnd().Match(line); + if (!match.Success) return false; + + var parts = match.Groups["ts"].Value.Split(':').Select(int.Parse).ToArray(); + var (h, m, sec) = parts.Length == 3 ? (parts[0], parts[1], parts[2]) : (0, parts[0], parts[1]); + if (sec > 59 || (parts.Length == 3 && m > 59)) return false; + + start = h * 3600 + m * 60 + sec; + title = match.Groups["title"].Value.Trim(); + return title.Length > 0; + } + + [GeneratedRegex(@"^[^\p{L}\p{N}]*?[(\[]?(?(?:\d{1,2}:)?\d{1,3}:\d{2})[)\]]?(?:\s*[-–—:|•.]+)?\s+(?.*[\p{L}\p{N}].*?)\s*$")] + private static partial Regex ChapterAtStart(); + + [GeneratedRegex(@"^\s*(?<title>.*?[\p{L}\p{N}].*?)\s*(?:[-–—:|•]+\s*)?[(\[]?(?<ts>(?:\d{1,2}:)?\d{1,3}:\d{2})[)\]]?\s*$")] + private static partial Regex ChapterAtEnd(); + // No handler is passed without a proxy, so HttpClient keeps its default proxy, which // reads HTTPS_PROXY, HTTP_PROXY and NO_PROXY and falls back to the system setting. internal static HttpClient CreateHttpClient(Uri? proxy) => proxy == null @@ -329,7 +402,7 @@ private static async Task<IReadOnlyList<Segment>> AddCaptionsAsync( } var captions = await client.Videos.ClosedCaptions.GetAsync(track, cancellationToken); - (output.transcriptRaw, output.transcript) = BuildTranscript(videoId.Value, captions.Captions); + (output.transcriptRaw, output.transcript) = BuildTranscript(videoId.Value, captions.Captions, output.chapters); output.captionStatus = CaptionStatus.Ok; output.captionLanguage = track.Language.Code; output.captionLanguageName = track.Language.Name; @@ -359,10 +432,13 @@ _ when ex.Message.Contains("not a bot", StringComparison.OrdinalIgnoreCase) => C _ => CaptionStatus.Error, }; - internal static (string Raw, string Markdown) BuildTranscript(string videoId, IEnumerable<ClosedCaption> captions) + internal static (string Raw, string Markdown) BuildTranscript( + string videoId, IEnumerable<ClosedCaption> captions, IReadOnlyList<Chapter>? chapters = null) { var rawSb = new StringBuilder(); var mdSb = new StringBuilder(); + chapters ??= []; + var nextChapter = 0; foreach (var caption in captions) { @@ -376,6 +452,17 @@ internal static (string Raw, string Markdown) BuildTranscript(string videoId, IE } rawSb.Append(text); + // Every chapter that has started gets a heading, even one with no captions of its own, + // so the Markdown outline always matches the chapter list. + var headingAdded = false; + while (nextChapter < chapters.Count && chapters[nextChapter].start <= caption.Offset.TotalSeconds) + { + // A blank line separates the heading from the list item before it, but not from another heading. + if (mdSb.Length > 0 && !headingAdded) mdSb.Append(Environment.NewLine); + mdSb.Append("## ").Append(chapters[nextChapter++].title).Append(Environment.NewLine).Append(Environment.NewLine); + headingAdded = true; + } + // Appending interpolated strings to a StringBuilder formats in place, with no temporary strings. mdSb.Append("- ["); AppendTimestamp(mdSb, caption.Offset); diff --git a/tests/Ytx.Tests/GoldenOutputTests.cs b/tests/Ytx.Tests/GoldenOutputTests.cs index 8ffdcc3..3403810 100644 --- a/tests/Ytx.Tests/GoldenOutputTests.cs +++ b/tests/Ytx.Tests/GoldenOutputTests.cs @@ -205,6 +205,11 @@ public void WriteOutput_MatchesGoldenIndentedJson() transcriptRaw = raw, transcript = markdown, segments = Program.BuildSegments(captions), + chapters = + [ + new Chapter { start = 0, end = 3723, title = "Intro" }, + new Chapter { start = 3723, end = null, title = "Caf\u00E9 talk" }, + ], }; var expected = """ @@ -245,6 +250,18 @@ public void WriteOutput_MatchesGoldenIndentedJson() "end": 360009, "text": "late" } + ], + "chapters": [ + { + "start": 0, + "end": 3723, + "title": "Intro" + }, + { + "start": 3723, + "end": null, + "title": "Café talk" + } ] } @@ -270,7 +287,7 @@ public void WriteOutput_MatchesGoldenCompactJson() var expected = "{\"url\":\"u\",\"videoId\":\"v\",\"title\":\"\\u2028\\u0085\\u3000\\uD83D\\uDE00\",\"channel\":\"\",\"channelId\":\"\"," + "\"uploadDate\":null,\"durationSeconds\":null,\"viewCount\":null,\"keywords\":[],\"description\":\"\"," + "\"captionStatus\":\"blocked\",\"captionLanguage\":null,\"captionLanguageName\":null,\"captionIsAutoGenerated\":null," - + "\"captionError\":\"Sign in to confirm you're not a bot\",\"transcriptRaw\":\"\",\"transcript\":\"\",\"segments\":null}" + Environment.NewLine; + + "\"captionError\":\"Sign in to confirm you're not a bot\",\"transcriptRaw\":\"\",\"transcript\":\"\",\"segments\":null,\"chapters\":[]}" + Environment.NewLine; AssertWrites(expected, output, compact: true); } diff --git a/tests/Ytx.Tests/ProgramTests.cs b/tests/Ytx.Tests/ProgramTests.cs index 1a68999..c99a4c7 100644 --- a/tests/Ytx.Tests/ProgramTests.cs +++ b/tests/Ytx.Tests/ProgramTests.cs @@ -313,6 +313,7 @@ public void CreateOutput_MapsMetadataAndCanonicalUrl() Assert.Equal(212, output.durationSeconds); Assert.Equal(42, output.viewCount); Assert.Equal(["k1", "k2"], output.keywords); + Assert.Empty(output.chapters); Assert.Equal(CaptionStatus.Skipped, output.captionStatus); Assert.Equal("", output.transcript); } @@ -345,6 +346,106 @@ public void ClassifyCaptionError_SeparatesBlockingFromOtherFailures(Exception ex Assert.Equal(expected, Program.ClassifyCaptionError(ex)); } + [Theory] + [InlineData("0:00 Intro", 0, "Intro")] + [InlineData("0:00 - Episode highlight", 0, "Episode highlight")] + [InlineData("(00:00) - Introduction", 0, "Introduction")] + [InlineData("[1:02:03] Deep dive", 3723, "Deep dive")] + [InlineData("• 3:12 | Setup", 192, "Setup")] + [InlineData("03:12: Setup", 192, "Setup")] + [InlineData("75:00 Long minutes", 4500, "Long minutes")] + [InlineData("Wrap-up - 12:34", 754, "Wrap-up")] + [InlineData("Questions (1:00:00)", 3600, "Questions")] + [InlineData(" 4:05 Spaces\r", 245, "Spaces")] + public void TryParseChapterLine_ReadsCommonLayouts(string line, int start, string title) + { + Assert.True(Program.TryParseChapterLine(line, out var actualStart, out var actualTitle)); + Assert.Equal((start, title), (actualStart, actualTitle)); + } + + [Theory] + [InlineData("No timestamp here")] + [InlineData("0:00")] + [InlineData("0:00 ---")] + [InlineData("1:60 Bad seconds")] + [InlineData("1:60:00 Bad minutes")] + [InlineData("")] + public void TryParseChapterLine_RejectsNonChapters(string line) + { + Assert.False(Program.TryParseChapterLine(line, out _, out _)); + } + + [Fact] + public void ParseChapters_ReturnsChaptersWithEnds() + { + var description = "Great talk.\n\nChapters:\n0:00 Intro\n1:30 Middle\n\n10:00 End\n\nFollow me at 99:00 on stream."; + + var chapters = Program.ParseChapters(description, TimeSpan.FromSeconds(700)); + + Assert.Collection(chapters, + c => Assert.Equal((0d, (double?)90, "Intro"), (c.start, c.end, c.title)), + c => Assert.Equal((90d, (double?)600, "Middle"), (c.start, c.end, c.title)), + c => Assert.Equal((600d, (double?)700, "End"), (c.start, c.end, c.title))); + } + + [Fact] + public void ParseChapters_LeavesLastEndNullWithoutDuration() + { + var chapters = Program.ParseChapters("0:00 A\n0:30 B\n1:00 C", null); + + Assert.Equal(3, chapters.Count); + Assert.Null(chapters[^1].end); + } + + [Fact] + public void ParseChapters_SkipsTimestampsBeforeTheListStartsAtZero() + { + var chapters = Program.ParseChapters("Live at 12:00 today\n0:00 A\n0:30 B\n1:00 C", TimeSpan.FromMinutes(5)); + + Assert.Equal(["A", "B", "C"], chapters.Select(c => c.title)); + } + + [Theory] + [InlineData("0:10 A\n0:30 B\n1:00 C")] // does not start at 0:00 + [InlineData("0:00 A\n0:30 B")] // fewer than three + [InlineData("0:00 A\n0:05 B\n1:00 C")] // a chapter shorter than 10 s + [InlineData("0:00 A\n0:30 B\n4:55 C")] // last chapter shorter than 10 s + [InlineData("0:00 A\n0:30 B\n9:00 C")] // starts after the video ends + [InlineData("0:00 A\nSome prose\n0:30 B\n1:00 C")] // the block is broken by prose + [InlineData("0:00 A\n0:30 B\n0:20 C\n1:00 D")] // times stop increasing + public void ParseChapters_FollowsYouTubeRules(string description) + { + Assert.Empty(Program.ParseChapters(description, TimeSpan.FromMinutes(5))); + } + + [Fact] + public void BuildTranscript_AddsAHeadingAtEachChapter() + { + var captions = new[] + { + new ClosedCaption("hello", TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(1), []), + new ClosedCaption("still intro", TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(1), []), + new ClosedCaption("after the gap", TimeSpan.FromSeconds(70), TimeSpan.FromSeconds(1), []), + }; + Chapter[] chapters = + [ + new() { start = 0, end = 30, title = "Intro" }, + new() { start = 30, end = 60, title = "Silent" }, + new() { start = 60, end = null, title = "Main" }, + ]; + + var (raw, markdown) = Program.BuildTranscript("AbCdEfGhIjK", captions, chapters); + + var nl = Environment.NewLine; + Assert.Equal("hello still intro after the gap", raw); + Assert.Equal( + $"## Intro{nl}{nl}- [00:01](https://www.youtube.com/watch?v=AbCdEfGhIjK&t=1s) hello{nl}" + + $"- [00:05](https://www.youtube.com/watch?v=AbCdEfGhIjK&t=5s) still intro{nl}{nl}" + + $"## Silent{nl}{nl}## Main{nl}{nl}" + + "- [01:10](https://www.youtube.com/watch?v=AbCdEfGhIjK&t=70s) after the gap", + markdown); + } + private static Output TranscriptOutput() => new() { transcript = "- [00:01](link) first",