diff --git a/KNOWLEDGE_BASE.md b/KNOWLEDGE_BASE.md index 66a5d66ec..b0c1628a0 100644 --- a/KNOWLEDGE_BASE.md +++ b/KNOWLEDGE_BASE.md @@ -236,7 +236,7 @@ Approved top-level commits about specific developments in technology, current af The desktop sidebar shows signed-in members up to five **Hot topics** above **Who to follow**, with bold headlines and a truncated recap underneath when one is available. These aggregate approved top-level commits posted in the past eight hours, with extra weight for trending discussions. Only each author's strongest post contributes to a topic's ranking, so repeated posts from one profile do not lift it. The list respects mutes and blocks. The sidebar no longer repeats your own profile summary; your profile is still available from the main navigation. -Hot topics can also receive a short AI-written recap based on approved, available top-level commits from the past eight hours and public replies in their threads, including nested replies. Both the AI usefulness check and the recap writer receive the reply context: a question can support a recap when its answers provide useful substance. The sample includes one strongest root post per author, up to one hundred roots, plus up to one hundred of the newest replies across those threads. Recaps aim to explain the underlying situation or event in one or two short sentences, rather than cataloguing individual posts. They are not independently verified reporting and can be reused for up to two hours. Only post and reply text and the headline are sent to the AI providers, without account metadata. Removing a linked top-level source commit, rejecting it, or deleting its author's account clears the associated recap. Removing a reply or its author's account does not clear an existing recap: its paraphrased contribution can remain until the recap expires, at most two hours after generation. Signed-in members can open a headline to read its available recap above semantic search results from the past week, shown newest first by default without an additional search. Expired recaps are omitted, and a recap is hidden if linked public threads include a post or reply from an author the viewer has muted or blocked. The results respect visibility, mutes and blocks too. +Hot topics can also receive a short AI-written recap based on approved, available top-level commits from the past eight hours and public replies in their threads, including nested replies. Both the AI usefulness check and the recap writer receive the reply context: a question can support a recap when its answers provide useful substance. The sample includes one strongest root post per author, up to one hundred roots, plus up to one hundred of the newest replies across those threads. Recaps aim to explain the underlying situation or event in one or two short sentences, rather than cataloguing individual posts. They are not independently verified reporting and can be reused for up to two hours. Only post and reply text and the headline are sent to the AI providers, without account metadata. Removing a linked top-level source commit, rejecting it, or deleting its author's account clears the associated recap. Removing a reply or its author's account does not clear an existing recap: its paraphrased contribution can remain until the recap expires, at most two hours after generation. Signed-in members can open a headline to read its available recap above commits linked to that headline and posted within the past week, shown newest first. This list does not use semantic search and does not count toward the commit search limit. Expired recaps are omitted, and a recap is hidden if linked public threads include a post or reply from an author the viewer has muted or blocked. The results respect visibility, mutes and blocks too. Sources: [composer](resources/js/components/commit-composer.tsx), [commit model](app/Models/Commit.php), [commit validation](app/Http/Requests/CreateCommitRequest.php), [text normalization](app/Services/CommitBodies.php), [submission throttle](app/Http/Middleware/ThrottleCommits.php). diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index ae397a5ea..df2e46465 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -248,25 +248,13 @@ private function bootRateLimitingDefaults(): void ]); // Every commit search pays for an embedding and a relevance verdict, so it gets a budget of its own... - RateLimiter::for('search', function (Request $request): Limit { - if ($request->routeIs('headline.show') && $request->header('X-Inertia-Partial-Component') === 'headline/show') { - $only = collect(explode(',', $request->header('X-Inertia-Partial-Data', '')))->filter(); - $except = explode(',', $request->header('X-Inertia-Partial-Except', '')); - - if (($only->isNotEmpty() && $only->doesntContain(fn (string $prop): bool => $prop === 'commits' || str_starts_with($prop, 'commits.'))) - || in_array('commits', $except, true)) { - return Limit::none(); - } - } - - return Limit::perMinute(15) - ->by($this->userId($request)) - ->response(fn (Request $request, array $headers): JsonResponse => response()->json( - ['message' => 'Slow down: too many searches. Give it a few seconds.'], - 429, - $headers, - )); - }); + RateLimiter::for('search', fn (Request $request): Limit => Limit::perMinute(15) + ->by($this->userId($request)) + ->response(fn (Request $request, array $headers): JsonResponse => response()->json( + ['message' => 'Slow down: too many searches. Give it a few seconds.'], + 429, + $headers, + ))); // Every commit page asks for its related commits, using the stored vector and a relevance verdict... RateLimiter::for('related-commits', fn (Request $request): Limit => Limit::perMinute(60)->by( diff --git a/app/Queries/CommitSearchQuery.php b/app/Queries/CommitSearchQuery.php index 32624af4b..9591747a1 100644 --- a/app/Queries/CommitSearchQuery.php +++ b/app/Queries/CommitSearchQuery.php @@ -8,7 +8,6 @@ use App\Contracts\Services\RelevanceProvider; use App\Models\Commit; use App\Models\User; -use Carbon\CarbonInterface; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Cache; use Illuminate\Support\Facades\DB; @@ -40,7 +39,7 @@ public function __construct(private EmbeddingProvider $embeddings, private Relev * * @return Collection */ - public function get(string|Commit $term, ?User $viewer, int $limit = 20, ?CarbonInterface $since = null): Collection + public function get(string|Commit $term, ?User $viewer, int $limit = 20): Collection { $source = $term instanceof Commit ? Commit::query()->whereKey($term->id)->first(['id', 'body', 'embedding', 'parent_id', 'root_id']) @@ -59,13 +58,13 @@ public function get(string|Commit $term, ?User $viewer, int $limit = 20, ?Carbon /** @var list $vector */ $vector = $source->embedding ?? Cache::remember('search-embedding:'.hash('sha256', $term), self::CACHE_SECONDS, fn (): array => $this->embeddings->embed($term)); - $candidates = DB::transaction(function () use ($source, $vector, $viewer, $limit, $since): Collection { - if ($source instanceof Commit || $since instanceof CarbonInterface) { + $candidates = DB::transaction(function () use ($source, $vector, $viewer, $limit): Collection { + if ($source instanceof Commit) { DB::statement('set local hnsw.iterative_scan = strict_order'); } return Commit::query() - ->whereIn('commits.id', $this->nearest($vector, $viewer, $limit * 2, $source, $since)) + ->whereIn('commits.id', $this->nearest($vector, $viewer, $limit * 2, $source)) ->visibleTo($viewer) ->withoutMutedAuthorsFor($viewer) ->withVisibleReplyCount($viewer) @@ -88,11 +87,10 @@ public function get(string|Commit $term, ?User $viewer, int $limit = 20, ?Carbon * * @param list $vector */ - private function nearest(array $vector, ?User $viewer, int $count, ?Commit $source, ?CarbonInterface $since): CommitBuilder + private function nearest(array $vector, ?User $viewer, int $count, ?Commit $source): CommitBuilder { return Commit::query() ->select('id') - ->when($since, fn (CommitBuilder $query, CarbonInterface $since): CommitBuilder => $query->where('created_at', '>=', $since)) ->when($source, fn (CommitBuilder $query, Commit $source): CommitBuilder => $query ->topLevel() ->whereKeyNot($source->threadRootId()) diff --git a/app/Queries/HeadlineCommitsQuery.php b/app/Queries/HeadlineCommitsQuery.php new file mode 100644 index 000000000..723e7fb62 --- /dev/null +++ b/app/Queries/HeadlineCommitsQuery.php @@ -0,0 +1,31 @@ + + */ + public function get(Headline $headline, User $viewer): Collection + { + return Commit::query() + ->whereAttachedTo($headline, 'headlines') + ->where('created_at', '>=', now()->subWeek()) + ->visibleTo($viewer) + ->withoutMutedAuthorsFor($viewer) + ->withVisibleReplyCount($viewer) + ->withContinuationId($viewer) + ->withCardDataFor($viewer) + ->newestFirst() + ->get() + ->toBase(); + } +} diff --git a/app/Queries/HeadlinePageQuery.php b/app/Queries/HeadlinePageQuery.php index 0881f1614..a6817bad5 100644 --- a/app/Queries/HeadlinePageQuery.php +++ b/app/Queries/HeadlinePageQuery.php @@ -12,7 +12,7 @@ final readonly class HeadlinePageQuery { - public function __construct(private CommitSearchQuery $search, private HeadlineSummaryQuery $summary) + public function __construct(private HeadlineCommitsQuery $commits, private HeadlineSummaryQuery $summary) { // } @@ -25,7 +25,7 @@ public function get(Headline $headline, User $viewer): array return [ 'name' => $headline->name, 'summary' => fn (): ?string => $this->summary->get($headline, $viewer), - 'commits' => fn (): Collection => $this->search->get($headline->name, $viewer, since: now()->subWeek()), + 'commits' => fn (): Collection => $this->commits->get($headline, $viewer), ]; } } diff --git a/routes/web.php b/routes/web.php index cdeb1dd6c..bf2b27185 100644 --- a/routes/web.php +++ b/routes/web.php @@ -358,7 +358,7 @@ Route::get('headlines/{headline}', [HeadlineController::class, 'show']) ->whereUuid('headline') - ->middleware(['auth', 'throttle:search']) + ->middleware('auth') ->name('headline.show'); Route::middleware('auth')->group(function (): void { diff --git a/tests/Feature/Controllers/HeadlineControllerTest.php b/tests/Feature/Controllers/HeadlineControllerTest.php index 6dfe192b3..f6ba089e4 100644 --- a/tests/Feature/Controllers/HeadlineControllerTest.php +++ b/tests/Feature/Controllers/HeadlineControllerTest.php @@ -3,6 +3,7 @@ declare(strict_types=1); use App\Contracts\Services\EmbeddingProvider; +use App\Contracts\Services\RelevanceProvider; use App\Http\Middleware\HandleInertiaRequests; use App\Models\Commit; use App\Models\Headline; @@ -14,23 +15,35 @@ $this->get(route('headline.show', $headline))->assertRedirectToRoute('home'); }); -it('shows a fresh recap and semantic matches from the past week, not just linked posts', function (): void { +it('shows a fresh recap and only linked commits from the past week, newest first without semantic search', function (): void { $headline = Headline::factory()->create(['name' => 'Database indexing', 'summary' => 'Developers compare indexing strategies.', 'summary_created_at' => now()->subHour()]); - $recent = Commit::factory()->create(['embedding' => embeddingVector(0), 'created_at' => now()->subWeek()]); - Commit::factory()->create(['embedding' => embeddingVector(0), 'created_at' => now()->subWeek()->subSecond()]); + $oldest = Commit::factory()->hasAttached($headline)->create(['created_at' => now()->subWeek()]); + $newest = Commit::factory()->hasAttached($headline)->create(['created_at' => now()->subHour()]); + $middle = Commit::factory()->hasAttached($headline)->create(['created_at' => now()->subDays(2)]); + Commit::factory()->hasAttached($headline)->create(['created_at' => now()->subWeek()->subSecond()]); + Commit::factory()->create(['body' => 'Database indexing', 'embedding' => embeddingVector(0)]); + Commit::factory()->hasAttached(Headline::factory()->create())->create(); + Commit::factory()->pending()->hasAttached($headline)->create(); + Commit::factory()->rejected()->hasAttached($headline)->create(); + Commit::factory()->hasAttached($headline)->create(['deleted_at' => now()]); + $this->mock(EmbeddingProvider::class)->shouldNotReceive('embed'); + $this->mock(RelevanceProvider::class)->shouldNotReceive('decideRelevance'); $this->actingAs(User::factory()->create())->get(route('headline.show', $headline)) ->assertOk()->assertInertia(fn ($page) => $page ->component('headline/show') ->where('name', 'Database indexing') ->where('summary', 'Developers compare indexing strategies.') - ->has('commits', 1)->where('commits.0.id', $recent->id)) + ->has('commits', 3) + ->where('commits.0.id', $newest->id) + ->where('commits.1.id', $middle->id) + ->where('commits.2.id', $oldest->id)) ->assertSee('', false); }); it('omits unavailable or expired recaps without hiding the discussions', function (?string $summary, ?string $created): void { $headline = Headline::factory()->create(['summary' => $summary, 'summary_created_at' => $created === null ? null : now()->modify($created)]); - $commit = Commit::factory()->create(['embedding' => embeddingVector(0)]); + $commit = Commit::factory()->hasAttached($headline)->create(); $this->actingAs(User::factory()->create())->get(route('headline.show', $headline)) ->assertOk()->assertInertia(fn ($page) => $page @@ -53,7 +66,7 @@ Commit::factory()->for($hidden)->hasAttached($headline)->create(['embedding' => embeddingVector(0)]); } - $visible = Commit::factory()->create(['embedding' => embeddingVector(0)]); + $visible = $isReply ? $root : Commit::factory()->hasAttached($headline)->create(); $this->actingAs($viewer)->get(route('headline.show', $headline)) ->assertOk()->assertInertia(fn ($page) => $page @@ -64,7 +77,7 @@ $this->actingAs(User::factory()->create())->get('/headlines/00000000-0000-4000-8000-000000000000')->assertNotFound(); }); -it('skips semantic search for unrelated and recap-only partial reloads', function (string $props): void { +it('omits discussions for unrelated and recap-only partial reloads', function (string $props): void { $headline = Headline::factory()->create(['summary' => 'A fresh recap.', 'summary_created_at' => now()]); $this->mock(EmbeddingProvider::class)->shouldNotReceive('embed'); @@ -104,7 +117,7 @@ $this->assertSoftDeleted($source); }); -it('rate limits headline semantic searches', function (): void { +it('does not charge headline browsing to the semantic search budget', function (): void { $headline = Headline::factory()->create(); $this->actingAs(User::factory()->create()); @@ -112,17 +125,20 @@ $this->get(route('headline.show', $headline))->assertOk(); } - $this->get(route('headline.show', $headline))->assertTooManyRequests(); + $this->get(route('headline.show', $headline))->assertOk(); + $this->getJson(route('commit-search.index', ['q' => 'indexing']))->assertOk(); }); -it('allows only non-search headline partial reloads after exhausting the shared search budget', function (): void { +it('allows headline pages and partial reloads after exhausting the semantic search budget', function (): void { $headline = Headline::factory()->create(['summary' => 'A fresh recap.', 'summary_created_at' => now()]); $this->actingAs(User::factory()->create()); for ($attempt = 0; $attempt < 15; $attempt++) { - $this->get(route('headline.show', $headline))->assertOk(); + $this->getJson(route('commit-search.index', ['q' => 'indexing']))->assertOk(); } + $this->getJson(route('commit-search.index', ['q' => 'indexing']))->assertTooManyRequests(); + $this->get(route('headline.show', $headline))->assertOk(); $this->withHeaders([ 'X-Inertia' => 'true', 'X-Inertia-Version' => (string) new HandleInertiaRequests()->version(request()), @@ -137,14 +153,14 @@ $this->withHeader('X-Inertia-Partial-Data', 'summary')->get(route('headline.show', $headline)) ->assertOk()->assertJsonPath('props.summary', 'A fresh recap.'); - $this->withHeader('X-Inertia-Partial-Component', 'search/index')->get(route('headline.show', $headline))->assertTooManyRequests(); + $this->withHeader('X-Inertia-Partial-Component', 'search/index')->get(route('headline.show', $headline))->assertOk(); $this->withHeader('X-Inertia-Partial-Component', 'headline/show'); foreach (['', 'commits', 'summary,commits', 'commits.0'] as $props) { - $this->withHeader('X-Inertia-Partial-Data', $props)->get(route('headline.show', $headline))->assertTooManyRequests(); + $this->withHeader('X-Inertia-Partial-Data', $props)->get(route('headline.show', $headline))->assertOk(); } - $this->withHeader('X-Inertia-Partial-Except', 'summary')->get(route('headline.show', $headline))->assertTooManyRequests(); + $this->withHeader('X-Inertia-Partial-Except', 'summary')->get(route('headline.show', $headline))->assertOk(); $this->withHeader('X-Inertia-Partial-Data', ''); $this->withHeader('X-Inertia-Partial-Except', 'commits')->get(route('headline.show', $headline)) ->assertOk()->assertJsonPath('props.summary', 'A fresh recap.')->assertJsonMissingPath('props.commits'); diff --git a/tests/Unit/Queries/CommitSearchQueryTest.php b/tests/Unit/Queries/CommitSearchQueryTest.php index 2fc027811..204d6f49e 100644 --- a/tests/Unit/Queries/CommitSearchQueryTest.php +++ b/tests/Unit/Queries/CommitSearchQueryTest.php @@ -47,13 +47,12 @@ expect(resolve(CommitSearchQuery::class)->get('a query', null, limit: 2))->toHaveCount(2); }); -it('filters posting time before limiting nearest candidates without restricting ordinary search', function (): void { +it('searches commits regardless of posting time', function (): void { $old = Commit::factory()->count(5)->create(['embedding' => embeddingVector(0), 'created_at' => now()->subWeek()->subSecond()]); $recent = Commit::factory()->create(['embedding' => embeddingVector(1), 'created_at' => now()->subWeek()]); $query = resolve(CommitSearchQuery::class); - expect($query->get('a query', null, limit: 2, since: now()->subWeek())->pluck('id')->all())->toBe([$recent->id]) - ->and($query->get('a query', null, limit: 10)->pluck('id')->all())->toEqualCanonicalizing([...$old->modelKeys(), $recent->id]); + expect($query->get('a query', null, limit: 10)->pluck('id')->all())->toEqualCanonicalizing([...$old->modelKeys(), $recent->id]); }); it('returns nothing, and asks for nothing, for a term shorter than three characters', function (string $term): void {