Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion stovepipe/controller/record/record.go
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
return err
}
for _, result := range results {
_, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
fact, _, err := c.recordValidationFact(ctx, store.GetValidationFactStore(), entity.ValidationFact{
URI: request.URI,
Project: result.Project,
Degree: result.Degree,
Expand All @@ -207,6 +207,15 @@ func (c *Controller) recordProjectFacts(ctx context.Context, store storage.Stora
if err != nil {
return err
}

entry, err := entity.NewProjectFactByDegree(fact)
if err != nil {
return err
}

if err := store.GetProjectFactByDegreeStore().Create(ctx, entry); err != nil && !errors.Is(err, storage.ErrAlreadyExists) {
return fmt.Errorf("failed to index project fact for request %q and project %q: %w", request.ID, fact.Project, err)
}
}
if len(results) == 0 {
return nil
Expand Down
56 changes: 56 additions & 0 deletions stovepipe/controller/record/record_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ type recordMocks struct {
reqStore *storagemock.MockRequestStore
queueStore *storagemock.MockQueueStore
factStore *storagemock.MockValidationFactStore
indexStore *storagemock.MockProjectFactByDegreeStore
store *storagemock.MockStorage
materializer *requestlogmock.MockMaterializer
sourceControl *sourcecontrolmock.MockSourceControl
Expand Down Expand Up @@ -148,6 +149,7 @@ func newControllerForTopic(t *testing.T, ctrl *gomock.Controller, topicKey consu
reqStore: storagemock.NewMockRequestStore(ctrl),
queueStore: storagemock.NewMockQueueStore(ctrl),
factStore: storagemock.NewMockValidationFactStore(ctrl),
indexStore: storagemock.NewMockProjectFactByDegreeStore(ctrl),
store: storagemock.NewMockStorage(ctrl),
materializer: requestlogmock.NewMockMaterializer(ctrl),
sourceControl: sourcecontrolmock.NewMockSourceControl(ctrl),
Expand All @@ -158,6 +160,7 @@ func newControllerForTopic(t *testing.T, ctrl *gomock.Controller, topicKey consu
m.store.EXPECT().GetRequestStore().Return(m.reqStore).AnyTimes()
m.store.EXPECT().GetQueueStore().Return(m.queueStore).AnyTimes()
m.store.EXPECT().GetValidationFactStore().Return(m.factStore).AnyTimes()
m.store.EXPECT().GetProjectFactByDegreeStore().Return(m.indexStore).AnyTimes()
m.materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).DoAndReturn(
func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
assert.Equal(t, entity.RequestEventValidationFactRecorded, log.Event)
Expand Down Expand Up @@ -402,6 +405,13 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) {
}).
Times(3)

var indexes []entity.ProjectFactByDegree
m.indexStore.EXPECT().Create(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, entry entity.ProjectFactByDegree) error {
indexes = append(indexes, entry)
return nil
}).Times(2)

var logs []entity.RequestLog
eventMaterializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).
DoAndReturn(func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
Expand All @@ -412,6 +422,12 @@ func TestProcess_RecordsNamedProjectStatusResults(t *testing.T) {

require.NoError(t, c.Process(queueContext(), delivery(t, ctrl, recordPayload(t, testID))))
require.Len(t, facts, 3)
require.Len(t, indexes, 2)
for i, fact := range facts[1:] {
entry, err := entity.NewProjectFactByDegree(fact)
require.NoError(t, err)
assert.Equal(t, entry, indexes[i])
}
assert.Equal(t, []string{"", "project-a", "project-b"}, []string{facts[0].Project, facts[1].Project, facts[2].Project})
for _, fact := range facts {
assert.Equal(t, testURI, fact.URI)
Expand Down Expand Up @@ -1217,3 +1233,43 @@ func TestProcess_QueueMismatchFails(t *testing.T) {

require.Error(t, c.Process(queueContext(), d))
}

func TestRecordProjectFactsRepairsIndexOnRetry(t *testing.T) {
for _, retryErr := range []error{nil, storage.ErrAlreadyExists} {
name := "index repaired"
if retryErr != nil {
name = "index already repaired"
}
t.Run(name, func(t *testing.T) {
ctrl := gomock.NewController(t)
c, m := newController(t, ctrl)
request := requestWithState(entity.RequestStateFailed)
resolverFactory := projectstatusmock.NewMockResolverFactory(ctrl)
resolver := projectstatusmock.NewMockResolver(ctrl)
c.projectStatusResolverFactory = resolverFactory
resolverFactory.EXPECT().For(projectstatus.ResolverConfig{QueueName: testQueue}).Return(resolver, nil).Times(2)
stored := entity.ValidationFact{URI: request.URI, Project: "p", Degree: 0.25, RequestID: request.ID, CreatedAt: 10}
entry, err := entity.NewProjectFactByDegree(stored)
require.NoError(t, err)
writeErr := errors.New("index write unavailable")
materializer := requestlogmock.NewMockMaterializer(ctrl)
c.materializer = materializer

gomock.InOrder(
resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return([]projectstatus.Result{{Project: "p", Degree: 0.25}}, nil),
m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(nil),
m.indexStore.EXPECT().Create(gomock.Any(), entry).Return(writeErr),
resolver.EXPECT().Resolve(gomock.Any(), request, request.TerminalBuildID).Return([]projectstatus.Result{{Project: "p", Degree: 0.75}}, nil),
m.factStore.EXPECT().Create(gomock.Any(), gomock.Any()).Return(storage.ErrAlreadyExists),
m.factStore.EXPECT().Get(gomock.Any(), request.URI, "p").Return(stored, nil),
m.indexStore.EXPECT().Create(gomock.Any(), entry).Return(retryErr),
materializer.EXPECT().PersistLog(gomock.Any(), m.store, gomock.Any()).DoAndReturn(func(_ context.Context, _ storage.Storage, log entity.RequestLog) error {
assert.Equal(t, entity.RequestEventProjectFactsRecorded, log.Event)
return nil
}),
)
require.ErrorIs(t, c.recordProjectFacts(queueContext(), m.store, request), writeErr)
require.NoError(t, c.recordProjectFacts(queueContext(), m.store, request))
})
}
}
2 changes: 2 additions & 0 deletions stovepipe/entity/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ go_library(
"ingest.go",
"list.go",
"open_issue.go",
"project_fact_by_degree.go",
"project_status.go",
"queue.go",
"queue_config.go",
Expand All @@ -32,6 +33,7 @@ go_test(
name = "go_default_test",
srcs = [
"build_test.go",
"project_fact_by_degree_test.go",
"queue_state_test.go",
"request_id_test.go",
"request_log_test.go",
Expand Down
39 changes: 39 additions & 0 deletions stovepipe/entity/project_fact_by_degree.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package entity

import (
"fmt"
"math"
)

// ProjectFactByDegree maps an ordered key to an immutable project fact within a queue.
type ProjectFactByDegree struct {
// URI is the commit containing the project result.
URI string
// Degree is the breakage degree in [0, 1], ordered highest first.
Degree float64
// Project identifies the fact and breaks equal-degree ties in ascending bytewise order.
Project string
}

// NewProjectFactByDegree derives a degree lookup entry from a named project's fact.
// Invalid degrees, empty URIs and whole-repository facts return an error.
func NewProjectFactByDegree(fact ValidationFact) (ProjectFactByDegree, error) {
if fact.URI == "" || fact.Project == "" || math.IsNaN(fact.Degree) || fact.Degree < DegreeGreen || fact.Degree > DegreeBroken {
return ProjectFactByDegree{}, fmt.Errorf("project fact degree lookup requires a URI, project and degree in [0, 1]")
}
return ProjectFactByDegree{URI: fact.URI, Degree: fact.Degree, Project: fact.Project}, nil
}
44 changes: 44 additions & 0 deletions stovepipe/entity/project_fact_by_degree_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright (c) 2026 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package entity

import (
"math"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestProjectFactByDegreePreservesDegree(t *testing.T) {
for _, degree := range []float64{1, math.Nextafter(1, 0), 0.75, math.Nextafter(0.5, 1), 0.5, math.SmallestNonzeroFloat64, 0, math.Copysign(0, -1)} {
entry, err := NewProjectFactByDegree(ValidationFact{URI: "commit", Project: "project", Degree: degree})
require.NoError(t, err)
assert.Equal(t, ProjectFactByDegree{URI: "commit", Project: "project", Degree: degree}, entry)
}
}

func TestProjectFactByDegreeRejectsInvalidFacts(t *testing.T) {
for _, fact := range []ValidationFact{
{Project: "p"}, {URI: "commit"},
{URI: "commit", Project: "p", Degree: math.NaN()},
{URI: "commit", Project: "p", Degree: math.Inf(1)},
{URI: "commit", Project: "p", Degree: -0.1},
{URI: "commit", Project: "p", Degree: 1.1},
} {
_, err := NewProjectFactByDegree(fact)
require.Error(t, err)
}
}
1 change: 1 addition & 0 deletions stovepipe/extension/storage/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go_library(
"build_issue_store.go",
"build_store.go",
"open_issue_store.go",
"project_fact_by_degree_store.go",
"queue_log_store.go",
"queue_store.go",
"queue_summary_store.go",
Expand Down
2 changes: 2 additions & 0 deletions stovepipe/extension/storage/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte

`QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md).

`ProjectFactByDegreeStore` is the ordered lookup mapping for named project facts. Its immutable `(queue, URI, degree DESC, project ASC)` primary key stores the original degree and orders by descending degree and ascending bytewise project ID, so listing is a bounded key-range read. It holds no duplicate validation payload; readers hydrate facts by their original keys and verify the mapping. The record controller ensures mappings from persisted winning facts before recording project completion, including retries after partial writes. Create the table before deploying writers. The table starts empty: only results processed by index-aware writers receive mappings, with no historical population or read-time fallback.

## Build issue observations

`BuildIssueStore` accumulates immutable observations identified by build, task, and unit within the bound queue. The build ID is supplied separately from the detected issue. Repeated observations return `ErrAlreadyExists`, which callers may treat as successful persistence. Empty unit IDs represent task-wide observations.
Expand Down
1 change: 1 addition & 0 deletions stovepipe/extension/storage/mock/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go_library(
"build_issue_store_mock.go",
"build_store_mock.go",
"open_issue_store_mock.go",
"project_fact_by_degree_store_mock.go",
"queue_log_store_mock.go",
"queue_store_mock.go",
"queue_summary_store_mock.go",
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions stovepipe/extension/storage/mock/storage_mock.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions stovepipe/extension/storage/mysql/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ go_library(
"build_issue_store.go",
"build_store.go",
"open_issue_store.go",
"project_fact_by_degree_store.go",
"queue_log_store.go",
"queue_store.go",
"queue_summary_store.go",
Expand Down Expand Up @@ -34,6 +35,7 @@ go_test(
"build_issue_store_test.go",
"build_store_test.go",
"open_issue_store_test.go",
"project_fact_by_degree_store_test.go",
"queue_store_test.go",
"queue_summary_store_test.go",
"request_acceptance_store_test.go",
Expand Down
Loading
Loading