From 027e999722d6ace8f53bedaf9e3f7045658c9624 Mon Sep 17 00:00:00 2001 From: Uriah Rokach Date: Sun, 9 Aug 2026 16:27:01 +0300 Subject: [PATCH 1/9] feat(monorepo): add plugin for per-sub-project DORA metrics in a monorepo Repos containing multiple logically separate projects (each deployed by its own CI job, PRs tagged by label) previously collapsed into a single set of DORA numbers, since DevLake's scope model is one-scope-per-repo. Adds a new metric plugin, monorepo, that runs after dora and attributes deployments (by CI job name) and merged pull requests (by label) to a configured sub-project, writing per-sub-project deployment and change lead time metrics to two new tables. Nothing existing is modified: dora and the core scope model are untouched, and PR coding/pickup/review time are reused from dora's own project_pr_metrics rather than recomputed. Includes unit tests, an e2e test with fixtures, and Grafana dashboards (mysql + postgresql) to view the output. --- .../plugins/monorepo/e2e/attribution_test.go | 101 +++ .../cicd_deployment_commits.csv | 7 + .../e2e/monorepo_attribution/cicd_tasks.csv | 9 + .../monorepo_attribution/project_mapping.csv | 5 + .../project_pr_metrics.csv | 7 + .../pull_request_labels.csv | 9 + .../monorepo_attribution/pull_requests.csv | 8 + .../monorepo_subproject_deployments.csv | 7 + .../monorepo_subproject_pr_metrics.csv | 5 + backend/plugins/monorepo/impl/impl.go | 170 +++++ .../20260809_add_init_tables.go | 44 ++ .../models/migrationscripts/register.go | 29 + .../monorepo/models/subproject_deployment.go | 53 ++ .../monorepo/models/subproject_pr_metric.go | 57 ++ backend/plugins/monorepo/monorepo.go | 43 ++ .../monorepo/tasks/deployment_attributor.go | 119 ++++ .../plugins/monorepo/tasks/pr_attributor.go | 275 ++++++++ .../monorepo/tasks/pr_attributor_test.go | 125 ++++ backend/plugins/monorepo/tasks/task_data.go | 154 +++++ .../plugins/monorepo/tasks/task_data_test.go | 220 ++++++ .../mysql/monorepo-subprojects.json | 340 ++++++++++ .../postgresql/monorepo-subprojects.json | 628 ++++++++++++++++++ 22 files changed, 2415 insertions(+) create mode 100644 backend/plugins/monorepo/e2e/attribution_test.go create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv create mode 100644 backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv create mode 100644 backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv create mode 100644 backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv create mode 100644 backend/plugins/monorepo/impl/impl.go create mode 100644 backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go create mode 100644 backend/plugins/monorepo/models/migrationscripts/register.go create mode 100644 backend/plugins/monorepo/models/subproject_deployment.go create mode 100644 backend/plugins/monorepo/models/subproject_pr_metric.go create mode 100644 backend/plugins/monorepo/monorepo.go create mode 100644 backend/plugins/monorepo/tasks/deployment_attributor.go create mode 100644 backend/plugins/monorepo/tasks/pr_attributor.go create mode 100644 backend/plugins/monorepo/tasks/pr_attributor_test.go create mode 100644 backend/plugins/monorepo/tasks/task_data.go create mode 100644 backend/plugins/monorepo/tasks/task_data_test.go create mode 100644 grafana/dashboards/mysql/monorepo-subprojects.json create mode 100644 grafana/dashboards/postgresql/monorepo-subprojects.json diff --git a/backend/plugins/monorepo/e2e/attribution_test.go b/backend/plugins/monorepo/e2e/attribution_test.go new file mode 100644 index 00000000000..b56d1ce4a07 --- /dev/null +++ b/backend/plugins/monorepo/e2e/attribution_test.go @@ -0,0 +1,101 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/code" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" + "github.com/stretchr/testify/assert" +) + +// TestMonorepoAttributionDataFlow exercises both subtasks against a monorepo containing +// serviceA and serviceB, each with its own deploy job. +// +// The fixtures deliberately include the cases that motivated this plugin: +// - pr2 (serviceB) merges at 09:00 while serviceA deploys at 10:00 and serviceB only at +// 12:00. DORA would link pr2 to the 10:00 deployment because it searches the whole +// repository; pr2 must instead link to 12:00. +// - pipeline3 runs both deploy jobs, so it must yield one row per sub-project. +// - a failed deployment and a staging deployment sit between pr1's merge and the +// deployment that actually shipped it, so neither may be linked. +func TestMonorepoAttributionDataFlow(t *testing.T) { + var plugin impl.Monorepo + dataflowTester := e2ehelper.NewDataFlowTester(t, "monorepo", plugin) + + subProjects := []tasks.SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } + matcher, err := tasks.NewSubProjectMatcher(subProjects) + assert.Nil(t, err) + + taskData := &tasks.MonorepoTaskData{ + Options: &tasks.MonorepoOptions{ + ProjectName: "monorepo", + SubProjects: subProjects, + }, + Matcher: matcher, + } + + // seed the domain layer + dataflowTester.FlushTabler(&crossdomain.ProjectMapping{}) + dataflowTester.FlushTabler(&devops.CICDTask{}) + dataflowTester.FlushTabler(&devops.CicdDeploymentCommit{}) + dataflowTester.FlushTabler(&code.PullRequest{}) + dataflowTester.FlushTabler(&code.PullRequestLabel{}) + dataflowTester.FlushTabler(&crossdomain.ProjectPrMetric{}) + + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_mapping.csv", &crossdomain.ProjectMapping{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_tasks.csv", &devops.CICDTask{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/cicd_deployment_commits.csv", &devops.CicdDeploymentCommit{}) + dataflowTester.ImportNullableCsvIntoTabler("./monorepo_attribution/pull_requests.csv", &code.PullRequest{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/pull_request_labels.csv", &code.PullRequestLabel{}) + dataflowTester.ImportCsvIntoTabler("./monorepo_attribution/project_pr_metrics.csv", &crossdomain.ProjectPrMetric{}) + + // deployments must be attributed first: the pull request subtask reads them back to + // work out which deployment shipped each merged pull request. + dataflowTester.FlushTabler(&models.SubProjectDeployment{}) + dataflowTester.Subtask(tasks.AttributeDeploymentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectDeployment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_deployments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + dataflowTester.FlushTabler(&models.SubProjectPrMetric{}) + dataflowTester.Subtask(tasks.AttributePullRequestsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.SubProjectPrMetric{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/monorepo_subproject_pr_metrics.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv new file mode 100644 index 00000000000..0bf5e350c27 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_deployment_commits.csv @@ -0,0 +1,7 @@ +id,cicd_deployment_id,cicd_scope_id,name,result,status,environment,repo_url,commit_sha,created_date,finished_date +dc1,pipeline1,cicd1,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitA1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc2,pipeline2,cicd1,deploy-serviceB,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitB1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +dc3,pipeline3,cicd1,deploy-both,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitAB,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +dc5,pipeline5,cicd2,deploy-serviceA,SUCCESS,DONE,PRODUCTION,https://gitlab.example.com/acme/other,commitOther,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +dc6,pipeline6,cicd1,deploy-serviceA,FAILURE,DONE,PRODUCTION,https://gitlab.example.com/acme/monorepo,commitFail,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +dc7,pipeline7,cicd1,deploy-serviceA,SUCCESS,DONE,STAGING,https://gitlab.example.com/acme/monorepo,commitStg,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv new file mode 100644 index 00000000000..9324d96ffb1 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/cicd_tasks.csv @@ -0,0 +1,9 @@ +id,name,pipeline_id,type,result,status,environment,cicd_scope_id,created_date,finished_date +task1,deploy-serviceA,pipeline1,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task1b,build,pipeline1,,SUCCESS,DONE,,cicd1,2026-08-01T09:40:00.000+00:00,2026-08-01T09:50:00.000+00:00 +task2,deploy-serviceB,pipeline2,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-01T11:50:00.000+00:00,2026-08-01T12:00:00.000+00:00 +task3a,deploy-serviceA,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task3b,deploy-serviceB,pipeline3,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd1,2026-08-02T09:50:00.000+00:00,2026-08-02T10:00:00.000+00:00 +task5,deploy-serviceA,pipeline5,DEPLOYMENT,SUCCESS,DONE,PRODUCTION,cicd2,2026-08-01T09:50:00.000+00:00,2026-08-01T10:00:00.000+00:00 +task6,deploy-serviceA,pipeline6,DEPLOYMENT,FAILURE,DONE,PRODUCTION,cicd1,2026-08-01T09:20:00.000+00:00,2026-08-01T09:30:00.000+00:00 +task7,deploy-serviceA,pipeline7,DEPLOYMENT,SUCCESS,DONE,STAGING,cicd1,2026-08-01T09:35:00.000+00:00,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv new file mode 100644 index 00000000000..c871e7cb114 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_mapping.csv @@ -0,0 +1,5 @@ +project_name,table,row_id +monorepo,cicd_scopes,cicd1 +monorepo,repos,repo1 +other,cicd_scopes,cicd2 +other,repos,repo2 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv new file mode 100644 index 00000000000..c60538d6507 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/project_pr_metrics.csv @@ -0,0 +1,7 @@ +id,project_name,pr_coding_time,pr_pickup_time,pr_review_time +pr1,monorepo,100,20,30 +pr2,monorepo,200,40,60 +pr3,monorepo,300,60,90 +pr4,monorepo,400,80,120 +pr5,monorepo,500,100,150 +pr7,monorepo,700,140,210 diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv new file mode 100644 index 00000000000..95b45e7f473 --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_request_labels.csv @@ -0,0 +1,9 @@ +pull_request_id,label_name +pr1,serviceA +pr2,serviceB +pr3,serviceB +pr3,serviceA +pr4,bug +pr5,serviceA +pr6,serviceA +pr7,serviceA diff --git a/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv new file mode 100644 index 00000000000..c1809224f2b --- /dev/null +++ b/backend/plugins/monorepo/e2e/monorepo_attribution/pull_requests.csv @@ -0,0 +1,8 @@ +id,base_repo_id,created_date,merged_date,merge_commit_sha +pr1,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitA1 +pr2,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitB1 +pr3,repo1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,commitAB +pr4,repo1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitBug +pr5,repo1,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00,commitLate +pr6,repo2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,commitOther +pr7,repo1,2026-08-01T08:00:00.000+00:00,NULL,commitOpen diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv new file mode 100644 index 00000000000..1504f88eff7 --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_deployments.csv @@ -0,0 +1,7 @@ +project_name,sub_project,cicd_deployment_id,commit_sha,job_name,result,environment,finished_date +monorepo,serviceA,pipeline1,commitA1,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-01T10:00:00.000+00:00 +monorepo,serviceB,pipeline2,commitB1,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-01T12:00:00.000+00:00 +monorepo,serviceA,pipeline3,commitAB,deploy-serviceA,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceB,pipeline3,commitAB,deploy-serviceB,SUCCESS,PRODUCTION,2026-08-02T10:00:00.000+00:00 +monorepo,serviceA,pipeline6,commitFail,deploy-serviceA,FAILURE,PRODUCTION,2026-08-01T09:30:00.000+00:00 +monorepo,serviceA,pipeline7,commitStg,deploy-serviceA,SUCCESS,STAGING,2026-08-01T09:45:00.000+00:00 diff --git a/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv new file mode 100644 index 00000000000..f40d9459bbf --- /dev/null +++ b/backend/plugins/monorepo/e2e/snapshot_tables/monorepo_subproject_pr_metrics.csv @@ -0,0 +1,5 @@ +project_name,pull_request_id,sub_project,coding_time,pickup_time,review_time,deploy_time,cycle_time,deployment_id,pr_created_date,pr_merged_date,deployed_date +monorepo,pr1,serviceA,100,20,30,60,220,pipeline1,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr2,serviceB,200,40,60,180,440,pipeline2,2026-08-01T08:00:00.000+00:00,2026-08-01T09:00:00.000+00:00,2026-08-01T12:00:00.000+00:00 +monorepo,pr3,serviceA,300,60,90,30,390,pipeline1,2026-08-01T08:30:00.000+00:00,2026-08-01T09:30:00.000+00:00,2026-08-01T10:00:00.000+00:00 +monorepo,pr5,serviceA,500,100,150,,560,,2026-08-03T08:00:00.000+00:00,2026-08-03T09:00:00.000+00:00, diff --git a/backend/plugins/monorepo/impl/impl.go b/backend/plugins/monorepo/impl/impl.go new file mode 100644 index 00000000000..32d5b52d901 --- /dev/null +++ b/backend/plugins/monorepo/impl/impl.go @@ -0,0 +1,170 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/monorepo/models" + "github.com/apache/incubator-devlake/plugins/monorepo/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/monorepo/tasks" +) + +// make sure interface is implemented +var _ interface { + plugin.PluginMeta + plugin.PluginTask + plugin.PluginModel + plugin.PluginMetric + plugin.PluginMigration + plugin.MetricPluginBlueprintV200 +} = (*Monorepo)(nil) + +type Monorepo struct{} + +func (p Monorepo) Description() string { + return "Split a monorepo into sub-projects and compute per-sub-project DORA metrics" +} + +func (p Monorepo) Name() string { + return "monorepo" +} + +func (p Monorepo) Dashboards() []plugin.GrafanaDashboard { + return nil +} + +func (p Monorepo) SvgIcon() string { + return ` + +` +} + +// RequiredDataEntities declares that deployments must be recognisable as CI/CD tasks of +// type Deployment, which is what sub-project attribution matches job names against. +func (p Monorepo) RequiredDataEntities() (data []map[string]interface{}, err errors.Error) { + return []map[string]interface{}{ + { + "model": "cicd_tasks", + "requiredFields": map[string]string{ + "column": "type", + "execptedValue": "Deployment", + }, + }, + }, nil +} + +func (p Monorepo) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + } +} + +func (p Monorepo) IsProjectMetric() bool { + return true +} + +// RunAfter ensures DORA has produced project_pr_metrics (coding/pickup/review times) and +// its deployment records before sub-project attribution reads them. +func (p Monorepo) RunAfter() ([]string, errors.Error) { + return []string{"dora"}, nil +} + +func (p Monorepo) Settings() interface{} { + return nil +} + +func (p Monorepo) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.AttributeDeploymentsMeta, + tasks.AttributePullRequestsMeta, + } +} + +func (p Monorepo) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + op, err := tasks.DecodeAndValidateTaskOptions(options) + if err != nil { + return nil, err + } + matcher, err := tasks.NewSubProjectMatcher(op.SubProjects) + if err != nil { + return nil, err + } + return &tasks.MonorepoTaskData{ + Options: op, + Matcher: matcher, + }, nil +} + +// RootPkgPath information lost when compiled as plugin(.so) +func (p Monorepo) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/monorepo" +} + +func (p Monorepo) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Monorepo) MakeMetricPluginPipelinePlanV200(projectName string, options json.RawMessage) (coreModels.PipelinePlan, errors.Error) { + op := &tasks.MonorepoOptions{} + if options != nil && string(options) != "\"\"" { + if err := json.Unmarshal(options, op); err != nil { + return nil, errors.Default.WrapRaw(err) + } + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New( + "the monorepo plugin requires a subProjects list in its metric plugin options") + } + // Validate eagerly so a bad regex is reported when the blueprint is saved rather + // than midway through a pipeline run. + if _, err := tasks.NewSubProjectMatcher(op.SubProjects); err != nil { + return nil, err + } + + subProjects := make([]map[string]interface{}, 0, len(op.SubProjects)) + for _, sp := range op.SubProjects { + subProjects = append(subProjects, map[string]interface{}{ + "name": sp.Name, + "prLabels": sp.PrLabels, + "deployJobPattern": sp.DeployJobPattern, + }) + } + + plan := coreModels.PipelinePlan{ + { + { + Plugin: "monorepo", + Options: map[string]interface{}{ + "projectName": projectName, + "subProjects": subProjects, + }, + Subtasks: []string{ + tasks.AttributeDeploymentsMeta.Name, + tasks.AttributePullRequestsMeta.Name, + }, + }, + }, + } + return plan, nil +} diff --git a/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go new file mode 100644 index 00000000000..ce485504013 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/20260809_add_init_tables.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var _ plugin.MigrationScript = (*addInitTables)(nil) + +type addInitTables struct{} + +func (script *addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &models.SubProjectDeployment{}, + &models.SubProjectPrMetric{}, + ) +} + +func (*addInitTables) Version() uint64 { return 20260809100000 } + +func (*addInitTables) Name() string { + return "create init tables for the monorepo plugin" +} diff --git a/backend/plugins/monorepo/models/migrationscripts/register.go b/backend/plugins/monorepo/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/monorepo/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/monorepo/models/subproject_deployment.go b/backend/plugins/monorepo/models/subproject_deployment.go new file mode 100644 index 00000000000..04f55279a06 --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_deployment.go @@ -0,0 +1,53 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectDeployment attributes a deployment to a single sub-project of a monorepo, +// based on the name of the CI job that performed the deployment. +// +// One deployment may produce several rows when a single pipeline runs the deploy jobs +// of several sub-projects. That is not double counting: each sub-project really was +// deployed by that pipeline. +type SubProjectDeployment struct { + common.NoPKModel + // The four primary key columns are deliberately kept narrow: MySQL caps a composite + // index at 3072 bytes, which is 768 characters under utf8mb4. + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + SubProject string `gorm:"primaryKey;type:varchar(100)"` + // CicdDeploymentId is the id of the deployment (a cicd_pipelines.id when the + // deployment was generated from a pipeline), taken from cicd_deployment_commits. + CicdDeploymentId string `gorm:"primaryKey;type:varchar(255)"` + // CommitSha is wide enough for a SHA-256 hash; the source column is varchar(255) but + // only ever holds a git object id. + CommitSha string `gorm:"primaryKey;type:varchar(64)"` + // JobName is the cicd_tasks.name that matched this sub-project's DeployJobPattern. + JobName string `gorm:"type:varchar(255)"` + Result string `gorm:"type:varchar(100)"` + Environment string `gorm:"type:varchar(255)"` + FinishedDate *time.Time +} + +func (SubProjectDeployment) TableName() string { + return "monorepo_subproject_deployments" +} diff --git a/backend/plugins/monorepo/models/subproject_pr_metric.go b/backend/plugins/monorepo/models/subproject_pr_metric.go new file mode 100644 index 00000000000..ab138230c8d --- /dev/null +++ b/backend/plugins/monorepo/models/subproject_pr_metric.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// SubProjectPrMetric holds the change-lead-time breakdown for a merged pull request, +// attributed to exactly one sub-project of a monorepo. +// +// CodingTime/PickupTime/ReviewTime are carried over from DORA's project_pr_metrics: +// they depend only on the pull request itself, so DORA already computes them correctly +// for a monorepo. Only DeployTime (and therefore CycleTime) is recomputed here, against +// the deployments of this sub-project rather than the whole repository's. +// +// All durations are in minutes, matching DORA's convention. +type SubProjectPrMetric struct { + common.NoPKModel + ProjectName string `gorm:"primaryKey;type:varchar(100)"` + PullRequestId string `gorm:"primaryKey;type:varchar(255)"` + SubProject string `gorm:"index;type:varchar(255)"` + + CodingTime *int64 + PickupTime *int64 + ReviewTime *int64 + DeployTime *int64 + CycleTime *int64 + + // DeploymentId is the sub-project deployment this PR was linked to, if any. + DeploymentId string `gorm:"type:varchar(255)"` + + PrCreatedDate *time.Time + PrMergedDate *time.Time + DeployedDate *time.Time +} + +func (SubProjectPrMetric) TableName() string { + return "monorepo_subproject_pr_metrics" +} diff --git a/backend/plugins/monorepo/monorepo.go b/backend/plugins/monorepo/monorepo.go new file mode 100644 index 00000000000..0b23cd9b74a --- /dev/null +++ b/backend/plugins/monorepo/monorepo.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/monorepo/impl" + "github.com/spf13/cobra" +) + +// PluginEntry exports for Framework to search and load +var PluginEntry impl.Monorepo //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "monorepo"} + + projectName := cmd.Flags().StringP("projectName", "p", "", "project name") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("projectName") + + cmd.Run = func(cmd *cobra.Command, args []string) { + runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{ + "projectName": *projectName, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/monorepo/tasks/deployment_attributor.go b/backend/plugins/monorepo/tasks/deployment_attributor.go new file mode 100644 index 00000000000..e9ac492b612 --- /dev/null +++ b/backend/plugins/monorepo/tasks/deployment_attributor.go @@ -0,0 +1,119 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributeDeploymentsMeta = plugin.SubTaskMeta{ + Name: "attributeDeployments", + EntryPoint: AttributeDeployments, + EnabledByDefault: true, + Description: "Attribute each deployment to a monorepo sub-project by the name of the CI job that deployed it", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, +} + +// deploymentJobRow is one (deployment, deploy job) pair as returned by the query below. +// +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type deploymentJobRow struct { + common.RawDataOrigin + CicdDeploymentId string + CommitSha string + Result string + Environment string + FinishedDate *time.Time + JobName string +} + +func AttributeDeployments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*MonorepoTaskData) + + // Rebuild from scratch: attribution depends on configuration that may have changed + // since the last run, so stale rows cannot be reconciled incrementally. + if err := db.Exec( + "DELETE FROM monorepo_subproject_deployments WHERE project_name = ?", + data.Options.ProjectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_deployments") + } + + // Only deployments generated from pipelines can be attributed: cicd_deployment_id is + // the pipeline id, which is what cicd_tasks rows hang off. Deployments imported + // straight from a provider's deployment API carry no job and are skipped. + clauses := []dal.Clause{ + dal.Select(`dc.cicd_deployment_id, dc.commit_sha, dc.result, dc.environment, + dc.finished_date, t.name AS job_name`), + dal.From("cicd_deployment_commits dc"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)"), + dal.Join("JOIN cicd_tasks t ON (t.pipeline_id = dc.cicd_deployment_id)"), + dal.Where("pm.project_name = ? AND t.type = ?", data.Options.ProjectName, devops.DEPLOYMENT), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: data.Options.ProjectName, + }, + Table: "cicd_deployment_commits", + }, + InputRowType: reflect.TypeOf(deploymentJobRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + row := inputRow.(*deploymentJobRow) + matched := data.Matcher.MatchDeployJob(row.JobName) + results := make([]interface{}, 0, len(matched)) + for _, subProject := range matched { + results = append(results, &models.SubProjectDeployment{ + ProjectName: data.Options.ProjectName, + SubProject: subProject, + CicdDeploymentId: row.CicdDeploymentId, + CommitSha: row.CommitSha, + JobName: row.JobName, + Result: row.Result, + Environment: row.Environment, + FinishedDate: row.FinishedDate, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor.go b/backend/plugins/monorepo/tasks/pr_attributor.go new file mode 100644 index 00000000000..405c80f7219 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor.go @@ -0,0 +1,275 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "math" + "reflect" + "sort" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/monorepo/models" +) + +var AttributePullRequestsMeta = plugin.SubTaskMeta{ + Name: "attributePullRequests", + EntryPoint: AttributePullRequests, + EnabledByDefault: true, + Description: "Attribute merged pull requests to monorepo sub-projects by label and compute their change lead time", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, +} + +// RawDataOrigin is embedded because DataConverter copies that field from the input row +// onto every result; without it the conversion panics. +type pullRequestRow struct { + common.RawDataOrigin + Id string + CreatedDate time.Time + MergedDate *time.Time +} + +type prLabelRow struct { + PullRequestId string + LabelName string +} + +// deployedAt is the minimum information needed to link a merged PR to a deployment. +type deployedAt struct { + Id string + FinishedDate time.Time +} + +func AttributePullRequests(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + data := taskCtx.GetData().(*MonorepoTaskData) + projectName := data.Options.ProjectName + + if err := db.Exec( + "DELETE FROM monorepo_subproject_pr_metrics WHERE project_name = ?", + projectName, + ); err != nil { + return errors.Default.Wrap(err, "error deleting previous monorepo_subproject_pr_metrics") + } + + labelsByPr, err := loadPrLabels(db, projectName) + if err != nil { + return err + } + // DORA already computes coding/pickup/review time correctly for a monorepo: they + // depend only on the pull request itself. Only the deploy leg needs recomputing. + doraMetrics, err := loadDoraPrMetrics(db, projectName) + if err != nil { + return err + } + deploymentsBySubProject, err := loadSubProjectDeployments(db, projectName) + if err != nil { + return err + } + logger.Info("monorepo: %d labelled PRs, %d DORA metric rows, %d sub-projects with deployments", + len(labelsByPr), len(doraMetrics), len(deploymentsBySubProject)) + + clauses := []dal.Clause{ + dal.Select("pr.id, pr.created_date, pr.merged_date"), + dal.From("pull_requests pr"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ? AND pr.merged_date IS NOT NULL", projectName), + } + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + unattributed := 0 + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: MonorepoApiParams{ + ProjectName: projectName, + }, + Table: "pull_requests", + }, + InputRowType: reflect.TypeOf(pullRequestRow{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + pr := inputRow.(*pullRequestRow) + subProject := data.Matcher.MatchPrLabels(labelsByPr[pr.Id]) + if subProject == "" { + // No sub-project claims this PR; it is simply out of scope here. + unattributed++ + return nil, nil + } + + metric := &models.SubProjectPrMetric{ + ProjectName: projectName, + PullRequestId: pr.Id, + SubProject: subProject, + PrCreatedDate: &pr.CreatedDate, + PrMergedDate: pr.MergedDate, + } + if dm := doraMetrics[pr.Id]; dm != nil { + metric.CodingTime = dm.PrCodingTime + metric.PickupTime = dm.PrPickupTime + metric.ReviewTime = dm.PrReviewTime + } + + if deployment := firstDeploymentAfter(deploymentsBySubProject[subProject], pr.MergedDate); deployment != nil { + metric.DeploymentId = deployment.Id + metric.DeployedDate = &deployment.FinishedDate + metric.DeployTime = computeTimeSpan(pr.MergedDate, &deployment.FinishedDate) + } + + // Mirrors DORA's definition: coding + (merged - created) + deploy. + var cycleTime int64 + if metric.CodingTime != nil { + cycleTime += *metric.CodingTime + } + if prDuring := computeTimeSpan(&pr.CreatedDate, pr.MergedDate); prDuring != nil { + cycleTime += *prDuring + } + if metric.DeployTime != nil { + cycleTime += *metric.DeployTime + } + metric.CycleTime = &cycleTime + + return []interface{}{metric}, nil + }, + }) + if err != nil { + return err + } + + if err := converter.Execute(); err != nil { + return err + } + if unattributed > 0 { + logger.Info("monorepo: %d merged PRs matched no sub-project label and were skipped", unattributed) + } + return nil +} + +// firstDeploymentAfter returns the earliest deployment that finished after mergedDate. +// +// This is an approximation: it assumes a merged change is shipped by the next successful +// production deployment of its sub-project. Hotfixes, cherry-picks, rollbacks and re-runs +// can break that assumption. Exact attribution would need each deployment's commit range +// from the refdiff plugin's commits_diffs table; this function is the seam where that +// swap would happen. +func firstDeploymentAfter(deployments []deployedAt, mergedDate *time.Time) *deployedAt { + if mergedDate == nil || len(deployments) == 0 { + return nil + } + // deployments is sorted by FinishedDate ascending. + i := sort.Search(len(deployments), func(i int) bool { + return deployments[i].FinishedDate.After(*mergedDate) + }) + if i >= len(deployments) { + return nil + } + return &deployments[i] +} + +func loadPrLabels(db dal.Dal, projectName string) (map[string][]string, errors.Error) { + var rows []prLabelRow + err := db.All(&rows, + dal.Select("prl.pull_request_id, prl.label_name"), + dal.From("pull_request_labels prl"), + dal.Join("JOIN pull_requests pr ON (pr.id = prl.pull_request_id)"), + dal.Join("JOIN project_mapping pm ON (pm.table = 'repos' AND pm.row_id = pr.base_repo_id)"), + dal.Where("pm.project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading pull request labels") + } + byPr := make(map[string][]string) + for _, r := range rows { + byPr[r.PullRequestId] = append(byPr[r.PullRequestId], r.LabelName) + } + return byPr, nil +} + +func loadDoraPrMetrics(db dal.Dal, projectName string) (map[string]*crossdomain.ProjectPrMetric, errors.Error) { + var rows []*crossdomain.ProjectPrMetric + err := db.All(&rows, + dal.From(&crossdomain.ProjectPrMetric{}), + dal.Where("project_name = ?", projectName), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading project_pr_metrics") + } + byPr := make(map[string]*crossdomain.ProjectPrMetric, len(rows)) + for _, r := range rows { + byPr[r.Id] = r + } + return byPr, nil +} + +// loadSubProjectDeployments returns the successful production deployments of each +// sub-project, sorted by finish time so they can be searched by merge date. +func loadSubProjectDeployments(db dal.Dal, projectName string) (map[string][]deployedAt, errors.Error) { + var rows []models.SubProjectDeployment + err := db.All(&rows, + dal.From(&models.SubProjectDeployment{}), + dal.Where( + "project_name = ? AND result = ? AND environment = ? AND finished_date IS NOT NULL", + projectName, devops.RESULT_SUCCESS, devops.PRODUCTION, + ), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "error loading monorepo_subproject_deployments") + } + bySubProject := make(map[string][]deployedAt) + for _, r := range rows { + bySubProject[r.SubProject] = append(bySubProject[r.SubProject], deployedAt{ + Id: r.CicdDeploymentId, + FinishedDate: *r.FinishedDate, + }) + } + for name := range bySubProject { + list := bySubProject[name] + sort.Slice(list, func(i, j int) bool { + return list[i].FinishedDate.Before(list[j].FinishedDate) + }) + bySubProject[name] = list + } + return bySubProject, nil +} + +// computeTimeSpan returns the whole minutes between start and end, or nil when either is +// missing or the span is negative. Mirrors the identical unexported helper in the DORA +// plugin (plugins/dora/tasks/change_lead_time_calculator.go) so the two produce the same +// numbers; it cannot be imported because it is not exported there. +func computeTimeSpan(start, end *time.Time) *int64 { + if start == nil || end == nil { + return nil + } + span := end.Sub(*start) + minutes := int64(math.Ceil(span.Minutes())) + if minutes < 0 { + return nil + } + return &minutes +} diff --git a/backend/plugins/monorepo/tasks/pr_attributor_test.go b/backend/plugins/monorepo/tasks/pr_attributor_test.go new file mode 100644 index 00000000000..c716d005a56 --- /dev/null +++ b/backend/plugins/monorepo/tasks/pr_attributor_test.go @@ -0,0 +1,125 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func at(hour int) time.Time { + return time.Date(2026, 8, 9, hour, 0, 0, 0, time.UTC) +} + +func TestFirstDeploymentAfter(t *testing.T) { + // Sorted ascending, as loadSubProjectDeployments guarantees. + deployments := []deployedAt{ + {Id: "deploy-08", FinishedDate: at(8)}, + {Id: "deploy-12", FinishedDate: at(12)}, + {Id: "deploy-18", FinishedDate: at(18)}, + } + + cases := []struct { + name string + deployments []deployedAt + mergedDate *time.Time + expectedId string + }{ + { + name: "picks the earliest deployment after the merge", + deployments: deployments, + mergedDate: ptrTime(at(10)), + expectedId: "deploy-12", + }, + { + name: "merge before every deployment picks the first", + deployments: deployments, + mergedDate: ptrTime(at(1)), + expectedId: "deploy-08", + }, + { + name: "merge after every deployment has none to link", + deployments: deployments, + mergedDate: ptrTime(at(20)), + expectedId: "", + }, + { + name: "a deployment finishing exactly at merge time does not count", + deployments: deployments, + mergedDate: ptrTime(at(12)), + expectedId: "deploy-18", + }, + { + name: "no deployments at all", + deployments: nil, + mergedDate: ptrTime(at(10)), + expectedId: "", + }, + { + name: "unmerged pull request", + deployments: deployments, + mergedDate: nil, + expectedId: "", + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := firstDeploymentAfter(tc.deployments, tc.mergedDate) + if tc.expectedId == "" { + assert.Nil(t, got) + return + } + assert.NotNil(t, got) + assert.Equal(t, tc.expectedId, got.Id) + }) + } +} + +func TestComputeTimeSpan(t *testing.T) { + start := at(10) + end := at(12) + + t.Run("whole minutes between two times", func(t *testing.T) { + got := computeTimeSpan(&start, &end) + assert.NotNil(t, got) + assert.Equal(t, int64(120), *got) + }) + + t.Run("partial minutes round up", func(t *testing.T) { + later := start.Add(90 * time.Second) + got := computeTimeSpan(&start, &later) + assert.NotNil(t, got) + assert.Equal(t, int64(2), *got) + }) + + t.Run("negative spans are discarded", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(&end, &start)) + }) + + t.Run("missing endpoints yield nil", func(t *testing.T) { + assert.Nil(t, computeTimeSpan(nil, &end)) + assert.Nil(t, computeTimeSpan(&start, nil)) + assert.Nil(t, computeTimeSpan(nil, nil)) + }) +} + +func ptrTime(t time.Time) *time.Time { + return &t +} diff --git a/backend/plugins/monorepo/tasks/task_data.go b/backend/plugins/monorepo/tasks/task_data.go new file mode 100644 index 00000000000..26b57b390f6 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data.go @@ -0,0 +1,154 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "regexp" + + "github.com/apache/incubator-devlake/core/errors" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +type MonorepoApiParams struct { + ProjectName string +} + +// SubProjectConfig declares one logical project living inside a monorepo. +type SubProjectConfig struct { + // Name identifies the sub-project in the output tables and dashboards. + Name string `json:"name" mapstructure:"name"` + // PrLabels are the pull request labels that mark a PR as belonging to this + // sub-project. Matching is exact and case-sensitive. + PrLabels []string `json:"prLabels" mapstructure:"prLabels"` + // DeployJobPattern is a regular expression matched against cicd_tasks.name to + // recognise this sub-project's deployment jobs, e.g. "^deploy-serviceA$". + DeployJobPattern string `json:"deployJobPattern" mapstructure:"deployJobPattern"` +} + +type MonorepoOptions struct { + ProjectName string `json:"projectName" mapstructure:"projectName"` + // SubProjects is ordered: when a pull request carries the labels of more than one + // sub-project, the earliest entry in this list wins. + SubProjects []SubProjectConfig `json:"subProjects" mapstructure:"subProjects"` +} + +type MonorepoTaskData struct { + Options *MonorepoOptions + Matcher *SubProjectMatcher +} + +// SubProjectMatcher resolves deployments and pull requests to sub-projects. It holds +// the compiled form of the configuration so the regexes are built once per task rather +// than once per row. +type SubProjectMatcher struct { + names []string + prLabels []map[string]struct{} + deployJobRes []*regexp.Regexp +} + +// NewSubProjectMatcher compiles the sub-project configuration, validating it along the way. +func NewSubProjectMatcher(subProjects []SubProjectConfig) (*SubProjectMatcher, errors.Error) { + m := &SubProjectMatcher{ + names: make([]string, 0, len(subProjects)), + prLabels: make([]map[string]struct{}, 0, len(subProjects)), + deployJobRes: make([]*regexp.Regexp, 0, len(subProjects)), + } + seen := make(map[string]struct{}, len(subProjects)) + for i, sp := range subProjects { + if sp.Name == "" { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: name is required", i)) + } + if _, dup := seen[sp.Name]; dup { + return nil, errors.BadInput.New(fmt.Sprintf("subProjects[%d]: duplicate name %q", i, sp.Name)) + } + seen[sp.Name] = struct{}{} + + var jobRe *regexp.Regexp + if sp.DeployJobPattern != "" { + compiled, err := regexp.Compile(sp.DeployJobPattern) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf( + "subProjects[%d] (%s): invalid deployJobPattern", i, sp.Name)) + } + jobRe = compiled + } + + labels := make(map[string]struct{}, len(sp.PrLabels)) + for _, l := range sp.PrLabels { + if l != "" { + labels[l] = struct{}{} + } + } + + m.names = append(m.names, sp.Name) + m.prLabels = append(m.prLabels, labels) + m.deployJobRes = append(m.deployJobRes, jobRe) + } + return m, nil +} + +// MatchDeployJob returns every sub-project whose DeployJobPattern matches jobName. +// +// More than one match is possible and is reported faithfully: a single pipeline running +// both deploy-serviceA and deploy-serviceB genuinely deploys two sub-projects. If a +// single job name matches two patterns, that indicates overlapping configuration. +func (m *SubProjectMatcher) MatchDeployJob(jobName string) []string { + var matched []string + for i, re := range m.deployJobRes { + if re != nil && re.MatchString(jobName) { + matched = append(matched, m.names[i]) + } + } + return matched +} + +// MatchPrLabels returns the single sub-project a pull request belongs to, or "" when no +// sub-project claims it. When several sub-projects match, the earliest one in the +// configured order wins — labels carry no size signal that could rank them otherwise. +func (m *SubProjectMatcher) MatchPrLabels(labels []string) string { + if len(labels) == 0 { + return "" + } + present := make(map[string]struct{}, len(labels)) + for _, l := range labels { + present[l] = struct{}{} + } + for i, wanted := range m.prLabels { + for l := range wanted { + if _, ok := present[l]; ok { + return m.names[i] + } + } + } + return "" +} + +func DecodeAndValidateTaskOptions(options map[string]interface{}) (*MonorepoOptions, errors.Error) { + var op MonorepoOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "error decoding monorepo task options") + } + if op.ProjectName == "" { + return nil, errors.BadInput.New("projectName is required for the monorepo plugin") + } + if len(op.SubProjects) == 0 { + return nil, errors.BadInput.New("at least one entry in subProjects is required for the monorepo plugin") + } + return &op, nil +} diff --git a/backend/plugins/monorepo/tasks/task_data_test.go b/backend/plugins/monorepo/tasks/task_data_test.go new file mode 100644 index 00000000000..2d6affcca05 --- /dev/null +++ b/backend/plugins/monorepo/tasks/task_data_test.go @@ -0,0 +1,220 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// twoServices is the canonical monorepo configuration used across these tests: +// serviceA is declared first, so it wins any tie. +func twoServices() []SubProjectConfig { + return []SubProjectConfig{ + { + Name: "serviceA", + PrLabels: []string{"serviceA"}, + DeployJobPattern: "^deploy-serviceA$", + }, + { + Name: "serviceB", + PrLabels: []string{"serviceB", "svc-b"}, + DeployJobPattern: "^deploy-serviceB$", + }, + } +} + +func TestMatchDeployJob(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + jobName string + expected []string + }{ + {"matches serviceA", "deploy-serviceA", []string{"serviceA"}}, + {"matches serviceB", "deploy-serviceB", []string{"serviceB"}}, + {"build job is not a deployment", "build-serviceA", nil}, + {"unrelated job matches nothing", "run-tests", nil}, + {"anchored pattern rejects a superstring", "deploy-serviceAB", nil}, + {"empty job name matches nothing", "", nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchDeployJob(tc.jobName)) + }) + } +} + +// A pipeline that runs both services' deploy jobs produces a row for each. The two jobs +// arrive as separate rows, so each resolves to exactly one sub-project. +func TestMatchDeployJob_PipelineDeployingBothServices(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA"}, matcher.MatchDeployJob("deploy-serviceA")) + assert.Equal(t, []string{"serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +// Overlapping patterns are reported faithfully rather than silently resolved, so a +// misconfiguration is visible in the data instead of hidden. +func TestMatchDeployJob_OverlappingPatterns(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "deploy"}, + {Name: "serviceB", DeployJobPattern: "^deploy-serviceB$"}, + }) + assert.Nil(t, err) + + assert.Equal(t, []string{"serviceA", "serviceB"}, matcher.MatchDeployJob("deploy-serviceB")) +} + +func TestMatchDeployJob_NoPatternNeverMatches(t *testing.T) { + matcher, err := NewSubProjectMatcher([]SubProjectConfig{ + {Name: "labelsOnly", PrLabels: []string{"labelsOnly"}}, + }) + assert.Nil(t, err) + + assert.Nil(t, matcher.MatchDeployJob("deploy-labelsOnly")) +} + +func TestMatchPrLabels(t *testing.T) { + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + + cases := []struct { + name string + labels []string + expected string + }{ + {"single matching label", []string{"serviceA"}, "serviceA"}, + {"alias label resolves to its sub-project", []string{"svc-b"}, "serviceB"}, + {"matching label among unrelated ones", []string{"bug", "serviceB", "urgent"}, "serviceB"}, + {"no matching label", []string{"bug", "urgent"}, ""}, + {"no labels at all", nil, ""}, + {"empty label slice", []string{}, ""}, + {"matching is case sensitive", []string{"servicea"}, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, matcher.MatchPrLabels(tc.labels)) + }) + } +} + +// A PR labelled for several sub-projects is assigned to exactly one: the earliest in +// configuration order. Labels carry no size signal, so declaration order is the tie-break. +func TestMatchPrLabels_TieBreakIsConfigOrder(t *testing.T) { + both := []string{"serviceB", "serviceA"} + + matcher, err := NewSubProjectMatcher(twoServices()) + assert.Nil(t, err) + assert.Equal(t, "serviceA", matcher.MatchPrLabels(both)) + + // Reversing the configuration reverses the winner, proving order drives the result + // rather than the order of labels on the PR. + reversed := []SubProjectConfig{twoServices()[1], twoServices()[0]} + reversedMatcher, err := NewSubProjectMatcher(reversed) + assert.Nil(t, err) + assert.Equal(t, "serviceB", reversedMatcher.MatchPrLabels(both)) +} + +func TestNewSubProjectMatcher_Validation(t *testing.T) { + cases := []struct { + name string + subProjects []SubProjectConfig + expectErr bool + }{ + { + name: "valid configuration", + subProjects: twoServices(), + }, + { + name: "empty configuration is allowed here, rejected by option decoding", + subProjects: nil, + }, + { + name: "missing name", + subProjects: []SubProjectConfig{{PrLabels: []string{"x"}}}, + expectErr: true, + }, + { + name: "duplicate names", + subProjects: []SubProjectConfig{ + {Name: "serviceA", DeployJobPattern: "^a$"}, + {Name: "serviceA", DeployJobPattern: "^b$"}, + }, + expectErr: true, + }, + { + name: "invalid deploy job regex", + subProjects: []SubProjectConfig{{Name: "serviceA", DeployJobPattern: "^deploy-(unclosed"}}, + expectErr: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + matcher, err := NewSubProjectMatcher(tc.subProjects) + if tc.expectErr { + assert.NotNil(t, err) + assert.Nil(t, matcher) + return + } + assert.Nil(t, err) + assert.NotNil(t, matcher) + }) + } +} + +func TestDecodeAndValidateTaskOptions(t *testing.T) { + t.Run("valid options", func(t *testing.T) { + op, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + "subProjects": []interface{}{ + map[string]interface{}{ + "name": "serviceA", + "prLabels": []interface{}{"serviceA"}, + "deployJobPattern": "^deploy-serviceA$", + }, + }, + }) + assert.Nil(t, err) + assert.Equal(t, "monorepo", op.ProjectName) + assert.Len(t, op.SubProjects, 1) + assert.Equal(t, "serviceA", op.SubProjects[0].Name) + assert.Equal(t, []string{"serviceA"}, op.SubProjects[0].PrLabels) + assert.Equal(t, "^deploy-serviceA$", op.SubProjects[0].DeployJobPattern) + }) + + t.Run("missing projectName is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "subProjects": []interface{}{ + map[string]interface{}{"name": "serviceA"}, + }, + }) + assert.NotNil(t, err) + }) + + t.Run("missing subProjects is rejected", func(t *testing.T) { + _, err := DecodeAndValidateTaskOptions(map[string]interface{}{ + "projectName": "monorepo", + }) + assert.NotNil(t, err) + }) +} diff --git a/grafana/dashboards/mysql/monorepo-subprojects.json b/grafana/dashboards/mysql/monorepo-subprojects.json new file mode 100644 index 00000000000..60d1fec5f70 --- /dev/null +++ b/grafana/dashboards/mysql/monorepo-subprojects.json @@ -0,0 +1,340 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "datasource", "uid": "grafana" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Per-sub-project DORA metrics for a monorepo, produced by the monorepo plugin. Deployments are attributed by CI job name, pull requests by label.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { "type": "datasource", "uid": "grafana" }, + "gridPos": { "h": 3, "w": 24, "x": 0, "y": 0 }, + "id": 10, + "options": { + "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, + "content": "## Monorepo Sub-Projects\n\nEach **sub-project** is a logical project inside a single Git repository. Deployments are attributed by **CI job name**, pull requests by **label**. Only Deployment Frequency and Lead Time for Changes are available — Change Failure Rate and Time to Restore require incident data, which this plugin does not attribute.", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "title": "", + "type": "text" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Number of successful production deployments per sub-project in the selected time range. A sub-project showing zero usually means its deployJobPattern matches no CI job.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "scaleDistribution": { "type": "linear" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 3 }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { "mode": "single", "sort": "none" }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT sub_project AS 'Sub-Project', COUNT(DISTINCT cicd_deployment_id) AS 'Deployments'\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "refId": "A", + "sql": { "columns": [{ "parameters": [], "type": "function" }], "groupBy": [{ "property": { "type": "string" }, "type": "groupBy" }], "limit": 50 } + } + ], + "title": "Deployment Count by Sub-Project", + "type": "barchart" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Successful production deployments over time, one series per sub-project. This is the metric that a plain DevLake setup cannot separate: without attribution every sub-project shows the whole repository's deployment count.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 9, "w": 12, "x": 12, "y": 3 }, + "id": 2, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n DATE(finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A" + } + ], + "title": "Deployment Frequency over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Average change lead time per sub-project, broken into its stages. Coding, pickup and review come from DORA unchanged; the deploy leg is recomputed against this sub-project's own deployments. All values in hours.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { + "align": "auto", + "cellOptions": { "type": "auto" }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Cycle Time (h)" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-text" } }, + { "id": "color", "value": { "fixedColor": "blue", "mode": "fixed" } } + ] + } + ] + }, + "gridPos": { "h": 9, "w": 12, "x": 0, "y": 12 }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n sub_project AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "refId": "A" + } + ], + "title": "Change Lead Time Breakdown by Sub-Project", + "type": "table" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Merged pull requests carrying none of the configured sub-project labels. These are invisible to every metric on this dashboard. A rising number means labelling discipline is slipping, not that the code is wrong.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "#EAB839", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 12, "y": 12 }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.table = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", + "refId": "A" + } + ], + "title": "Unattributed PRs", + "type": "stat" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Merged pull requests that have been attributed to a sub-project but that no deployment has shipped yet, or whose shipping deployment could not be identified.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "noValue": "0", + "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 6, "x": 18, "y": 12 }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", + "refId": "A" + } + ], + "title": "Awaiting Deployment", + "type": "stat" + }, + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution — useful for checking a deployJobPattern is matching what you expect.", + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": true, "inspect": false }, + "mappings": [], + "noValue": "-", + "thresholds": { "mode": "absolute", "steps": [{ "color": "text", "value": null }] } + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Result" }, + "properties": [ + { "id": "custom.cellOptions", "value": { "type": "color-text" } }, + { + "id": "mappings", + "value": [ + { "options": { "SUCCESS": { "color": "green", "index": 0 } }, "type": "value" }, + { "options": { "FAILURE": { "color": "red", "index": 1 } }, "type": "value" } + ] + } + ] + } + ] + }, + "gridPos": { "h": 10, "w": 24, "x": 0, "y": 21 }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, + "showHeader": true, + "sortBy": [{ "desc": true, "displayName": "Finished" }] + }, + "targets": [ + { + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n finished_date AS 'Finished',\n sub_project AS 'Sub-Project',\n job_name AS 'CI Job',\n environment AS 'Environment',\n result AS 'Result',\n cicd_deployment_id AS 'Deployment',\n commit_sha AS 'Commit'\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "refId": "A" + } + ], + "title": "Attributed Deployments", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": ["monorepo", "dora"], + "templating": { + "list": [ + { + "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, + "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, + "definition": "select distinct name from projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "select distinct name from projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { "from": "now-90d", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "Monorepo Sub-Projects", + "uid": "monorepo-subprojects", + "version": 1, + "weekStart": "" +} diff --git a/grafana/dashboards/postgresql/monorepo-subprojects.json b/grafana/dashboards/postgresql/monorepo-subprojects.json new file mode 100644 index 00000000000..d9a6f7086b3 --- /dev/null +++ b/grafana/dashboards/postgresql/monorepo-subprojects.json @@ -0,0 +1,628 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Per-sub-project DORA metrics for a monorepo, produced by the monorepo plugin. Deployments are attributed by CI job name, pull requests by label.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "## Monorepo Sub-Projects\n\nEach **sub-project** is a logical project inside a single Git repository. Deployments are attributed by **CI job name**, pull requests by **label**. Only Deployment Frequency and Lead Time for Changes are available \u2014 Change Failure Rate and Time to Restore require incident data, which this plugin does not attribute.", + "mode": "markdown" + }, + "pluginVersion": "10.1.0", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Number of successful production deployments per sub-project in the selected time range. A sub-project showing zero usually means its deployJobPattern matches no CI job.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 3 + }, + "id": 1, + "options": { + "barRadius": 0, + "barWidth": 0.7, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT sub_project AS \"Sub-Project\", COUNT(DISTINCT cicd_deployment_id) AS \"Deployments\"\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Deployment Count by Sub-Project", + "type": "barchart" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Successful production deployments over time, one series per sub-project. This is the metric that a plain DevLake setup cannot separate: without attribution every sub-project shows the whole repository's deployment count.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisLabel": "deployments", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n date_trunc('day', finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "refId": "A" + } + ], + "title": "Deployment Frequency over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Average change lead time per sub-project, broken into its stages. Coding, pickup and review come from DORA unchanged; the deploy leg is recomputed against this sub-project's own deployments. All values in hours.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Cycle Time (h)" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 12 + }, + "id": 3, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n sub_project AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "refId": "A" + } + ], + "title": "Change Lead Time Breakdown by Sub-Project", + "type": "table" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Merged pull requests carrying none of the configured sub-project labels. These are invisible to every metric on this dashboard. A rising number means labelling discipline is slipping, not that the code is wrong.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "#EAB839", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", + "refId": "A" + } + ], + "title": "Unattributed PRs", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Merged pull requests that have been attributed to a sub-project but that no deployment has shipped yet, or whose shipping deployment could not be identified.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", + "refId": "A" + } + ], + "title": "Awaiting Deployment", + "type": "stat" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution \u2014 useful for checking a deployJobPattern is matching what you expect.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Result" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + }, + { + "id": "mappings", + "value": [ + { + "options": { + "SUCCESS": { + "color": "green", + "index": 0 + } + }, + "type": "value" + }, + { + "options": { + "FAILURE": { + "color": "red", + "index": 1 + } + }, + "type": "value" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Finished" + } + ] + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n finished_date AS \"Finished\",\n sub_project AS \"Sub-Project\",\n job_name AS \"CI Job\",\n environment AS \"Environment\",\n result AS \"Result\",\n cicd_deployment_id AS \"Deployment\",\n commit_sha AS \"Commit\"\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "refId": "A" + } + ], + "title": "Attributed Deployments", + "type": "table" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "monorepo", + "dora" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "devlake-postgres-api" + }, + "definition": "select distinct name from projects", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project", + "options": [], + "query": "select distinct name from projects", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Monorepo Sub-Projects", + "uid": "monorepo-subprojects", + "version": 1, + "weekStart": "" +} From 37a0b6e9a0a71f2030e328e97e158b652ae6a5fb Mon Sep 17 00:00:00 2001 From: Uriah Rokach Date: Sun, 16 Aug 2026 10:58:36 +0300 Subject: [PATCH 2/9] feat(monorepo): add config-ui panel for editing sub-project settings Adds a Project Settings panel for the monorepo plugin, letting users define sub-projects (name, PR labels, deploy job pattern) directly in config-ui rather than only via the raw API, following the existing pattern used for the linker plugin's fields. --- .../routes/project/detail/settings-panel.tsx | 109 ++++++++++++++++++ config-ui/src/routes/project/detail/styled.ts | 13 +++ 2 files changed, 122 insertions(+) diff --git a/config-ui/src/routes/project/detail/settings-panel.tsx b/config-ui/src/routes/project/detail/settings-panel.tsx index b7a78946466..4302f226430 100644 --- a/config-ui/src/routes/project/detail/settings-panel.tsx +++ b/config-ui/src/routes/project/detail/settings-panel.tsx @@ -18,6 +18,7 @@ import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; +import { CloseOutlined, PlusOutlined } from '@ant-design/icons'; import { Flex, Space, Card, Modal, Input, Checkbox, Button } from 'antd'; import API from '@/api'; @@ -30,6 +31,15 @@ import * as S from './styled'; const RegexPrIssueDefaultValue = '(?mi)(Closes)[\\s]*.*(((and )?#\\d+[ ]*)+)'; +interface ISubProject { + name: string; + // Comma-separated in the UI; split into an array on save. + prLabels: string; + deployJobPattern: string; +} + +const emptySubProject: ISubProject = { name: '', prLabels: '', deployJobPattern: '' }; + interface Props { project: IProject; onRefresh: () => void; @@ -47,6 +57,10 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const [issueTrace, setIssueTrace] = useState({ enable: false, }); + const [monorepo, setMonorepo] = useState<{ enable: boolean; subProjects: ISubProject[] }>({ + enable: false, + subProjects: [emptySubProject], + }); const [operating, setOperating] = useState(false); const [open, setOpen] = useState(false); @@ -56,6 +70,7 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { const dora = project.metrics.find((ms) => ms.pluginName === 'dora'); const linker = project.metrics.find((ms) => ms.pluginName === 'linker'); const issueTrace = project.metrics.find((ms) => ms.pluginName === 'issue_trace'); + const monorepo = project.metrics.find((ms) => ms.pluginName === 'monorepo'); setName(project.name); setDora({ @@ -68,8 +83,35 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { setIssueTrace({ enable: issueTrace?.enable ?? false, }); + const subProjects = monorepo?.pluginOption?.subProjects; + setMonorepo({ + enable: monorepo?.enable ?? false, + subProjects: + Array.isArray(subProjects) && subProjects.length + ? subProjects.map((sp: any) => ({ + name: sp.name ?? '', + prLabels: Array.isArray(sp.prLabels) ? sp.prLabels.join(',') : '', + deployJobPattern: sp.deployJobPattern ?? '', + })) + : [emptySubProject], + }); }, [project]); + const handleAddSubProject = () => { + setMonorepo({ ...monorepo, subProjects: [...monorepo.subProjects, { ...emptySubProject }] }); + }; + + const handleDeleteSubProject = (index: number) => { + setMonorepo({ ...monorepo, subProjects: monorepo.subProjects.filter((_, i) => i !== index) }); + }; + + const handleUpdateSubProject = (index: number, field: keyof ISubProject, value: string) => { + setMonorepo({ + ...monorepo, + subProjects: monorepo.subProjects.map((sp, i) => (i === index ? { ...sp, [field]: value } : sp)), + }); + }; + const handleUpdate = async () => { const [success] = await operator( () => @@ -94,6 +136,22 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { pluginOption: {}, enable: issueTrace.enable, }, + { + pluginName: 'monorepo', + pluginOption: { + subProjects: monorepo.subProjects + .filter((sp) => sp.name.trim()) + .map((sp) => ({ + name: sp.name.trim(), + prLabels: sp.prLabels + .split(',') + .map((l) => l.trim()) + .filter((l) => l), + deployJobPattern: sp.deployJobPattern.trim(), + })), + }, + enable: monorepo.enable, + }, ], }), { @@ -191,6 +249,57 @@ export const SettingsPanel = ({ project, onRefresh }: Props) => { } description="Parse the issue status and assignee history from issue changelogs. Currently, only Jira issues are supported." /> + setMonorepo({ ...monorepo, enable: e.target.checked })} + > + Enable Monorepo Sub-Projects + + } + description={ + + Split a single repository into several logical sub-projects for DORA-style metrics. Deployments are + matched by CI job name, pull requests by label. When a pull request carries more than one + sub-project's label, the first matching sub-project in the list below wins. + + + } + > + {monorepo.enable && ( + + {monorepo.subProjects.map((sp, i) => ( +
+ handleUpdateSubProject(i, 'name', e.target.value)} + /> + handleUpdateSubProject(i, 'prLabels', e.target.value)} + /> + handleUpdateSubProject(i, 'deployJobPattern', e.target.value)} + /> + {monorepo.subProjects.length > 1 && ( +
+ ))} + + + )} + + {reservedSubProjectName && ( + + )} )} - From 80816002fed4cef0e71a8b26d2535eb9648a20a1 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Thu, 27 Aug 2026 23:21:31 +0300 Subject: [PATCH 8/9] feat(dashboards): migrate monorepo-subprojects dashboard to core tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrites all 6 panels of monorepo-subprojects.json (MySQL and PostgreSQL) to read from the new core tables (cicd_deployment_commits joined through cicd_deployment_subprojects, and project_pr_metrics) instead of the deprecated monorepo_subproject_deployments/monorepo_subproject_pr_metrics compat tables, per design §5.2's recommendation to migrate this dashboard rather than keep it on the compat tables. Deployment/PR-count panels use COALESCE(sub_project, 'All') so single-repo (non-monorepo) projects still render a sensible group. The "Unattributed merged PRs" panel simplifies to a direct pull_requests.sub_project = 'unattributed' filter now that attribution lives on the PR row itself. The "Attributed Deployments" detail table drops its CI-job-name column, since that information isn't reconstructable from the mapping table alone without re-deriving the regex match in SQL; its description is updated to say so. Note: this is the one dashboard fully migrated in this change. The broader dashboard file list in design §5.2 (Gitlab.json, dora-details-*, engineering-throughput-and-cycle-time*, engineering-overview.json, dora-by-team.json) was surveyed but deliberately NOT modified - see the implementation report for why. Co-Authored-By: Claude Sonnet 5 --- grafana/dashboards/mysql/monorepo-subprojects.json | 14 +++++++------- .../postgresql/monorepo-subprojects.json | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/grafana/dashboards/mysql/monorepo-subprojects.json b/grafana/dashboards/mysql/monorepo-subprojects.json index 60d1fec5f70..0ad2757d998 100644 --- a/grafana/dashboards/mysql/monorepo-subprojects.json +++ b/grafana/dashboards/mysql/monorepo-subprojects.json @@ -76,7 +76,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sub_project AS 'Sub-Project', COUNT(DISTINCT cicd_deployment_id) AS 'Deployments'\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "rawSql": "SELECT COALESCE(ds.sub_project, 'All') AS 'Sub-Project', COUNT(DISTINCT dc.cicd_deployment_id) AS 'Deployments'\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1\nORDER BY 2 DESC", "refId": "A", "sql": { "columns": [{ "parameters": [], "type": "function" }], "groupBy": [{ "property": { "type": "string" }, "type": "groupBy" }], "limit": 50 } } @@ -126,7 +126,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n DATE(finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "rawSql": "SELECT\n DATE(dc.finished_date) AS time,\n COALESCE(ds.sub_project, 'All') AS metric,\n COUNT(DISTINCT dc.cicd_deployment_id) AS value\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" } ], @@ -172,7 +172,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n sub_project AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "rawSql": "SELECT\n COALESCE(sub_project, 'All') AS 'Sub-Project',\n COUNT(*) AS 'PRs',\n ROUND(AVG(pr_coding_time) / 60, 1) AS 'Coding (h)',\n ROUND(AVG(pr_pickup_time) / 60, 1) AS 'Pickup (h)',\n ROUND(AVG(pr_review_time) / 60, 1) AS 'Review (h)',\n ROUND(AVG(pr_deploy_time) / 60, 1) AS 'Deploy (h)',\n ROUND(AVG(pr_cycle_time) / 60, 1) AS 'Cycle Time (h)'\nFROM project_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], @@ -214,7 +214,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.table = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", + "rawSql": "SELECT COUNT(*) AS 'Unattributed merged PRs'\nFROM pull_requests pr\nJOIN project_mapping pm ON (pm.`table` = 'repos' AND pm.row_id = pr.base_repo_id)\nWHERE pr.merged_date IS NOT NULL\n AND pr.sub_project = 'unattributed'\n AND ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(pr.merged_date)", "refId": "A" } ], @@ -249,7 +249,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", + "rawSql": "SELECT COUNT(*) AS 'Merged, not yet deployed'\nFROM project_pr_metrics\nWHERE (deployment_commit_id = '' OR deployment_commit_id IS NULL)\n AND sub_project IS NOT NULL\n AND ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(pr_merged_date)", "refId": "A" } ], @@ -258,7 +258,7 @@ }, { "datasource": { "type": "mysql", "uid": "devlake-mysql-api" }, - "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution — useful for checking a deployJobPattern is matching what you expect.", + "description": "Every attributed deployment, newest first, sourced from cicd_deployment_commits joined through the core cicd_deployment_subprojects mapping table.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, @@ -297,7 +297,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n finished_date AS 'Finished',\n sub_project AS 'Sub-Project',\n job_name AS 'CI Job',\n environment AS 'Environment',\n result AS 'Result',\n cicd_deployment_id AS 'Deployment',\n commit_sha AS 'Commit'\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name IN (${project:singlequote}))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "rawSql": "SELECT\n dc.finished_date AS 'Finished',\n COALESCE(ds.sub_project, 'All') AS 'Sub-Project',\n dc.environment AS 'Environment',\n dc.result AS 'Result',\n dc.cicd_deployment_id AS 'Deployment',\n dc.commit_sha AS 'Commit'\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.`table` = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE ('${project:csv}' = '' OR pm.project_name IN (${project:singlequote}))\n AND $__timeFilter(dc.finished_date)\nORDER BY dc.finished_date DESC\nLIMIT 200", "refId": "A" } ], diff --git a/grafana/dashboards/postgresql/monorepo-subprojects.json b/grafana/dashboards/postgresql/monorepo-subprojects.json index d9a6f7086b3..372860d58c1 100644 --- a/grafana/dashboards/postgresql/monorepo-subprojects.json +++ b/grafana/dashboards/postgresql/monorepo-subprojects.json @@ -128,7 +128,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT sub_project AS \"Sub-Project\", COUNT(DISTINCT cicd_deployment_id) AS \"Deployments\"\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY sub_project\nORDER BY 2 DESC", + "rawSql": "SELECT COALESCE(ds.sub_project, 'All') AS \"Sub-Project\", COUNT(DISTINCT dc.cicd_deployment_id) AS \"Deployments\"\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1\nORDER BY 2 DESC", "refId": "A", "sql": { "columns": [ @@ -234,7 +234,7 @@ "editorMode": "code", "format": "time_series", "rawQuery": true, - "rawSql": "SELECT\n date_trunc('day', finished_date) AS time,\n sub_project AS metric,\n COUNT(DISTINCT cicd_deployment_id) AS value\nFROM monorepo_subproject_deployments\nWHERE result = 'SUCCESS'\n AND environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nGROUP BY 1, 2\nORDER BY 1", + "rawSql": "SELECT\n date_trunc('day', dc.finished_date) AS time,\n COALESCE(ds.sub_project, 'All') AS metric,\n COUNT(DISTINCT dc.cicd_deployment_id) AS value\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE dc.result = 'SUCCESS'\n AND dc.environment = 'PRODUCTION'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nGROUP BY 1, 2\nORDER BY 1", "refId": "A" } ], @@ -324,7 +324,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n sub_project AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM monorepo_subproject_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY sub_project\nORDER BY 1", + "rawSql": "SELECT\n COALESCE(sub_project, 'All') AS \"Sub-Project\",\n COUNT(*) AS \"PRs\",\n ROUND((AVG(pr_coding_time) / 60)::numeric, 1) AS \"Coding (h)\",\n ROUND((AVG(pr_pickup_time) / 60)::numeric, 1) AS \"Pickup (h)\",\n ROUND((AVG(pr_review_time) / 60)::numeric, 1) AS \"Review (h)\",\n ROUND((AVG(pr_deploy_time) / 60)::numeric, 1) AS \"Deploy (h)\",\n ROUND((AVG(pr_cycle_time) / 60)::numeric, 1) AS \"Cycle Time (h)\"\nFROM project_pr_metrics\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)\nGROUP BY 1\nORDER BY 1", "refId": "A" } ], @@ -394,7 +394,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm\n ON pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id\nLEFT JOIN monorepo_subproject_pr_metrics m\n ON m.pull_request_id = pr.id AND m.project_name = pm.project_name\nWHERE pr.merged_date IS NOT NULL\n AND m.pull_request_id IS NULL\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", + "rawSql": "SELECT COUNT(*) AS \"Unattributed merged PRs\"\nFROM pull_requests pr\nJOIN project_mapping pm ON (pm.\"table\" = 'repos' AND pm.row_id = pr.base_repo_id)\nWHERE pr.merged_date IS NOT NULL\n AND pr.sub_project = 'unattributed'\n AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr.merged_date)", "refId": "A" } ], @@ -456,7 +456,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM monorepo_subproject_pr_metrics\nWHERE deployment_id = ''\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", + "rawSql": "SELECT COUNT(*) AS \"Merged, not yet deployed\"\nFROM project_pr_metrics\nWHERE (deployment_commit_id = '' OR deployment_commit_id IS NULL)\n AND sub_project IS NOT NULL\n AND ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(pr_merged_date)", "refId": "A" } ], @@ -468,7 +468,7 @@ "type": "grafana-postgresql-datasource", "uid": "devlake-postgres-api" }, - "description": "Every attributed deployment, newest first. The job name column shows which CI job caused the attribution \u2014 useful for checking a deployJobPattern is matching what you expect.", + "description": "Every attributed deployment, newest first, sourced from cicd_deployment_commits joined through the core cicd_deployment_subprojects mapping table.", "fieldConfig": { "defaults": { "color": { @@ -568,7 +568,7 @@ "editorMode": "code", "format": "table", "rawQuery": true, - "rawSql": "SELECT\n finished_date AS \"Finished\",\n sub_project AS \"Sub-Project\",\n job_name AS \"CI Job\",\n environment AS \"Environment\",\n result AS \"Result\",\n cicd_deployment_id AS \"Deployment\",\n commit_sha AS \"Commit\"\nFROM monorepo_subproject_deployments\nWHERE ('${project:csv}' = '' OR project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(finished_date)\nORDER BY finished_date DESC\nLIMIT 200", + "rawSql": "SELECT\n dc.finished_date AS \"Finished\",\n COALESCE(ds.sub_project, 'All') AS \"Sub-Project\",\n dc.environment AS \"Environment\",\n dc.result AS \"Result\",\n dc.cicd_deployment_id AS \"Deployment\",\n dc.commit_sha AS \"Commit\"\nFROM cicd_deployment_commits dc\nJOIN project_mapping pm ON (pm.\"table\" = 'cicd_scopes' AND pm.row_id = dc.cicd_scope_id)\nLEFT JOIN cicd_deployment_subprojects ds\n ON ds.cicd_deployment_id = dc.cicd_deployment_id AND ds.project_name = pm.project_name\nWHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[]))\n AND $__timeFilter(dc.finished_date)\nORDER BY dc.finished_date DESC\nLIMIT 200", "refId": "A" } ], From c708d1697ed9cbc0401602e1fc5eff486a72ae05 Mon Sep 17 00:00:00 2001 From: Yotam Soudry Date: Thu, 27 Aug 2026 23:40:07 +0300 Subject: [PATCH 9/9] feat(dashboards): add sub_project grouping to single-metric PR time series MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the mechanical COALESCE(sub_project, 'All') transform (design §5.1/§5.3) to every panel across Gitlab.json, engineering-overview.json and engineering-throughput-and-cycle-time.json (MySQL + PostgreSQL) that is safely convertible: a time series with exactly one time column and one numeric value column, which Grafana's long-format convention already renders as a single line and will now render as one line per sub-project once a text grouping column is added. sub_project is inserted as the second SELECT column (matching the design doc's own example) and added to GROUP BY; original value column names/aliases are preserved. Panels deliberately left untouched, and why: - Grafana "stat" panels (single KPI tiles): their reduceOptions is configured with values:false, meaning they collapse all returned rows into one number via "last non-null". Adding a GROUP BY would silently make the tile show an arbitrary single sub-project's value instead of the whole-project total - fixing that needs a reduceOptions change too, which needs Grafana to verify renders correctly. - Wide-format time series with multiple value columns (e.g. "PRs Opened/Merged" - two metric columns in one row) and barchart/table panels already grouping by something else (e.g. "Top 20 Contributors by Merged PRs" groups by author) - adding sub_project would multiply or reshape the series in ways that need a Grafana-side call, not a mechanical SQL edit. - Every dora-details-*.json and dora-by-team.json panel, and most of engineering-throughput-and-cycle-time-team-view.json: multi-CTE queries with window functions (percent_rank, row_number, calendar-generation CTEs for medians). Retrofitting a grouping dimension into these correctly means restructuring most of the CTEs, which is real engineering work that needs a live Grafana instance to verify - not safe to do blind. Co-Authored-By: Claude Sonnet 5 --- grafana/dashboards/mysql/Gitlab.json | 2 +- grafana/dashboards/mysql/engineering-overview.json | 4 ++-- .../mysql/engineering-throughput-and-cycle-time.json | 4 ++-- grafana/dashboards/postgresql/Gitlab.json | 2 +- grafana/dashboards/postgresql/engineering-overview.json | 4 ++-- .../postgresql/engineering-throughput-and-cycle-time.json | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/grafana/dashboards/mysql/Gitlab.json b/grafana/dashboards/mysql/Gitlab.json index 3a7043b69ec..b0dec184bf6 100644 --- a/grafana/dashboards/mysql/Gitlab.json +++ b/grafana/dashboards/mysql/Gitlab.json @@ -969,7 +969,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "-- The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field `original_status`\nSELECT\n DATE_ADD(date(created_date), INTERVAL -$interval(date(created_date))+1 DAY) as time,\n count(distinct case when status = 'CLOSED' then id else null end)/count(distinct case when status in ('CLOSED', 'MERGED') then id else null end) as ratio\nFROM pull_requests\nWHERE\n $__timeFilter(created_date)\n and base_repo_id in (${repo_id})\ngroup by 1\n", + "rawSql": "-- The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field `original_status`\nSELECT\n DATE_ADD(date(created_date), INTERVAL -$interval(date(created_date))+1 DAY) as time,\n COALESCE(sub_project, 'All') as sub_project,\n count(distinct case when status = 'CLOSED' then id else null end)/count(distinct case when status in ('CLOSED', 'MERGED') then id else null end) as ratio\nFROM pull_requests\nWHERE\n $__timeFilter(created_date)\n and base_repo_id in (${repo_id})\ngroup by 1, 2", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/mysql/engineering-overview.json b/grafana/dashboards/mysql/engineering-overview.json index ad6d5dfefb6..6e4f2be7c07 100644 --- a/grafana/dashboards/mysql/engineering-overview.json +++ b/grafana/dashboards/mysql/engineering-overview.json @@ -1483,7 +1483,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY) as time,\n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere pm.project_name in (${project})\nand $__timeFilter(created_date)\nand created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time\n\n", + "rawSql": "select\n DATE_ADD(date(created_date), INTERVAL -DAY(date(created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n 100*count(distinct case when pr.id in (select pull_request_id from pull_request_issues) then pr.id else null end)/count(distinct pr.id) as unlinked_pr_rate\nfrom pull_requests pr\njoin project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere pm.project_name in (${project})\nand $__timeFilter(created_date)\nand created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time, pr.sub_project", "refId": "A", "select": [ [ @@ -1969,7 +1969,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "select\n DATE_ADD(date(pr.created_date), INTERVAL -DAY(date(pr.created_date))+1 DAY) as time,\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440) as pr_time_to_merge_in_days\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and $__timeFilter(pr.created_date)\n and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time\norder by time", + "rawSql": "select\n DATE_ADD(date(pr.created_date), INTERVAL -DAY(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n AVG(TIMESTAMPDIFF(MINUTE, pr.created_date, pr.merged_date) / 1440) as pr_time_to_merge_in_days\nfrom\n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nwhere\n pm.project_name in (${project}) and\n pr.merged_date is not null\n and $__timeFilter(pr.created_date)\n and pr.created_date >= DATE_ADD(DATE_ADD($__timeFrom(), INTERVAL -DAY($__timeFrom())+1 DAY), INTERVAL +1 MONTH)\ngroup by time, pr.sub_project\norder by time", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json index fd62b4e053f..3db70532cd3 100644 --- a/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json +++ b/grafana/dashboards/mysql/engineering-throughput-and-cycle-time.json @@ -622,7 +622,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n count(distinct prc.id)/count(distinct pr.id) as \"PR Review Depth\"\nFROM \n pull_requests pr\n left join pull_request_comments prc on pr.id = prc.pull_request_id\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\ngroup by 1\n", + "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n count(distinct prc.id)/count(distinct pr.id) as \"PR Review Depth\"\nFROM \n pull_requests pr\n left join pull_request_comments prc on pr.id = prc.pull_request_id\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\ngroup by 1, 2", "refId": "A", "select": [ [ @@ -1004,7 +1004,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n sum(case when pr.id not in (SELECT pull_request_id FROM pull_request_comments) then 1 else 0 end) as \"PRs Merged w/o Review\"\nFROM \n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\nGROUP BY 1\nORDER BY 1", + "rawSql": "SELECT\n DATE_ADD(date(pr.created_date), INTERVAL -$interval(date(pr.created_date))+1 DAY) as time,\n COALESCE(pr.sub_project, 'All') as sub_project,\n sum(case when pr.id not in (SELECT pull_request_id FROM pull_request_comments) then 1 else 0 end) as \"PRs Merged w/o Review\"\nFROM \n pull_requests pr\n join project_mapping pm on pr.base_repo_id = pm.row_id and pm.table = 'repos' \nWHERE\n $__timeFilter(pr.created_date)\n and pm.project_name in (${project})\n and pr.merged_date is not null\nGROUP BY 1, 2\nORDER BY 1", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/Gitlab.json b/grafana/dashboards/postgresql/Gitlab.json index af78160fc19..d038db58a23 100644 --- a/grafana/dashboards/postgresql/Gitlab.json +++ b/grafana/dashboards/postgresql/Gitlab.json @@ -969,7 +969,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1", + "rawSql": "/* The PR/MR statuses are standardized to 'OPEN', 'MERGED' and 'CLOSED'. You can check out the original status from the field \"original_status\" */ SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(created_date AS DATE)) - 1) END + 1) AS time, COALESCE(sub_project, 'All') AS sub_project, CAST(COUNT(DISTINCT CASE WHEN status = 'CLOSED' THEN id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT CASE WHEN status IN ('CLOSED', 'MERGED') THEN id ELSE NULL END), 0) AS ratio FROM pull_requests WHERE $__timeFilter(created_date) AND ('${repo_id:csv}' = '' OR base_repo_id::text = ANY(ARRAY[${repo_id:singlequote}]::text[])) GROUP BY 1, 2", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/engineering-overview.json b/grafana/dashboards/postgresql/engineering-overview.json index 6d22a879f9c..287973048ca 100644 --- a/grafana/dashboards/postgresql/engineering-overview.json +++ b/grafana/dashboards/postgresql/engineering-overview.json @@ -1483,7 +1483,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\"", + "rawSql": "SELECT CAST(created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(created_date AS DATE)) + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, CAST(100 * COUNT(DISTINCT CASE WHEN pr.id IN (SELECT pull_request_id FROM pull_request_issues) THEN pr.id ELSE NULL END) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS unlinked_pr_rate FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND $__timeFilter(created_date) AND created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", pr.sub_project", "refId": "A", "select": [ [ @@ -1969,7 +1969,7 @@ "metricColumn": "none", "queryType": "randomWalk", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) + 1) AS time, AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS pr_time_to_merge_in_days FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.created_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\" ORDER BY time NULLS FIRST", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, AVG(CAST((EXTRACT(EPOCH FROM (pr.merged_date - pr.created_date))/60) AS NUMERIC) / NULLIF(1440, 0)) AS pr_time_to_merge_in_days FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL AND $__timeFilter(pr.created_date) AND pr.created_date >= $__timeFrom()::timestamp + INTERVAL '1 DAY' * (-EXTRACT(DAY FROM CAST($__timeFrom() AS DATE)) + 1) + INTERVAL '1 MONTH' GROUP BY \"time\", pr.sub_project ORDER BY time NULLS FIRST", "refId": "A", "select": [ [ diff --git a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json index 174ad11e25e..6abd47e0afd 100644 --- a/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json +++ b/grafana/dashboards/postgresql/engineering-throughput-and-cycle-time.json @@ -622,7 +622,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, CAST(COUNT(DISTINCT prc.id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS \"PR Review Depth\" FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, CAST(COUNT(DISTINCT prc.id) AS NUMERIC) / NULLIF(COUNT(DISTINCT pr.id), 0) AS \"PR Review Depth\" FROM pull_requests AS pr LEFT JOIN pull_request_comments AS prc ON pr.id = prc.pull_request_id JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1, 2", "refId": "A", "select": [ [ @@ -1004,7 +1004,7 @@ "hide": false, "metricColumn": "none", "rawQuery": true, - "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, SUM(CASE WHEN NOT pr.id IN (SELECT pull_request_id FROM pull_request_comments) THEN 1 ELSE 0 END) AS \"PRs Merged w/o Review\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1 ORDER BY 1 NULLS FIRST", + "rawSql": "SELECT CAST(pr.created_date AS DATE) + INTERVAL '1 DAY' * (-CASE WHEN '$interval' = 'DAYOFMONTH' THEN EXTRACT(DAY FROM CAST(pr.created_date AS DATE)) ELSE (EXTRACT(ISODOW FROM CAST(pr.created_date AS DATE)) - 1) END + 1) AS time, COALESCE(pr.sub_project, 'All') AS sub_project, SUM(CASE WHEN NOT pr.id IN (SELECT pull_request_id FROM pull_request_comments) THEN 1 ELSE 0 END) AS \"PRs Merged w/o Review\" FROM pull_requests AS pr JOIN project_mapping AS pm ON pr.base_repo_id = pm.row_id AND pm.table = 'repos' WHERE $__timeFilter(pr.created_date) AND ('${project:csv}' = '' OR pm.project_name::text = ANY(ARRAY[${project:singlequote}]::text[])) AND NOT pr.merged_date IS NULL GROUP BY 1, 2 ORDER BY 1 NULLS FIRST", "refId": "A", "select": [ [