diff --git a/services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe b/services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe index bc26c5639f..be8ad3d9b2 100644 --- a/services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe +++ b/services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe @@ -1,5 +1,49 @@ DESCRIPTION > Compacts activities from same issue into one, keeping necessary information in a single row. Helps to serve issue-wide widgets in the development tab. + Opened and closed events are unfiltered, so issues opened or closed by bots (e.g. stale bots) are included, the same way + `pull_requests_analyzed` includes bot activity. Issues are still limited to enabled repositories, matching the cleaned buckets. + The unfiltered buckets are refreshed once a day, so the opened and closed events also include the realtime MV + (`activityRelations_enrich_snapshot_MV_ds`), which holds the last few days of activity changes, including the ones the daily + bucket merge has not taken in yet. This keeps issues as fresh as the hourly cleaned buckets. The MV is read whole: looking up + each bucket's latest snapshot scans the bucket datasources and is too slow for the push-time check. + The first non-author response still comes from the bot-filtered `activityRelations_deduplicated_cleaned_bucket_union`, so an + automated bot comment does not count as a response. + +NODE issue_events_delta +DESCRIPTION > + Issue opened and closed events from the realtime MV, keeping the most recently updated version of each activity. + +SQL > + SELECT + activityId, + sourceId, + sourceParentId, + timestamp, + segmentId, + channel, + platform, + memberId, + organizationId, + type + FROM activityRelations_enrich_snapshot_MV_ds + WHERE type IN ('issues-opened', 'issues-closed') + ORDER BY updatedAt DESC + LIMIT 1 BY segmentId, timestamp, type, platform, channel, sourceId + +NODE issues_opened_events +DESCRIPTION > + Issue opened events: the daily unfiltered buckets, with any activity that also appears in the MV replaced by its MV version. + +SQL > + SELECT activityId, sourceId, timestamp, segmentId, channel, platform, memberId, organizationId + FROM activityRelations_enriched_deduplicated_bucket_union + WHERE + type = 'issues-opened' + AND activityId NOT IN (SELECT activityId FROM issue_events_delta WHERE type = 'issues-opened') + UNION ALL + SELECT activityId, sourceId, timestamp, segmentId, channel, platform, memberId, organizationId + FROM issue_events_delta + WHERE type = 'issues-opened' NODE issues_opened SQL > @@ -12,14 +56,36 @@ SQL > platform, memberId, organizationId - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'issues-opened' AND toYear(timestamp) >= 1971 + FROM issues_opened_events + WHERE + toYear(timestamp) >= 1971 + AND ( + ( + platform IN ('git', 'gerrit', 'github', 'gitlab') + AND (lower(channel), segmentId) + IN (SELECT lower(channel), segmentId FROM repos_to_channels) + ) + OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab') + ) NODE issues_closed +DESCRIPTION > + First close per issue from the daily buckets and the MV together. A close present in both counts the same under MIN, so no + deduplication is needed here. + SQL > SELECT sourceParentId, MIN(timestamp) AS closedAt - FROM activityRelations_deduplicated_cleaned_bucket_union - WHERE type = 'issues-closed' AND sourceParentId != '' AND toYear(timestamp) >= 1971 + FROM + ( + SELECT sourceParentId, timestamp + FROM activityRelations_enriched_deduplicated_bucket_union + WHERE type = 'issues-closed' + UNION ALL + SELECT sourceParentId, timestamp + FROM issue_events_delta + WHERE type = 'issues-closed' + ) + WHERE sourceParentId != '' AND toYear(timestamp) >= 1971 GROUP BY sourceParentId NODE issues_comment_non_author diff --git a/services/libs/tinybird/pipes/issues_count.pipe b/services/libs/tinybird/pipes/issues_count.pipe new file mode 100644 index 0000000000..58a06d20b0 --- /dev/null +++ b/services/libs/tinybird/pipes/issues_count.pipe @@ -0,0 +1,93 @@ +DESCRIPTION > + - `issues_count.pipe` serves the opened and closed counts of the "Development - Issues resolution" widget. + - Counts issues from `issues_analyzed`, which holds one row per issue, so each issue is counted once per state. + - Includes issues opened or closed by bots (e.g. stale bots), since `issues_analyzed` is built from the unfiltered activity source. + - `state` picks the lifecycle timestamp the count is based on: `openedAt` or `closedAt`. Issues without that timestamp are not counted. + - **When `granularity` is NOT provided, returns a single total count** across the entire time range. + - **When `granularity` is provided, returns time-series data** aggregated by period (daily, weekly, monthly, quarterly, yearly), using `generate_timeseries` so periods without issues return 0. + - The pipe scopes data either to a single project (via `segments_filtered`) or to every project in a + collection (via `segments_filtered_by_collection`). Exactly one of `project`/`collectionSlug` must be given. + - Parameters: + - `project`: Project slug (e.g., 'k8s', 'tensorflow') - inherited from `segments_filtered`. Required unless `collectionSlug` is given. + - `collectionSlug`: Collection slug (e.g., 'cncf') - inherited from `segments_filtered_by_collection`. Required unless `project` is given. + - `repos`: Optional array of repository URLs for filtering (e.g., ['https://github.com/kubernetes/kubernetes']) + - `state`: Optional string, one of 'opened' (default) or 'closed' + - `startDate`: Optional DateTime filter for the `state` timestamp after this value (e.g., '2024-01-01 00:00:00') + - `endDate`: Optional DateTime filter for the `state` timestamp before this value (e.g., '2024-12-31 23:59:59') + - `platform`: Optional string filter for source platform (e.g., 'github', 'gitlab') + - `granularity`: Optional string for time aggregation ('daily', 'weekly', 'monthly', 'quarterly', 'yearly') + - Response: + - Without granularity: `issueCount` (total count) + - With granularity: `startDate`, `endDate`, and `issueCount` for each time period + +TOKEN "insights-app-token" READ + +TAGS "Development metrics", "Widget", "Issues" + +NODE issues_by_state +SQL > + % + SELECT + id, + {% if String( + state, 'opened', description="One of 'opened' or 'closed'", required=False + ) == 'closed' %} closedAt + {% else %} openedAt + {% end %} AS eventAt + FROM issues_analyzed + WHERE + {% if defined(collectionSlug) %} + segmentId IN (SELECT segmentId FROM segments_filtered_by_collection) + {% else %} segmentId = (SELECT segmentId FROM segments_filtered) + {% end %} AND channel IN (SELECT channel FROM repos_to_channels) AND isNotNull(eventAt) + {% if defined(platform) %} + AND platform + = {{ + String( + platform, + description="Filter by platform (e.g., 'github', 'gitlab')", + required=False, + ) + }} + {% end %} + {% if defined(startDate) %} + AND eventAt + > {{ DateTime(startDate, description="Filter state timestamp after", required=False) }} + {% end %} + {% if defined(endDate) %} + AND eventAt + < {{ DateTime(endDate, description="Filter state timestamp before", required=False) }} + {% end %} + +NODE timeseries_generation_for_issues_count +SQL > + % + {% if defined(granularity) %} + SELECT ds."startDate", ds."endDate", countIf(iss.id != '') AS "issueCount" + FROM generate_timeseries ds + LEFT JOIN + issues_by_state iss + ON CASE + WHEN {{ granularity }} = 'daily' + THEN toDate(iss.eventAt) + WHEN {{ granularity }} = 'weekly' + THEN toStartOfWeek(iss.eventAt) + WHEN {{ granularity }} = 'monthly' + THEN toStartOfMonth(iss.eventAt) + WHEN {{ granularity }} = 'quarterly' + THEN toStartOfQuarter(iss.eventAt) + WHEN {{ granularity }} = 'yearly' + THEN toStartOfYear(iss.eventAt) + END + = ds."startDate" + GROUP BY ds."startDate", ds."endDate" + ORDER BY ds."startDate" + {% else %} SELECT 1 + {% end %} + +NODE issues_count_merged +SQL > + % + {% if not defined(granularity) %} SELECT count() AS "issueCount" FROM issues_by_state + {% else %} SELECT * FROM timeseries_generation_for_issues_count + {% end %} diff --git a/services/libs/tinybird/pipes/pull_requests_count.pipe b/services/libs/tinybird/pipes/pull_requests_count.pipe new file mode 100644 index 0000000000..2cf97f47ea --- /dev/null +++ b/services/libs/tinybird/pipes/pull_requests_count.pipe @@ -0,0 +1,97 @@ +DESCRIPTION > + - `pull_requests_count.pipe` serves the opened, merged and closed counts of the "Development - Pull requests" widget. + - Counts pull requests (GitHub PRs, GitLab MRs and Gerrit changesets) from `pull_requests_filtered`. `pull_requests_analyzed` holds one row per PR, so each PR is counted once per state. + - Includes PRs opened, merged or closed by bots. A PR merged by a bot account (e.g. Prow or a merge queue) counts as merged, since `pull_requests_analyzed` takes `mergedAt` from the merge event whoever performed it. + - `state` picks the lifecycle timestamp the count is based on: `openedAt`, `mergedAt` or `closedAt`. PRs without that timestamp are not counted. + - **When `granularity` is NOT provided, returns a single total count** across the entire time range. + - **When `granularity` is provided, returns time-series data** aggregated by period (daily, weekly, monthly, quarterly, yearly), using `generate_timeseries` so periods without PRs return 0. + - Parameters: + - `project`: Project slug (e.g., 'k8s', 'tensorflow') - inherited from `segments_filtered`. Required unless `collectionSlug` is given. + - `collectionSlug`: Collection slug (e.g., 'cncf') - inherited from `segments_filtered_by_collection`. Required unless `project` is given. + - `repos`: Optional array of repository URLs for filtering (e.g., ['https://github.com/kubernetes/kubernetes']) + - `state`: Optional string, one of 'opened' (default), 'merged' or 'closed' + - `startDate`: Optional DateTime filter for the `state` timestamp after this value (e.g., '2024-01-01 00:00:00') + - `endDate`: Optional DateTime filter for the `state` timestamp before this value (e.g., '2024-12-31 23:59:59') + - `platform`: Optional string filter for source platform (e.g., 'github', 'gitlab', 'gerrit') + - `granularity`: Optional string for time aggregation ('daily', 'weekly', 'monthly', 'quarterly', 'yearly') + - Response: + - Without granularity: `pullRequestCount` (total count) + - With granularity: `startDate`, `endDate`, and `pullRequestCount` for each time period + +TOKEN "insights-app-token" READ + +TAGS "Development metrics", "Widget", "Pull requests" + +NODE pull_requests_by_state +SQL > + % + SELECT + prf.id, + {% if String( + state, + 'opened', + description="One of 'opened', 'merged' or 'closed'", + required=False, + ) == 'merged' %} prf.mergedAt + {% elif String( + state, + 'opened', + description="One of 'opened', 'merged' or 'closed'", + required=False, + ) == 'closed' %} prf.closedAt + {% else %} prf.openedAt + {% end %} AS eventAt + FROM pull_requests_filtered prf + WHERE + isNotNull(eventAt) + {% if defined(platform) %} + AND prf.platform + = {{ + String( + platform, + description="Filter by platform (e.g., 'gerrit', 'github', 'gitlab')", + required=False, + ) + }} + {% end %} + {% if defined(startDate) %} + AND eventAt + > {{ DateTime(startDate, description="Filter state timestamp after", required=False) }} + {% end %} + {% if defined(endDate) %} + AND eventAt + < {{ DateTime(endDate, description="Filter state timestamp before", required=False) }} + {% end %} + +NODE timeseries_generation_for_pull_requests_count +SQL > + % + {% if defined(granularity) %} + SELECT ds."startDate", ds."endDate", countIf(prs.id != '') AS "pullRequestCount" + FROM generate_timeseries ds + LEFT JOIN + pull_requests_by_state prs + ON CASE + WHEN {{ granularity }} = 'daily' + THEN toDate(prs.eventAt) + WHEN {{ granularity }} = 'weekly' + THEN toStartOfWeek(prs.eventAt) + WHEN {{ granularity }} = 'monthly' + THEN toStartOfMonth(prs.eventAt) + WHEN {{ granularity }} = 'quarterly' + THEN toStartOfQuarter(prs.eventAt) + WHEN {{ granularity }} = 'yearly' + THEN toStartOfYear(prs.eventAt) + END + = ds."startDate" + GROUP BY ds."startDate", ds."endDate" + ORDER BY ds."startDate" + {% else %} SELECT 1 + {% end %} + +NODE pull_requests_count_merged +SQL > + % + {% if not defined(granularity) %} SELECT count() AS "pullRequestCount" FROM pull_requests_by_state + {% else %} SELECT * FROM timeseries_generation_for_pull_requests_count + {% end %}