fix: count bot-merged prs and bot-closed issues in development pipes

services/libs/tinybird/pipes/issue_analysis_copy_pipe.pipe

@@ -1,5 +1,49 @@
 DESCRIPTION >
     Compacts activities from same issue into one, keeping necessary information in a single row. Helps to serve issue-wide widgets in the development tab.
+    Opened and closed events are unfiltered, so issues opened or closed by bots (e.g. stale bots) are included, the same way
+    `pull_requests_analyzed` includes bot activity. Issues are still limited to enabled repositories, matching the cleaned buckets.
+    The unfiltered buckets are refreshed once a day, so the opened and closed events also include the realtime MV
+    (`activityRelations_enrich_snapshot_MV_ds`), which holds the last few days of activity changes, including the ones the daily
+    bucket merge has not taken in yet. This keeps issues as fresh as the hourly cleaned buckets. The MV is read whole: looking up
+    each bucket's latest snapshot scans the bucket datasources and is too slow for the push-time check.
+    The first non-author response still comes from the bot-filtered `activityRelations_deduplicated_cleaned_bucket_union`, so an
+    automated bot comment does not count as a response.
+
+NODE issue_events_delta
+DESCRIPTION >
+    Issue opened and closed events from the realtime MV, keeping the most recently updated version of each activity.
+
+SQL >
+    SELECT
+        activityId,
+        sourceId,
+        sourceParentId,
+        timestamp,
+        segmentId,
+        channel,
+        platform,
+        memberId,
+        organizationId,
+        type
+    FROM activityRelations_enrich_snapshot_MV_ds
+    WHERE type IN ('issues-opened', 'issues-closed')
+    ORDER BY updatedAt DESC
+    LIMIT 1 BY segmentId, timestamp, type, platform, channel, sourceId
+
+NODE issues_opened_events
+DESCRIPTION >
+    Issue opened events: the daily unfiltered buckets, with any activity that also appears in the MV replaced by its MV version.
+
+SQL >
+    SELECT activityId, sourceId, timestamp, segmentId, channel, platform, memberId, organizationId
+    FROM activityRelations_enriched_deduplicated_bucket_union
+    WHERE
+        type = 'issues-opened'
+        AND activityId NOT IN (SELECT activityId FROM issue_events_delta WHERE type = 'issues-opened')
+    UNION ALL
+    SELECT activityId, sourceId, timestamp, segmentId, channel, platform, memberId, organizationId
+    FROM issue_events_delta
+    WHERE type = 'issues-opened'
 
 NODE issues_opened
 SQL >
@@ -12,14 +56,36 @@ SQL >
         platform,
         memberId,
         organizationId
-    FROM activityRelations_deduplicated_cleaned_bucket_union
-    WHERE type = 'issues-opened' AND toYear(timestamp) >= 1971
+    FROM issues_opened_events
+    WHERE
+        toYear(timestamp) >= 1971
+        AND (
+            (
+                platform IN ('git', 'gerrit', 'github', 'gitlab')
+                AND (lower(channel), segmentId)
+                IN (SELECT lower(channel), segmentId FROM repos_to_channels)
+            )
+            OR platform NOT IN ('git', 'gerrit', 'github', 'gitlab')
+        )
 
 NODE issues_closed
+DESCRIPTION >
+    First close per issue from the daily buckets and the MV together. A close present in both counts the same under MIN, so no
+    deduplication is needed here.
+
 SQL >
     SELECT sourceParentId, MIN(timestamp) AS closedAt
-    FROM activityRelations_deduplicated_cleaned_bucket_union
-    WHERE type = 'issues-closed' AND sourceParentId != '' AND toYear(timestamp) >= 1971
+    FROM
+        (
+            SELECT sourceParentId, timestamp
+            FROM activityRelations_enriched_deduplicated_bucket_union
+            WHERE type = 'issues-closed'
+            UNION ALL
+            SELECT sourceParentId, timestamp
+            FROM issue_events_delta
+            WHERE type = 'issues-closed'
+        )
+    WHERE sourceParentId != '' AND toYear(timestamp) >= 1971
     GROUP BY sourceParentId
 
 NODE issues_comment_non_author

services/libs/tinybird/pipes/issues_count.pipe

@@ -0,0 +1,93 @@
+DESCRIPTION >
+    - `issues_count.pipe` serves the opened and closed counts of the "Development - Issues resolution" widget.
+    - Counts issues from `issues_analyzed`, which holds one row per issue, so each issue is counted once per state.
+    - Includes issues opened or closed by bots (e.g. stale bots), since `issues_analyzed` is built from the unfiltered activity source.
+    - `state` picks the lifecycle timestamp the count is based on: `openedAt` or `closedAt`. Issues without that timestamp are not counted.
+    - **When `granularity` is NOT provided, returns a single total count** across the entire time range.
+    - **When `granularity` is provided, returns time-series data** aggregated by period (daily, weekly, monthly, quarterly, yearly), using `generate_timeseries` so periods without issues return 0.
+    - The pipe scopes data either to a single project (via `segments_filtered`) or to every project in a
+    collection (via `segments_filtered_by_collection`). Exactly one of `project`/`collectionSlug` must be given.
+    - Parameters:
+    - `project`: Project slug (e.g., 'k8s', 'tensorflow') - inherited from `segments_filtered`. Required unless `collectionSlug` is given.
+    - `collectionSlug`: Collection slug (e.g., 'cncf') - inherited from `segments_filtered_by_collection`. Required unless `project` is given.
+    - `repos`: Optional array of repository URLs for filtering (e.g., ['https://github.com/kubernetes/kubernetes'])
+    - `state`: Optional string, one of 'opened' (default) or 'closed'
+    - `startDate`: Optional DateTime filter for the `state` timestamp after this value (e.g., '2024-01-01 00:00:00')
+    - `endDate`: Optional DateTime filter for the `state` timestamp before this value (e.g., '2024-12-31 23:59:59')
+    - `platform`: Optional string filter for source platform (e.g., 'github', 'gitlab')
+    - `granularity`: Optional string for time aggregation ('daily', 'weekly', 'monthly', 'quarterly', 'yearly')
+    - Response:
+    - Without granularity: `issueCount` (total count)
+    - With granularity: `startDate`, `endDate`, and `issueCount` for each time period
+
+TOKEN "insights-app-token" READ
+
+TAGS "Development metrics", "Widget", "Issues"
+
+NODE issues_by_state
+SQL >
+    %
+    SELECT
+        id,
+        {% if String(
+            state, 'opened', description="One of 'opened' or 'closed'", required=False
+        ) == 'closed' %} closedAt
+        {% else %} openedAt
+        {% end %} AS eventAt
+    FROM issues_analyzed
+    WHERE
+        {% if defined(collectionSlug) %}
+            segmentId IN (SELECT segmentId FROM segments_filtered_by_collection)
+        {% else %} segmentId = (SELECT segmentId FROM segments_filtered)
+        {% end %} AND channel IN (SELECT channel FROM repos_to_channels) AND isNotNull(eventAt)
+        {% if defined(platform) %}
+            AND platform
+            = {{
+                String(
+                    platform,
+                    description="Filter by platform (e.g., 'github', 'gitlab')",
+                    required=False,
+                )
+            }}
+        {% end %}
+        {% if defined(startDate) %}
+            AND eventAt
+            > {{ DateTime(startDate, description="Filter state timestamp after", required=False) }}
+        {% end %}
+        {% if defined(endDate) %}
+            AND eventAt
+            < {{ DateTime(endDate, description="Filter state timestamp before", required=False) }}
+        {% end %}
+
+NODE timeseries_generation_for_issues_count
+SQL >
+    %
+    {% if defined(granularity) %}
+        SELECT ds."startDate", ds."endDate", countIf(iss.id != '') AS "issueCount"
+        FROM generate_timeseries ds
+        LEFT JOIN
+            issues_by_state iss
+            ON CASE
+                WHEN {{ granularity }} = 'daily'
+                THEN toDate(iss.eventAt)
+                WHEN {{ granularity }} = 'weekly'
+                THEN toStartOfWeek(iss.eventAt)
+                WHEN {{ granularity }} = 'monthly'
+                THEN toStartOfMonth(iss.eventAt)
+                WHEN {{ granularity }} = 'quarterly'
+                THEN toStartOfQuarter(iss.eventAt)
+                WHEN {{ granularity }} = 'yearly'
+                THEN toStartOfYear(iss.eventAt)
+            END
+            = ds."startDate"
+        GROUP BY ds."startDate", ds."endDate"
+        ORDER BY ds."startDate"
+    {% else %} SELECT 1
+    {% end %}
+
+NODE issues_count_merged
+SQL >
+    %
+    {% if not defined(granularity) %} SELECT count() AS "issueCount" FROM issues_by_state
+    {% else %} SELECT * FROM timeseries_generation_for_issues_count
+    {% end %}

services/libs/tinybird/pipes/pull_requests_count.pipe

@@ -0,0 +1,97 @@
+DESCRIPTION >
+    - `pull_requests_count.pipe` serves the opened, merged and closed counts of the "Development - Pull requests" widget.
+    - Counts pull requests (GitHub PRs, GitLab MRs and Gerrit changesets) from `pull_requests_filtered`. `pull_requests_analyzed` holds one row per PR, so each PR is counted once per state.
+    - Includes PRs opened, merged or closed by bots. A PR merged by a bot account (e.g. Prow or a merge queue) counts as merged, since `pull_requests_analyzed` takes `mergedAt` from the merge event whoever performed it.
+    - `state` picks the lifecycle timestamp the count is based on: `openedAt`, `mergedAt` or `closedAt`. PRs without that timestamp are not counted.
+    - **When `granularity` is NOT provided, returns a single total count** across the entire time range.
+    - **When `granularity` is provided, returns time-series data** aggregated by period (daily, weekly, monthly, quarterly, yearly), using `generate_timeseries` so periods without PRs return 0.
+    - Parameters:
+    - `project`: Project slug (e.g., 'k8s', 'tensorflow') - inherited from `segments_filtered`. Required unless `collectionSlug` is given.
+    - `collectionSlug`: Collection slug (e.g., 'cncf') - inherited from `segments_filtered_by_collection`. Required unless `project` is given.
+    - `repos`: Optional array of repository URLs for filtering (e.g., ['https://github.com/kubernetes/kubernetes'])
+    - `state`: Optional string, one of 'opened' (default), 'merged' or 'closed'
+    - `startDate`: Optional DateTime filter for the `state` timestamp after this value (e.g., '2024-01-01 00:00:00')
+    - `endDate`: Optional DateTime filter for the `state` timestamp before this value (e.g., '2024-12-31 23:59:59')
+    - `platform`: Optional string filter for source platform (e.g., 'github', 'gitlab', 'gerrit')
+    - `granularity`: Optional string for time aggregation ('daily', 'weekly', 'monthly', 'quarterly', 'yearly')
+    - Response:
+    - Without granularity: `pullRequestCount` (total count)
+    - With granularity: `startDate`, `endDate`, and `pullRequestCount` for each time period
+
+TOKEN "insights-app-token" READ
+
+TAGS "Development metrics", "Widget", "Pull requests"
+
+NODE pull_requests_by_state
+SQL >
+    %
+    SELECT
+        prf.id,
+        {% if String(
+            state,
+            'opened',
+            description="One of 'opened', 'merged' or 'closed'",
+            required=False,
+        ) == 'merged' %} prf.mergedAt
+        {% elif String(
+            state,
+            'opened',
+            description="One of 'opened', 'merged' or 'closed'",
+            required=False,
+        ) == 'closed' %} prf.closedAt
+        {% else %} prf.openedAt
+        {% end %} AS eventAt
+    FROM pull_requests_filtered prf
+    WHERE
+        isNotNull(eventAt)
+        {% if defined(platform) %}
+            AND prf.platform
+            = {{
+                String(
+                    platform,
+                    description="Filter by platform (e.g., 'gerrit', 'github', 'gitlab')",
+                    required=False,
+                )
+            }}
+        {% end %}
+        {% if defined(startDate) %}
+            AND eventAt
+            > {{ DateTime(startDate, description="Filter state timestamp after", required=False) }}
+        {% end %}
+        {% if defined(endDate) %}
+            AND eventAt
+            < {{ DateTime(endDate, description="Filter state timestamp before", required=False) }}
+        {% end %}
+
+NODE timeseries_generation_for_pull_requests_count
+SQL >
+    %
+    {% if defined(granularity) %}
+        SELECT ds."startDate", ds."endDate", countIf(prs.id != '') AS "pullRequestCount"
+        FROM generate_timeseries ds
+        LEFT JOIN
+            pull_requests_by_state prs
+            ON CASE
+                WHEN {{ granularity }} = 'daily'
+                THEN toDate(prs.eventAt)
+                WHEN {{ granularity }} = 'weekly'
+                THEN toStartOfWeek(prs.eventAt)
+                WHEN {{ granularity }} = 'monthly'
+                THEN toStartOfMonth(prs.eventAt)
+                WHEN {{ granularity }} = 'quarterly'
+                THEN toStartOfQuarter(prs.eventAt)
+                WHEN {{ granularity }} = 'yearly'
+                THEN toStartOfYear(prs.eventAt)
+            END
+            = ds."startDate"
+        GROUP BY ds."startDate", ds."endDate"
+        ORDER BY ds."startDate"
+    {% else %} SELECT 1
+    {% end %}
+
+NODE pull_requests_count_merged
+SQL >
+    %
+    {% if not defined(granularity) %} SELECT count() AS "pullRequestCount" FROM pull_requests_by_state
+    {% else %} SELECT * FROM timeseries_generation_for_pull_requests_count
+    {% end %}