feat(redmine): add Redmine plugin and dashboard

.github/dependabot.yml

@@ -120,6 +120,7 @@ updates:
       - /backend/python
       - /backend/python/pydevlake
       - /backend/python/plugins/azuredevops
+      - /backend/python/plugins/redmine
       - /backend/python/test/fakeplugin
       - /grafana/scripts
     schedule:

backend/python/plugins/redmine/README.md

@@ -0,0 +1,65 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Redmine
+
+Apache DevLake plugin for ingesting data from [Redmine](https://www.redmine.org/) through the [REST API](https://www.redmine.org/projects/redmine/wiki/Rest_api). Each DevLake scope is a Redmine project.
+
+Collected data is normalized into DevLake's ticket domain so it can be used in Grafana dashboards and cross-domain metrics.
+
+## Collected data
+
+| Redmine resource | Raw table | Tool table | Domain tables |
+| --- | --- | --- | --- |
+| Issues | `_raw_redmine_issues` | `_tool_redmine_issues` | `issues`, `board_issues`, `issue_assignees`, `issue_labels`, `issue_relationships` |
+| Journals (optional) | `_raw_redmine_issuejournals` | `_tool_redmine_journals` | `issue_comments`, `issue_changelogs` |
+| Time entries (optional) | `_raw_redmine_timeentries` | `_tool_redmine_timeentries` | `issue_worklogs` |
+| Projects (scope) | — | `_tool_redmine_projects` | `boards` |
+
+Custom field values stay on `_tool_redmine_issues.custom_fields_json`.
+
+Issues are collected incrementally by `updated_on`. A full sync applies the project sync policy `timeAfter` as a Redmine `created_on` filter.
+
+## Configuration
+
+1. Enable the REST API on the Redmine instance.
+2. Create an API key under **My account → API access key** for a user who can see the projects to sync.
+3. In Config UI, add a Redmine connection with the instance base URL and that API key.
+4. Select projects and, if needed, map tracker and status names. Journals and time entries are off unless enabled on the scope config. Time entries require the Redmine permission **View time entries**.
+
+### Default mappings
+
+When a scope config does not override a name, the plugin uses these defaults. Any other tracker becomes `TASK`. Any other status becomes `OTHER`.
+
+| Redmine tracker | DevLake type |
+| --- | --- |
+| Bug | INCIDENT |
+| Feature | REQUIREMENT |
+| Support | TASK |
+| Task | TASK |
+
+| Redmine status | DevLake status |
+| --- | --- |
+| New | TODO |
+| In Progress | IN_PROGRESS |
+| Feedback | IN_PROGRESS |
+| Resolved | DONE |
+| Closed | DONE |
+| Rejected | DONE |
+
+## Dashboard
+
+After a blueprint run, open the Redmine dashboard in Grafana. MySQL and PostgreSQL definitions are in `grafana/dashboards/mysql/Redmine.json` and `grafana/dashboards/postgresql/Redmine.json`.

backend/python/plugins/redmine/build.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+cd "$(dirname "$0")"
+sh ../../uv.sh sync .

backend/python/plugins/redmine/pyproject.toml

@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[tool.poetry]
+name = "redmine"
+version = "0.1.0"
+description = "DevLake plugin for Redmine"
+authors = ["Roger Abelenda <3179183+rabelenda@users.noreply.github.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.11,<3.12"
+pydevlake = { path = "../../pydevlake", develop = true }
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^9.1.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

backend/python/plugins/redmine/redmine/__init__.py

@@ -0,0 +1,14 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

backend/python/plugins/redmine/redmine/api.py

@@ -0,0 +1,181 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Iterator, Optional, Union
+
+from pydevlake.api import API, Paginator, PagedResponse, Request, request_hook
+
+
+_PAGE_SIZE = 100
+_ISSUE_SORT_UPDATED_ON = 'updated_on:desc'
+
+
+class RedminePaginator(Paginator):
+    def get_items(self, response) -> list:
+        for value in response.json.values():
+            if isinstance(value, list):
+                return value
+        return []
+
+    def get_next_page_id(self, response) -> Optional[int]:
+        data = response.json
+        items = self.get_items(response)
+        total = data.get('total_count', 0)
+        offset = data.get('offset', 0)
+        next_offset = offset + len(items)
+        return next_offset if next_offset < total else None
+
+    def set_next_page_param(self, request, next_page_id):
+        request.query_args['offset'] = next_page_id
+
+
+class RedmineAPI(API):
+    paginator = RedminePaginator()
+
+    @property
+    def base_url(self) -> str:
+        return str(self.connection.endpoint).rstrip('/')
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_):
+        pass
+
+    @request_hook
+    def authenticate(self, request: Request):
+        request.headers['X-Redmine-API-Key'] = self.connection.token.get_secret_value()
+
+    def test_connection(self):
+        self.get('projects.json', limit=1)
+
+    def projects(self):
+        return self.get('projects.json', limit=_PAGE_SIZE)
+
+    def issues(
+        self,
+        project_id: str,
+        include_journals: bool = False,
+        include_relations: bool = False,
+        created_on: Optional[str] = None,
+        updated_since: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> Iterator[dict]:
+        yield from self.iter_issues(
+            project_id,
+            include_journals=include_journals,
+            include_relations=include_relations,
+            created_on=created_on,
+            updated_since=updated_since,
+            limit=limit,
+        )
+
+    def iter_issues(
+        self,
+        project_id: str,
+        include_journals: bool = False,
+        include_relations: bool = False,
+        created_on: Optional[str] = None,
+        updated_since: Optional[str] = None,
+        limit: Optional[int] = None,
+    ) -> Iterator[dict]:
+        params = self._issue_params(
+            project_id,
+            include_journals=include_journals,
+            include_relations=include_relations,
+            created_on=created_on,
+        )
+        if updated_since:
+            yield from self._iter_issues(
+                params,
+                limit=limit,
+                stop_before_updated_on=updated_since,
+                sort=_ISSUE_SORT_UPDATED_ON,
+            )
+            return
+        if limit is not None:
+            yield from self._iter_issues(params, limit=limit)
+            return
+        yield from self.get('issues.json', **params)
+
+    def issue(self, issue_id: Union[int, str], include_journals: bool = False) -> dict:
+        params = {}
+        if include_journals:
+            params['include'] = 'journals'
+        response = self.get(f'issues/{issue_id}.json', **params)
+        return response.json['issue']
+
+    @staticmethod
+    def _issue_params(
+        project_id: str,
+        include_journals: bool = False,
+        include_relations: bool = False,
+        created_on: Optional[str] = None,
+    ) -> dict:
+        params = {'project_id': project_id, 'status_id': '*', 'limit': _PAGE_SIZE}
+        includes = []
+        if include_journals:
+            includes.append('journals')
+        if include_relations:
+            includes.append('relations')
+        if includes:
+            params['include'] = ','.join(includes)
+        if created_on:
+            params['created_on'] = created_on
+        return params
+
+    @staticmethod
+    def _should_stop_issue(issue: dict, stop_before_updated_on: Optional[str]) -> bool:
+        if stop_before_updated_on:
+            updated_on = issue.get('updated_on')
+            if updated_on and updated_on <= stop_before_updated_on:
+                return True
+        return False
+
+    def _iter_issues(
+        self,
+        params: dict,
+        limit: Optional[int] = None,
+        stop_before_updated_on: Optional[str] = None,
+        sort: Optional[str] = None,
+    ) -> Iterator[dict]:
+        query = dict(params)
+        if sort:
+            query['sort'] = sort
+        yielded = 0
+        offset = 0
+        while limit is None or yielded < limit:
+            query['limit'] = _PAGE_SIZE if limit is None else min(_PAGE_SIZE, limit - yielded)
+            if offset:
+                query['offset'] = offset
+            elif 'offset' in query:
+                del query['offset']
+            paged = self.get('issues.json', **query)
+            items = paged.items or []
+            if not items:
+                break
+            for item in items:
+                if self._should_stop_issue(item, stop_before_updated_on):
+                    return
+                yield item
+                yielded += 1
+                if limit is not None and yielded >= limit:
+                    return
+            if paged.next_page_request is None:
+                break
+            offset += len(items)
+
+    def time_entries(self, project_id: str):
+        return self.get('time_entries.json', project_id=project_id, limit=_PAGE_SIZE)