diff --git a/backend/Dockerfile.local b/backend/Dockerfile.local index 2c8b7ef1dcb..d4fab0a1480 100644 --- a/backend/Dockerfile.local +++ b/backend/Dockerfile.local @@ -73,6 +73,12 @@ RUN apt-get update && apt-get install -y \ libssh2-1 \ libssl3 \ ca-certificates \ + # gitextractor shells out to the git CLI unless + # USE_GO_GIT_IN_GIT_EXTRACTOR is set, so the binary must be present or every + # clone fails with "git: executable file not found in $PATH". + # debian:bookworm-slim does not ship it; the official image inherits it from + # its python base. + git \ && rm -rf /var/lib/apt/lists/* # Copy libgit2 diff --git a/backend/plugins/kiro/api/blueprint_v200.go b/backend/plugins/kiro/api/blueprint_v200.go new file mode 100644 index 00000000000..ceea9cbaee5 --- /dev/null +++ b/backend/plugins/kiro/api/blueprint_v200.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +func MakeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + + plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + + // No domain layer scopes: this plugin writes only to _tool_kiro_* tables. + // Cross-tool AI modelling in the domain layer is separate work. + return plan, []plugin.Scope{}, nil +} + +func makeDataSourcePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig], + connection *models.KiroConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + slice := scopeDetail.Scope + + op := &tasks.KiroOptions{ + ConnectionId: slice.ConnectionId, + ScopeId: slice.Id, + AccountId: slice.AccountId, + Year: slice.Year, + Month: slice.Month, + } + + // An empty entity list enables every subtask; the three streams are + // always collected together because they describe the same activity. + task, err := helper.MakePipelinePlanTask("kiro", subtaskMetas, []string{}, op) + if err != nil { + return nil, err + } + + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + plan[i] = append(stage, task) + } + return plan, nil +} diff --git a/backend/plugins/kiro/api/connection.go b/backend/plugins/kiro/api/connection.go new file mode 100644 index 00000000000..075274b1d56 --- /dev/null +++ b/backend/plugins/kiro/api/connection.go @@ -0,0 +1,163 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// PostConnections creates a new connection. +// @Summary create kiro connection +// @Description Create kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + // Wrapped as BadInput so a struct-tag validation failure reports 400 rather + // than 500 - the difference between "fix your input" and "the server broke". + if err := api.Decode(input.Body, connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.Create(connection, input); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// PatchConnection updates an existing connection. +// @Summary patch kiro connection +// @Description Patch kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Param body body models.KiroConnection true "json body" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + if err := (&models.KiroConnection{}).MergeFromRequest(connection, input.Body); err != nil { + return nil, errors.Convert(err) + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil +} + +// DeleteConnection removes a connection. +// @Summary delete a kiro connection +// @Description Delete a kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} srvhelper.DsRefs "References exist to this connection" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + conn := &models.KiroConnection{} + output, err := connectionHelper.Delete(conn, input) + if err != nil { + return output, err + } + output.Body = conn.Sanitize() + return output, nil +} + +// ListConnections lists all connections. +// @Summary get all kiro connections +// @Description Get all kiro connections +// @Tags plugins/kiro +// @Success 200 {object} []models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connections []models.KiroConnection + if err := connectionHelper.List(&connections); err != nil { + return nil, err + } + for i := range connections { + connections[i] = connections[i].Sanitize() + } + return &plugin.ApiResourceOutput{Body: connections}, nil +} + +// GetConnection returns one connection. +// @Summary get kiro connection detail +// @Description Get kiro connection detail +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} models.KiroConnection +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil +} + +// validateConnection checks the fields collection cannot proceed without. +// +// Identity Store fields are deliberately not required: they only resolve display +// names, and identity for joining to git history comes from the report's +// User_Email column. Requiring them would block a working setup. +func validateConnection(conn *models.KiroConn) error { + if conn.AccessKeyId == "" { + return errors.BadInput.New("AccessKeyId is required") + } + if conn.SecretAccessKey == "" { + return errors.BadInput.New("SecretAccessKey is required") + } + if conn.Region == "" { + return errors.BadInput.New("Region is required") + } + if conn.Bucket == "" { + return errors.BadInput.New("Bucket is required") + } + // A partial Identity Store configuration is a mistake worth reporting: it + // silently yields no display names, which looks like a data problem rather + // than a configuration one. + if (conn.IdentityStoreId == "") != (conn.IdentityStoreRegion == "") { + return errors.BadInput.New("IdentityStoreId and IdentityStoreRegion must be set together") + } + return nil +} diff --git a/backend/plugins/kiro/api/init.go b/backend/plugins/kiro/api/init.go new file mode 100644 index 00000000000..e4b614a743d --- /dev/null +++ b/backend/plugins/kiro/api/init.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/go-playground/validator/v10" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + vld *validator.Validate + connectionHelper *api.ConnectionApiHelper + basicRes context.BasicRes + // Scope config is NoScopeConfig: the report CSV's meaning is fixed by AWS + // and uniform across an organization, so there is nothing per-scope to + // configure. + dsHelper *api.DsHelper[models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig] +) + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + connectionHelper = api.NewConnectionHelper( + basicRes, + vld, + p.Name(), + ) + + dsHelper = api.NewDataSourceHelper[ + models.KiroConnection, models.KiroS3Slice, srvhelper.NoScopeConfig, + ]( + basicRes, + p.Name(), + // Searchable scope fields. + []string{"accountId", "name"}, + func(c models.KiroConnection) models.KiroConnection { return c.Sanitize() }, + func(s models.KiroS3Slice) models.KiroS3Slice { return s.Sanitize() }, + nil, + ) + + // Scope browsing and search are implemented directly in remote_api.go rather + // than through the shared DsRemoteApiScopeList/Search helpers. Those build an + // HTTP client from the connection first, and that constructor runs a DNS + // check on the endpoint - a bucket name is not a hostname, so it fails. The + // helpers assume an HTTP data source; this one is S3. +} diff --git a/backend/plugins/kiro/api/remote_api.go b/backend/plugins/kiro/api/remote_api.go new file mode 100644 index 00000000000..1be7635558f --- /dev/null +++ b/backend/plugins/kiro/api/remote_api.go @@ -0,0 +1,338 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "strconv" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// listKiroRemoteScopes browses the export layout as a tree. +// +// Three levels, mirroring Kiro's own S3 partitioning: +// +// (root) -> one group per AWS account with exported data +// {account} -> one group per year, plus a whole-year scope +// {account}/{y} -> one selectable scope per month +// +// Everything comes from S3 rather than user input. That is the point: a +// hand-typed prefix cannot be validated from the outcome, because a typo and a +// month with no data both produce a successful run that collects nothing. +func listKiroRemoteScopes(connection *models.KiroConnection, groupId string) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + if connection == nil { + return nil, errors.BadInput.New("connection is required") + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + + accountId, year, err := parseGroupId(groupId) + if err != nil { + return nil, err + } + + switch { + case accountId == "": + return listAccountGroups(discovery) + case year == 0: + return listYearGroups(discovery, accountId) + default: + return listMonthScopes(discovery, accountId, year) + } +} + +// listAccountGroups is the tree root: the accounts that actually have exports. +func listAccountGroups(discovery *tasks.Discovery) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(accounts)) + for _, accountId := range accounts { + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + Id: accountId, + Name: accountId, + FullName: accountId, + }) + } + return entries, nil +} + +// listYearGroups lists the years under an account. +// +// Each year is offered both as a group to expand and as a directly selectable +// scope, because a nil month means "collect the whole year" - which is how a +// year-long backfill is expressed without creating twelve scopes by hand. +func listYearGroups(discovery *tasks.Discovery, accountId string) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + years, err := discovery.ListYears(accountId) + if err != nil { + return nil, err + } + + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(years)*2) + for _, year := range years { + groupId := fmt.Sprintf("%s/%04d", accountId, year) + parent := accountId + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_GROUP, + ParentId: &parent, + Id: groupId, + Name: fmt.Sprintf("%04d", year), + FullName: groupId, + }) + + wholeYear := &models.KiroS3Slice{AccountId: accountId, Year: year} + *wholeYear = wholeYear.Sanitize() + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: wholeYear.Id, + Name: fmt.Sprintf("%04d (whole year)", year), + FullName: wholeYear.ScopeName(), + Data: wholeYear, + }) + } + return entries, nil +} + +// listMonthScopes lists the months that hold data for an account and year. +func listMonthScopes(discovery *tasks.Discovery, accountId string, year int) ( + []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], errors.Error, +) { + months, err := discovery.ListMonths(accountId, year) + if err != nil { + return nil, err + } + + parent := fmt.Sprintf("%s/%04d", accountId, year) + entries := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0, len(months)) + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + entries = append(entries, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: &parent, + Id: slice.Id, + Name: fmt.Sprintf("%04d-%02d", year, month), + FullName: slice.ScopeName(), + Data: slice, + }) + } + return entries, nil +} + +// searchKiroRemoteScopes filters the discovered months by substring. +// +// Matching is against "{account} {year}-{month}", so "2026-07" or an account +// number both work. The search space is one listing per year, small enough to +// scan without an index. +func searchKiroRemoteScopes( + connection *models.KiroConnection, + query string, + page int, + pageSize int, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + err errors.Error, +) { + empty := []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + if connection == nil { + return empty, nil + } + query = strings.ToLower(strings.TrimSpace(query)) + if query == "" { + return empty, nil + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + return nil, err + } + accounts, err := discovery.ListAccounts() + if err != nil { + return nil, err + } + + matches := make([]dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], 0) + for _, accountId := range accounts { + years, yearErr := discovery.ListYears(accountId) + if yearErr != nil { + return nil, yearErr + } + for _, year := range years { + months, monthErr := discovery.ListMonths(accountId, year) + if monthErr != nil { + return nil, monthErr + } + for _, month := range months { + m := month + slice := &models.KiroS3Slice{AccountId: accountId, Year: year, Month: &m} + *slice = slice.Sanitize() + + if !strings.Contains(strings.ToLower(slice.ScopeName()), query) && + !strings.Contains(strings.ToLower(slice.Id), query) { + continue + } + matches = append(matches, dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + Id: slice.Id, + Name: slice.ScopeName(), + FullName: slice.ScopeName(), + Data: slice, + }) + } + } + } + + return paginate(matches, page, pageSize), nil +} + +// paginate applies the requested page window to an in-memory result set. +func paginate( + entries []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice], + page int, pageSize int, +) []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice] { + if page <= 0 { + page = 1 + } + if pageSize <= 0 { + pageSize = 50 + } + start := (page - 1) * pageSize + if start >= len(entries) { + return []dsmodels.DsRemoteApiScopeListEntry[models.KiroS3Slice]{} + } + end := start + pageSize + if end > len(entries) { + end = len(entries) + } + return entries[start:end] +} + +// parseGroupId splits a tree node id into its parts. +// +// "" is the root, "{account}" is an account node, "{account}/{year}" is a year +// node. +func parseGroupId(groupId string) (accountId string, year int, err errors.Error) { + trimmed := strings.Trim(strings.TrimSpace(groupId), "/") + if trimmed == "" { + return "", 0, nil + } + + parts := strings.Split(trimmed, "/") + switch len(parts) { + case 1: + return parts[0], 0, nil + case 2: + parsedYear, convErr := strconv.Atoi(parts[1]) + if convErr != nil { + return "", 0, errors.BadInput.New("invalid year in groupId: " + groupId) + } + return parts[0], parsedYear, nil + default: + return "", 0, errors.BadInput.New("unrecognized groupId: " + groupId) + } +} + +// RemoteScopes browses the Kiro export layout in S3. +// +// Implemented directly rather than through the shared scope-list helper. That +// helper builds an HTTP client from the connection first, and its constructor +// runs a DNS check on the endpoint - which fails here, because a bucket name is +// not a hostname. The helper is built for HTTP data sources; this one is S3. +// @Summary list available kiro scopes discovered from S3 +// @Description Browse accounts, years and months that actually have exported data +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param groupId query string false "account id, or account/year" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + children, err := listKiroRemoteScopes(connection, input.Query.Get("groupId")) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} + +// SearchRemoteScopes finds discovered scopes by substring. +// +// Implemented directly rather than through the shared search helper: that +// helper's callback receives only an HTTP ApiClient, and discovery here needs +// the connection itself to build an S3 client. +// @Summary search kiro scopes discovered from S3 +// @Description Search the discovered months by account or year-month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param search query string false "search" +// @Param page query int false "page number" +// @Param pageSize query int false "page size per page" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.KiroS3Slice] "the parentIds are always null" +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/search-remote-scopes [GET] +func SearchRemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, err + } + + page, _ := strconv.Atoi(input.Query.Get("page")) + pageSize, _ := strconv.Atoi(input.Query.Get("pageSize")) + + children, err := searchKiroRemoteScopes(connection, input.Query.Get("search"), page, pageSize) + if err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: dsmodels.DsRemoteApiScopeList[models.KiroS3Slice]{Children: children}, + }, nil +} diff --git a/backend/plugins/kiro/api/s3_slice_api.go b/backend/plugins/kiro/api/s3_slice_api.go new file mode 100644 index 00000000000..f8a05fb185f --- /dev/null +++ b/backend/plugins/kiro/api/s3_slice_api.go @@ -0,0 +1,117 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +type PutScopesReqBody = helper.PutScopesReqBody[models.KiroS3Slice] +type ScopeDetail = srvhelper.ScopeDetail[models.KiroS3Slice, srvhelper.NoScopeConfig] + +// PutScopes creates or updates Kiro collection scopes. +// @Summary create or update kiro scopes +// @Description Create or update kiro scopes, each covering one AWS account for one month +// @Tags plugins/kiro +// @Accept application/json +// @Param connectionId path int true "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// GetScopeList returns the scopes for a connection. +// @Summary get kiro scopes +// @Description get kiro scopes +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param pageSize query int false "page size" +// @Param page query int false "page number" +// @Param blueprints query bool false "include blueprint references" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope returns a single scope. +// @Summary get one kiro scope +// @Description get one kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// PatchScope updates a scope. +// @Summary patch a kiro scope +// @Description patch a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param scope body models.KiroS3Slice true "json" +// @Success 200 {object} models.KiroS3Slice +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// DeleteScope removes a scope and optionally its collected data. +// @Summary delete a kiro scope +// @Description delete a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} + +// GetScopeLatestSyncState reports the most recent sync for a scope. +// @Summary get the latest sync state of a kiro scope +// @Description get the latest sync state of a kiro scope +// @Tags plugins/kiro +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Success 200 {object} []models.LatestSyncState +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{connectionId}/scopes/{scopeId}/latest-sync-state [GET] +func GetScopeLatestSyncState(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeLatestSyncState(input) +} diff --git a/backend/plugins/kiro/api/test_connection.go b/backend/plugins/kiro/api/test_connection.go new file mode 100644 index 00000000000..8df055e582f --- /dev/null +++ b/backend/plugins/kiro/api/test_connection.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/http" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +// TestConnection validates a connection that has not been saved yet. +// @Summary test kiro connection +// @Description Test kiro connection +// @Tags plugins/kiro +// @Param body body models.KiroConn true "json body" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.KiroConnection + // Wrapped as BadInput: a struct-tag validation failure is the caller's + // problem, and an unwrapped Decode error surfaces as HTTP 500 - which sends + // the user looking at server logs instead of at their own form. + if err := api.Decode(input.Body, &connection, vld); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := validateConnection(&connection.KiroConn); err != nil { + return nil, errors.BadInput.Wrap(err, "connection validation failed") + } + if err := testConnection(&connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(&connection), + Status: http.StatusOK, + }, nil +} + +// TestExistingConnection validates a saved connection, optionally with overrides +// from the request body. +// @Summary test existing kiro connection +// @Description Test existing kiro connection +// @Tags plugins/kiro +// @Param id path int true "connection ID" +// @Success 200 {object} ConnectionReport +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/kiro/connections/{id}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.KiroConnection{} + if err := connectionHelper.First(connection, input.Params); err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := api.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, errors.BadInput.Wrap(err, "invalid connection payload") + } + if err := testConnection(connection); err != nil { + return nil, err + } + return &plugin.ApiResourceOutput{ + Body: buildConnectionReport(connection), + Status: http.StatusOK, + }, nil +} + +// ConnectionReport describes what the connection can actually see. +// +// A bare success/failure verdict is not enough to tell whether a configuration +// is right, because a wrong prefix and a genuinely empty period produce the same +// outcome: collection runs, finds nothing, and reports success. Returning the +// discovered accounts and per-stream object counts makes the difference visible +// before any scope is created. +type ConnectionReport struct { + ReportBucket string `json:"reportBucket"` + PromptLogBucket string `json:"promptLogBucket"` + // Accounts are the AWS account ids found under the report prefix. An empty + // list is the clearest sign that the bucket or prefix is wrong. + Accounts []string `json:"accounts"` + // Streams reports object counts for the most recent discovered period. + Streams []tasks.StreamCount `json:"streams"` + // Hint explains an empty result in plain terms. + Hint string `json:"hint,omitempty"` +} + +// connectionReportCountLimit caps counting so the check stays fast on a bucket +// holding hundreds of thousands of objects. The exact number does not matter for +// verifying a path - only whether it is zero. +const connectionReportCountLimit = 500 + +// buildConnectionReport probes the layout and summarizes what was found. +// +// Errors are folded into the report rather than returned: the connection itself +// is already known to work at this point, and a discovery failure is more useful +// shown as an empty result with a hint than as a failed request. +func buildConnectionReport(connection *models.KiroConnection) *ConnectionReport { + report := &ConnectionReport{ + ReportBucket: connection.Bucket, + PromptLogBucket: connection.GetPromptLogBucket(), + Accounts: []string{}, + Streams: []tasks.StreamCount{}, + } + + discovery, err := tasks.NewDiscovery(connection) + if err != nil { + report.Hint = "could not initialise S3 discovery: " + err.Error() + return report + } + + accounts, err := discovery.ListAccounts() + if err != nil { + report.Hint = "could not list accounts under the report prefix: " + err.Error() + return report + } + report.Accounts = accounts + + if len(accounts) == 0 { + report.Hint = fmt.Sprintf( + "no account directories under s3://%s/%s/AWSLogs/ - check the bucket and report prefix", + connection.Bucket, connection.GetReportPrefix()) + return report + } + + // Probe the newest period that exists, since that is where data is most + // likely to be and therefore the most informative check. + accountId := accounts[len(accounts)-1] + years, err := discovery.ListYears(accountId) + if err != nil || len(years) == 0 { + report.Hint = fmt.Sprintf( + "account %s has no year directories - check the region in the connection", accountId) + return report + } + year := years[len(years)-1] + + var month *int + if months, monthErr := discovery.ListMonths(accountId, year); monthErr == nil && len(months) > 0 { + latest := months[len(months)-1] + month = &latest + } + + report.Streams = discovery.CountStreams(accountId, year, month, connectionReportCountLimit) + + period := fmt.Sprintf("%04d", year) + if month != nil { + period = fmt.Sprintf("%04d-%02d", year, *month) + } + report.Hint = fmt.Sprintf("counts are for account %s, period %s", accountId, period) + return report +} + +// testConnection issues a real request against every bucket in use. +// +// Constructing a client proves nothing - the AWS SDK builds one happily from +// invalid credentials, so a test that stops there reports success for a +// connection that cannot read anything. A single-key list is the cheapest call +// that actually exercises credentials and bucket permissions. +// +// Both buckets are checked when reports and logs are separated, since they may +// carry different KMS keys and IAM conditions; a connection that can read +// reports but not logs would otherwise pass and then silently collect nothing. +func testConnection(connection *models.KiroConnection) errors.Error { + clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return err + } + + for _, bucket := range clients.Buckets() { + client := clients.Report + if bucket == clients.PromptLog.Bucket { + client = clients.PromptLog + } + if _, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + MaxKeys: aws.Int64(1), + }); listErr != nil { + return errors.BadInput.Wrap(listErr, "cannot access s3 bucket "+bucket) + } + } + + return nil +} diff --git a/backend/plugins/kiro/impl/impl.go b/backend/plugins/kiro/impl/impl.go new file mode 100644 index 00000000000..c6b19708aeb --- /dev/null +++ b/backend/plugins/kiro/impl/impl.go @@ -0,0 +1,208 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/kiro/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.DataSourcePluginBlueprintV200 +} = (*Kiro)(nil) + +// Kiro collects Kiro enterprise usage exports from S3. +// +// This is a separate plugin rather than an evolution of the retired predecessor: +// the old format is frozen, and a plugin that keeps evolving should not depend +// on frozen code. No implementation code is shared, following the same split +// as bitbucket and bitbucket_server. +type Kiro struct{} + +func (p Kiro) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Kiro) Name() string { + return "kiro" +} + +func (p Kiro) Description() string { + return "collect Kiro usage reports and interaction logs from S3" +} + +func (p Kiro) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/kiro" +} + +// GetTablesInfo must list every model or plugins/table_info_test.go fails. +func (p Kiro) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.KiroConnection{}, + &models.KiroS3Slice{}, + &models.KiroS3FileMeta{}, + &models.KiroUserReport{}, + &models.KiroUserModelMessage{}, + &models.KiroChatLog{}, + &models.KiroCompletionLog{}, + } +} + +func (p Kiro) Connection() dal.Tabler { + return &models.KiroConnection{} +} + +func (p Kiro) Scope() plugin.ToolLayerScope { + return &models.KiroS3Slice{} +} + +// ScopeConfig returns nil: the export format is defined by AWS and uniform +// across an organization, so there is nothing per-scope to configure. +func (p Kiro) ScopeConfig() dal.Tabler { + return nil +} + +func (p Kiro) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +// SubTaskMetas lists discovery first, then one extractor per stream. The +// extractors declare their dependency on discovery, so the split is safe and +// gives each stream its own progress reporting - useful when a scope holds tens +// of thousands of log objects and one needs to know which stream is slow. +func (p Kiro) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectKiroS3FilesMeta, + tasks.ExtractKiroUserReportMeta, + tasks.ExtractKiroChatLogMeta, + tasks.ExtractKiroCompletionLogMeta, + } +} + +func (p Kiro) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.KiroOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, err + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("connectionId is required") + } + if op.AccountId == "" { + return nil, errors.BadInput.New("accountId is required") + } + if op.Year <= 0 { + return nil, errors.BadInput.New("year is required") + } + + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + connection := &models.KiroConnection{} + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, err + } + + s3Clients, err := tasks.NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + + // Identity Store is optional and only supplies display names, so a failure + // here degrades presentation rather than collection. + identityClient, identityErr := tasks.NewKiroIdentityClient(connection) + if identityErr != nil { + taskCtx.GetLogger().Warn(identityErr, "identity store unavailable, proceeding without display names") + identityClient = nil + } + + timePath := fmt.Sprintf("%04d", op.Year) + if op.Month != nil { + timePath = fmt.Sprintf("%04d/%02d", op.Year, *op.Month) + } + + return &tasks.KiroTaskData{ + Options: &op, + Connection: connection, + S3Clients: s3Clients, + IdentityClient: identityClient, + Prefixes: tasks.BuildPrefixes(connection, op.AccountId, timePath), + }, nil +} + +func (p Kiro) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Kiro) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "GET": api.GetConnection, + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + // Scope discovery: lists the accounts, years and months that actually + // have exported data, so a scope is selected instead of hand-entered. + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/search-remote-scopes": { + "GET": api.SearchRemoteScopes, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes/:scopeId/latest-sync-state": { + "GET": api.GetScopeLatestSyncState, + }, + } +} diff --git a/backend/plugins/kiro/impl/table_info_test.go b/backend/plugins/kiro/impl/table_info_test.go new file mode 100644 index 00000000000..cb195602d63 --- /dev/null +++ b/backend/plugins/kiro/impl/table_info_test.go @@ -0,0 +1,37 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/apache/incubator-devlake/helpers/unithelper" +) + +// The repo-wide plugins/table_info_test.go performs this same check for every +// plugin, but building that package requires gitextractor and therefore a +// specific libgit2. Running it here as well keeps the feedback local: a model +// added without registering it in GetTablesInfo fails immediately rather than +// only in CI. +func TestKiroTableInfo(t *testing.T) { + checker := unithelper.NewTableInfoChecker(unithelper.TableInfoCheckerConfig{}) + checker.FeedIn("../models", Kiro{}.GetTablesInfo) + if err := checker.Verify(); err != nil { + t.Error(err) + } +} diff --git a/backend/plugins/kiro/kiro.go b/backend/plugins/kiro/kiro.go new file mode 100644 index 00000000000..27c8dff3032 --- /dev/null +++ b/backend/plugins/kiro/kiro.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "github.com/spf13/cobra" + + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/kiro/impl" +) + +var PluginEntry impl.Kiro + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "kiro"} + connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "kiro connection id") + accountId := cmd.Flags().StringP("accountId", "a", "", "AWS account id that Kiro exports for") + year := cmd.Flags().IntP("year", "y", 0, "year to collect") + month := cmd.Flags().IntP("month", "m", 0, "month to collect; omit to collect the whole year") + + _ = cmd.MarkFlagRequired("connectionId") + _ = cmd.MarkFlagRequired("accountId") + _ = cmd.MarkFlagRequired("year") + + cmd.Run = func(cmd *cobra.Command, args []string) { + options := map[string]interface{}{ + "connectionId": *connectionId, + "accountId": *accountId, + "year": *year, + } + // A zero month means the whole year, matching the scope model where a + // nil month widens collection. + if *month > 0 { + options["month"] = *month + } + runner.DirectRun(cmd, args, PluginEntry, options, "") + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/kiro/models/chat_log.go b/backend/plugins/kiro/models/chat_log.go index ecd943696fe..cde329f517d 100644 --- a/backend/plugins/kiro/models/chat_log.go +++ b/backend/plugins/kiro/models/chat_log.go @@ -44,7 +44,7 @@ type KiroChatLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` // IdentityStoreId is the stripped prefix, retained for auditability. IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // ChatTriggerType is MANUAL or INLINE_CHAT per the docs; only MANUAL has // been observed. Not validated against a fixed set. ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` diff --git a/backend/plugins/kiro/models/completion_log.go b/backend/plugins/kiro/models/completion_log.go index 0d09c28e0ad..cfd633e5335 100644 --- a/backend/plugins/kiro/models/completion_log.go +++ b/backend/plugins/kiro/models/completion_log.go @@ -40,7 +40,7 @@ type KiroCompletionLog struct { UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` // FileName has no path component; see the type comment. FileName string `gorm:"type:varchar(255);index" json:"fileName"` diff --git a/backend/plugins/kiro/models/migrationscripts/archived/init.go b/backend/plugins/kiro/models/migrationscripts/archived/init.go index 5c1c9a3224f..53422db0499 100644 --- a/backend/plugins/kiro/models/migrationscripts/archived/init.go +++ b/backend/plugins/kiro/models/migrationscripts/archived/init.go @@ -127,7 +127,7 @@ type KiroChatLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` ChatTriggerType string `gorm:"type:varchar(20)" json:"chatTriggerType"` ModelId *string `gorm:"type:varchar(100)" json:"modelId"` HasPrompt bool `gorm:"index" json:"hasPrompt"` @@ -152,7 +152,7 @@ type KiroCompletionLog struct { RequestId string `gorm:"primaryKey;type:varchar(64)"` UserId string `gorm:"type:varchar(64);index" json:"userId"` IdentityStoreId string `gorm:"type:varchar(32)" json:"identityStoreId"` - Timestamp time.Time `gorm:"type:datetime(6);index" json:"timestamp"` + Timestamp time.Time `gorm:"precision:6;index" json:"timestamp"` FileName string `gorm:"type:varchar(255);index" json:"fileName"` FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"` HasCustomization bool `json:"hasCustomization"` diff --git a/backend/plugins/kiro/models/timestamp_schema_test.go b/backend/plugins/kiro/models/timestamp_schema_test.go new file mode 100644 index 00000000000..4bc479b1c62 --- /dev/null +++ b/backend/plugins/kiro/models/timestamp_schema_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "sync" + "testing" + + "github.com/apache/incubator-devlake/plugins/kiro/models/migrationscripts/archived" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + mysqlDriver "gorm.io/driver/mysql" + postgresDriver "gorm.io/driver/postgres" + "gorm.io/gorm/schema" +) + +func TestLogTimestampUsesPortableMicrosecondPrecision(t *testing.T) { + logModels := []struct { + name string + model any + }{ + {name: "chat runtime model", model: &KiroChatLog{}}, + {name: "completion runtime model", model: &KiroCompletionLog{}}, + {name: "chat migration model", model: &archived.KiroChatLog{}}, + {name: "completion migration model", model: &archived.KiroCompletionLog{}}, + } + + for _, logModel := range logModels { + t.Run(logModel.name, func(t *testing.T) { + sch, err := schema.Parse(logModel.model, &sync.Map{}, schema.NamingStrategy{}) + require.NoError(t, err) + + field := sch.LookUpField("Timestamp") + require.NotNil(t, field) + assert.Equal(t, 6, field.Precision) + assert.Equal(t, "datetime(6) NULL", mysqlDriver.New(mysqlDriver.Config{}).DataTypeOf(field)) + assert.Equal(t, "timestamptz(6)", postgresDriver.New(postgresDriver.Config{}).DataTypeOf(field)) + }) + } +} diff --git a/backend/plugins/kiro/tasks/discovery.go b/backend/plugins/kiro/tasks/discovery.go new file mode 100644 index 00000000000..1bd6a134759 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery.go @@ -0,0 +1,158 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "strconv" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Discovery walks the report prefix to find what data actually exists. +// +// Kiro's S3 layout encodes every scope dimension as a path segment: +// +// {reportPrefix}/AWSLogs/{accountId}/KiroLogs/user_report/{region}/{year}/{month}/ +// +// so a scope never has to be typed by hand. That matters beyond convenience: a +// mistyped prefix produces exactly the same outcome as a month with no data - +// collection succeeds and finds nothing - so hand-entered paths cannot be +// verified from the result. +type Discovery struct { + clients *KiroS3Clients + connection *models.KiroConnection +} + +func NewDiscovery(connection *models.KiroConnection) (*Discovery, errors.Error) { + clients, err := NewKiroS3Clients(connection) + if err != nil { + return nil, err + } + return &Discovery{clients: clients, connection: connection}, nil +} + +// reportRoot is the prefix holding the per-account directories. +func (d *Discovery) reportRoot() string { + return fmt.Sprintf("%s/AWSLogs", d.connection.GetReportPrefix()) +} + +// accountReportPrefix is where one account's report months live. +func (d *Discovery) accountReportPrefix(accountId string) string { + return fmt.Sprintf("%s/AWSLogs/%s/KiroLogs/user_report/%s", + d.connection.GetReportPrefix(), accountId, d.connection.Region) +} + +// ListAccounts returns the AWS account ids that have exported data. +// +// Kiro requires a bucket per account holding subscriptions and does not support +// cross-account buckets, so in practice this is usually one entry - but reading +// it from S3 removes the chance of a typo in a 12-digit number. +func (d *Discovery) ListAccounts() ([]string, errors.Error) { + return d.clients.Report.ListSubPrefixes(d.reportRoot()) +} + +// ListYears returns the years with report data for an account. +func (d *Discovery) ListYears(accountId string) ([]int, errors.Error) { + names, err := d.clients.Report.ListSubPrefixes(d.accountReportPrefix(accountId)) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// ListMonths returns the months with report data for an account and year. +func (d *Discovery) ListMonths(accountId string, year int) ([]int, errors.Error) { + prefix := fmt.Sprintf("%s/%04d", d.accountReportPrefix(accountId), year) + names, err := d.clients.Report.ListSubPrefixes(prefix) + if err != nil { + return nil, err + } + return parseNumericSegments(names), nil +} + +// StreamCount is how many collectable objects one stream holds. +type StreamCount struct { + FileType string `json:"fileType"` + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Count int `json:"count"` + // AtLeast is true when counting stopped at the cap, so Count is a floor. + AtLeast bool `json:"atLeast"` + // Error explains why a stream could not be counted, e.g. missing + // permission on the log bucket while the report bucket is readable. + Error string `json:"error,omitempty"` +} + +// CountStreams reports the object count for each of the three streams. +// +// This is the answer to "is my configuration right?". Reporting counts per +// stream distinguishes the three cases that otherwise look identical: a wrong +// prefix (zero everywhere), a genuinely dormant stream (zero for one type - +// inline completions stopped being produced under agentic usage), and a +// permissions gap on one bucket (an error for the log streams only). +// +// countLimit caps the work per stream; pass 0 to count everything. +func (d *Discovery) CountStreams(accountId string, year int, month *int, countLimit int) []StreamCount { + timePath := fmt.Sprintf("%04d", year) + if month != nil { + timePath = fmt.Sprintf("%04d/%02d", year, *month) + } + + specs := BuildPrefixes(d.connection, accountId, timePath) + results := make([]StreamCount, 0, len(specs)) + + for _, spec := range specs { + client := d.clients.ForFileType(spec.FileType) + result := StreamCount{ + FileType: spec.FileType, + Bucket: client.Bucket, + Prefix: spec.Prefix, + } + count, atLeast, err := client.CountObjects(spec.Prefix, countLimit) + if err != nil { + // Recorded rather than returned: one unreadable stream should still + // leave the others' counts visible, since that contrast is what + // identifies a per-bucket permission problem. + result.Error = err.Error() + } else { + result.Count = count + result.AtLeast = atLeast + } + results = append(results, result) + } + + return results +} + +// parseNumericSegments keeps only the segments that are numbers, in order. +// +// S3 prefixes are strings, and a non-numeric directory would otherwise surface +// as a year or month. +func parseNumericSegments(names []string) []int { + values := make([]int, 0, len(names)) + for _, name := range names { + value, err := strconv.Atoi(name) + if err != nil { + continue + } + values = append(values, value) + } + return values +} diff --git a/backend/plugins/kiro/tasks/discovery_test.go b/backend/plugins/kiro/tasks/discovery_test.go new file mode 100644 index 00000000000..785f79cbd58 --- /dev/null +++ b/backend/plugins/kiro/tasks/discovery_test.go @@ -0,0 +1,211 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// prefixMockS3 answers listings from a canned prefix tree, and records the +// requests so tests can assert that a delimiter was used. +type prefixMockS3 struct { + // commonPrefixes maps a queried prefix to the child prefixes returned. + commonPrefixes map[string][]string + // objects maps a queried prefix to the object keys beneath it. + objects map[string][]string + seenDelimiter []string + seenPrefix []string +} + +func (m *prefixMockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + prefix := "" + if input.Prefix != nil { + prefix = *input.Prefix + } + m.seenPrefix = append(m.seenPrefix, prefix) + if input.Delimiter != nil { + m.seenDelimiter = append(m.seenDelimiter, *input.Delimiter) + } else { + m.seenDelimiter = append(m.seenDelimiter, "") + } + + out := &s3.ListObjectsV2Output{IsTruncated: aws.Bool(false)} + for _, child := range m.commonPrefixes[prefix] { + full := prefix + child + "/" + out.CommonPrefixes = append(out.CommonPrefixes, &s3.CommonPrefix{Prefix: aws.String(full)}) + } + for _, key := range m.objects[prefix] { + out.Contents = append(out.Contents, &s3.Object{Key: aws.String(prefix + key)}) + } + return out, nil +} + +func (m *prefixMockS3) GetObject(*s3.GetObjectInput) (*s3.GetObjectOutput, error) { + return nil, nil +} + +func discoveryFixture(svc S3API) *Discovery { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + client := &KiroS3Client{S3: svc, Bucket: conn.Bucket} + return &Discovery{ + clients: &KiroS3Clients{Report: client, PromptLog: client}, + connection: conn, + } +} + +// Kiro encodes every scope dimension as a path segment, which is what allows a +// scope to be selected rather than typed. This matters beyond convenience: a +// mistyped prefix and a month with no data produce the same outcome - a +// successful run that collects nothing - so a hand-entered path cannot be +// verified from the result. +func TestDiscovery_ListsLayoutFromS3(t *testing.T) { + // The real layout, as confirmed against the live bucket. + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/": {"123456789012"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026"}, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/": {"02", "03", "07"}, + }} + discovery := discoveryFixture(svc) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Equal(t, []string{"123456789012"}, accounts) + + years, err := discovery.ListYears("123456789012") + require.Nil(t, err) + assert.Equal(t, []int{2026}, years) + + months, err := discovery.ListMonths("123456789012", 2026) + require.Nil(t, err) + // Zero-padded segments must parse to plain integers, and only months that + // actually hold data are offered. + assert.Equal(t, []int{2, 3, 7}, months) + + // A delimiter is required: without it S3 returns every object under the + // prefix instead of just the segment names, which on a log prefix would be + // hundreds of thousands of keys. + for _, delimiter := range svc.seenDelimiter { + assert.Equal(t, "/", delimiter) + } +} + +func TestDiscovery_IgnoresNonNumericSegments(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/": {"2026", "unexpected", "2025"}, + }} + years, err := discoveryFixture(svc).ListYears("123456789012") + require.Nil(t, err) + // S3 prefixes are strings; a stray directory must not surface as a year. + assert.Equal(t, []int{2025, 2026}, years) +} + +func TestDiscovery_EmptyLayout(t *testing.T) { + // An empty account list is the clearest signal that the bucket or report + // prefix is wrong, so it must come back as an empty result rather than an + // error. + discovery := discoveryFixture(&prefixMockS3{}) + + accounts, err := discovery.ListAccounts() + require.Nil(t, err) + assert.Empty(t, accounts) +} + +// Per-stream counts separate three situations that a single pass/fail verdict +// cannot: a wrong prefix (zero everywhere), a dormant stream (zero for one type, +// which is what inline completions look like under agentic usage), and a +// permission gap on one bucket. +func TestDiscovery_CountStreams(t *testing.T) { + base := "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07/" + logBase := "logging/AWSLogs/123456789012/KiroLogs/" + svc := &prefixMockS3{objects: map[string][]string{ + base: {"KIRO_CLI_x_user_report_1.csv", "KIRO_IDE_x_user_report_1.csv"}, + logBase + "GenerateAssistantResponse/us-east-1/2026/07/": {"a.json.gz", "b.json.gz", "c.json.gz"}, + // GenerateCompletions intentionally absent: dormant, not misconfigured. + }} + + month := 7 + counts := discoveryFixture(svc).CountStreams("123456789012", 2026, &month, 0) + require.Len(t, counts, 3) + + byType := map[string]StreamCount{} + for _, c := range counts { + byType[c.FileType] = c + } + assert.Equal(t, 2, byType[models.FileTypeReport].Count) + assert.Equal(t, 3, byType[models.FileTypeChatLog].Count) + assert.Equal(t, 0, byType[models.FileTypeCompletionLog].Count) + + // The prefix is reported alongside the count so a zero can be checked + // against the bucket directly. + assert.Contains(t, byType[models.FileTypeReport].Prefix, "user_report/us-east-1/2026/07") + for _, c := range counts { + assert.NotEmpty(t, c.Bucket) + assert.Empty(t, c.Error) + } +} + +func TestCountObjects(t *testing.T) { + prefix := "p/" + svc := &prefixMockS3{objects: map[string][]string{ + // Only .csv and .json.gz are collectable; the count must match what + // would actually be collected, not every object present. + prefix: {"a.csv", "b.json.gz", "c.txt", "d", "e.zip"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "b"} + + count, atLeast, err := client.CountObjects(prefix, 0) + require.Nil(t, err) + assert.Equal(t, 2, count) + assert.False(t, atLeast) + + // Counting stops at the cap so the check stays cheap on a large bucket; the + // flag marks the value as a floor. + count, atLeast, err = client.CountObjects(prefix, 1) + require.Nil(t, err) + assert.Equal(t, 1, count) + assert.True(t, atLeast) +} + +func TestListSubPrefixes_TrailingSlashHandling(t *testing.T) { + svc := &prefixMockS3{commonPrefixes: map[string][]string{ + "a/b/": {"x", "y"}, + }} + client := &KiroS3Client{S3: svc, Bucket: "bkt"} + + // A caller-supplied prefix may or may not end in a slash; both must resolve + // to the same listing, and the child names come back without the slash. + withSlash, err := client.ListSubPrefixes("a/b/") + require.Nil(t, err) + withoutSlash, err := client.ListSubPrefixes("a/b") + require.Nil(t, err) + + assert.Equal(t, []string{"x", "y"}, withSlash) + assert.Equal(t, withSlash, withoutSlash) +} diff --git a/backend/plugins/kiro/tasks/extractor.go b/backend/plugins/kiro/tasks/extractor.go new file mode 100644 index 00000000000..10867515005 --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor.go @@ -0,0 +1,204 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// parseFunc turns one downloaded object into batches of rows ready for +// insertion. It is a pure function so it can run on a worker goroutine without +// touching the database. +// +// The result is a slice of batches rather than one flat slice because GORM +// derives the target table from the element type: a mixed []interface{} fails +// with "Table not set". A report CSV yields two different models, so each model +// gets its own homogeneous batch. +type parseFunc func(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) + +// rowBatch is a set of rows that all belong to the same table. +type rowBatch struct { + // rows must be a typed slice (e.g. []*models.KiroChatLog), not + // []interface{}, so GORM can resolve the table. + rows interface{} + count int +} + +// parsedFile is a worker's output, handed to the main goroutine for persistence. +type parsedFile struct { + meta *models.KiroS3FileMeta + batches []rowBatch + err errors.Error +} + +// rowCount totals the rows across every batch, for the cursor's record count. +func (p parsedFile) rowCount() int { + total := 0 + for _, batch := range p.batches { + total += batch.count + } + return total +} + +// pendingFiles returns the files still awaiting extraction for a scope. +// +// Files that have already failed MaxAttempts times are excluded. Without that +// bound, one permanently malformed object is retried on every run: the log fills +// with the same error and the scope never reaches a finished state. +func pendingFiles(db dal.Dal, connectionId uint64, scopeId string, fileType string) ([]models.KiroS3FileMeta, errors.Error) { + var files []models.KiroS3FileMeta + err := db.All(&files, + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND scope_id = ? AND file_type = ? AND processed = ? AND attempt_count < ?", + connectionId, scopeId, fileType, false, models.MaxAttempts), + ) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to query pending kiro files") + } + return files, nil +} + +// extractFiles downloads and parses every pending file of one type, then stores +// the results. +// +// Downloads run concurrently but all database writes happen on the calling +// goroutine. That split is a correctness requirement, not an optimization: +// twenty workers inserting into the same table concurrently deadlock on MySQL +// gap locks, and DevLake's retry layer would turn those deadlocks into +// intermittent failures that only appear under load. +func extractFiles(taskCtx plugin.SubTaskContext, fileType string, parse parseFunc) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + files, err := pendingFiles(db, data.Options.ConnectionId, data.Options.ScopeId, fileType) + if err != nil { + return err + } + if len(files) == 0 { + logger.Info("no pending %s files for scope %s", fileType, data.Options.ScopeId) + return nil + } + + logger.Info("extracting %d %s files", len(files), fileType) + taskCtx.SetProgress(0, len(files)) + + client := data.S3Clients.ForFileType(fileType) + results := make(chan parsedFile, len(files)) + + scheduler, err := helper.NewWorkerScheduler( + taskCtx.GetContext(), + data.WorkerCount(), + // The tick is a global rate limiter, not a per-worker one: every task + // waits for one tick before running, so this value caps total + // throughput regardless of pool size. A one-second tick would pin the + // whole run to one file per second and leave the pool idle - measured + // at 13 seconds for 13 files. Keep it well below the per-object fetch + // time so the workers, not the ticker, set the pace. + time.Millisecond, + logger, + ) + if err != nil { + return err + } + defer scheduler.Release() + + for i := range files { + meta := files[i] + scheduler.SubmitBlocking(func() errors.Error { + // Workers only fetch and parse. Anything touching the database + // happens after WaitAsync below. + body, getErr := client.GetObjectBytes(meta.S3Path) + if getErr != nil { + results <- parsedFile{meta: &meta, err: getErr} + return nil + } + batches, parseErr := parse(body, data.Options.ConnectionId, data.Options.ScopeId) + results <- parsedFile{meta: &meta, batches: batches, err: parseErr} + return nil + }) + } + + if waitErr := scheduler.WaitAsync(); waitErr != nil { + return waitErr + } + close(results) + + for result := range results { + if saveErr := persistFile(db, result); saveErr != nil { + return saveErr + } + taskCtx.IncProgress(1) + } + + return nil +} + +// persistFile stores one file's rows and updates its cursor entry. +// +// A parse or download failure is recorded rather than aborting the run: one bad +// object should not block the rest of the scope. The reason is written to +// error_message so a short month can be explained with a query instead of +// guesswork. +func persistFile(db dal.Dal, result parsedFile) errors.Error { + meta := result.meta + + if result.err != nil { + meta.AttemptCount++ + meta.ErrorMessage = result.err.Error() + // Processed stays false so the file is retried, up to MaxAttempts. + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to record kiro extraction failure") + } + return nil + } + + // Each batch is written separately so GORM sees a typed slice and can + // resolve the target table. + for _, batch := range result.batches { + if batch.count == 0 { + continue + } + if err := db.CreateOrUpdate(batch.rows); err != nil { + // A write failure counts as an attempt too, otherwise a row that + // cannot be stored would be fetched forever. + meta.AttemptCount++ + meta.ErrorMessage = err.Error() + if updateErr := db.Update(meta); updateErr != nil { + return errors.Default.Wrap(updateErr, "failed to record kiro write failure") + } + return nil + } + } + + now := time.Now() + meta.Processed = true + meta.ProcessedTime = &now + meta.RecordCount = result.rowCount() + meta.ErrorMessage = "" + if err := db.Update(meta); err != nil { + return errors.Default.Wrap(err, "failed to mark kiro file processed") + } + return nil +} diff --git a/backend/plugins/kiro/tasks/extractor_test.go b/backend/plugins/kiro/tasks/extractor_test.go new file mode 100644 index 00000000000..5eec6051e3b --- /dev/null +++ b/backend/plugins/kiro/tasks/extractor_test.go @@ -0,0 +1,184 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// GORM resolves the destination table from a slice's element type, so every +// batch must be a typed slice. Passing []interface{} fails at runtime with +// "Table not set" - a real bug caught only by running against a database, since +// it type-checks fine. These assertions pin the requirement so a future +// refactor back to a flat interface slice fails here instead of in production. +func assertTypedBatch[T any](t *testing.T, batch rowBatch, wantCount int) { + t.Helper() + rows, ok := batch.rows.([]T) + require.True(t, ok, "batch must carry a typed slice, got %T", batch.rows) + assert.Len(t, rows, wantCount) + assert.Equal(t, wantCount, batch.count) +} + +// The adapters exist so one parse produces everything a file yields; these +// verify the adaptation rather than the parsing, which is covered directly. +func TestParseUserReportRows_ReturnsTwoTypedBatches(t *testing.T) { + data := loadReportFixture(t, "02_standard_cli.csv") + + batches, err := parseUserReportRows(data, testConnectionId, testScopeId) + require.Nil(t, err) + + // Two batches, not one mixed slice: a report CSV yields two different + // models, and GORM cannot infer a table from a heterogeneous slice. + require.Len(t, batches, 2) + assertTypedBatch[*models.KiroUserReport](t, batches[0], 1) + assertTypedBatch[*models.KiroUserModelMessage](t, batches[1], 1) +} + +func TestParseLogRows(t *testing.T) { + t.Run("chat log", func(t *testing.T) { + batches, err := parseChatLogRows(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroChatLog](t, batches[0], 2) + }) + + t.Run("completion log", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + // A record with no completions still produces a row: those records are the + // denominator for what Kiro offered versus what was taken. + t.Run("empty completions still produce a row", func(t *testing.T) { + batches, err := parseCompletionLogRows(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, batches, 1) + assertTypedBatch[*models.KiroCompletionLog](t, batches[0], 1) + }) + + t.Run("parse failure propagates", func(t *testing.T) { + _, err := parseChatLogRows([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) +} + +// The scheduler's tick is a global rate limiter rather than a per-worker one: +// every submitted task waits one tick before running, so the interval caps total +// throughput no matter how large the pool is. A one-second tick pinned a real run +// to one file per second - 13 seconds for 13 files, with 20 workers idle. +func TestSchedulerTickDoesNotThrottleThePool(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + tick := regexp.MustCompile(`NewWorkerScheduler\((?s:.*?)time\.(\w+),`).FindStringSubmatch(string(source)) + require.NotNil(t, tick, "expected a tick interval passed to NewWorkerScheduler") + assert.Equal(t, "Millisecond", tick[1], + "a coarser tick throttles the whole pool to one object per tick") +} + +// Twenty workers inserting into one table concurrently deadlock on MySQL gap +// locks, and DevLake's retry layer turns those deadlocks into failures that only +// appear under load - the hardest kind to reproduce. The worker closure must +// therefore stay free of database access, which is asserted structurally because +// no unit test can observe the absence of a write. +func TestExtractorWorkersDoNotTouchTheDatabase(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + body := string(source) + workerStart := regexp.MustCompile(`scheduler\.SubmitBlocking\(func\(\) errors\.Error \{`).FindStringIndex(body) + require.NotNil(t, workerStart, "expected a worker closure submitted to the scheduler") + + // Take the closure body up to the WaitAsync call, which is where the main + // goroutine resumes and persistence begins. + waitIdx := regexp.MustCompile(`scheduler\.WaitAsync\(\)`).FindStringIndex(body) + require.NotNil(t, waitIdx) + require.Less(t, workerStart[1], waitIdx[0]) + workerRegion := body[workerStart[1]:waitIdx[0]] + + for _, forbidden := range []string{"db.Create", "db.Update", "db.CreateOrUpdate", "db.All", "db.First", "db.Exec"} { + assert.NotContains(t, workerRegion, forbidden, + "worker goroutines must not access the database; %s belongs on the main goroutine", forbidden) + } +} + +// A permanently malformed object would otherwise be retried on every run, +// filling the log with the same error while the scope never reaches a finished +// state. +func TestPendingFilesQueryBoundsRetries(t *testing.T) { + source, err := os.ReadFile("extractor.go") + require.NoError(t, err) + + clauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, clauses) + + var found bool + for _, clause := range clauses { + if regexp.MustCompile(`attempt_count\s*<`).MatchString(clause[1]) { + found = true + // The same query must also scope to the connection and scope, or one + // scope's run would pick up another's files. + assert.Contains(t, clause[1], "connection_id") + assert.Contains(t, clause[1], "scope_id") + assert.Contains(t, clause[1], "file_type") + } + } + assert.True(t, found, "the pending-files query must bound attempt_count") +} + +func TestMaxAttemptsIsBounded(t *testing.T) { + // A cap that is zero or negative would stop all extraction; an unbounded one + // would never converge. + assert.Greater(t, models.MaxAttempts, 0) + assert.LessOrEqual(t, models.MaxAttempts, 10) +} + +// Every extractor must declare the collector as a dependency: without the file +// metadata rows there is nothing to extract, and DevLake would otherwise be free +// to schedule extraction before discovery. +func TestExtractorSubTaskMetas(t *testing.T) { + for _, meta := range []plugin.SubTaskMeta{ + ExtractKiroUserReportMeta, + ExtractKiroChatLogMeta, + ExtractKiroCompletionLogMeta, + } { + t.Run(meta.Name, func(t *testing.T) { + assert.NotEmpty(t, meta.Name) + assert.NotNil(t, meta.EntryPoint) + assert.True(t, meta.EnabledByDefault) + + var dependsOnCollector bool + for _, dep := range meta.Dependencies { + if dep.Name == CollectKiroS3FilesMeta.Name { + dependsOnCollector = true + } + } + assert.True(t, dependsOnCollector, "extraction depends on file discovery having run") + }) + } +} diff --git a/backend/plugins/kiro/tasks/identity_client.go b/backend/plugins/kiro/tasks/identity_client.go new file mode 100644 index 00000000000..8e6ed34c2fd --- /dev/null +++ b/backend/plugins/kiro/tasks/identity_client.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/identitystore" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// IdentityStoreAPI is the subset of the Identity Store API used here. +type IdentityStoreAPI interface { + DescribeUser(input *identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) +} + +// KiroIdentityClient resolves user ids to human-readable display names. +// +// This is entirely optional. Identity for joining to git history comes from the +// User_Email column of the report CSV, so a missing or misconfigured Identity +// Store degrades only the display name, never the data pipeline. +type KiroIdentityClient struct { + IdentityStore IdentityStoreAPI + StoreId string +} + +// NewKiroIdentityClient returns nil when Identity Store is not configured, +// which callers treat as "no display names" rather than as an error. +func NewKiroIdentityClient(connection *models.KiroConnection) (*KiroIdentityClient, error) { + if connection.IdentityStoreId == "" || connection.IdentityStoreRegion == "" { + return nil, nil + } + + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.IdentityStoreRegion), + Credentials: credentials.NewStaticCredentials( + connection.AccessKeyId, + connection.SecretAccessKey, + "", + ), + }) + if err != nil { + return nil, err + } + + return &KiroIdentityClient{ + IdentityStore: identitystore.New(sess), + StoreId: connection.IdentityStoreId, + }, nil +} + +// ResolveDisplayName looks up a display name, returning nil when it cannot be +// determined. +// +// nil rather than the raw user id: the column exists for human readability, and +// storing an id there would make it look like a resolved name. +func (client *KiroIdentityClient) ResolveDisplayName(userId string) (*string, error) { + if client == nil || client.IdentityStore == nil || userId == "" { + return nil, nil + } + + result, err := client.IdentityStore.DescribeUser(&identitystore.DescribeUserInput{ + IdentityStoreId: aws.String(client.StoreId), + UserId: aws.String(userId), + }) + if err != nil { + // Surfaced for logging, but callers proceed without a display name. + return nil, err + } + + if result.DisplayName != nil && *result.DisplayName != "" { + name := *result.DisplayName + return &name, nil + } + return nil, nil +} diff --git a/backend/plugins/kiro/tasks/log_extractor.go b/backend/plugins/kiro/tasks/log_extractor.go new file mode 100644 index 00000000000..476b06add9c --- /dev/null +++ b/backend/plugins/kiro/tasks/log_extractor.go @@ -0,0 +1,84 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var ( + _ plugin.SubTaskEntryPoint = ExtractKiroChatLog + _ plugin.SubTaskEntryPoint = ExtractKiroCompletionLog +) + +var ExtractKiroChatLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroChatLog", + EntryPoint: ExtractKiroChatLog, + EnabledByDefault: true, + Description: "Extract Kiro chat interactions from GenerateAssistantResponse logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +var ExtractKiroCompletionLogMeta = plugin.SubTaskMeta{ + Name: "extractKiroCompletionLog", + EntryPoint: ExtractKiroCompletionLog, + EnabledByDefault: true, + Description: "Extract Kiro inline suggestions from GenerateCompletions logs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroChatLog loads the chat interaction logs for this scope. +// +// This is the high-volume stream: one object per interaction, roughly 700 bytes +// each, which for a single active user reaches several hundred objects a day. +// The cost is request count rather than bytes, which is what the worker pool +// addresses. +func ExtractKiroChatLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeChatLog, parseChatLogRows) +} + +// ExtractKiroCompletionLog loads the inline suggestion logs for this scope. +// +// This stream is dormant under agentic usage - the sampled history stops in +// March - but the objects remain in S3, so a backfill picks them up and a team +// that enables IDE inline completion starts producing them again. +func ExtractKiroCompletionLog(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeCompletionLog, parseCompletionLogRows) +} + +// Both adapters return a single typed batch: the slice keeps its concrete +// element type so GORM can resolve the table. +func parseChatLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseChatLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} + +func parseCompletionLogRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + logs, err := ParseCompletionLog(data, connectionId, scopeId) + if err != nil { + return nil, err + } + return []rowBatch{{rows: logs, count: len(logs)}}, nil +} diff --git a/backend/plugins/kiro/tasks/log_parser.go b/backend/plugins/kiro/tasks/log_parser.go new file mode 100644 index 00000000000..3de9a66fdea --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser.go @@ -0,0 +1,256 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "path/filepath" + "strings" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// Markers used by the prompt heuristics. These read a prompt's text for signs +// of how the developer invoked Kiro; they are only meaningful when a prompt is +// actually present. +const ( + steeringMarker = ".kiro/steering" + specMarker = ".kiro/specs" +) + +// Every field below is a pointer or checked for presence because Kiro omits +// empty keys entirely rather than emitting a null. Across 4530 sampled records, +// five documented fields never appeared at all and modelId appeared on only 45%, +// so "absent" and "zero" have to stay distinguishable. +type chatLogFile struct { + Records []struct { + Request *struct { + Prompt *string `json:"prompt"` + ChatTriggerType *string `json:"chatTriggerType"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + ModelId *string `json:"modelId"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateAssistantResponseEventRequest"` + Response *struct { + AssistantResponse *string `json:"assistantResponse"` + FollowupPrompts *string `json:"followupPrompts"` + RequestId *string `json:"requestId"` + MessageMetadata *struct { + ConversationId *string `json:"conversationId"` + UtteranceId *string `json:"utteranceId"` + } `json:"messageMetadata"` + } `json:"generateAssistantResponseEventResponse"` + } `json:"records"` +} + +type completionLogFile struct { + Records []struct { + Request *struct { + LeftContext *string `json:"leftContext"` + RightContext *string `json:"rightContext"` + FileName *string `json:"fileName"` + UserId *string `json:"userId"` + TimeStamp *string `json:"timeStamp"` + CustomizationArn *string `json:"customizationArn"` + } `json:"generateCompletionsEventRequest"` + Response *struct { + Completions []string `json:"completions"` + RequestId *string `json:"requestId"` + } `json:"generateCompletionsEventResponse"` + } `json:"records"` +} + +// ParseChatLog parses a gzipped GenerateAssistantResponse log. +// +// Neither the prompt nor the assistant response text is persisted: derived +// features are computed here and the originals are dropped, which keeps +// proprietary code and personal content out of the warehouse. +// +// A file holds one or two records, so the array is always walked. +func ParseChatLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroChatLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed chatLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal chat log") + } + + var result []*models.KiroChatLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + // Without both halves there is no usable interaction. + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + // requestId is the primary key; a record without one cannot be + // stored or deduplicated. + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + log := &models.KiroChatLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + ChatTriggerType: deref(record.Request.ChatTriggerType), + // Only ~45% of records carry a model id, so it stays nil when + // absent. Attributing model usage from this column would skew any + // share-of-usage figure; user_model_messages is authoritative. + ModelId: record.Request.ModelId, + ResponseLength: len(deref(record.Response.AssistantResponse)), + // Presence, not content: the follow-up text itself is not stored. + HasFollowupPrompts: record.Response.FollowupPrompts != nil, + } + + prompt := deref(record.Request.Prompt) + if prompt != "" { + // A non-empty prompt means the user spoke this turn. + sum := sha256.Sum256([]byte(prompt)) + hash := hex.EncodeToString(sum[:]) + hasSteering := strings.Contains(prompt, steeringMarker) + isSpecMode := strings.Contains(prompt, specMarker) + + log.HasPrompt = true + log.PromptLength = len(prompt) + log.PromptSha256 = &hash + log.HasSteering = &hasSteering + log.IsSpecMode = &isSpecMode + } + // When the prompt is empty the agent continued on its own. The hash and + // both heuristics stay nil: hashing the empty string would give roughly + // 71% of rows one identical hash and destroy the rework signal, and a + // false heuristic would be indistinguishable from a real negative. + + if md := record.Response.MessageMetadata; md != nil { + log.ConversationId = md.ConversationId + log.UtteranceId = md.UtteranceId + } + + result = append(result, log) + } + + return result, nil +} + +// ParseCompletionLog parses a gzipped GenerateCompletions log. +// +// The counters are named Returned* rather than Accepted*: a record is written +// when the suggestion is requested, and an empty completions array is common, so +// these measure what Kiro offered rather than what was taken. Records with no +// completions are still stored - they are the denominator. +func ParseCompletionLog(gzData []byte, connectionId uint64, scopeId string) ([]*models.KiroCompletionLog, errors.Error) { + raw, err := gunzip(gzData) + if err != nil { + return nil, err + } + + var parsed completionLogFile + if jsonErr := json.Unmarshal(raw, &parsed); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal completion log") + } + + var result []*models.KiroCompletionLog + for _, record := range parsed.Records { + if record.Request == nil || record.Response == nil { + continue + } + requestId := deref(record.Response.RequestId) + if requestId == "" { + continue + } + + userId, identityStoreId := SplitUserId(deref(record.Request.UserId)) + + timestamp, tsErr := ParseKiroTime(deref(record.Request.TimeStamp)) + if tsErr != nil { + return nil, tsErr + } + + // fileName is a bare file name with no directory path, so it cannot be + // resolved to a unique repository file. + fileName := deref(record.Request.FileName) + + charCount := 0 + lineCount := 0 + for _, completion := range record.Response.Completions { + charCount += len(completion) + lineCount += strings.Count(completion, "\n") + 1 + } + + result = append(result, &models.KiroCompletionLog{ + ConnectionId: connectionId, + ScopeId: scopeId, + RequestId: requestId, + UserId: userId, + IdentityStoreId: identityStoreId, + Timestamp: timestamp, + FileName: fileName, + FileExtension: strings.TrimPrefix(filepath.Ext(fileName), "."), + // Present on completion records but never on chat records. + HasCustomization: record.Request.CustomizationArn != nil, + CompletionsCount: len(record.Response.Completions), + ReturnedCharCount: charCount, + ReturnedLineCount: lineCount, + LeftContextLength: len(deref(record.Request.LeftContext)), + RightContextLength: len(deref(record.Request.RightContext)), + }) + } + + return result, nil +} + +func gunzip(gzData []byte) ([]byte, errors.Error) { + reader, err := gzip.NewReader(bytes.NewReader(gzData)) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to open gzip reader") + } + defer reader.Close() + + raw, err := io.ReadAll(reader) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to decompress log file") + } + return raw, nil +} + +func deref(s *string) string { + if s == nil { + return "" + } + return *s +} diff --git a/backend/plugins/kiro/tasks/log_parser_test.go b/backend/plugins/kiro/tasks/log_parser_test.go new file mode 100644 index 00000000000..f883569fecd --- /dev/null +++ b/backend/plugins/kiro/tasks/log_parser_test.go @@ -0,0 +1,319 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "compress/gzip" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixtures are real log files whose text content was replaced but whose exact +// string lengths and line structure were preserved, because those are the +// values the parser derives and this test asserts. +func loadLogFixture(t *testing.T, name string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "logs", name)) + require.NoError(t, err) + return data +} + +func gzipBytes(t *testing.T, s string) []byte { + t.Helper() + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + _, err := w.Write([]byte(s)) + require.NoError(t, err) + require.NoError(t, w.Close()) + return buf.Bytes() +} + +// An empty prompt means the agent continued on its own after a tool call rather +// than the user speaking. This is the majority case - roughly 71% of sampled +// records - and it must not be confused with a user turn. +func TestParseChatLog_EmptyPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_01_empty_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.False(t, l.HasPrompt) + assert.Equal(t, 0, l.PromptLength) + // Hashing the empty string would give ~71% of rows one shared hash and + // destroy the rework signal, so it stays NULL. + assert.Nil(t, l.PromptSha256) + // Both heuristics read prompt text, so without a prompt they are unknown + // rather than false. + assert.Nil(t, l.HasSteering) + assert.Nil(t, l.IsSpecMode) + + assert.Equal(t, "64d13ea7-dff5-4563-9285-6a9e351e87a0", l.RequestId) + assert.Equal(t, "MANUAL", l.ChatTriggerType) + require.NotNil(t, l.ModelId) + assert.Equal(t, "claude-opus-5", *l.ModelId) + assert.Equal(t, 259, l.ResponseLength) + + // Log records always carry the identity-store prefix; stripping it is what + // lets this join against the report table. + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) + assert.Equal(t, "d-1234567890", l.IdentityStoreId) + + // Nanosecond input truncated for DATETIME(6). + assert.Equal(t, 2026, l.Timestamp.Year()) + assert.Equal(t, 23, l.Timestamp.Hour()) + assert.Equal(t, 0, l.Timestamp.Nanosecond()%1000) + + // Documented but never observed, so they stay NULL - their absence is why + // the S3 logs cannot group interactions into sessions. + assert.Nil(t, l.ConversationId) + assert.Nil(t, l.UtteranceId) +} + +func TestParseChatLog_WithPrompt(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_02_with_prompt.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.True(t, l.HasPrompt) + assert.Equal(t, 1363, l.PromptLength) + require.NotNil(t, l.PromptSha256) + assert.Len(t, *l.PromptSha256, 64) + // With a prompt present the heuristics carry a real verdict. + require.NotNil(t, l.HasSteering) + require.NotNil(t, l.IsSpecMode) + assert.Equal(t, 64, l.ResponseLength) +} + +// Kiro packs one or two records per file, so the array is always walked. +func TestParseChatLog_TwoRecords(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_03_two_records.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 2) + + assert.NotEqual(t, logs[0].RequestId, logs[1].RequestId) + // A zero-length assistant response is real data, not a parse failure. + assert.Equal(t, 0, logs[0].ResponseLength) + assert.Equal(t, 117, logs[1].ResponseLength) +} + +// modelId is absent on more than half of all records. Storing an empty string +// instead of NULL would invent a phantom model accounting for most of the +// traffic. +func TestParseChatLog_MissingModelId(t *testing.T) { + logs, err := ParseChatLog(loadLogFixture(t, "chat_04_no_model_id.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Nil(t, l.ModelId, "an absent modelId must be NULL, never an empty string") + // This record does carry followupPrompts, which a single-day sample had + // suggested never appears. + assert.True(t, l.HasFollowupPrompts) + assert.True(t, l.HasPrompt) + assert.Equal(t, 175, l.PromptLength) +} + +func TestParseChatLog_PromptHashing(t *testing.T) { + // The same prompt text must hash identically, which is what makes repeated + // submission detectable as a rework signal. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"fix the retry logic","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29.027400929Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"%s"}}]}` + + first, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-1")), testConnectionId, testScopeId) + require.Nil(t, err) + second, err := ParseChatLog(gzipBytes(t, fmt.Sprintf(body, "req-2")), testConnectionId, testScopeId) + require.Nil(t, err) + + require.Len(t, first, 1) + require.Len(t, second, 1) + require.NotNil(t, first[0].PromptSha256) + require.NotNil(t, second[0].PromptSha256) + assert.Equal(t, *first[0].PromptSha256, *second[0].PromptSha256) + assert.NotEqual(t, first[0].RequestId, second[0].RequestId) +} + +func TestParseChatLog_Heuristics(t *testing.T) { + tests := []struct { + name string + prompt string + wantHasSteering bool + wantIsSpecMode bool + }{ + {"steering reference", "please follow .kiro/steering/style.md", true, false}, + {"spec reference", "implement .kiro/specs/auth/tasks.md", false, true}, + {"both", "read .kiro/steering and .kiro/specs", true, true}, + {"neither", "just fix this bug", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"` + tt.prompt + `","chatTriggerType":"MANUAL",` + + `"userId":"d-abc.user-1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + require.NotNil(t, logs[0].HasSteering) + require.NotNil(t, logs[0].IsSpecMode) + assert.Equal(t, tt.wantHasSteering, *logs[0].HasSteering) + assert.Equal(t, tt.wantIsSpecMode, *logs[0].IsSpecMode) + }) + } +} + +func TestParseCompletionLog_NonEmpty(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_01_non_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + + l := logs[0] + assert.Equal(t, "e0d1e760-a907-47d4-baa5-93df0f38b274", l.RequestId) + // A bare file name with no directory path, so it cannot be resolved to a + // unique repository file. + assert.Equal(t, "mcp.json", l.FileName) + assert.Equal(t, "json", l.FileExtension) + assert.Equal(t, 1, l.CompletionsCount) + assert.Equal(t, 347, l.ReturnedCharCount) + assert.Equal(t, 15, l.ReturnedLineCount) + assert.Equal(t, 5324, l.LeftContextLength) + assert.Equal(t, 12, l.RightContextLength) + assert.False(t, l.HasCustomization) + assert.Equal(t, "11111111-1111-4111-8111-111111111111", l.UserId) +} + +// A completion record is written when the suggestion is requested, not when it +// is accepted, so an empty array is normal. The record is the denominator and +// must still be stored. +func TestParseCompletionLog_EmptyCompletions(t *testing.T) { + logs, err := ParseCompletionLog(loadLogFixture(t, "completion_02_empty.json.gz"), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1, "a record with no completions is still stored") + + l := logs[0] + assert.Equal(t, 0, l.CompletionsCount) + assert.Equal(t, 0, l.ReturnedCharCount) + assert.Equal(t, 0, l.ReturnedLineCount) + // Context was still sent, which is what distinguishes this from a truncated + // record. + assert.Equal(t, 5323, l.LeftContextLength) +} + +func TestParseCompletionLog_LineCounting(t *testing.T) { + tests := []struct { + name string + completions string + wantLines int + wantChars int + }{ + {"single line", `["abc"]`, 1, 3}, + {"two lines", `["a\nb"]`, 2, 3}, + {"trailing newline counts the empty final line", `["a\n"]`, 2, 2}, + {"two completions summed", `["a\nb","c"]`, 3, 4}, + {"empty array", `[]`, 0, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","leftContext":"","rightContext":"",` + + `"userId":"d-abc.user-1","timeStamp":"2026-03-19T13:49:58Z"},` + + `"generateCompletionsEventResponse":{"completions":` + tt.completions + `,"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, tt.wantLines, logs[0].ReturnedLineCount) + assert.Equal(t, tt.wantChars, logs[0].ReturnedCharCount) + }) + } +} + +func TestParseCompletionLog_Customization(t *testing.T) { + // customizationArn appears on completion records but never on chat records. + body := `{"records":[{"generateCompletionsEventRequest":` + + `{"fileName":"f.ts","userId":"d-abc.u1","timeStamp":"2026-03-19T13:49:58Z",` + + `"customizationArn":"arn:aws:codewhisperer:us-east-1:1:customization/abc"},` + + `"generateCompletionsEventResponse":{"completions":[],"requestId":"r1"}}]}` + logs, err := ParseCompletionLog(gzipBytes(t, body), testConnectionId, testScopeId) + require.Nil(t, err) + require.Len(t, logs, 1) + assert.True(t, logs[0].HasCustomization) +} + +func TestParseLog_EdgeCases(t *testing.T) { + t.Run("empty records array", func(t *testing.T) { + logs, err := ParseChatLog(gzipBytes(t, `{"records":[]}`), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record without requestId is skipped", func(t *testing.T) { + // requestId is the primary key, so such a record cannot be stored or + // deduplicated. + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("record missing the response half is skipped", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"x","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.Nil(t, err) + assert.Empty(t, logs) + }) + + t.Run("bad timestamp is an error", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"not-a-time"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + _, err := ParseChatLog(gzipBytes(t, body), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("non-gzip input is an error", func(t *testing.T) { + _, err := ParseChatLog([]byte("not gzipped"), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("malformed json is an error", func(t *testing.T) { + _, err := ParseChatLog(gzipBytes(t, `{"records":`), testConnectionId, testScopeId) + assert.NotNil(t, err) + }) + + t.Run("connection and scope are propagated", func(t *testing.T) { + body := `{"records":[{"generateAssistantResponseEventRequest":` + + `{"prompt":"","userId":"d-abc.u1","timeStamp":"2026-07-27T23:03:29Z"},` + + `"generateAssistantResponseEventResponse":{"assistantResponse":"ok","requestId":"r1"}}]}` + logs, err := ParseChatLog(gzipBytes(t, body), 42, "scope-x") + require.Nil(t, err) + require.Len(t, logs, 1) + assert.Equal(t, uint64(42), logs[0].ConnectionId) + assert.Equal(t, "scope-x", logs[0].ScopeId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_client.go b/backend/plugins/kiro/tasks/s3_client.go new file mode 100644 index 00000000000..a0e2d19ba5a --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client.go @@ -0,0 +1,217 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "io" + "sort" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// S3API is the subset of the S3 API this plugin uses, declared as an interface +// so collectors and extractors can be tested without AWS. +type S3API interface { + ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) + GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) +} + +// KiroS3Client is bound to exactly one bucket. +// +// Kiro recommends keeping interaction logs in a bucket separate from the +// activity reports, and the two may carry different KMS keys and IAM +// conditions. One client per bucket keeps those permission boundaries distinct, +// so an access failure points at a specific bucket instead of an ambiguous +// request. +type KiroS3Client struct { + S3 S3API + Bucket string +} + +// KiroS3Clients holds the report and log clients for a connection. When no +// separate log bucket is configured both fields address the same bucket, so +// single-bucket and dual-bucket setups take the same code path everywhere else. +type KiroS3Clients struct { + Report *KiroS3Client + PromptLog *KiroS3Client +} + +// NewKiroS3Clients builds the client pair for a connection. +func NewKiroS3Clients(connection *models.KiroConnection) (*KiroS3Clients, errors.Error) { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(connection.Region), + Credentials: credentials.NewStaticCredentials(connection.AccessKeyId, connection.SecretAccessKey, ""), + }) + if err != nil { + return nil, errors.Convert(err) + } + + // A single S3 service client can address both buckets; the split is at the + // KiroS3Client level, which pins the bucket name. + svc := s3.New(sess) + + return &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: connection.Bucket}, + PromptLog: &KiroS3Client{S3: svc, Bucket: connection.GetPromptLogBucket()}, + }, nil +} + +// ForFileType returns the client that owns a given file type. +func (c *KiroS3Clients) ForFileType(fileType string) *KiroS3Client { + if fileType == models.FileTypeReport { + return c.Report + } + return c.PromptLog +} + +// Buckets returns the distinct buckets in use - one entry when reports and logs +// share a bucket, two when they do not. +func (c *KiroS3Clients) Buckets() []string { + if c.Report.Bucket == c.PromptLog.Bucket { + return []string{c.Report.Bucket} + } + return []string{c.Report.Bucket, c.PromptLog.Bucket} +} + +// ListSubPrefixes returns the immediate child "directories" under a prefix. +// +// Kiro's export layout is fully self-describing - accounts, years and months all +// appear as path segments - so this is what lets a scope be picked from what +// actually exists instead of typed by hand. A mistyped prefix is otherwise +// indistinguishable from a month with no data: collection succeeds and finds +// nothing either way. +// +// Uses a delimiter so S3 returns only the segment names, not every object +// beneath them. +func (c *KiroS3Client) ListSubPrefixes(prefix string) ([]string, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + var names []string + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + Delimiter: aws.String("/"), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, errors.Convert(err) + } + + for _, common := range output.CommonPrefixes { + if common.Prefix == nil { + continue + } + // Strip the queried prefix and the trailing slash to leave just the + // segment name. + name := strings.TrimSuffix(strings.TrimPrefix(*common.Prefix, prefix), "/") + if name != "" { + names = append(names, name) + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + sort.Strings(names) + return names, nil +} + +// CountObjects reports how many collectable objects sit under a prefix. +// +// This is what turns "did I get the path right?" into an answerable question: +// the connection test reports these counts per stream, so a wrong prefix shows +// as zero before any scope is created. +// +// Counting stops at limit to keep the check cheap; the returned bool reports +// whether more objects remain. +func (c *KiroS3Client) CountObjects(prefix string, limit int) (int, bool, errors.Error) { + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + + count := 0 + var continuationToken *string + for { + output, err := c.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(c.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return 0, false, errors.Convert(err) + } + + for _, object := range output.Contents { + if object.Key == nil { + continue + } + // Same filter the collector applies, so the count reflects what + // would actually be collected rather than every object present. + if !strings.HasSuffix(*object.Key, ".csv") && !strings.HasSuffix(*object.Key, ".json.gz") { + continue + } + count++ + if limit > 0 && count >= limit { + return count, true, nil + } + } + + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + + return count, false, nil +} + +// GetObjectBytes downloads an object in full. +// +// Objects are small - roughly 700 bytes for a chat log, a few KB for a +// completion log, and under 1 KB for a report CSV - so streaming would add +// complexity without saving memory. +func (c *KiroS3Client) GetObjectBytes(key string) ([]byte, errors.Error) { + output, err := c.S3.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(c.Bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, errors.Convert(err) + } + defer output.Body.Close() + + data, readErr := io.ReadAll(output.Body) + if readErr != nil { + return nil, errors.Convert(readErr) + } + return data, nil +} diff --git a/backend/plugins/kiro/tasks/s3_client_test.go b/backend/plugins/kiro/tasks/s3_client_test.go new file mode 100644 index 00000000000..08c910fcf20 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_client_test.go @@ -0,0 +1,237 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/aws/aws-sdk-go/service/identitystore" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// mockS3 records the bucket each call addressed, so tests can prove a request +// went to the right one. +type mockS3 struct { + getObjectBody string + getObjectErr error + listOutputs []*s3.ListObjectsV2Output + listCallIdx int + seenGetBuckets []string + seenListBuckets []string + seenGetKeys []string +} + +func (m *mockS3) GetObject(input *s3.GetObjectInput) (*s3.GetObjectOutput, error) { + m.seenGetBuckets = append(m.seenGetBuckets, *input.Bucket) + m.seenGetKeys = append(m.seenGetKeys, *input.Key) + if m.getObjectErr != nil { + return nil, m.getObjectErr + } + return &s3.GetObjectOutput{ + Body: io.NopCloser(bytes.NewReader([]byte(m.getObjectBody))), + }, nil +} + +func (m *mockS3) ListObjectsV2(input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + m.seenListBuckets = append(m.seenListBuckets, *input.Bucket) + if m.listCallIdx >= len(m.listOutputs) { + return &s3.ListObjectsV2Output{}, nil + } + out := m.listOutputs[m.listCallIdx] + m.listCallIdx++ + return out, nil +} + +// The fallback rules mean a single-bucket deployment (what real exports look +// like today) and the dual-bucket layout Kiro recommends both work without +// branching in collectors or extractors. +func TestKiroS3Clients_BucketRouting(t *testing.T) { + t.Run("single bucket routes both file kinds to the same bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "one-bucket"}, + } + + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "one-bucket", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + // Deduplicated, so a connection test checks access once rather than twice. + assert.Equal(t, []string{"one-bucket"}, clients.Buckets()) + }) + + t.Run("separate buckets route by file type", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + + assert.Equal(t, "reports", clients.ForFileType(models.FileTypeReport).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeChatLog).Bucket) + assert.Equal(t, "logs", clients.ForFileType(models.FileTypeCompletionLog).Bucket) + assert.Equal(t, []string{"reports", "logs"}, clients.Buckets()) + }) + + // An unrecognized file type must not silently read from the report bucket, + // where it would find nothing; log data is the larger and more likely case. + t.Run("unknown file type falls to the log bucket", func(t *testing.T) { + svc := &mockS3{} + clients := &KiroS3Clients{ + Report: &KiroS3Client{S3: svc, Bucket: "reports"}, + PromptLog: &KiroS3Client{S3: svc, Bucket: "logs"}, + } + assert.Equal(t, "logs", clients.ForFileType("something-new").Bucket) + }) +} + +func TestNewKiroS3Clients_FallbackFromConnection(t *testing.T) { + t.Run("no prompt log bucket falls back to the report bucket", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "kiro-export-test", clients.Report.Bucket) + assert.Equal(t, "kiro-export-test", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 1) + }) + + t.Run("explicit prompt log bucket is used", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports-bucket", + PromptLogBucket: "logs-bucket", + }} + clients, err := NewKiroS3Clients(conn) + require.Nil(t, err) + assert.Equal(t, "reports-bucket", clients.Report.Bucket) + assert.Equal(t, "logs-bucket", clients.PromptLog.Bucket) + assert.Len(t, clients.Buckets(), 2) + }) +} + +func TestKiroS3Client_GetObjectBytes(t *testing.T) { + t.Run("reads the body and addresses the bound bucket", func(t *testing.T) { + svc := &mockS3{getObjectBody: "hello"} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + data, err := client.GetObjectBytes("some/key.csv") + require.Nil(t, err) + assert.Equal(t, "hello", string(data)) + assert.Equal(t, []string{"my-bucket"}, svc.seenGetBuckets) + assert.Equal(t, []string{"some/key.csv"}, svc.seenGetKeys) + }) + + t.Run("propagates an S3 error", func(t *testing.T) { + svc := &mockS3{getObjectErr: errors.New("access denied")} + client := &KiroS3Client{S3: svc, Bucket: "my-bucket"} + + _, err := client.GetObjectBytes("k") + assert.NotNil(t, err) + }) +} + +// mockIdentityStore lets the optional display-name path be exercised without +// AWS. +type mockIdentityStore struct { + displayName *string + err error +} + +func (m *mockIdentityStore) DescribeUser(*identitystore.DescribeUserInput) (*identitystore.DescribeUserOutput, error) { + if m.err != nil { + return nil, m.err + } + return &identitystore.DescribeUserOutput{DisplayName: m.displayName}, nil +} + +func TestKiroIdentityClient_ResolveDisplayName(t *testing.T) { + name := "Some Developer" + + t.Run("resolves a display name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &name}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + require.NotNil(t, got) + assert.Equal(t, name, *got) + }) + + // The column exists for human readability; falling back to the raw id would + // make an unresolved value look like a resolved one. + t.Run("empty display name yields nil, not the user id", func(t *testing.T) { + empty := "" + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{displayName: &empty}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + // Identity Store is optional, so an unconfigured client must be safe to + // call rather than something every caller has to nil-check. + t.Run("nil client is safe to call", func(t *testing.T) { + var client *KiroIdentityClient + got, err := client.ResolveDisplayName("user-1") + require.NoError(t, err) + assert.Nil(t, got) + }) + + t.Run("error surfaces but yields no name", func(t *testing.T) { + client := &KiroIdentityClient{IdentityStore: &mockIdentityStore{err: errors.New("throttled")}, StoreId: "d-1"} + got, err := client.ResolveDisplayName("user-1") + assert.Error(t, err) + assert.Nil(t, got) + }) +} + +func TestNewKiroIdentityClient_OptionalConfiguration(t *testing.T) { + // Missing configuration is not an error: collection works fully without + // display names because identity comes from the report's email column. + for _, tt := range []struct { + name string + conn models.KiroConn + }{ + {"neither set", models.KiroConn{}}, + {"only store id", models.KiroConn{IdentityStoreId: "d-1"}}, + {"only region", models.KiroConn{IdentityStoreRegion: "us-east-1"}}, + } { + t.Run(tt.name, func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: tt.conn}) + require.NoError(t, err) + assert.Nil(t, client) + }) + } + + t.Run("fully configured returns a client", func(t *testing.T) { + client, err := NewKiroIdentityClient(&models.KiroConnection{KiroConn: models.KiroConn{ + IdentityStoreId: "d-1234567890", + IdentityStoreRegion: "us-east-1", + }}) + require.NoError(t, err) + require.NotNil(t, client) + assert.Equal(t, "d-1234567890", client.StoreId) + }) +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector.go b/backend/plugins/kiro/tasks/s3_file_collector.go new file mode 100644 index 00000000000..552292e3737 --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector.go @@ -0,0 +1,171 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "path" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = CollectKiroS3Files + +// CollectKiroS3FilesMeta discovers which S3 objects exist for a scope. +var CollectKiroS3FilesMeta = plugin.SubTaskMeta{ + Name: "collectKiroS3Files", + EntryPoint: CollectKiroS3Files, + EnabledByDefault: true, + Description: "List Kiro export objects in S3 and record them for extraction", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +// CollectKiroS3Files lists every relevant object under the scope's prefixes and +// records the ones not seen before. +// +// Work is batched per listing page rather than per object. A single S3 page +// holds up to 1000 keys, so one SELECT and one INSERT per page replaces two +// round trips per file - at tens of thousands of objects a day that is the +// difference between dozens of queries and tens of thousands. +func CollectKiroS3Files(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*KiroTaskData) + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + + taskCtx.SetProgress(0, -1) + + for _, spec := range data.Prefixes { + client := data.S3Clients.ForFileType(spec.FileType) + prefix := spec.Prefix + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + logger.Info("scanning s3://%s/%s for %s files", client.Bucket, prefix, spec.FileType) + + var continuationToken *string + for { + output, listErr := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + Bucket: aws.String(client.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if listErr != nil { + return errors.Convert(listErr) + } + + candidates := collectCandidates(output, client.Bucket, spec, data.Options) + inserted, saveErr := saveNewFileMeta(db, data.Options.ConnectionId, candidates) + if saveErr != nil { + return saveErr + } + taskCtx.IncProgress(inserted) + + // IsTruncated is a pointer; dereferencing it unguarded panics on an + // empty response. + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + continuationToken = output.NextContinuationToken + } + } + + return nil +} + +// collectCandidates turns one listing page into file metadata rows. +// +// Only .csv and .json.gz are kept. That filter also excludes the small +// extension-less objects AWS writes at the KiroLogs root as permission probes. +func collectCandidates(output *s3.ListObjectsV2Output, bucket string, spec PrefixSpec, options *KiroOptions) []*models.KiroS3FileMeta { + candidates := make([]*models.KiroS3FileMeta, 0, len(output.Contents)) + for _, object := range output.Contents { + if object.Key == nil { + continue + } + key := *object.Key + if !strings.HasSuffix(key, ".csv") && !strings.HasSuffix(key, ".json.gz") { + continue + } + candidates = append(candidates, &models.KiroS3FileMeta{ + ConnectionId: options.ConnectionId, + S3Path: key, + // Basename only. The full key lives in S3Path, which is sized for + // it; putting a full key here would eventually overflow the column. + FileName: path.Base(key), + Bucket: bucket, + ScopeId: options.ScopeId, + FileType: spec.FileType, + Processed: false, + }) + } + return candidates +} + +// saveNewFileMeta inserts the rows that are not already recorded, returning how +// many were added. +// +// The existence check queries by (connection_id, s3_path), which is exactly the +// primary key. Querying on an unindexed column here would turn each page into a +// full table scan and the task would never finish at scale. +func saveNewFileMeta(db dal.Dal, connectionId uint64, candidates []*models.KiroS3FileMeta) (int, errors.Error) { + if len(candidates) == 0 { + return 0, nil + } + + paths := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + paths = append(paths, candidate.S3Path) + } + + var existingRows []models.KiroS3FileMeta + err := db.All(&existingRows, + dal.Select("s3_path"), + dal.From(&models.KiroS3FileMeta{}), + dal.Where("connection_id = ? AND s3_path IN ?", connectionId, paths), + ) + if err != nil { + return 0, errors.Default.Wrap(err, "failed to query existing kiro file metadata") + } + + existing := make(map[string]struct{}, len(existingRows)) + for _, row := range existingRows { + existing[row.S3Path] = struct{}{} + } + + fresh := make([]*models.KiroS3FileMeta, 0, len(candidates)) + for _, candidate := range candidates { + if _, seen := existing[candidate.S3Path]; seen { + continue + } + fresh = append(fresh, candidate) + } + if len(fresh) == 0 { + return 0, nil + } + + if err := db.Create(fresh); err != nil { + return 0, errors.Default.Wrap(err, "failed to record kiro file metadata") + } + return len(fresh), nil +} diff --git a/backend/plugins/kiro/tasks/s3_file_collector_test.go b/backend/plugins/kiro/tasks/s3_file_collector_test.go new file mode 100644 index 00000000000..e45b26044fb --- /dev/null +++ b/backend/plugins/kiro/tasks/s3_file_collector_test.go @@ -0,0 +1,215 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "os" + "regexp" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +func listOutput(truncated bool, keys ...string) *s3.ListObjectsV2Output { + contents := make([]*s3.Object, 0, len(keys)) + for _, k := range keys { + key := k + contents = append(contents, &s3.Object{Key: &key}) + } + return &s3.ListObjectsV2Output{ + Contents: contents, + IsTruncated: aws.Bool(truncated), + } +} + +func TestCollectCandidates_FileFiltering(t *testing.T) { + spec := PrefixSpec{FileType: models.FileTypeChatLog} + options := &KiroOptions{ConnectionId: 1, ScopeId: "s1"} + + t.Run("keeps csv and json.gz only", func(t *testing.T) { + output := listOutput(false, + "p/report.csv", + "p/log.json.gz", + // AWS writes small extension-less objects at the KiroLogs root as + // permission probes; they must not enter the work queue. + "p/26404955-bf00-40d3-b713-43d18edf0638", + "p/notes.txt", + "p/archive.zip", + ) + candidates := collectCandidates(output, "bkt", spec, options) + require.Len(t, candidates, 2) + assert.Equal(t, "p/report.csv", candidates[0].S3Path) + assert.Equal(t, "p/log.json.gz", candidates[1].S3Path) + }) + + t.Run("stores basename separately from the full key", func(t *testing.T) { + key := "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07/27/23/" + + "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz" + candidates := collectCandidates(listOutput(false, key), "bkt", spec, options) + require.Len(t, candidates, 1) + + // The full key goes in S3Path, which is sized for it. FileName holds + // only the basename - a full key there would eventually overflow. + assert.Equal(t, key, candidates[0].S3Path) + assert.Equal(t, "123456789012_GenerateAssistantResponse_202607272303_3tbIeIrGJNDFbfVx.json.gz", + candidates[0].FileName) + assert.Less(t, len(candidates[0].FileName), 255) + }) + + t.Run("records bucket, scope and file type", func(t *testing.T) { + candidates := collectCandidates(listOutput(false, "p/a.csv"), "my-bucket", spec, options) + require.Len(t, candidates, 1) + assert.Equal(t, "my-bucket", candidates[0].Bucket) + assert.Equal(t, "s1", candidates[0].ScopeId) + assert.Equal(t, models.FileTypeChatLog, candidates[0].FileType) + assert.Equal(t, uint64(1), candidates[0].ConnectionId) + assert.False(t, candidates[0].Processed) + }) + + t.Run("nil key is skipped", func(t *testing.T) { + output := &s3.ListObjectsV2Output{ + Contents: []*s3.Object{{Key: nil}, {Key: aws.String("p/a.csv")}}, + IsTruncated: aws.Bool(false), + } + candidates := collectCandidates(output, "bkt", spec, options) + assert.Len(t, candidates, 1) + }) + + t.Run("empty page yields nothing", func(t *testing.T) { + candidates := collectCandidates(listOutput(false), "bkt", spec, options) + assert.Empty(t, candidates) + }) +} + +func TestBuildPrefixes(t *testing.T) { + // Paths verified against real exports. + t.Run("single bucket layout", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "kiro-export-test", + ReportPrefix: "user-report", + PromptLogPrefix: "logging", + }} + prefixes := BuildPrefixes(conn, "123456789012", "2026/07") + + require.Len(t, prefixes, 3) + assert.Equal(t, + "user-report/AWSLogs/123456789012/KiroLogs/user_report/us-east-1/2026/07", + prefixes[0].Prefix) + assert.Equal(t, models.FileTypeReport, prefixes[0].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateAssistantResponse/us-east-1/2026/07", + prefixes[1].Prefix) + assert.Equal(t, models.FileTypeChatLog, prefixes[1].FileType) + assert.Equal(t, + "logging/AWSLogs/123456789012/KiroLogs/GenerateCompletions/us-east-1/2026/07", + prefixes[2].Prefix) + assert.Equal(t, models.FileTypeCompletionLog, prefixes[2].FileType) + + for _, p := range prefixes { + assert.Equal(t, "kiro-export-test", p.Bucket) + } + }) + + t.Run("defaults apply when prefixes are unset", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "b", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Contains(t, prefixes[0].Prefix, "user-report/AWSLogs/acct/KiroLogs/user_report") + assert.Contains(t, prefixes[1].Prefix, "logging/AWSLogs/acct/KiroLogs/GenerateAssistantResponse") + }) + + t.Run("separate buckets are assigned per file type", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{ + Region: "us-east-1", + Bucket: "reports", + PromptLogBucket: "logs", + }} + prefixes := BuildPrefixes(conn, "acct", "2026/07") + assert.Equal(t, "reports", prefixes[0].Bucket) + assert.Equal(t, "logs", prefixes[1].Bucket) + assert.Equal(t, "logs", prefixes[2].Bucket) + }) + + // A nil month widens the scope to the whole year, which is how a year-long + // backfill is expressed. + t.Run("year-only time path", func(t *testing.T) { + conn := &models.KiroConnection{KiroConn: models.KiroConn{Region: "us-east-1", Bucket: "b"}} + prefixes := BuildPrefixes(conn, "acct", "2026") + assert.True(t, regexp.MustCompile(`/us-east-1/2026$`).MatchString(prefixes[0].Prefix)) + }) +} + +func TestWorkerCount(t *testing.T) { + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{}}).WorkerCount()) + assert.Equal(t, 5, (&KiroTaskData{Options: &KiroOptions{WorkerCount: 5}}).WorkerCount()) + // A zero or negative override is ignored rather than disabling concurrency. + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{Options: &KiroOptions{WorkerCount: -1}}).WorkerCount()) + assert.Equal(t, DefaultWorkerCount, (&KiroTaskData{}).WorkerCount()) +} + +// This guards the predecessor defect that motivated the primary key choice: it queried +// its cursor table by s3_path while keying it on file_name, so the lookup falls +// back to scanning every row for the connection and collection never finishes at +// scale. The failure mode is a task that hangs rather than an error, so it is +// worth asserting structurally instead of hoping a reviewer notices. +func TestFileMetaQueriesMatchPrimaryKey(t *testing.T) { + pkColumns := primaryKeyColumns(t, "s3_file_meta.go") + require.Equal(t, []string{"ConnectionId", "S3Path"}, pkColumns, + "the cursor table must be keyed on the connection and the full object path") + + source, err := os.ReadFile("s3_file_collector.go") + require.NoError(t, err) + + whereClauses := regexp.MustCompile(`dal\.Where\(\s*"([^"]+)"`).FindAllStringSubmatch(string(source), -1) + require.NotEmpty(t, whereClauses, "expected at least one filtered query") + + for _, clause := range whereClauses { + condition := clause[1] + assert.Contains(t, condition, "connection_id", + "every cursor query must filter on connection_id, the first key column") + assert.Contains(t, condition, "s3_path", + "every cursor query must filter on s3_path, the second key column") + assert.NotContains(t, condition, "file_name", + "file_name is not indexed and must never appear in a lookup") + } +} + +// primaryKeyColumns extracts the fields tagged as primary keys from a model +// file, in declaration order - which is also the order of the composite index. +func primaryKeyColumns(t *testing.T, modelFile string) []string { + t.Helper() + source, err := os.ReadFile("../models/" + modelFile) + require.NoError(t, err) + + fieldRe := regexp.MustCompile(`(?m)^\s*([A-Z][A-Za-z0-9]*)\s+\S+\s+` + "`" + `[^` + "`" + `]*primaryKey[^` + "`" + `]*` + "`") + matches := fieldRe.FindAllStringSubmatch(string(source), -1) + + columns := make([]string, 0, len(matches)) + for _, m := range matches { + columns = append(columns, m[1]) + } + return columns +} diff --git a/backend/plugins/kiro/tasks/task_data.go b/backend/plugins/kiro/tasks/task_data.go new file mode 100644 index 00000000000..c8b504f1d18 --- /dev/null +++ b/backend/plugins/kiro/tasks/task_data.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +// DefaultWorkerCount is how many objects are fetched concurrently. +// +// Collection is bound by request count, not bandwidth: a single user can +// produce ~600 log objects on a busy day, each under a kilobyte. S3 sustains +// thousands of GETs per second per prefix, so 20 is well within limits while +// cutting a peak day from tens of minutes to a couple. +const DefaultWorkerCount = 20 + +// KiroOptions are the blueprint-supplied task options. +type KiroOptions struct { + ConnectionId uint64 `json:"connectionId"` + ScopeId string `json:"scopeId"` + AccountId string `json:"accountId"` + Year int `json:"year"` + Month *int `json:"month"` + // WorkerCount overrides DefaultWorkerCount when set above zero. + WorkerCount int `json:"workerCount"` +} + +// PrefixSpec is one S3 location to scan, along with the kind of file found +// there and which bucket holds it. +type PrefixSpec struct { + Bucket string + Prefix string + FileType string +} + +// KiroTaskData is shared by every subtask in a run. +type KiroTaskData struct { + Options *KiroOptions + Connection *models.KiroConnection + S3Clients *KiroS3Clients + IdentityClient *KiroIdentityClient + // Prefixes are the locations to scan, precomputed so the collector does not + // re-derive paths. + Prefixes []PrefixSpec +} + +// WorkerCount returns the effective concurrency for this run. +func (d *KiroTaskData) WorkerCount() int { + if d.Options != nil && d.Options.WorkerCount > 0 { + return d.Options.WorkerCount + } + return DefaultWorkerCount +} + +// BuildPrefixes derives the three S3 locations a scope covers. +// +// Layout confirmed against real exports: +// +// {bucket}/{reportPrefix}/AWSLogs/{acct}/KiroLogs/user_report/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateAssistantResponse/{region}/{y}/{m} +// {bucket}/{logPrefix}/AWSLogs/{acct}/KiroLogs/GenerateCompletions/{region}/{y}/{m} +// +// The report path's hour segment is always 00 (reports are written at 02:00 +// UTC) while log paths carry a real hour, but neither is included here: the +// prefix stops at the month so a scope lists the whole period in one sweep. +func BuildPrefixes(connection *models.KiroConnection, accountId string, timePath string) []PrefixSpec { + region := connection.Region + reportBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetReportPrefix(), accountId) + logBase := fmt.Sprintf("%s/AWSLogs/%s/KiroLogs", connection.GetPromptLogPrefix(), accountId) + + return []PrefixSpec{ + { + Bucket: connection.Bucket, + Prefix: fmt.Sprintf("%s/user_report/%s/%s", reportBase, region, timePath), + FileType: models.FileTypeReport, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateAssistantResponse/%s/%s", logBase, region, timePath), + FileType: models.FileTypeChatLog, + }, + { + Bucket: connection.GetPromptLogBucket(), + Prefix: fmt.Sprintf("%s/GenerateCompletions/%s/%s", logBase, region, timePath), + FileType: models.FileTypeCompletionLog, + }, + } +} diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz new file mode 100644 index 00000000000..a713fdaf95d Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_01_empty_prompt.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz new file mode 100644 index 00000000000..36fb2f6eed1 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_02_with_prompt.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz new file mode 100644 index 00000000000..37faba6e2e8 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_03_two_records.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz b/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz new file mode 100644 index 00000000000..21551bd3f68 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/chat_04_no_model_id.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz new file mode 100644 index 00000000000..c25e2a154b1 Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/completion_01_non_empty.json.gz differ diff --git a/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz b/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz new file mode 100644 index 00000000000..a914b79e55f Binary files /dev/null and b/backend/plugins/kiro/tasks/testdata/logs/completion_02_empty.json.gz differ diff --git a/backend/plugins/kiro/tasks/user_report_extractor.go b/backend/plugins/kiro/tasks/user_report_extractor.go new file mode 100644 index 00000000000..ddef74a1cd5 --- /dev/null +++ b/backend/plugins/kiro/tasks/user_report_extractor.go @@ -0,0 +1,63 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/kiro/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractKiroUserReport + +var ExtractKiroUserReportMeta = plugin.SubTaskMeta{ + Name: "extractKiroUserReport", + EntryPoint: ExtractKiroUserReport, + EnabledByDefault: true, + Description: "Extract daily per-user activity from Kiro report CSVs", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Dependencies: []*plugin.SubTaskMeta{&CollectKiroS3FilesMeta}, +} + +// ExtractKiroUserReport loads the report CSVs discovered for this scope. +// +// Reports are written once per day per client type, so there are only a few +// hundred per year - the concurrency that matters for logs is irrelevant here, +// but reusing extractFiles keeps the retry and bookkeeping behaviour identical +// across all three streams. +func ExtractKiroUserReport(taskCtx plugin.SubTaskContext) errors.Error { + return extractFiles(taskCtx, models.FileTypeReport, parseUserReportRows) +} + +// parseUserReportRows adapts ParseUserReport to the extractor's batch interface. +// +// Both tables come from one parse because the per-model counts are columns of the +// same CSV row; splitting them into two passes would mean reading every file +// twice. They are returned as two batches rather than one mixed slice because +// GORM resolves the target table from the slice's element type. +func parseUserReportRows(data []byte, connectionId uint64, scopeId string) ([]rowBatch, errors.Error) { + reports, modelMessages, err := ParseUserReport(data, connectionId, scopeId) + if err != nil { + return nil, err + } + + return []rowBatch{ + {rows: reports, count: len(reports)}, + {rows: modelMessages, count: len(modelMessages)}, + }, nil +} diff --git a/backend/plugins/schema_e2e/migration_schema_test.go b/backend/plugins/schema_e2e/migration_schema_test.go index 0fbc02893bc..5cdf115f960 100644 --- a/backend/plugins/schema_e2e/migration_schema_test.go +++ b/backend/plugins/schema_e2e/migration_schema_test.go @@ -70,6 +70,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" @@ -121,6 +122,7 @@ func allGoPlugins() []plugin.PluginMeta { issueTrace.IssueTrace{}, jenkins.Jenkins{}, jira.Jira{}, + kiro.Kiro{}, linear.Linear{}, linker.Linker{}, opsgenie.Opsgenie{}, diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index c3262153dfc..afe5fa1a31d 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -46,6 +46,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + kiro "github.com/apache/incubator-devlake/plugins/kiro/impl" linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" @@ -115,6 +116,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("linker/models", linker.Linker{}.GetTablesInfo) checker.FeedIn("issue_trace/models", issueTrace.IssueTrace{}.GetTablesInfo) checker.FeedIn("q_dev/models", q_dev.QDev{}.GetTablesInfo) + checker.FeedIn("kiro/models", kiro.Kiro{}.GetTablesInfo) checker.FeedIn("gh-copilot/models", copilot.GhCopilot{}.GetTablesInfo) err := checker.Verify() if err != nil { diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index 259d01bb0f1..441cda5cb2f 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -33,6 +33,7 @@ import { GitLabConfig } from './gitlab'; import { IncidentioConfig } from './incidentio'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { KiroConfig } from './kiro'; import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; @@ -64,6 +65,7 @@ export const pluginConfigs: IPluginConfig[] = [ IncidentioConfig, JenkinsConfig, JiraConfig, + KiroConfig, LinearConfig, PagerDutyConfig, RootlyConfig, diff --git a/config-ui/src/plugins/register/kiro/assets/icon.svg b/config-ui/src/plugins/register/kiro/assets/icon.svg new file mode 100644 index 00000000000..503114f140a --- /dev/null +++ b/config-ui/src/plugins/register/kiro/assets/icon.svg @@ -0,0 +1,62 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/config-ui/src/plugins/register/kiro/config.tsx b/config-ui/src/plugins/register/kiro/config.tsx new file mode 100644 index 00000000000..d848e8a852e --- /dev/null +++ b/config-ui/src/plugins/register/kiro/config.tsx @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const KiroConfig: IPluginConfig = { + plugin: 'kiro', + name: 'Kiro', + icon: ({ color }) => , + sort: 12, + connection: { + docLink: 'https://kiro.dev/docs/enterprise/monitor-and-track/user-activity/', + initialValues: { + name: '', + region: 'us-east-1', + bucket: '', + reportPrefix: 'user-report', + promptLogBucket: '', + promptLogPrefix: 'logging', + identityStoreId: '', + identityStoreRegion: '', + }, + fields: [ + 'name', + { + key: 'region', + label: 'AWS Region', + subLabel: + 'The region where your Kiro profile was installed. The exports live under this region in the S3 path, so it must match exactly.', + }, + { + key: 'bucket', + label: 'S3 Bucket', + subLabel: 'Bucket holding the daily user activity report CSVs.', + }, + { + key: 'reportPrefix', + label: 'Report Prefix', + subLabel: 'Prefix within the bucket, before AWSLogs/. Leave as user-report unless you configured another.', + defaultValue: 'user-report', + }, + { + key: 'promptLogBucket', + label: 'Prompt Log Bucket (optional)', + subLabel: + 'Only needed if interaction logs go to a different bucket, which Kiro recommends. Leave empty to reuse the bucket above.', + }, + { + key: 'promptLogPrefix', + label: 'Prompt Log Prefix', + subLabel: 'Prefix for the interaction logs. Leave as logging unless you configured another.', + defaultValue: 'logging', + }, + { + key: 'accessKeyId', + label: 'AWS Access Key ID', + }, + { + key: 'secretAccessKey', + label: 'AWS Secret Access Key', + }, + { + key: 'identityStoreId', + label: 'IAM Identity Center Store ID (optional)', + subLabel: + 'Only resolves display names. User identity comes from the report’s User_Email column, so collection works without this. If set, the region below is required too.', + }, + { + key: 'identityStoreRegion', + label: 'IAM Identity Center Region (optional)', + subLabel: 'May differ from the S3 region. Required if a store ID is set.', + }, + ], + }, + dataScope: { + // No custom render: the default picker calls the plugin's remote-scopes + // endpoint, which browses the export layout in S3 as accounts -> years -> + // months and lists only periods that actually hold data. Hand-entering a + // prefix cannot be verified from the outcome, because a typo and an empty + // month both produce a run that succeeds and collects nothing. + title: 'Accounts & Periods', + }, + scopeConfig: { + entities: ['CROSS'], + transformation: {}, + }, +}; diff --git a/config-ui/src/plugins/register/kiro/index.ts b/config-ui/src/plugins/register/kiro/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/kiro/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/e2e/kiro-full-flow.spec.ts b/e2e/kiro-full-flow.spec.ts new file mode 100644 index 00000000000..88234a18844 --- /dev/null +++ b/e2e/kiro-full-flow.spec.ts @@ -0,0 +1,252 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { test, expect, request, Page } from '@playwright/test'; +import * as path from 'path'; +import * as fs from 'fs'; + +const API = 'http://localhost:8080'; +const UI = 'http://localhost:4000'; +const GRAFANA = 'http://localhost:3002'; +const SCREENSHOT_DIR = path.join(__dirname, 'screenshots'); + +// The full-flow test needs an existing Kiro connection with valid credentials. +// Keep environment-specific ids out of the repository. +const EXISTING_CONNECTION_ID = Number(process.env.KIRO_CONNECTION_ID || 0); +const KIRO_ACCOUNT_ID = process.env.KIRO_ACCOUNT_ID || ''; + +const state: { + connectionId: number; + scopeId: string; + blueprintId: number; + pipelineId: number; +} = { connectionId: EXISTING_CONNECTION_ID, scopeId: '', blueprintId: 0, pipelineId: 0 }; + +fs.mkdirSync(SCREENSHOT_DIR, { recursive: true }); + +async function grafanaLogin(page: Page) { + await page.goto(`${GRAFANA}/grafana/login`); + await page.waitForLoadState('networkidle'); + if (page.url().includes('/login')) { + await page.locator('input[name="user"]').fill('admin'); + await page.locator('input[name="password"]').fill('admin'); + await page.locator('button[type="submit"]').click(); + await page.waitForTimeout(2000); + // Handle "change password" prompt if shown + const skipBtn = page.locator('a:has-text("Skip")'); + if (await skipBtn.isVisible({ timeout: 2000 }).catch(() => false)) { + await skipBtn.click(); + } + await page.waitForTimeout(1000); + } +} + +async function openGrafanaDashboard(page: Page, uid: string, screenshotPath: string) { + await grafanaLogin(page); + await page.goto(`${GRAFANA}/grafana/d/${uid}?orgId=1&from=now-90d&to=now`); + + // Wait for first panel data to load + try { + await page.waitForResponse( + (resp) => resp.url().includes('/api/ds/query') && resp.status() === 200, + { timeout: 30000 } + ); + } catch { /* some dashboards may not fire queries immediately */ } + + // Wait for rendering to settle + await page.waitForTimeout(5000); + + // Take viewport screenshot (top section) + await page.screenshot({ path: screenshotPath.replace('.png', '-top.png') }); + + // Scroll down and take more sections + const scrollHeight = await page.evaluate(() => document.body.scrollHeight); + let section = 1; + for (let y = 900; y < scrollHeight; y += 900) { + await page.evaluate((scrollY) => window.scrollTo(0, scrollY), y); + await page.waitForTimeout(3000); + section++; + await page.screenshot({ path: screenshotPath.replace('.png', `-section${section}.png`) }); + } + + // Also take full page screenshot + await page.evaluate(() => window.scrollTo(0, 0)); + await page.waitForTimeout(2000); + await page.screenshot({ path: screenshotPath, fullPage: true }); +} + +test.describe.serial('Kiro Plugin Full Flow', () => { + + test('Step 1: Verify Existing Connection via API', async () => { + expect(EXISTING_CONNECTION_ID, 'set KIRO_CONNECTION_ID').toBeGreaterThan(0); + expect(KIRO_ACCOUNT_ID, 'set KIRO_ACCOUNT_ID').not.toBe(''); + const api = await request.newContext({ baseURL: API }); + + const resp = await api.get(`/plugins/kiro/connections/${state.connectionId}`); + expect(resp.ok()).toBeTruthy(); + const conn = await resp.json(); + console.log(`Using connection: id=${conn.id}, name=${conn.name}, bucket=${conn.bucket}`); + + const testResp = await api.post(`/plugins/kiro/connections/${state.connectionId}/test`); + const testBody = await testResp.json(); + console.log('Test connection:', { accounts: testBody.accounts, streams: testBody.streams, hint: testBody.hint }); + expect(testResp.ok()).toBeTruthy(); + }); + + test('Step 2: View Config-UI Home', async ({ page }) => { + await page.goto(UI); + await page.waitForLoadState('networkidle'); + await page.waitForTimeout(1000); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '01-config-ui-home.png'), fullPage: true }); + console.log('Screenshot: Config-UI home'); + }); + + test('Step 3: Create Scope (S3 Slice) via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.put(`/plugins/kiro/connections/${state.connectionId}/scopes`, { + data: { + data: [ + { + accountId: KIRO_ACCOUNT_ID, + basePath: '', + year: 2026, + month: 3, + }, + ], + }, + }); + + const body = await resp.json(); + console.log('Scope created:', resp.status()); + expect(resp.ok()).toBeTruthy(); + state.scopeId = body[0]?.id; + expect(state.scopeId).toBeTruthy(); + console.log(`Scope id: ${state.scopeId}`); + }); + + test('Step 4: Create Blueprint via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post('/blueprints', { + data: { + name: `e2e-blueprint-${Date.now()}`, + mode: 'NORMAL', + enable: true, + cronConfig: '0 0 * * *', + isManual: true, + connections: [ + { + pluginName: 'kiro', + connectionId: state.connectionId, + scopes: [{ scopeId: state.scopeId }], + }, + ], + }, + }); + + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.blueprintId = body.id; + console.log(`Blueprint created: id=${state.blueprintId}`); + }); + + test('Step 5: Trigger Pipeline via API', async () => { + const api = await request.newContext({ baseURL: API }); + + const resp = await api.post(`/blueprints/${state.blueprintId}/trigger`, { data: {} }); + const body = await resp.json(); + expect(resp.ok()).toBeTruthy(); + state.pipelineId = body.id; + console.log(`Pipeline triggered: id=${state.pipelineId}`); + }); + + test('Step 6: Wait for Pipeline to Complete', async () => { + const api = await request.newContext({ baseURL: API }); + const maxWait = 120000; + const start = Date.now(); + let status = ''; + + while (Date.now() - start < maxWait) { + const resp = await api.get(`/pipelines/${state.pipelineId}`); + const pipeline = await resp.json(); + status = pipeline.status; + console.log(`Pipeline status: ${status} (${Math.round((Date.now() - start) / 1000)}s)`); + if (['TASK_COMPLETED', 'TASK_FAILED', 'TASK_PARTIAL'].includes(status)) break; + await new Promise((r) => setTimeout(r, 3000)); + } + + // Print task details + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + if (tasksResp.ok()) { + const { tasks } = await tasksResp.json(); + for (const t of tasks || []) { + console.log(` Task ${t.id}: ${t.status}${t.failedSubTask ? ` (failed: ${t.failedSubTask})` : ''}`); + if (t.message) console.log(` Error: ${t.message.substring(0, 300)}`); + } + } + + expect(status).toBe('TASK_COMPLETED'); + }); + + test('Step 7: Verify Data via MySQL', async () => { + const api = await request.newContext({ baseURL: API }); + + // Use pipeline tasks to confirm data was processed + const tasksResp = await api.get(`/pipelines/${state.pipelineId}/tasks`); + const { tasks } = await tasksResp.json(); + expect(tasks[0].status).toBe('TASK_COMPLETED'); + console.log(`Pipeline completed in ${tasks[0].spentSeconds}s`); + }); + + test('Step 8: Grafana - Kiro Usage Dashboard (new format)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_user_report', path.join(SCREENSHOT_DIR, '02-dashboard-user-report.png')); + console.log('Screenshot: Kiro Usage Dashboard'); + }); + + test('Step 9: Grafana - Kiro Feature Metrics', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_feature_metrics', path.join(SCREENSHOT_DIR, '03-dashboard-feature-metrics.png')); + console.log('Screenshot: Kiro Feature Metrics'); + }); + + test('Step 10: Grafana - Kiro AI Activity Insights (logging)', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_logging', path.join(SCREENSHOT_DIR, '04-dashboard-logging.png')); + console.log('Screenshot: Kiro AI Activity Insights'); + }); + + test('Step 11: Grafana - Kiro Executive Dashboard', async ({ page }) => { + await openGrafanaDashboard(page, 'kiro_executive', path.join(SCREENSHOT_DIR, '05-dashboard-executive.png')); + console.log('Screenshot: Kiro Executive Dashboard'); + }); + + test('Step 12: View Pipeline in Config-UI', async ({ page }) => { + // Navigate to the API proxy route for pipelines + await page.goto(`${UI}/api/pipelines?pageSize=5`); + await page.waitForLoadState('networkidle'); + await page.screenshot({ path: path.join(SCREENSHOT_DIR, '06-config-ui-pipelines.png'), fullPage: true }); + console.log('Screenshot: Pipelines API response'); + }); + + test('Step 13: Cleanup', async () => { + const api = await request.newContext({ baseURL: API }); + if (state.blueprintId) { + await api.delete(`/blueprints/${state.blueprintId}`); + console.log(`Deleted blueprint ${state.blueprintId}`); + } + console.log('Cleanup complete'); + }); +});