diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index b85e6441d3..8833a564a5 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -113,13 +113,11 @@ "githubIssueReporter": { "appId": "CROWD_GITHUB_ISSUE_REPORTER_APP_ID", "privateKey": "CROWD_GITHUB_ISSUE_REPORTER_PRIVATE_KEY", - "installationId": "CROWD_GITHUB_ISSUE_REPORTER_INSTALLATION_ID" - }, - "jiraIssueReporter": { - "apiUrl": "CROWD_JIRA_ISSUE_REPORTER_API_URL", - "apiTokenEmail": "CROWD_JIRA_ISSUE_REPORTER_API_TOKEN_EMAIL", - "token": "CROWD_JIRA_ISSUE_REPORTER_API_TOKEN", - "projectKey": "CROWD_JIRA_ISSUE_REPORTER_PROJECT_KEY" + "installationId": "CROWD_GITHUB_ISSUE_REPORTER_INSTALLATION_ID", + "repo": "CROWD_GITHUB_ISSUE_REPORTER_REPO", + "projectNumber": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_NUMBER", + "projectTeam": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_TEAM", + "projectStatus": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_STATUS" }, "stackexchange": { "key": "CROWD_STACKEXCHANGE_KEY" diff --git a/backend/config/default.json b/backend/config/default.json index 2e2b332cd1..ac0367b5eb 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -47,7 +47,12 @@ "encryption": {}, "openStatusApi": {}, "gitlab": {}, - "jiraIssueReporter": {}, + "githubIssueReporter": { + "repo": "linuxfoundation/lfx-cdp-insights-ops", + "projectNumber": "30", + "projectTeam": "Data Quality / Support", + "projectStatus": "Backlog" + }, "snowflake": {}, "nango": {}, "linuxFoundation": {} diff --git a/backend/src/conf/index.ts b/backend/src/conf/index.ts index 9d0b1dabc8..b7a5ef23a3 100644 --- a/backend/src/conf/index.ts +++ b/backend/src/conf/index.ts @@ -5,7 +5,7 @@ import { ISearchSyncApiConfig } from '@crowd/opensearch' import { IQueueClientConfig } from '@crowd/queue' import { IRedisConfiguration } from '@crowd/redis' import { ITemporalConfig } from '@crowd/temporal' -import { IGithubIssueReporterConfiguration, IJiraIssueReporterConfiguration } from '@crowd/types' +import { IGithubIssueReporterConfiguration } from '@crowd/types' import { ApiConfiguration, @@ -117,9 +117,6 @@ export const GITHUB_CONFIG: GithubConfiguration = config.get('githubIssueReporter') -export const JIRA_ISSUE_REPORTER_CONFIG: IJiraIssueReporterConfiguration = - config.get('jiraIssueReporter') - export const NANGO_CONFIG: NangoConfiguration = config.get('nango') export const ENRICHMENT_CONFIG: EnrichmentConfiguration = diff --git a/backend/src/services/dataIssueService.ts b/backend/src/services/dataIssueService.ts index 9afa7dec76..ea940de646 100644 --- a/backend/src/services/dataIssueService.ts +++ b/backend/src/services/dataIssueService.ts @@ -1,8 +1,4 @@ -import axios from 'axios' - -import { JIRA_ISSUE_REPORTER_CONFIG } from '@/conf' import SequelizeRepository from '@/database/repositories/sequelizeRepository' -import { createHeading, createParagraph } from '@crowd/common' import { OrganizationField, findOrgById } from '@crowd/data-access-layer' import { createDataIssue } from '@crowd/data-access-layer/src/data_issues' import { MemberField, findMemberById } from '@crowd/data-access-layer/src/members' @@ -10,8 +6,11 @@ import { PgPromiseQueryExecutor } from '@crowd/data-access-layer/src/queryExecut import { LoggerBase } from '@crowd/logging' import { DataIssueEntity } from '@crowd/types' +import { createInternalIssue } from './helpers/githubIssueReporter' import { IServiceOptions } from './IServiceOptions' +const DATA_ISSUE_LABELS = ['product-support', 'needs-triage', 'data-issue'] + export interface IDataIssueCreatePayload { entity: DataIssueEntity profileUrl: string @@ -22,19 +21,6 @@ export interface IDataIssueCreatePayload { createdById: string } -interface IJiraCreateIssueResponse { - id: string - key: string - self: string - transitions: { - status: string - errorCollection: { - errorMessages: string[] - errors: object - } - } -} - export default class DataIssueService extends LoggerBase { private readonly qx: PgPromiseQueryExecutor @@ -71,58 +57,30 @@ export default class DataIssueService extends LoggerBase { reportedBy = `${user.email}` } - try { - const result = await axios.post( - `${JIRA_ISSUE_REPORTER_CONFIG.apiUrl}/issue`, - { - fields: { - project: { - key: JIRA_ISSUE_REPORTER_CONFIG.projectKey, - }, - summary: `[Data Issue] ${entityName} (${data.entity[0].toUpperCase()}${data.entity - .slice(1) - .toLowerCase()})`, - description: { - version: 1, - type: 'doc', - content: [ - createHeading('Entity'), - createParagraph(entityName), - createHeading('Profile'), - createParagraph(data.profileUrl, true), - createHeading('Data Issue'), - createParagraph(data.dataIssue), - createHeading('Description'), - createParagraph(data.description), - createHeading('Reported by'), - createParagraph(reportedBy), - ], - }, - issuetype: { - name: 'Task', - }, - labels: ['data-issue'], - }, - }, - { - headers: { - Authorization: `Basic ${Buffer.from( - `${JIRA_ISSUE_REPORTER_CONFIG.apiTokenEmail}:${JIRA_ISSUE_REPORTER_CONFIG.token}`, - ).toString('base64')}`, - }, - }, - ) - - const res = await createDataIssue(qx, { - ...data, - issueUrl: result.data.self, - createdById: user.id, - }) + const title = `[Data Issue] ${entityName} (${data.entity[0].toUpperCase()}${data.entity + .slice(1) + .toLowerCase()})` + const body = [ + `### Entity\n${entityName}`, + `### Profile\n${data.profileUrl}`, + `### Data Issue\n${data.dataIssue}`, + `### Description\n${data.description}`, + `### Reported by\n${reportedBy}`, + ].join('\n\n') - return res + let issueUrl: string + try { + const issue = await createInternalIssue(this.log, title, body, DATA_ISSUE_LABELS) + issueUrl = issue.html_url } catch (error) { - this.log.info(error) - throw new Error('Error during session create!') + this.log.error(error, 'Failed to create GitHub issue for data issue report!') + throw new Error('Failed to report the data issue!') } + + return createDataIssue(qx, { + ...data, + issueUrl, + createdById: user.id, + }) } } diff --git a/backend/src/services/helpers/githubIssueReporter.ts b/backend/src/services/helpers/githubIssueReporter.ts new file mode 100644 index 0000000000..7f6787b1e5 --- /dev/null +++ b/backend/src/services/helpers/githubIssueReporter.ts @@ -0,0 +1,161 @@ +import { createAppAuth } from '@octokit/auth-app' +import { graphql } from '@octokit/graphql' +import { request } from '@octokit/request' + +import { GITHUB_ISSUE_REPORTER_CONFIG } from '@/conf' +import { Logger } from '@crowd/logging' + +interface ICreatedIssue { + html_url: string + node_id: string +} + +interface IProjectField { + id?: string + name?: string + options?: { id: string; name: string }[] +} + +const PROJECT_QUERY = `query($owner: String!, $number: Int!) { + organization(login: $owner) { + projectV2(number: $number) { + id + fields(first: 50) { nodes { ... on ProjectV2SingleSelectField { id name options { id name } } } } + } + } +}` + +const ADD_ITEM_MUTATION = `mutation($projectId: ID!, $contentId: ID!) { + addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { item { id } } +}` + +const SET_FIELD_MUTATION = `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) { + updateProjectV2ItemFieldValue(input: { + projectId: $projectId, itemId: $itemId, fieldId: $fieldId, value: { singleSelectOptionId: $optionId } + }) { projectV2Item { id } } +}` + +function normalizePrivateKey(rawKey: string): string { + return rawKey.includes('BEGIN') + ? rawKey.replace(/\\n/g, '\n') + : Buffer.from(rawKey, 'base64').toString('utf8') +} + +async function getInstallationToken(): Promise { + const { appId, privateKey, installationId } = GITHUB_ISSUE_REPORTER_CONFIG + + if (!appId || !privateKey || !installationId) { + throw new Error('GitHub issue reporter credentials are not configured') + } + + const auth = createAppAuth({ appId, privateKey: normalizePrivateKey(privateKey) }) + const { token } = await auth({ type: 'installation', installationId: Number(installationId) }) + + return token +} + +export async function createInternalIssue( + log: Logger, + title: string, + body: string, + labels: string[] = [], +): Promise { + const { repo, projectNumber, projectTeam, projectStatus } = GITHUB_ISSUE_REPORTER_CONFIG + + const token = await getInstallationToken() + const headers = { authorization: `token ${token}` } + const gql = graphql.defaults({ headers }) + + // Resolve the project fields before creating the issue, so a misconfigured project fails + // without leaving an issue outside the project. + const [owner] = repo.split('/') + const { organization } = await gql<{ + organization: { projectV2: { id: string; fields: { nodes: IProjectField[] } } | null } + }>(PROJECT_QUERY, { owner, number: Number(projectNumber) }) + const project = organization.projectV2 + if (!project) { + throw new Error(`Project ${owner}#${projectNumber} not found`) + } + const fieldValues = ( + [ + ['Team', projectTeam], + ['Status', projectStatus], + ] as const + ).map(([fieldName, optionName]) => { + const field = project.fields.nodes.find((f) => f.name === fieldName) + const option = field?.options?.find((o) => o.name === optionName) + if (!field?.id || !option) { + throw new Error(`Project field "${fieldName}" has no option "${optionName}"`) + } + return { fieldId: field.id, optionId: option.id } + }) + + await ensureLabels(headers, repo, labels) + const { data: issue } = await request(`POST /repos/${repo}/issues`, { + headers, + title, + body, + labels, + }) + + // Both mutations are idempotent, so transient failures are retried. + try { + await withRetry(async () => { + const { addProjectV2ItemById } = await gql<{ + addProjectV2ItemById: { item: { id: string } } + }>(ADD_ITEM_MUTATION, { projectId: project.id, contentId: issue.node_id }) + + for (const { fieldId, optionId } of fieldValues) { + await gql(SET_FIELD_MUTATION, { + projectId: project.id, + itemId: addProjectV2ItemById.item.id, + fieldId, + optionId, + }) + } + }) + } catch (error) { + log.error(error, `Created ${issue.html_url} but could not add it to the project!`) + } + + return issue +} + +interface IRequestFailure { + status?: number + response?: { data?: { errors?: { code?: string }[] } } +} + +async function ensureLabels( + headers: Record, + repo: string, + labels: string[], +): Promise { + for (const name of labels) { + try { + await request(`GET /repos/${repo}/labels/${encodeURIComponent(name)}`, { headers }) + } catch (error) { + if ((error as IRequestFailure).status !== 404) throw error + try { + await request(`POST /repos/${repo}/labels`, { headers, name, color: 'ededed' }) + } catch (createError) { + // A concurrent report may have just created it. + const { status, response } = createError as IRequestFailure + const exists = + status === 422 && response?.data?.errors?.some((e) => e.code === 'already_exists') + if (!exists) throw createError + } + } + } +} + +async function withRetry(fn: () => Promise, attempts = 3, delayMs = 500): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await fn() + } catch (error) { + if (attempt >= attempts) throw error + await new Promise((resolve) => setTimeout(resolve, delayMs * attempt)) + } + } +} diff --git a/services/libs/common/src/index.ts b/services/libs/common/src/index.ts index a2bda4f6cc..95730ed17f 100644 --- a/services/libs/common/src/index.ts +++ b/services/libs/common/src/index.ts @@ -46,7 +46,6 @@ export * from './github/errors' export * from './displayName' export * from './organization' export * from './country' -export * from './jira' export * from './email' export * from './bot' export * from './llm' diff --git a/services/libs/common/src/jira.ts b/services/libs/common/src/jira.ts deleted file mode 100644 index 877aa6f72b..0000000000 --- a/services/libs/common/src/jira.ts +++ /dev/null @@ -1,35 +0,0 @@ -// Helper functions to create ADF nodes in Jira -export function createHeading(text: string, level = 3) { - return { - type: 'heading', - attrs: { level }, - content: [{ type: 'text', text }], - } -} - -export function createParagraph(text: string, isUrl = false) { - if (isUrl) { - return { - type: 'paragraph', - content: [ - { - type: 'text', - text, - marks: [ - { - type: 'link', - attrs: { - href: text, - }, - }, - ], - }, - ], - } - } - - return { - type: 'paragraph', - content: [{ type: 'text', text }], - } -} diff --git a/services/libs/types/src/dataIssues.ts b/services/libs/types/src/dataIssues.ts index c905397592..4c345d0c10 100644 --- a/services/libs/types/src/dataIssues.ts +++ b/services/libs/types/src/dataIssues.ts @@ -18,11 +18,8 @@ export interface IGithubIssueReporterConfiguration { privateKey: string installationId: string webhookSecret: string -} - -export interface IJiraIssueReporterConfiguration { - apiUrl: string - token: string - apiTokenEmail: string - projectKey: string + repo: string + projectNumber: string + projectTeam: string + projectStatus: string }