Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions backend/config/custom-environment-variables.json
Original file line number Diff line number Diff line change
Expand Up @@ -113,13 +113,11 @@
"githubIssueReporter": {
"appId": "CROWD_GITHUB_ISSUE_REPORTER_APP_ID",
"privateKey": "CROWD_GITHUB_ISSUE_REPORTER_PRIVATE_KEY",
"installationId": "CROWD_GITHUB_ISSUE_REPORTER_INSTALLATION_ID"
},
"jiraIssueReporter": {
"apiUrl": "CROWD_JIRA_ISSUE_REPORTER_API_URL",
"apiTokenEmail": "CROWD_JIRA_ISSUE_REPORTER_API_TOKEN_EMAIL",
"token": "CROWD_JIRA_ISSUE_REPORTER_API_TOKEN",
"projectKey": "CROWD_JIRA_ISSUE_REPORTER_PROJECT_KEY"
"installationId": "CROWD_GITHUB_ISSUE_REPORTER_INSTALLATION_ID",
"repo": "CROWD_GITHUB_ISSUE_REPORTER_REPO",
"projectNumber": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_NUMBER",
"projectTeam": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_TEAM",
"projectStatus": "CROWD_GITHUB_ISSUE_REPORTER_PROJECT_STATUS"
},
"stackexchange": {
"key": "CROWD_STACKEXCHANGE_KEY"
Expand Down
7 changes: 6 additions & 1 deletion backend/config/default.json
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,12 @@
"encryption": {},
"openStatusApi": {},
"gitlab": {},
"jiraIssueReporter": {},
"githubIssueReporter": {
"repo": "linuxfoundation/lfx-cdp-insights-ops",
"projectNumber": "30",
"projectTeam": "Data Quality / Support",
"projectStatus": "Backlog"
},
"snowflake": {},
"nango": {},
"linuxFoundation": {}
Expand Down
5 changes: 1 addition & 4 deletions backend/src/conf/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { ISearchSyncApiConfig } from '@crowd/opensearch'
import { IQueueClientConfig } from '@crowd/queue'
import { IRedisConfiguration } from '@crowd/redis'
import { ITemporalConfig } from '@crowd/temporal'
import { IGithubIssueReporterConfiguration, IJiraIssueReporterConfiguration } from '@crowd/types'
import { IGithubIssueReporterConfiguration } from '@crowd/types'

import {
ApiConfiguration,
Expand Down Expand Up @@ -117,9 +117,6 @@ export const GITHUB_CONFIG: GithubConfiguration = config.get<GithubConfiguration
export const GITHUB_ISSUE_REPORTER_CONFIG: IGithubIssueReporterConfiguration =
config.get<IGithubIssueReporterConfiguration>('githubIssueReporter')

export const JIRA_ISSUE_REPORTER_CONFIG: IJiraIssueReporterConfiguration =
config.get<IJiraIssueReporterConfiguration>('jiraIssueReporter')

export const NANGO_CONFIG: NangoConfiguration = config.get<NangoConfiguration>('nango')

export const ENRICHMENT_CONFIG: EnrichmentConfiguration =
Expand Down
92 changes: 25 additions & 67 deletions backend/src/services/dataIssueService.ts
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
import axios from 'axios'

import { JIRA_ISSUE_REPORTER_CONFIG } from '@/conf'
import SequelizeRepository from '@/database/repositories/sequelizeRepository'
import { createHeading, createParagraph } from '@crowd/common'
import { OrganizationField, findOrgById } from '@crowd/data-access-layer'
import { createDataIssue } from '@crowd/data-access-layer/src/data_issues'
import { MemberField, findMemberById } from '@crowd/data-access-layer/src/members'
import { PgPromiseQueryExecutor } from '@crowd/data-access-layer/src/queryExecutor'
import { LoggerBase } from '@crowd/logging'
import { DataIssueEntity } from '@crowd/types'

import { createInternalIssue } from './helpers/githubIssueReporter'
import { IServiceOptions } from './IServiceOptions'

const DATA_ISSUE_LABELS = ['product-support', 'needs-triage', 'data-issue']

export interface IDataIssueCreatePayload {
entity: DataIssueEntity
profileUrl: string
Expand All @@ -22,19 +21,6 @@ export interface IDataIssueCreatePayload {
createdById: string
}

interface IJiraCreateIssueResponse {
id: string
key: string
self: string
transitions: {
status: string
errorCollection: {
errorMessages: string[]
errors: object
}
}
}

export default class DataIssueService extends LoggerBase {
private readonly qx: PgPromiseQueryExecutor

Expand Down Expand Up @@ -71,58 +57,30 @@ export default class DataIssueService extends LoggerBase {
reportedBy = `${user.email}`
}

try {
const result = await axios.post<IJiraCreateIssueResponse>(
`${JIRA_ISSUE_REPORTER_CONFIG.apiUrl}/issue`,
{
fields: {
project: {
key: JIRA_ISSUE_REPORTER_CONFIG.projectKey,
},
summary: `[Data Issue] ${entityName} (${data.entity[0].toUpperCase()}${data.entity
.slice(1)
.toLowerCase()})`,
description: {
version: 1,
type: 'doc',
content: [
createHeading('Entity'),
createParagraph(entityName),
createHeading('Profile'),
createParagraph(data.profileUrl, true),
createHeading('Data Issue'),
createParagraph(data.dataIssue),
createHeading('Description'),
createParagraph(data.description),
createHeading('Reported by'),
createParagraph(reportedBy),
],
},
issuetype: {
name: 'Task',
},
labels: ['data-issue'],
},
},
{
headers: {
Authorization: `Basic ${Buffer.from(
`${JIRA_ISSUE_REPORTER_CONFIG.apiTokenEmail}:${JIRA_ISSUE_REPORTER_CONFIG.token}`,
).toString('base64')}`,
},
},
)

const res = await createDataIssue(qx, {
...data,
issueUrl: result.data.self,
createdById: user.id,
})
const title = `[Data Issue] ${entityName} (${data.entity[0].toUpperCase()}${data.entity
.slice(1)
.toLowerCase()})`
const body = [
`### Entity\n${entityName}`,
`### Profile\n${data.profileUrl}`,
`### Data Issue\n${data.dataIssue}`,
`### Description\n${data.description}`,
`### Reported by\n${reportedBy}`,
].join('\n\n')

return res
let issueUrl: string
try {
const issue = await createInternalIssue(this.log, title, body, DATA_ISSUE_LABELS)
issueUrl = issue.html_url
} catch (error) {
this.log.info(error)
throw new Error('Error during session create!')
this.log.error(error, 'Failed to create GitHub issue for data issue report!')
throw new Error('Failed to report the data issue!')
}

return createDataIssue(qx, {
...data,
issueUrl,
createdById: user.id,
})
}
}
161 changes: 161 additions & 0 deletions backend/src/services/helpers/githubIssueReporter.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
import { createAppAuth } from '@octokit/auth-app'
import { graphql } from '@octokit/graphql'
import { request } from '@octokit/request'

import { GITHUB_ISSUE_REPORTER_CONFIG } from '@/conf'
import { Logger } from '@crowd/logging'

interface ICreatedIssue {
html_url: string
node_id: string
}

interface IProjectField {
id?: string
name?: string
options?: { id: string; name: string }[]
}

const PROJECT_QUERY = `query($owner: String!, $number: Int!) {
organization(login: $owner) {
projectV2(number: $number) {
id
fields(first: 50) { nodes { ... on ProjectV2SingleSelectField { id name options { id name } } } }
}
}
}`

const ADD_ITEM_MUTATION = `mutation($projectId: ID!, $contentId: ID!) {
addProjectV2ItemById(input: { projectId: $projectId, contentId: $contentId }) { item { id } }
}`

const SET_FIELD_MUTATION = `mutation($projectId: ID!, $itemId: ID!, $fieldId: ID!, $optionId: String!) {
updateProjectV2ItemFieldValue(input: {
projectId: $projectId, itemId: $itemId, fieldId: $fieldId, value: { singleSelectOptionId: $optionId }
}) { projectV2Item { id } }
}`

function normalizePrivateKey(rawKey: string): string {
return rawKey.includes('BEGIN')
? rawKey.replace(/\\n/g, '\n')
: Buffer.from(rawKey, 'base64').toString('utf8')
}

async function getInstallationToken(): Promise<string> {
const { appId, privateKey, installationId } = GITHUB_ISSUE_REPORTER_CONFIG

if (!appId || !privateKey || !installationId) {
throw new Error('GitHub issue reporter credentials are not configured')
}

const auth = createAppAuth({ appId, privateKey: normalizePrivateKey(privateKey) })
const { token } = await auth({ type: 'installation', installationId: Number(installationId) })

return token
}

export async function createInternalIssue(
log: Logger,
title: string,
body: string,
labels: string[] = [],
): Promise<ICreatedIssue> {
const { repo, projectNumber, projectTeam, projectStatus } = GITHUB_ISSUE_REPORTER_CONFIG

const token = await getInstallationToken()
const headers = { authorization: `token ${token}` }
const gql = graphql.defaults({ headers })

// Resolve the project fields before creating the issue, so a misconfigured project fails
// without leaving an issue outside the project.
const [owner] = repo.split('/')
const { organization } = await gql<{
organization: { projectV2: { id: string; fields: { nodes: IProjectField[] } } | null }
}>(PROJECT_QUERY, { owner, number: Number(projectNumber) })
const project = organization.projectV2
if (!project) {
throw new Error(`Project ${owner}#${projectNumber} not found`)
}
const fieldValues = (
[
['Team', projectTeam],
['Status', projectStatus],
] as const
).map(([fieldName, optionName]) => {
const field = project.fields.nodes.find((f) => f.name === fieldName)
const option = field?.options?.find((o) => o.name === optionName)
if (!field?.id || !option) {
throw new Error(`Project field "${fieldName}" has no option "${optionName}"`)
}
return { fieldId: field.id, optionId: option.id }
})

await ensureLabels(headers, repo, labels)
const { data: issue } = await request(`POST /repos/${repo}/issues`, {
headers,
title,
body,
labels,
})

// Both mutations are idempotent, so transient failures are retried.
try {
await withRetry(async () => {
const { addProjectV2ItemById } = await gql<{
addProjectV2ItemById: { item: { id: string } }
}>(ADD_ITEM_MUTATION, { projectId: project.id, contentId: issue.node_id })

for (const { fieldId, optionId } of fieldValues) {
await gql(SET_FIELD_MUTATION, {
projectId: project.id,
itemId: addProjectV2ItemById.item.id,
fieldId,
optionId,
})
}
})
Comment thread
joanagmaia marked this conversation as resolved.
} catch (error) {
log.error(error, `Created ${issue.html_url} but could not add it to the project!`)
}

return issue
}

interface IRequestFailure {
status?: number
response?: { data?: { errors?: { code?: string }[] } }
}

async function ensureLabels(
headers: Record<string, string>,
repo: string,
labels: string[],
): Promise<void> {
for (const name of labels) {
try {
await request(`GET /repos/${repo}/labels/${encodeURIComponent(name)}`, { headers })
} catch (error) {
if ((error as IRequestFailure).status !== 404) throw error
try {
await request(`POST /repos/${repo}/labels`, { headers, name, color: 'ededed' })
} catch (createError) {
// A concurrent report may have just created it.
const { status, response } = createError as IRequestFailure
const exists =
status === 422 && response?.data?.errors?.some((e) => e.code === 'already_exists')
if (!exists) throw createError
}
}
}
}

async function withRetry<T>(fn: () => Promise<T>, attempts = 3, delayMs = 500): Promise<T> {
for (let attempt = 1; ; attempt++) {
try {
return await fn()
} catch (error) {
if (attempt >= attempts) throw error
await new Promise((resolve) => setTimeout(resolve, delayMs * attempt))
}
}
}
1 change: 0 additions & 1 deletion services/libs/common/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ export * from './github/errors'
export * from './displayName'
export * from './organization'
export * from './country'
export * from './jira'
export * from './email'
export * from './bot'
export * from './llm'
Expand Down
35 changes: 0 additions & 35 deletions services/libs/common/src/jira.ts

This file was deleted.

11 changes: 4 additions & 7 deletions services/libs/types/src/dataIssues.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,8 @@ export interface IGithubIssueReporterConfiguration {
privateKey: string
installationId: string
webhookSecret: string
}

export interface IJiraIssueReporterConfiguration {
apiUrl: string
token: string
apiTokenEmail: string
projectKey: string
repo: string
projectNumber: string
projectTeam: string
projectStatus: string
}
Loading