diff --git a/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql new file mode 100644 index 0000000000..83a09a3d34 --- /dev/null +++ b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql @@ -0,0 +1,29 @@ +with deleted as ( + update "memberIdentities" + set "deletedAt" = now(), + "updatedAt" = now() + where "deletedAt" is null + and ( + ( + platform = 'meetings' + and type in ('email', 'username') + and lower(value) in ( + 'meeting1@seidengroup.com', + 'meetings@dpdk.org', + 'zoom-admin@nephio.org', + 'zoom0@hyperledger.org', + 'zoom2@hyperledger.org', + 'zoom2@lfai.foundation', + 'zoom@hyperledger.org', + 'zoom@longnowboston.org', + 'zoom@openwallet.foundation' + ) + ) + or (platform = 'cvent' and type = 'username' and lower(value) in ('na', 'none', 'null')) + or (platform = 'lfid' and type = 'username' and lower(value) = 'undefined') + ) + returning "memberId" +) +update members +set "updatedAt" = now() +where id in (select "memberId" from deleted); diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.test.ts b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts new file mode 100644 index 0000000000..9c59351cb9 --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest' + +import { cleanPersonalEmail, isSharedMeetingAccountEmail } from './personalEmail' + +describe('isSharedMeetingAccountEmail', () => { + it.each(['zoom@hyperledger.org', 'Zoom2@LFAI.foundation', 'meetings@dpdk.org'])( + 'flags the known shared meeting host account %s', + (email) => { + expect(isSharedMeetingAccountEmail(email)).toBe(true) + }, + ) + + it.each([ + 'zoom3@hyperledger.org', + 'zoom@antonpalmqvist.com', + 'james.wang@zoom.us', + 'noreply@github.com', + 'mail@jkroepke.de', + ])('does not flag %s, which is not on the known list', (email) => { + expect(isSharedMeetingAccountEmail(email)).toBe(false) + }) +}) + +describe('cleanPersonalEmail', () => { + it('trims and keeps a personal email', () => { + expect(cleanPersonalEmail(' ada@lovelace.dev ')).toBe('ada@lovelace.dev') + }) + + it('drops a known shared meeting host account', () => { + expect(cleanPersonalEmail(' zoom@openwallet.foundation ')).toBeNull() + }) + + it('drops a placeholder email value', () => { + expect(cleanPersonalEmail('NONE')).toBeNull() + }) +}) diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.ts b/services/apps/snowflake_connectors/src/core/personalEmail.ts new file mode 100644 index 0000000000..d7d1c97798 --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/personalEmail.ts @@ -0,0 +1,22 @@ +import { cleanIdentityValue } from './transformerBase' + +const SHARED_MEETING_ACCOUNT_EMAILS = new Set([ + 'meeting1@seidengroup.com', + 'meetings@dpdk.org', + 'zoom-admin@nephio.org', + 'zoom0@hyperledger.org', + 'zoom2@hyperledger.org', + 'zoom2@lfai.foundation', + 'zoom@hyperledger.org', + 'zoom@longnowboston.org', + 'zoom@openwallet.foundation', +]) + +export function isSharedMeetingAccountEmail(email: string): boolean { + return SHARED_MEETING_ACCOUNT_EMAILS.has(email.toLowerCase()) +} + +export function cleanPersonalEmail(raw: unknown): string | null { + const email = cleanIdentityValue(raw) + return email && !isSharedMeetingAccountEmail(email) ? email : null +} diff --git a/services/apps/snowflake_connectors/src/core/transformerBase.test.ts b/services/apps/snowflake_connectors/src/core/transformerBase.test.ts new file mode 100644 index 0000000000..72b8d8fa7a --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/transformerBase.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' + +import { cleanIdentityValue } from './transformerBase' + +describe('cleanIdentityValue', () => { + it.each(['NONE', 'none', ' None ', 'NULL', 'undefined', 'NA', 'n/a', '-'])( + 'treats the placeholder %j as missing', + (placeholder) => { + expect(cleanIdentityValue(placeholder)).toBeNull() + }, + ) + + it.each([null, undefined, '', ' '])('treats %j as missing', (empty) => { + expect(cleanIdentityValue(empty)).toBeNull() + }) + + it('trims a real LF user id', () => { + expect(cleanIdentityValue(' 0031T00003abcDE ')).toBe('0031T00003abcDE') + }) + + it('keeps usernames that only contain a placeholder word', () => { + expect(cleanIdentityValue('nonesuch')).toBe('nonesuch') + }) + + it('stringifies a numeric user id', () => { + expect(cleanIdentityValue(12345)).toBe('12345') + }) +}) diff --git a/services/apps/snowflake_connectors/src/core/transformerBase.ts b/services/apps/snowflake_connectors/src/core/transformerBase.ts index f4c6cd8b9c..a602176344 100644 --- a/services/apps/snowflake_connectors/src/core/transformerBase.ts +++ b/services/apps/snowflake_connectors/src/core/transformerBase.ts @@ -9,6 +9,16 @@ import { IActivityData, IMemberData, MemberIdentityType, PlatformType } from '@c const log = getServiceChildLogger('transformer') +const PLACEHOLDER_IDENTITY_VALUES = new Set(['none', 'null', 'undefined', 'na', 'n/a', '-']) + +export function cleanIdentityValue(raw: unknown): string | null { + const value = typeof raw === 'string' ? raw.trim() : typeof raw === 'number' ? String(raw) : '' + if (!value || PLACEHOLDER_IDENTITY_VALUES.has(value.toLowerCase())) { + return null + } + return value +} + export interface SegmentRef { slug: string sourceId: string diff --git a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts index 42da0504eb..f23de1f6eb 100644 --- a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts @@ -8,7 +8,12 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('committeesCommitteesTransformer') @@ -16,7 +21,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { readonly platform = PlatformType.COMMITTEES transformRow(row: Record): TransformedActivity | null { - const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null + const email = cleanPersonalEmail(row.CONTACTEMAIL__C) if (!email) { log.warn( { sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C }, @@ -27,7 +32,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { const committeeId = (row.COMMITTEE_ID as string).trim() const fivetranDeleted = row.FIVETRAN_DELETED as boolean - const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null + const lfUsername = cleanIdentityValue(row.LF_USERNAME) const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null @@ -41,7 +46,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { ? CommitteesActivityType.REMOVED_FROM_COMMITTEE : CommitteesActivityType.ADDED_TO_COMMITTEE - const sourceId = (row.PRIMARY_SOURCE_USER_ID as string | null)?.trim() || undefined + const sourceId = cleanIdentityValue(row.PRIMARY_SOURCE_USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, platformUsername: null, diff --git a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts index 5fdea82fb0..3ad2f84448 100644 --- a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts @@ -9,7 +9,12 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('cventTransformer') @@ -17,12 +22,12 @@ export class CventTransformer extends TransformerBase { readonly platform = PlatformType.CVENT transformRow(row: Record): TransformedActivity | null { - const userName = (row.USER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null + const userName = cleanIdentityValue(row.USER_NAME) + const lfUsername = cleanIdentityValue(row.LFID) const fullName = (row.FULL_NAME as string | null)?.trim() || null const firstName = (row.FIRST_NAME as string | null)?.trim() || null const lastName = (row.LAST_NAME as string | null)?.trim() || null - const email = (row.EMAIL as string).trim() + const email = cleanPersonalEmail(row.EMAIL) if (!email) { log.warn( { registrationId: row.REGISTRATION_ID, userName, lfUsername }, @@ -32,7 +37,7 @@ export class CventTransformer extends TransformerBase { } const registrationId = (row.REGISTRATION_ID as string)?.trim() - const sourceId = (row.USER_ID as string | null) || undefined + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const displayName = fullName || diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts new file mode 100644 index 0000000000..96cfb54a4b --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from 'vitest' + +import { buildSourceQuery } from './buildSourceQuery' + +describe('buildSourceQuery', () => { + it.each([undefined, '2026-10-19 00:00:00'])( + 'keeps placeholder LF user ids out of the LF SSO lookup (since %j)', + (sinceTimestamp) => { + const query = buildSourceQuery(sinceTimestamp) + + expect(query).toContain( + "UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')", + ) + expect(query).toContain( + "UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')", + ) + }, + ) + + it('lets the LF SSO lookup fill in when the row itself carries a placeholder SSO', () => { + expect(buildSourceQuery()).toContain( + "CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-') THEN t.INVITEE_LF_SSO END,\n sso.LF_SSO", + ) + }) +}) diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts index 43a606ba0e..2e9f5b4e94 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts @@ -22,12 +22,16 @@ const ORG_ACCOUNTS = ` WHERE website IS NOT NULL )` +const PLACEHOLDER_VALUES = `('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')` + const LF_SSO_LOOKUP = ` lf_sso_lookup AS ( SELECT INVITEE_LF_USER_ID AS LF_USER_ID, MAX(INVITEE_LF_SSO) AS LF_SSO FROM ANALYTICS.SILVER_FACT.MEETING_ATTENDANCE WHERE INVITEE_LF_SSO IS NOT NULL AND INVITEE_LF_USER_ID IS NOT NULL + AND UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES} + AND UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ${PLACEHOLDER_VALUES} GROUP BY INVITEE_LF_USER_ID )` @@ -45,7 +49,10 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { t.MEETING_DATE, t.MEETING_TIME, t.INVITEE_FULL_NAME, - COALESCE(t.INVITEE_LF_SSO, sso.LF_SSO) AS INVITEE_LF_SSO, + COALESCE( + CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES} THEN t.INVITEE_LF_SSO END, + sso.LF_SSO + ) AS INVITEE_LF_SSO, t.INVITEE_LF_USER_ID, t.INVITEE_EMAIL, t.INVITEE_ATTENDED, diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts index 4484506da5..5dfe747404 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts @@ -8,7 +8,12 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('meetingAttendanceTransformer') @@ -33,14 +38,14 @@ export class MeetingAttendanceTransformer extends TransformerBase { readonly platform = PlatformType.MEETINGS transformRow(row: Record): TransformedActivity | TransformedActivity[] | null { - const email = (row.INVITEE_EMAIL as string | null)?.trim() || null + const email = cleanPersonalEmail(row.INVITEE_EMAIL) if (!email) { log.debug({ primaryKey: row.PRIMARY_KEY }, 'Skipping row: missing email') return null } - const lfUsername = (row.INVITEE_LF_SSO as string | null)?.trim() || null - const memberSourceId = (row.INVITEE_LF_USER_ID as string | null)?.trim() || undefined + const lfUsername = cleanIdentityValue(row.INVITEE_LF_SSO) + const memberSourceId = cleanIdentityValue(row.INVITEE_LF_USER_ID) ?? undefined const generatedSourceId = (row.GENERATED_SOURCE_ID as string | null)?.trim() || undefined const userId = memberSourceId ?? generatedSourceId const displayName = (row.INVITEE_FULL_NAME as string | null)?.trim() || email diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index 8662c8981d..28a779cc7e 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -2,14 +2,15 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCertificatesTransformer') export class TncCertificatesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.debug({ certificateId: row.CERTIFICATE_ID }, 'Skipping row: missing email') return null @@ -17,8 +18,8 @@ export class TncCertificatesTransformer extends TncTransformerBase { const certificateId = (row.CERTIFICATE_ID as string)?.trim() const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 36f755f197..93dbc73792 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -2,14 +2,15 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCoursesTransformer') export class TncCoursesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.warn( { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL }, @@ -25,8 +26,8 @@ export class TncCoursesTransformer extends TncTransformerBase { } const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.INTERNAL_TI_USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.INTERNAL_TI_USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index a3bad55e1d..cdddc49a0b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -2,14 +2,15 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { cleanPersonalEmail } from '../../../core/personalEmail' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncEnrollmentsTransformer') export class TncEnrollmentsTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.warn( { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL }, @@ -20,8 +21,8 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { const enrollmentId = (row.ENROLLMENT_ID as string)?.trim() const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email,