From cfa97cabd8529b859830f0a3728069e5426a890f Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:07:14 +0100 Subject: [PATCH 1/6] fix: treat placeholder lf user ids from the meetings source as missing Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- .../src/core/transformerBase.test.ts | 28 +++++++++++++++++++ .../src/core/transformerBase.ts | 10 +++++++ .../buildSourceQuery.test.ts | 19 +++++++++++++ .../meeting-attendance/buildSourceQuery.ts | 4 +++ .../meeting-attendance/transformer.ts | 10 +++++-- 5 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 services/apps/snowflake_connectors/src/core/transformerBase.test.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts diff --git a/services/apps/snowflake_connectors/src/core/transformerBase.test.ts b/services/apps/snowflake_connectors/src/core/transformerBase.test.ts new file mode 100644 index 0000000000..72b8d8fa7a --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/transformerBase.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' + +import { cleanIdentityValue } from './transformerBase' + +describe('cleanIdentityValue', () => { + it.each(['NONE', 'none', ' None ', 'NULL', 'undefined', 'NA', 'n/a', '-'])( + 'treats the placeholder %j as missing', + (placeholder) => { + expect(cleanIdentityValue(placeholder)).toBeNull() + }, + ) + + it.each([null, undefined, '', ' '])('treats %j as missing', (empty) => { + expect(cleanIdentityValue(empty)).toBeNull() + }) + + it('trims a real LF user id', () => { + expect(cleanIdentityValue(' 0031T00003abcDE ')).toBe('0031T00003abcDE') + }) + + it('keeps usernames that only contain a placeholder word', () => { + expect(cleanIdentityValue('nonesuch')).toBe('nonesuch') + }) + + it('stringifies a numeric user id', () => { + expect(cleanIdentityValue(12345)).toBe('12345') + }) +}) diff --git a/services/apps/snowflake_connectors/src/core/transformerBase.ts b/services/apps/snowflake_connectors/src/core/transformerBase.ts index f4c6cd8b9c..a602176344 100644 --- a/services/apps/snowflake_connectors/src/core/transformerBase.ts +++ b/services/apps/snowflake_connectors/src/core/transformerBase.ts @@ -9,6 +9,16 @@ import { IActivityData, IMemberData, MemberIdentityType, PlatformType } from '@c const log = getServiceChildLogger('transformer') +const PLACEHOLDER_IDENTITY_VALUES = new Set(['none', 'null', 'undefined', 'na', 'n/a', '-']) + +export function cleanIdentityValue(raw: unknown): string | null { + const value = typeof raw === 'string' ? raw.trim() : typeof raw === 'number' ? String(raw) : '' + if (!value || PLACEHOLDER_IDENTITY_VALUES.has(value.toLowerCase())) { + return null + } + return value +} + export interface SegmentRef { slug: string sourceId: string diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts new file mode 100644 index 0000000000..8aa52a0cb2 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' + +import { buildSourceQuery } from './buildSourceQuery' + +describe('buildSourceQuery', () => { + it.each([undefined, '2026-10-19 00:00:00'])( + 'keeps placeholder LF user ids out of the LF SSO lookup (since %j)', + (sinceTimestamp) => { + const query = buildSourceQuery(sinceTimestamp) + + expect(query).toContain( + "UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')", + ) + expect(query).toContain( + "UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')", + ) + }, + ) +}) diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts index 43a606ba0e..0761e7976c 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts @@ -22,12 +22,16 @@ const ORG_ACCOUNTS = ` WHERE website IS NOT NULL )` +const PLACEHOLDER_VALUES = `('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')` + const LF_SSO_LOOKUP = ` lf_sso_lookup AS ( SELECT INVITEE_LF_USER_ID AS LF_USER_ID, MAX(INVITEE_LF_SSO) AS LF_SSO FROM ANALYTICS.SILVER_FACT.MEETING_ATTENDANCE WHERE INVITEE_LF_SSO IS NOT NULL AND INVITEE_LF_USER_ID IS NOT NULL + AND UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES} + AND UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ${PLACEHOLDER_VALUES} GROUP BY INVITEE_LF_USER_ID )` diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts index 4484506da5..76c92b14a9 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts @@ -8,7 +8,11 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('meetingAttendanceTransformer') @@ -39,8 +43,8 @@ export class MeetingAttendanceTransformer extends TransformerBase { return null } - const lfUsername = (row.INVITEE_LF_SSO as string | null)?.trim() || null - const memberSourceId = (row.INVITEE_LF_USER_ID as string | null)?.trim() || undefined + const lfUsername = cleanIdentityValue(row.INVITEE_LF_SSO) + const memberSourceId = cleanIdentityValue(row.INVITEE_LF_USER_ID) ?? undefined const generatedSourceId = (row.GENERATED_SOURCE_ID as string | null)?.trim() || undefined const userId = memberSourceId ?? generatedSourceId const displayName = (row.INVITEE_FULL_NAME as string | null)?.trim() || email From 1d3439a2fcf8a1fcd1a52040ac945d4d62aa7cb2 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:17:18 +0100 Subject: [PATCH 2/6] fix: treat placeholder identity values from all snowflake connectors as missing Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- .../committees/committees/transformer.ts | 12 ++++++++---- .../cvent/event-registrations/transformer.ts | 14 +++++++++----- .../meetings/meeting-attendance/transformer.ts | 2 +- .../integrations/tnc/certificates/transformer.ts | 8 ++++---- .../src/integrations/tnc/courses/transformer.ts | 8 ++++---- .../integrations/tnc/enrollments/transformer.ts | 8 ++++---- 6 files changed, 30 insertions(+), 22 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts index 42da0504eb..fffaf67a31 100644 --- a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts @@ -8,7 +8,11 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('committeesCommitteesTransformer') @@ -16,7 +20,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { readonly platform = PlatformType.COMMITTEES transformRow(row: Record): TransformedActivity | null { - const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null + const email = cleanIdentityValue(row.CONTACTEMAIL__C) if (!email) { log.warn( { sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C }, @@ -27,7 +31,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { const committeeId = (row.COMMITTEE_ID as string).trim() const fivetranDeleted = row.FIVETRAN_DELETED as boolean - const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null + const lfUsername = cleanIdentityValue(row.LF_USERNAME) const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null @@ -41,7 +45,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { ? CommitteesActivityType.REMOVED_FROM_COMMITTEE : CommitteesActivityType.ADDED_TO_COMMITTEE - const sourceId = (row.PRIMARY_SOURCE_USER_ID as string | null)?.trim() || undefined + const sourceId = cleanIdentityValue(row.PRIMARY_SOURCE_USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, platformUsername: null, diff --git a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts index 5fdea82fb0..657f6aa65b 100644 --- a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts @@ -9,7 +9,11 @@ import { PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { + TransformedActivity, + TransformerBase, + cleanIdentityValue, +} from '../../../core/transformerBase' const log = getServiceChildLogger('cventTransformer') @@ -17,12 +21,12 @@ export class CventTransformer extends TransformerBase { readonly platform = PlatformType.CVENT transformRow(row: Record): TransformedActivity | null { - const userName = (row.USER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null + const userName = cleanIdentityValue(row.USER_NAME) + const lfUsername = cleanIdentityValue(row.LFID) const fullName = (row.FULL_NAME as string | null)?.trim() || null const firstName = (row.FIRST_NAME as string | null)?.trim() || null const lastName = (row.LAST_NAME as string | null)?.trim() || null - const email = (row.EMAIL as string).trim() + const email = cleanIdentityValue(row.EMAIL) if (!email) { log.warn( { registrationId: row.REGISTRATION_ID, userName, lfUsername }, @@ -32,7 +36,7 @@ export class CventTransformer extends TransformerBase { } const registrationId = (row.REGISTRATION_ID as string)?.trim() - const sourceId = (row.USER_ID as string | null) || undefined + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const displayName = fullName || diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts index 76c92b14a9..ff34b07269 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts @@ -37,7 +37,7 @@ export class MeetingAttendanceTransformer extends TransformerBase { readonly platform = PlatformType.MEETINGS transformRow(row: Record): TransformedActivity | TransformedActivity[] | null { - const email = (row.INVITEE_EMAIL as string | null)?.trim() || null + const email = cleanIdentityValue(row.INVITEE_EMAIL) if (!email) { log.debug({ primaryKey: row.PRIMARY_KEY }, 'Skipping row: missing email') return null diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index 8662c8981d..61d5182a3d 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -2,14 +2,14 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCertificatesTransformer') export class TncCertificatesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanIdentityValue(row.USER_EMAIL) if (!email) { log.debug({ certificateId: row.CERTIFICATE_ID }, 'Skipping row: missing email') return null @@ -17,8 +17,8 @@ export class TncCertificatesTransformer extends TncTransformerBase { const certificateId = (row.CERTIFICATE_ID as string)?.trim() const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 36f755f197..c573b0963e 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -2,14 +2,14 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCoursesTransformer') export class TncCoursesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanIdentityValue(row.USER_EMAIL) if (!email) { log.warn( { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL }, @@ -25,8 +25,8 @@ export class TncCoursesTransformer extends TncTransformerBase { } const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.INTERNAL_TI_USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.INTERNAL_TI_USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index a3bad55e1d..b0367aa761 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -2,14 +2,14 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' -import { TransformedActivity } from '../../../core/transformerBase' +import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncEnrollmentsTransformer') export class TncEnrollmentsTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = (row.USER_EMAIL as string | null)?.trim() || null + const email = cleanIdentityValue(row.USER_EMAIL) if (!email) { log.warn( { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL }, @@ -20,8 +20,8 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { const enrollmentId = (row.ENROLLMENT_ID as string)?.trim() const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null - const lfUsername = (row.LFID as string | null)?.trim() || null - const sourceId = (row.USER_ID as string | null) || undefined + const lfUsername = cleanIdentityValue(row.LFID) + const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined const identities = this.buildMemberIdentities({ email, From 76f223a7743fd96658737a0a6848ea99564ae176 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:41:15 +0100 Subject: [PATCH 3/6] fix: skip shared, list and automation emails in snowflake connectors Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- .../src/core/personalEmail.test.ts | 59 ++++++++++++++ .../src/core/personalEmail.ts | 76 +++++++++++++++++++ .../committees/committees/transformer.ts | 3 +- .../cvent/event-registrations/transformer.ts | 3 +- .../meeting-attendance/transformer.ts | 3 +- .../tnc/certificates/transformer.ts | 3 +- .../integrations/tnc/courses/transformer.ts | 3 +- .../tnc/enrollments/transformer.ts | 3 +- 8 files changed, 147 insertions(+), 6 deletions(-) create mode 100644 services/apps/snowflake_connectors/src/core/personalEmail.test.ts create mode 100644 services/apps/snowflake_connectors/src/core/personalEmail.ts diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.test.ts b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts new file mode 100644 index 0000000000..76b74db1eb --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from 'vitest' + +import { cleanPersonalEmail, isSharedOrAutomatedEmail } from './personalEmail' + +describe('isSharedOrAutomatedEmail', () => { + it.each([ + 'zoom@hyperledger.org', + 'zoom2@lfai.foundation', + 'zoom-admin@nephio.org', + 'Meetings@dpdk.org', + 'meeting1@seidengroup.com', + ])('flags the shared meeting host account %s', (email) => { + expect(isSharedOrAutomatedEmail(email)).toBe(true) + }) + + it.each(['cdf-toc@lists.cd.foundation', 'wg@groups.io', 'kubecon-team@googlegroups.com'])( + 'flags the mailing list %s', + (email) => { + expect(isSharedOrAutomatedEmail(email)).toBe(true) + }, + ) + + it.each([ + 'noreply@github.com', + 'no-reply+events@pushfeedback.com', + 'student1@example.com', + 'cb-integration-bot@linuxfoundation.org', + 'jenkins@cd.foundation', + ])('flags the noreply, placeholder or automation address %s', (email) => { + expect(isSharedOrAutomatedEmail(email)).toBe(true) + }) + + it.each([ + 'james.wang@zoom.us', + 'zoom201719@gmail.com', + 'zoom2arun@gmail.com', + 'mail@jkroepke.de', + 'hello@stevegiguere.com', + 'hans.bot@yenlo.com', + 'varun.pandey.bot@gmail.com', + 'vad1mo@users.noreply.github.com', + ])('keeps the personal address %s', (email) => { + expect(isSharedOrAutomatedEmail(email)).toBe(false) + }) +}) + +describe('cleanPersonalEmail', () => { + it('trims and keeps a personal email', () => { + expect(cleanPersonalEmail(' ada@lovelace.dev ')).toBe('ada@lovelace.dev') + }) + + it('drops a shared meeting host account', () => { + expect(cleanPersonalEmail('zoom@hyperledger.org')).toBeNull() + }) + + it('drops a placeholder email value', () => { + expect(cleanPersonalEmail('NONE')).toBeNull() + }) +}) diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.ts b/services/apps/snowflake_connectors/src/core/personalEmail.ts new file mode 100644 index 0000000000..22e1f536bd --- /dev/null +++ b/services/apps/snowflake_connectors/src/core/personalEmail.ts @@ -0,0 +1,76 @@ +import { cleanIdentityValue } from './transformerBase' + +const NOREPLY_LOCAL_PARTS = new Set(['noreply', 'no-reply', 'donotreply', 'do-not-reply']) + +const PLACEHOLDER_EMAIL_DOMAINS = new Set([ + 'example.com', + 'example.org', + 'example.net', + 'example.edu', +]) + +const MAILING_LIST_DOMAINS = new Set(['groups.io', 'googlegroups.com']) + +const FREEMAIL_DOMAINS = new Set([ + 'gmail.com', + 'googlemail.com', + 'outlook.com', + 'hotmail.com', + 'live.com', + 'yahoo.com', + 'icloud.com', + 'protonmail.com', + 'proton.me', + 'qq.com', + '163.com', +]) + +const AUTOMATION_LOCAL_PARTS = new Set([ + 'dependabot', + 'renovate', + 'ci', + 'jenkins', + 'github-actions', + 'actions', +]) + +const MEETING_ACCOUNT_LOCAL_PART = /^(zoom|meetings?)(\d+|-[a-z0-9]+)?$/ + +function isMailingList(domain: string): boolean { + return domain.startsWith('lists.') || MAILING_LIST_DOMAINS.has(domain) +} + +function isMeetingAccount(localPart: string, domain: string): boolean { + return MEETING_ACCOUNT_LOCAL_PART.test(localPart) && !FREEMAIL_DOMAINS.has(domain) +} + +function isAutomation(localPart: string): boolean { + return ( + AUTOMATION_LOCAL_PARTS.has(localPart) || + localPart.endsWith('-bot') || + localPart.endsWith('[bot]') + ) +} + +export function isSharedOrAutomatedEmail(email: string): boolean { + const value = email.toLowerCase() + const atIndex = value.lastIndexOf('@') + if (atIndex < 0) { + return false + } + const localPart = value.slice(0, atIndex).split('+')[0] + const domain = value.slice(atIndex + 1) + + return ( + NOREPLY_LOCAL_PARTS.has(localPart) || + PLACEHOLDER_EMAIL_DOMAINS.has(domain) || + isMailingList(domain) || + isMeetingAccount(localPart, domain) || + isAutomation(localPart) + ) +} + +export function cleanPersonalEmail(raw: unknown): string | null { + const email = cleanIdentityValue(raw) + return email && !isSharedOrAutomatedEmail(email) ? email : null +} diff --git a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts index fffaf67a31..f23de1f6eb 100644 --- a/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts @@ -8,6 +8,7 @@ import { PlatformType, } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, TransformerBase, @@ -20,7 +21,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase { readonly platform = PlatformType.COMMITTEES transformRow(row: Record): TransformedActivity | null { - const email = cleanIdentityValue(row.CONTACTEMAIL__C) + const email = cleanPersonalEmail(row.CONTACTEMAIL__C) if (!email) { log.warn( { sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C }, diff --git a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts index 657f6aa65b..3ad2f84448 100644 --- a/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts @@ -9,6 +9,7 @@ import { PlatformType, } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, TransformerBase, @@ -26,7 +27,7 @@ export class CventTransformer extends TransformerBase { const fullName = (row.FULL_NAME as string | null)?.trim() || null const firstName = (row.FIRST_NAME as string | null)?.trim() || null const lastName = (row.LAST_NAME as string | null)?.trim() || null - const email = cleanIdentityValue(row.EMAIL) + const email = cleanPersonalEmail(row.EMAIL) if (!email) { log.warn( { registrationId: row.REGISTRATION_ID, userName, lfUsername }, diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts index ff34b07269..5dfe747404 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts @@ -8,6 +8,7 @@ import { PlatformType, } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, TransformerBase, @@ -37,7 +38,7 @@ export class MeetingAttendanceTransformer extends TransformerBase { readonly platform = PlatformType.MEETINGS transformRow(row: Record): TransformedActivity | TransformedActivity[] | null { - const email = cleanIdentityValue(row.INVITEE_EMAIL) + const email = cleanPersonalEmail(row.INVITEE_EMAIL) if (!email) { log.debug({ primaryKey: row.PRIMARY_KEY }, 'Skipping row: missing email') return null diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index 61d5182a3d..28a779cc7e 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -2,6 +2,7 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' @@ -9,7 +10,7 @@ const log = getServiceChildLogger('tncCertificatesTransformer') export class TncCertificatesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = cleanIdentityValue(row.USER_EMAIL) + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.debug({ certificateId: row.CERTIFICATE_ID }, 'Skipping row: missing email') return null diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index c573b0963e..93dbc73792 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -2,6 +2,7 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' @@ -9,7 +10,7 @@ const log = getServiceChildLogger('tncCoursesTransformer') export class TncCoursesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = cleanIdentityValue(row.USER_EMAIL) + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.warn( { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL }, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index b0367aa761..cdddc49a0b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -2,6 +2,7 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations' import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types' +import { cleanPersonalEmail } from '../../../core/personalEmail' import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase' import { TncTransformerBase } from '../tncTransformerBase' @@ -9,7 +10,7 @@ const log = getServiceChildLogger('tncEnrollmentsTransformer') export class TncEnrollmentsTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { - const email = cleanIdentityValue(row.USER_EMAIL) + const email = cleanPersonalEmail(row.USER_EMAIL) if (!email) { log.warn( { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL }, From 8ba96533be87678a09062cad5b52aafa08dcc2bb Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:41:17 +0100 Subject: [PATCH 4/6] fix: soft delete known shared and placeholder snowflake member identities Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- ...oft-delete-shared-snowflake-identities.sql | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql diff --git a/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql new file mode 100644 index 0000000000..4d568306c2 --- /dev/null +++ b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql @@ -0,0 +1,80 @@ +with deleted as ( + update "memberIdentities" + set "deletedAt" = now(), + "updatedAt" = now() + where "deletedAt" is null + and ( + ( + platform in ('meetings', 'cvent', 'tnc', 'committees') + and type in ('email', 'username') + and lower(value) in ( + 'cb-integration-bot@linuxfoundation.org', + 'cdf-toc@lists.cd.foundation', + 'cdm-tokenized-assets@lists.finos.org', + 'dev-analytics-bot@linuxfoundation.org', + 'doe@example.com', + 'formation@lists.neonephos.org', + 'groupsio-analytics-bot@linuxfoundation.org', + 'groupsio-automation-bot@linuxfoundation.org', + 'ishu@example.com', + 'jenkins@cd.foundation', + 'jenkins-zoom-users@googlegroups.com', + 'johndoe.train@example.com', + 'jpf-okteto-kubecon-eu-2023-533434511717@googlegroups.com', + 'kernelci@lists.linux.dev', + 'kernelci-members@groups.io', + 'kernelci-tsc@groups.io', + 'kubernetes-sig-network-leads@googlegroups.com', + 'main@lists.openchainproject.org', + 'martin.xu@example.com', + 'meeting1@seidengroup.com', + 'meetings@dpdk.org', + 'meetings@lists.magmacore.org', + 'meetings@openjsf.org', + 'meetings@prestodb.io', + 'meetings@socalblender.org', + 'noreply@github.com', + 'noreply@pushfeedback.com', + 'noreply@zza.hu', + 'notifications@lists.overturemaps.org', + 'opengris@lists.finos.org', + 'operator-framework@googlegroups.com', + 'portal_helps@lists.onap.org', + 'rise-system-libraries-wg@lists.riseproject.dev', + 'sonic-outreach@lists.sonicfoundation.dev', + 'student1@example.com', + 'test@example.com', + 'wg-linux-distros@lists.openmainframeproject.org', + 'wpadmin@example.com', + 'zoom0@hyperledger.org', + 'zoom2@hyperledger.org', + 'zoom2@lfai.foundation', + 'zoom2@r-consortium.org', + 'zoom3@hyperledger.org', + 'zoom8@lfaidata.foundation', + 'zoom-admin@nephio.org', + 'zoom_anari@lists.khronos.org', + 'zoom@ccbny.com', + 'zoom@dent.dev', + 'zoom@district5030.org', + 'zoom@elisa.tech', + 'zoom@hyperledger.org', + 'zoom-jl@fd.io', + 'zoom@lfai.foundation', + 'zoom@longnowboston.org', + 'zoom-nodejs@openjsf.org', + 'zoom@openidl.org', + 'zoom@openids.io', + 'zoom@openwallet.foundation', + 'zoom@r-consortium.org', + 'zoom+sig@riscv.org' + ) + ) + or (platform = 'cvent' and type = 'username' and lower(value) in ('na', 'none', 'null')) + or (platform = 'lfid' and type = 'username' and lower(value) = 'undefined') + ) + returning "memberId" +) +update members +set "updatedAt" = now() +where id in (select "memberId" from deleted); From e83a8e77372c6ef16aa01874aed4ebf19ac11c11 Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:43:20 +0100 Subject: [PATCH 5/6] fix: let the lf sso lookup fill in when a meetings row has a placeholder sso Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- .../meetings/meeting-attendance/buildSourceQuery.test.ts | 6 ++++++ .../meetings/meeting-attendance/buildSourceQuery.ts | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts index 8aa52a0cb2..96cfb54a4b 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts @@ -16,4 +16,10 @@ describe('buildSourceQuery', () => { ) }, ) + + it('lets the LF SSO lookup fill in when the row itself carries a placeholder SSO', () => { + expect(buildSourceQuery()).toContain( + "CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-') THEN t.INVITEE_LF_SSO END,\n sso.LF_SSO", + ) + }) }) diff --git a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts index 0761e7976c..2e9f5b4e94 100644 --- a/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts @@ -49,7 +49,10 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { t.MEETING_DATE, t.MEETING_TIME, t.INVITEE_FULL_NAME, - COALESCE(t.INVITEE_LF_SSO, sso.LF_SSO) AS INVITEE_LF_SSO, + COALESCE( + CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES} THEN t.INVITEE_LF_SSO END, + sso.LF_SSO + ) AS INVITEE_LF_SSO, t.INVITEE_LF_USER_ID, t.INVITEE_EMAIL, t.INVITEE_ATTENDED, From 0f5592fb8a1c08c3c7cf4ba61063cd27965a9dbc Mon Sep 17 00:00:00 2001 From: Joana Maia Date: Tue, 20 Oct 2026 11:45:26 +0100 Subject: [PATCH 6/6] fix: limit the snowflake email blocklist to shared meeting accounts found in bad merges Refs linuxfoundation/lfx-cdp-insights-ops#853 Signed-off-by: Joana Maia --- ...oft-delete-shared-snowflake-identities.sql | 57 +------------ .../src/core/personalEmail.test.ts | 47 +++-------- .../src/core/personalEmail.ts | 80 +++---------------- 3 files changed, 28 insertions(+), 156 deletions(-) diff --git a/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql index 4d568306c2..83a09a3d34 100644 --- a/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql +++ b/backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql @@ -5,69 +5,18 @@ with deleted as ( where "deletedAt" is null and ( ( - platform in ('meetings', 'cvent', 'tnc', 'committees') + platform = 'meetings' and type in ('email', 'username') and lower(value) in ( - 'cb-integration-bot@linuxfoundation.org', - 'cdf-toc@lists.cd.foundation', - 'cdm-tokenized-assets@lists.finos.org', - 'dev-analytics-bot@linuxfoundation.org', - 'doe@example.com', - 'formation@lists.neonephos.org', - 'groupsio-analytics-bot@linuxfoundation.org', - 'groupsio-automation-bot@linuxfoundation.org', - 'ishu@example.com', - 'jenkins@cd.foundation', - 'jenkins-zoom-users@googlegroups.com', - 'johndoe.train@example.com', - 'jpf-okteto-kubecon-eu-2023-533434511717@googlegroups.com', - 'kernelci@lists.linux.dev', - 'kernelci-members@groups.io', - 'kernelci-tsc@groups.io', - 'kubernetes-sig-network-leads@googlegroups.com', - 'main@lists.openchainproject.org', - 'martin.xu@example.com', 'meeting1@seidengroup.com', 'meetings@dpdk.org', - 'meetings@lists.magmacore.org', - 'meetings@openjsf.org', - 'meetings@prestodb.io', - 'meetings@socalblender.org', - 'noreply@github.com', - 'noreply@pushfeedback.com', - 'noreply@zza.hu', - 'notifications@lists.overturemaps.org', - 'opengris@lists.finos.org', - 'operator-framework@googlegroups.com', - 'portal_helps@lists.onap.org', - 'rise-system-libraries-wg@lists.riseproject.dev', - 'sonic-outreach@lists.sonicfoundation.dev', - 'student1@example.com', - 'test@example.com', - 'wg-linux-distros@lists.openmainframeproject.org', - 'wpadmin@example.com', + 'zoom-admin@nephio.org', 'zoom0@hyperledger.org', 'zoom2@hyperledger.org', 'zoom2@lfai.foundation', - 'zoom2@r-consortium.org', - 'zoom3@hyperledger.org', - 'zoom8@lfaidata.foundation', - 'zoom-admin@nephio.org', - 'zoom_anari@lists.khronos.org', - 'zoom@ccbny.com', - 'zoom@dent.dev', - 'zoom@district5030.org', - 'zoom@elisa.tech', 'zoom@hyperledger.org', - 'zoom-jl@fd.io', - 'zoom@lfai.foundation', 'zoom@longnowboston.org', - 'zoom-nodejs@openjsf.org', - 'zoom@openidl.org', - 'zoom@openids.io', - 'zoom@openwallet.foundation', - 'zoom@r-consortium.org', - 'zoom+sig@riscv.org' + 'zoom@openwallet.foundation' ) ) or (platform = 'cvent' and type = 'username' and lower(value) in ('na', 'none', 'null')) diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.test.ts b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts index 76b74db1eb..9c59351cb9 100644 --- a/services/apps/snowflake_connectors/src/core/personalEmail.test.ts +++ b/services/apps/snowflake_connectors/src/core/personalEmail.test.ts @@ -1,46 +1,23 @@ import { describe, expect, it } from 'vitest' -import { cleanPersonalEmail, isSharedOrAutomatedEmail } from './personalEmail' +import { cleanPersonalEmail, isSharedMeetingAccountEmail } from './personalEmail' -describe('isSharedOrAutomatedEmail', () => { - it.each([ - 'zoom@hyperledger.org', - 'zoom2@lfai.foundation', - 'zoom-admin@nephio.org', - 'Meetings@dpdk.org', - 'meeting1@seidengroup.com', - ])('flags the shared meeting host account %s', (email) => { - expect(isSharedOrAutomatedEmail(email)).toBe(true) - }) - - it.each(['cdf-toc@lists.cd.foundation', 'wg@groups.io', 'kubecon-team@googlegroups.com'])( - 'flags the mailing list %s', +describe('isSharedMeetingAccountEmail', () => { + it.each(['zoom@hyperledger.org', 'Zoom2@LFAI.foundation', 'meetings@dpdk.org'])( + 'flags the known shared meeting host account %s', (email) => { - expect(isSharedOrAutomatedEmail(email)).toBe(true) + expect(isSharedMeetingAccountEmail(email)).toBe(true) }, ) it.each([ - 'noreply@github.com', - 'no-reply+events@pushfeedback.com', - 'student1@example.com', - 'cb-integration-bot@linuxfoundation.org', - 'jenkins@cd.foundation', - ])('flags the noreply, placeholder or automation address %s', (email) => { - expect(isSharedOrAutomatedEmail(email)).toBe(true) - }) - - it.each([ + 'zoom3@hyperledger.org', + 'zoom@antonpalmqvist.com', 'james.wang@zoom.us', - 'zoom201719@gmail.com', - 'zoom2arun@gmail.com', + 'noreply@github.com', 'mail@jkroepke.de', - 'hello@stevegiguere.com', - 'hans.bot@yenlo.com', - 'varun.pandey.bot@gmail.com', - 'vad1mo@users.noreply.github.com', - ])('keeps the personal address %s', (email) => { - expect(isSharedOrAutomatedEmail(email)).toBe(false) + ])('does not flag %s, which is not on the known list', (email) => { + expect(isSharedMeetingAccountEmail(email)).toBe(false) }) }) @@ -49,8 +26,8 @@ describe('cleanPersonalEmail', () => { expect(cleanPersonalEmail(' ada@lovelace.dev ')).toBe('ada@lovelace.dev') }) - it('drops a shared meeting host account', () => { - expect(cleanPersonalEmail('zoom@hyperledger.org')).toBeNull() + it('drops a known shared meeting host account', () => { + expect(cleanPersonalEmail(' zoom@openwallet.foundation ')).toBeNull() }) it('drops a placeholder email value', () => { diff --git a/services/apps/snowflake_connectors/src/core/personalEmail.ts b/services/apps/snowflake_connectors/src/core/personalEmail.ts index 22e1f536bd..d7d1c97798 100644 --- a/services/apps/snowflake_connectors/src/core/personalEmail.ts +++ b/services/apps/snowflake_connectors/src/core/personalEmail.ts @@ -1,76 +1,22 @@ import { cleanIdentityValue } from './transformerBase' -const NOREPLY_LOCAL_PARTS = new Set(['noreply', 'no-reply', 'donotreply', 'do-not-reply']) - -const PLACEHOLDER_EMAIL_DOMAINS = new Set([ - 'example.com', - 'example.org', - 'example.net', - 'example.edu', -]) - -const MAILING_LIST_DOMAINS = new Set(['groups.io', 'googlegroups.com']) - -const FREEMAIL_DOMAINS = new Set([ - 'gmail.com', - 'googlemail.com', - 'outlook.com', - 'hotmail.com', - 'live.com', - 'yahoo.com', - 'icloud.com', - 'protonmail.com', - 'proton.me', - 'qq.com', - '163.com', +const SHARED_MEETING_ACCOUNT_EMAILS = new Set([ + 'meeting1@seidengroup.com', + 'meetings@dpdk.org', + 'zoom-admin@nephio.org', + 'zoom0@hyperledger.org', + 'zoom2@hyperledger.org', + 'zoom2@lfai.foundation', + 'zoom@hyperledger.org', + 'zoom@longnowboston.org', + 'zoom@openwallet.foundation', ]) -const AUTOMATION_LOCAL_PARTS = new Set([ - 'dependabot', - 'renovate', - 'ci', - 'jenkins', - 'github-actions', - 'actions', -]) - -const MEETING_ACCOUNT_LOCAL_PART = /^(zoom|meetings?)(\d+|-[a-z0-9]+)?$/ - -function isMailingList(domain: string): boolean { - return domain.startsWith('lists.') || MAILING_LIST_DOMAINS.has(domain) -} - -function isMeetingAccount(localPart: string, domain: string): boolean { - return MEETING_ACCOUNT_LOCAL_PART.test(localPart) && !FREEMAIL_DOMAINS.has(domain) -} - -function isAutomation(localPart: string): boolean { - return ( - AUTOMATION_LOCAL_PARTS.has(localPart) || - localPart.endsWith('-bot') || - localPart.endsWith('[bot]') - ) -} - -export function isSharedOrAutomatedEmail(email: string): boolean { - const value = email.toLowerCase() - const atIndex = value.lastIndexOf('@') - if (atIndex < 0) { - return false - } - const localPart = value.slice(0, atIndex).split('+')[0] - const domain = value.slice(atIndex + 1) - - return ( - NOREPLY_LOCAL_PARTS.has(localPart) || - PLACEHOLDER_EMAIL_DOMAINS.has(domain) || - isMailingList(domain) || - isMeetingAccount(localPart, domain) || - isAutomation(localPart) - ) +export function isSharedMeetingAccountEmail(email: string): boolean { + return SHARED_MEETING_ACCOUNT_EMAILS.has(email.toLowerCase()) } export function cleanPersonalEmail(raw: unknown): string | null { const email = cleanIdentityValue(raw) - return email && !isSharedOrAutomatedEmail(email) ? email : null + return email && !isSharedMeetingAccountEmail(email) ? email : null }