fix: keep placeholder ids and shared meeting accounts from snowflake out of member identities

backend/src/database/migrations/V1792492400__soft-delete-shared-snowflake-identities.sql

@@ -0,0 +1,29 @@
+with deleted as (
+    update "memberIdentities"
+    set "deletedAt" = now(),
+        "updatedAt" = now()
+    where "deletedAt" is null
+      and (
+          (
+              platform = 'meetings'
+              and type in ('email', 'username')
+              and lower(value) in (
+                  'meeting1@seidengroup.com',
+                  'meetings@dpdk.org',
+                  'zoom-admin@nephio.org',
+                  'zoom0@hyperledger.org',
+                  'zoom2@hyperledger.org',
+                  'zoom2@lfai.foundation',
+                  'zoom@hyperledger.org',
+                  'zoom@longnowboston.org',
+                  'zoom@openwallet.foundation'
+              )
+          )
+          or (platform = 'cvent' and type = 'username' and lower(value) in ('na', 'none', 'null'))
+          or (platform = 'lfid' and type = 'username' and lower(value) = 'undefined')
+      )
+    returning "memberId"
+)
+update members
+set "updatedAt" = now()
+where id in (select "memberId" from deleted);

services/apps/snowflake_connectors/src/core/personalEmail.test.ts

@@ -0,0 +1,36 @@
+import { describe, expect, it } from 'vitest'
+
+import { cleanPersonalEmail, isSharedMeetingAccountEmail } from './personalEmail'
+
+describe('isSharedMeetingAccountEmail', () => {
+  it.each(['zoom@hyperledger.org', 'Zoom2@LFAI.foundation', 'meetings@dpdk.org'])(
+    'flags the known shared meeting host account %s',
+    (email) => {
+      expect(isSharedMeetingAccountEmail(email)).toBe(true)
+    },
+  )
+
+  it.each([
+    'zoom3@hyperledger.org',
+    'zoom@antonpalmqvist.com',
+    'james.wang@zoom.us',
+    'noreply@github.com',
+    'mail@jkroepke.de',
+  ])('does not flag %s, which is not on the known list', (email) => {
+    expect(isSharedMeetingAccountEmail(email)).toBe(false)
+  })
+})
+
+describe('cleanPersonalEmail', () => {
+  it('trims and keeps a personal email', () => {
+    expect(cleanPersonalEmail(' ada@lovelace.dev ')).toBe('ada@lovelace.dev')
+  })
+
+  it('drops a known shared meeting host account', () => {
+    expect(cleanPersonalEmail(' zoom@openwallet.foundation ')).toBeNull()
+  })
+
+  it('drops a placeholder email value', () => {
+    expect(cleanPersonalEmail('NONE')).toBeNull()
+  })
+})

services/apps/snowflake_connectors/src/core/personalEmail.ts

@@ -0,0 +1,22 @@
+import { cleanIdentityValue } from './transformerBase'
+
+const SHARED_MEETING_ACCOUNT_EMAILS = new Set([
+  'meeting1@seidengroup.com',
+  'meetings@dpdk.org',
+  'zoom-admin@nephio.org',
+  'zoom0@hyperledger.org',
+  'zoom2@hyperledger.org',
+  'zoom2@lfai.foundation',
+  'zoom@hyperledger.org',
+  'zoom@longnowboston.org',
+  'zoom@openwallet.foundation',
+])
+
+export function isSharedMeetingAccountEmail(email: string): boolean {
+  return SHARED_MEETING_ACCOUNT_EMAILS.has(email.toLowerCase())
+}
+
+export function cleanPersonalEmail(raw: unknown): string | null {
+  const email = cleanIdentityValue(raw)
+  return email && !isSharedMeetingAccountEmail(email) ? email : null
+}

services/apps/snowflake_connectors/src/core/transformerBase.test.ts

@@ -0,0 +1,28 @@
+import { describe, expect, it } from 'vitest'
+
+import { cleanIdentityValue } from './transformerBase'
+
+describe('cleanIdentityValue', () => {
+  it.each(['NONE', 'none', ' None ', 'NULL', 'undefined', 'NA', 'n/a', '-'])(
+    'treats the placeholder %j as missing',
+    (placeholder) => {
+      expect(cleanIdentityValue(placeholder)).toBeNull()
+    },
+  )
+
+  it.each([null, undefined, '', '   '])('treats %j as missing', (empty) => {
+    expect(cleanIdentityValue(empty)).toBeNull()
+  })
+
+  it('trims a real LF user id', () => {
+    expect(cleanIdentityValue(' 0031T00003abcDE ')).toBe('0031T00003abcDE')
+  })
+
+  it('keeps usernames that only contain a placeholder word', () => {
+    expect(cleanIdentityValue('nonesuch')).toBe('nonesuch')
+  })
+
+  it('stringifies a numeric user id', () => {
+    expect(cleanIdentityValue(12345)).toBe('12345')
+  })
+})

services/apps/snowflake_connectors/src/core/transformerBase.ts

@@ -9,6 +9,16 @@ import { IActivityData, IMemberData, MemberIdentityType, PlatformType } from '@c
 
 const log = getServiceChildLogger('transformer')
 
+const PLACEHOLDER_IDENTITY_VALUES = new Set(['none', 'null', 'undefined', 'na', 'n/a', '-'])
+
+export function cleanIdentityValue(raw: unknown): string | null {
+  const value = typeof raw === 'string' ? raw.trim() : typeof raw === 'number' ? String(raw) : ''
+  if (!value || PLACEHOLDER_IDENTITY_VALUES.has(value.toLowerCase())) {
+    return null
+  }
+  return value
+}
+
 export interface SegmentRef {
   slug: string
   sourceId: string

services/apps/snowflake_connectors/src/integrations/committees/committees/transformer.ts

@@ -8,15 +8,20 @@ import {
   PlatformType,
 } from '@crowd/types'
 
-import { TransformedActivity, TransformerBase } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import {
+  TransformedActivity,
+  TransformerBase,
+  cleanIdentityValue,
+} from '../../../core/transformerBase'
 
 const log = getServiceChildLogger('committeesCommitteesTransformer')
 
 export class CommitteesCommitteesTransformer extends TransformerBase {
   readonly platform = PlatformType.COMMITTEES
 
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
-    const email = (row.CONTACTEMAIL__C as string | null)?.trim() || null
+    const email = cleanPersonalEmail(row.CONTACTEMAIL__C)
     if (!email) {
       log.warn(
         { sfid: row.SFID, committeeId: row.COMMITTEE_ID, rawEmail: row.CONTACTEMAIL__C },
@@ -27,7 +32,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase {
 
     const committeeId = (row.COMMITTEE_ID as string).trim()
     const fivetranDeleted = row.FIVETRAN_DELETED as boolean
-    const lfUsername = (row.LF_USERNAME as string | null)?.trim() || null
+    const lfUsername = cleanIdentityValue(row.LF_USERNAME)
     const suFullName = (row.SU_FULL_NAME as string | null)?.trim() || null
     const suFirstName = (row.SU_FIRST_NAME as string | null)?.trim() || null
     const suLastName = (row.SU_LAST_NAME as string | null)?.trim() || null
@@ -41,7 +46,7 @@ export class CommitteesCommitteesTransformer extends TransformerBase {
       ? CommitteesActivityType.REMOVED_FROM_COMMITTEE
       : CommitteesActivityType.ADDED_TO_COMMITTEE
 
-    const sourceId = (row.PRIMARY_SOURCE_USER_ID as string | null)?.trim() || undefined
+    const sourceId = cleanIdentityValue(row.PRIMARY_SOURCE_USER_ID) ?? undefined
     const identities = this.buildMemberIdentities({
       email,
       platformUsername: null,

services/apps/snowflake_connectors/src/integrations/cvent/event-registrations/transformer.ts

@@ -9,20 +9,25 @@ import {
   PlatformType,
 } from '@crowd/types'
 
-import { TransformedActivity, TransformerBase } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import {
+  TransformedActivity,
+  TransformerBase,
+  cleanIdentityValue,
+} from '../../../core/transformerBase'
 
 const log = getServiceChildLogger('cventTransformer')
 
 export class CventTransformer extends TransformerBase {
   readonly platform = PlatformType.CVENT
 
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
-    const userName = (row.USER_NAME as string | null)?.trim() || null
-    const lfUsername = (row.LFID as string | null)?.trim() || null
+    const userName = cleanIdentityValue(row.USER_NAME)
+    const lfUsername = cleanIdentityValue(row.LFID)
     const fullName = (row.FULL_NAME as string | null)?.trim() || null
     const firstName = (row.FIRST_NAME as string | null)?.trim() || null
     const lastName = (row.LAST_NAME as string | null)?.trim() || null
-    const email = (row.EMAIL as string).trim()
+    const email = cleanPersonalEmail(row.EMAIL)
     if (!email) {
       log.warn(
         { registrationId: row.REGISTRATION_ID, userName, lfUsername },
@@ -32,7 +37,7 @@ export class CventTransformer extends TransformerBase {
     }
 
     const registrationId = (row.REGISTRATION_ID as string)?.trim()
-    const sourceId = (row.USER_ID as string | null) || undefined
+    const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined
 
     const displayName =
       fullName ||

services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.test.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from 'vitest'
+
+import { buildSourceQuery } from './buildSourceQuery'
+
+describe('buildSourceQuery', () => {
+  it.each([undefined, '2026-10-19 00:00:00'])(
+    'keeps placeholder LF user ids out of the LF SSO lookup (since %j)',
+    (sinceTimestamp) => {
+      const query = buildSourceQuery(sinceTimestamp)
+
+      expect(query).toContain(
+        "UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')",
+      )
+      expect(query).toContain(
+        "UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')",
+      )
+    },
+  )
+
+  it('lets the LF SSO lookup fill in when the row itself carries a placeholder SSO', () => {
+    expect(buildSourceQuery()).toContain(
+      "CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-') THEN t.INVITEE_LF_SSO END,\n      sso.LF_SSO",
+    )
+  })
+})

services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/buildSourceQuery.ts

@@ -22,12 +22,16 @@ const ORG_ACCOUNTS = `
     WHERE website IS NOT NULL
   )`
 
+const PLACEHOLDER_VALUES = `('', 'NONE', 'NULL', 'UNDEFINED', 'NA', 'N/A', '-')`
+
 const LF_SSO_LOOKUP = `
   lf_sso_lookup AS (
     SELECT INVITEE_LF_USER_ID AS LF_USER_ID, MAX(INVITEE_LF_SSO) AS LF_SSO
     FROM ANALYTICS.SILVER_FACT.MEETING_ATTENDANCE
     WHERE INVITEE_LF_SSO IS NOT NULL
       AND INVITEE_LF_USER_ID IS NOT NULL
+      AND UPPER(TRIM(INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES}
+      AND UPPER(TRIM(INVITEE_LF_USER_ID)) NOT IN ${PLACEHOLDER_VALUES}
     GROUP BY INVITEE_LF_USER_ID
   )`
 
@@ -45,7 +49,10 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => {
     t.MEETING_DATE,
     t.MEETING_TIME,
     t.INVITEE_FULL_NAME,
-    COALESCE(t.INVITEE_LF_SSO, sso.LF_SSO) AS INVITEE_LF_SSO,
+    COALESCE(
+      CASE WHEN UPPER(TRIM(t.INVITEE_LF_SSO)) NOT IN ${PLACEHOLDER_VALUES} THEN t.INVITEE_LF_SSO END,
+      sso.LF_SSO
+    ) AS INVITEE_LF_SSO,
     t.INVITEE_LF_USER_ID,
     t.INVITEE_EMAIL,
     t.INVITEE_ATTENDED,

services/apps/snowflake_connectors/src/integrations/meetings/meeting-attendance/transformer.ts

@@ -8,7 +8,12 @@ import {
   PlatformType,
 } from '@crowd/types'
 
-import { TransformedActivity, TransformerBase } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import {
+  TransformedActivity,
+  TransformerBase,
+  cleanIdentityValue,
+} from '../../../core/transformerBase'
 
 const log = getServiceChildLogger('meetingAttendanceTransformer')
 
@@ -33,14 +38,14 @@ export class MeetingAttendanceTransformer extends TransformerBase {
   readonly platform = PlatformType.MEETINGS
 
   transformRow(row: Record<string, unknown>): TransformedActivity | TransformedActivity[] | null {
-    const email = (row.INVITEE_EMAIL as string | null)?.trim() || null
+    const email = cleanPersonalEmail(row.INVITEE_EMAIL)
     if (!email) {
       log.debug({ primaryKey: row.PRIMARY_KEY }, 'Skipping row: missing email')
       return null
     }
 
-    const lfUsername = (row.INVITEE_LF_SSO as string | null)?.trim() || null
-    const memberSourceId = (row.INVITEE_LF_USER_ID as string | null)?.trim() || undefined
+    const lfUsername = cleanIdentityValue(row.INVITEE_LF_SSO)
+    const memberSourceId = cleanIdentityValue(row.INVITEE_LF_USER_ID) ?? undefined
     const generatedSourceId = (row.GENERATED_SOURCE_ID as string | null)?.trim() || undefined
     const userId = memberSourceId ?? generatedSourceId
     const displayName = (row.INVITEE_FULL_NAME as string | null)?.trim() || email

services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts

@@ -2,23 +2,24 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations'
 import { getServiceChildLogger } from '@crowd/logging'
 import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types'
 
-import { TransformedActivity } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase'
 import { TncTransformerBase } from '../tncTransformerBase'
 
 const log = getServiceChildLogger('tncCertificatesTransformer')
 
 export class TncCertificatesTransformer extends TncTransformerBase {
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
-    const email = (row.USER_EMAIL as string | null)?.trim() || null
+    const email = cleanPersonalEmail(row.USER_EMAIL)
     if (!email) {
       log.debug({ certificateId: row.CERTIFICATE_ID }, 'Skipping row: missing email')
       return null
     }
 
     const certificateId = (row.CERTIFICATE_ID as string)?.trim()
     const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null
-    const lfUsername = (row.LFID as string | null)?.trim() || null
-    const sourceId = (row.USER_ID as string | null) || undefined
+    const lfUsername = cleanIdentityValue(row.LFID)
+    const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined
 
     const identities = this.buildMemberIdentities({
       email,

services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts

@@ -2,14 +2,15 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations'
 import { getServiceChildLogger } from '@crowd/logging'
 import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types'
 
-import { TransformedActivity } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase'
 import { TncTransformerBase } from '../tncTransformerBase'
 
 const log = getServiceChildLogger('tncCoursesTransformer')
 
 export class TncCoursesTransformer extends TncTransformerBase {
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
-    const email = (row.USER_EMAIL as string | null)?.trim() || null
+    const email = cleanPersonalEmail(row.USER_EMAIL)
     if (!email) {
       log.warn(
         { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL },
@@ -25,8 +26,8 @@ export class TncCoursesTransformer extends TncTransformerBase {
     }
 
     const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null
-    const lfUsername = (row.LFID as string | null)?.trim() || null
-    const sourceId = (row.INTERNAL_TI_USER_ID as string | null) || undefined
+    const lfUsername = cleanIdentityValue(row.LFID)
+    const sourceId = cleanIdentityValue(row.INTERNAL_TI_USER_ID) ?? undefined
 
     const identities = this.buildMemberIdentities({
       email,

services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts

@@ -2,14 +2,15 @@ import { TNC_GRID, TncActivityType } from '@crowd/integrations'
 import { getServiceChildLogger } from '@crowd/logging'
 import { IActivityData, MemberAttributeName, PlatformType } from '@crowd/types'
 
-import { TransformedActivity } from '../../../core/transformerBase'
+import { cleanPersonalEmail } from '../../../core/personalEmail'
+import { TransformedActivity, cleanIdentityValue } from '../../../core/transformerBase'
 import { TncTransformerBase } from '../tncTransformerBase'
 
 const log = getServiceChildLogger('tncEnrollmentsTransformer')
 
 export class TncEnrollmentsTransformer extends TncTransformerBase {
   transformRow(row: Record<string, unknown>): TransformedActivity | null {
-    const email = (row.USER_EMAIL as string | null)?.trim() || null
+    const email = cleanPersonalEmail(row.USER_EMAIL)
     if (!email) {
       log.warn(
         { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL },
@@ -20,8 +21,8 @@ export class TncEnrollmentsTransformer extends TncTransformerBase {
 
     const enrollmentId = (row.ENROLLMENT_ID as string)?.trim()
     const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null
-    const lfUsername = (row.LFID as string | null)?.trim() || null
-    const sourceId = (row.USER_ID as string | null) || undefined
+    const lfUsername = cleanIdentityValue(row.LFID)
+    const sourceId = cleanIdentityValue(row.USER_ID) ?? undefined
 
     const identities = this.buildMemberIdentities({
       email,