Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 7 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,14 +180,18 @@ counts as PII".
| Names (`first_name`, contact `_title`, …) | `person` pseudonym |
| Company names, tax/registration ids | `company` / `value` pseudonym |
| Emails, phones, IBANs (typed or by field name) | format-preserving pseudonyms |
| Addresses | keep postal code / city / country; pseudonymize street; drop the rest |
| Addresses (`address`, `relation_address`) | keep postal code / city / country; pseudonymize street; drop the rest |
| Birthdates, PII-flagged dates | truncate to year |
| Payment items | pseudonymize IBAN/holder; keep bank name / BIC |
| User-relation attributes (`relation_user`, e.g. `contact_owner`) | pseudonymize name/email, **redact credentials** (`token`, `password`, …), keep ids/status/settings |
| Consent attributes (`consent`, e.g. `consent_email_marketing`) | pseudonymize the contact identifier (email/phone), keep consent status and events |
| Curated free-text (note/message content, opportunity description, …) | `[REDACTED]` |
| Message parties (`from`, `to`, `cc`, `bcc`, `reply_to`) | pseudonymize names, scrub addresses |
| Activity `caller` (incl. nested `EpilotAuth` claims) | same rules as user-relation attributes |
| Curated free-text (note/message content and titles, thread topic/title, opportunity description, …) | `[REDACTED]` |
| File names (`file` `filename` / `_title`) | `value` pseudonym |
| Entity fields not in the schema (e.g. `journey_data`) | field-name heuristics of `anonymizeUnknown` |
| `data_classification: 'pii'` fields | pseudonym (single-line) / `[REDACTED]` (multiline) |
| Everything else | unchanged |
| Everything else | emails / phones / IBANs embedded in strings are scrubbed, the rest is unchanged |

## Development

Expand Down
14 changes: 14 additions & 0 deletions src/defaults.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ export const STRATEGY_BY_ATTRIBUTE_TYPE: Record<string, AnonymizationStrategy> =
// user-relation attributes embed full epilot user objects (name, email, and often an auth
// `token`) — never pass them through raw
relation_user: 'user',
// address relations (e.g. opportunity `delivery_address`) carry the referenced address
// values alongside the `$relation_ref`
relation_address: 'address',
// consent attributes embed the contact identifier (email/phone) the consent is keyed on
consent: 'consent',
};
Expand Down Expand Up @@ -73,6 +76,13 @@ export const DEFAULT_PII_FIELDS: Record<string, AnonymizationStrategy> = {
'message:text': 'redact',
'message:html': 'redact',
'thread:topic': 'redact',
// computed titles of messages/threads are the subject/topic
'message:_title': 'redact',
'thread:_title': 'redact',
// uploaded file names regularly contain customer names (e.g. "Rechnung_Mustermann.pdf");
// `_title` is the file name too, so both map to the same pseudonym
'file:filename': 'value',
'file:_title': 'value',
// common user-relation attribute names — safety net for when the schema (and thus the
// `relation_user` type) is unavailable; type-based classification covers custom-named ones
'*:contact_owner': 'user',
Expand Down Expand Up @@ -100,6 +110,10 @@ export const USER_FIELD_STRATEGY: Record<string, AnonymizationStrategy> = {
given_name: 'person',
family_name: 'person',
phone: 'phone',
// identity claims of the caller's auth token (activity `caller.EpilotAuth`)
username: 'email',
'cognito:username': 'email',
preferred_username: 'email',
// credentials that must never appear in an anonymized response
token: 'redact',
access_token: 'redact',
Expand Down
25 changes: 18 additions & 7 deletions src/entity.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
} from './defaults';
import type { Pseudonymizer } from './pseudonymizer';
import type { AnonymizationStrategy, Attribute, EntityLike, Schema } from './types';
import { anonymizeUnknown } from './unknown';
import { isPlainObject, mapEntries } from './utils';

/**
Expand Down Expand Up @@ -93,7 +94,7 @@ export const anonymizeEntity = <T extends EntityLike>(params: {
continue;
}

// message sender/recipient names are always pseudonymized, regardless of schema
// message sender/recipient names (incl. cc/bcc) are always pseudonymized, regardless of schema
// classification or overrides (not even `data_classification: 'public'` opts them out)
if (slug === 'message' && MESSAGE_PARTY_KEYS.includes(key)) {
result[key] = anonymizeMessageParty(value, pseudonymizer);
Expand Down Expand Up @@ -133,16 +134,25 @@ export const anonymizeEntity = <T extends EntityLike>(params: {
// - kept by default (unclassified custom fields, computed titles, attribute types we have
// not special-cased): apply the catch-all scrub so a stray email/phone/IBAN embedded in
// the value cannot leak. Only matched identifier substrings are replaced.
result[key] = isExplicitlyKept({ slug, key, attribute, overrides })
? value
: mapStringLeaves(value, (str) => pseudonymizer.scrubText(str));
if (isExplicitlyKept({ slug, key, attribute, overrides })) {
result[key] = value;
continue;
}

// Fields the schema does not define (e.g. `journey_data` captured by a journey) are
// arbitrary JSON: apply the field-name heuristics of the non-schema mode as well. Only
// when the schema is known, so a missing schema cannot turn every `name` into a pseudonym.
result[key] =
schema && !attribute && !key.startsWith('_') && (isPlainObject(value) || Array.isArray(value))
? anonymizeUnknown({ value, pseudonymizer, overrides })
: mapStringLeaves(value, (str) => pseudonymizer.scrubText(str));
}

return result as T;
};

/** message entity attributes holding sender (`from`) / recipient (`to`) parties */
const MESSAGE_PARTY_KEYS = ['from', 'to'];
/** message entity attributes holding sender / recipient parties */
const MESSAGE_PARTY_KEYS = ['from', 'to', 'cc', 'bcc', 'reply_to'];

/**
* Anonymize a message party (`{ name, address }`, a single object or an array of them).
Expand Down Expand Up @@ -262,7 +272,8 @@ const anonymizeAddressItem = (item: unknown, pseudonymizer: Pseudonymizer): unkn

return Object.fromEntries(
Object.entries(item).flatMap(([key, val]) => {
if (ADDRESS_KEEP_KEYS.includes(key)) {
// relation metadata (`$relation_ref` of address relations) is not personal data
if (ADDRESS_KEEP_KEYS.includes(key) || SYSTEM_KEEP_KEYS.includes(key)) {
return [[key, val]];
}
if (key === 'street' && typeof val === 'string') {
Expand Down
31 changes: 16 additions & 15 deletions src/response.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { anonymizeEntity } from './entity';
import { anonymizeEntity, applyStrategy } from './entity';
import type { Pseudonymizer } from './pseudonymizer';
import type { AnonymizationStrategy, AnonymizerLogger, Schema, SchemaResolver } from './types';
import { isPlainObject, mapEntries } from './utils';
Expand Down Expand Up @@ -102,6 +102,12 @@ const transformNode = (node: unknown, context: TransformContext): unknown => {
return node.map((item) => transformNode(item, context));
}

// catch-all: identifiers embedded in strings outside any recognized shape (activity payloads
// written by other services, nested JSON, …) are scrubbed too. Idempotent on pseudonyms.
if (typeof node === 'string') {
return node ? context.pseudonymizer.scrubText(node) : node;
}

if (!isPlainObject(node)) {
return node;
}
Expand Down Expand Up @@ -183,19 +189,12 @@ const transformActivity = (record: Record<string, unknown>, context: TransformCo
return pseudonymizer.scrubText(value);
}
if (key === 'caller' && isPlainObject(value)) {
return mapEntries(value, (callerValue, callerKey) => {
if (typeof callerValue !== 'string') {
return callerValue;
}
if (['name', 'display_name', 'full_name'].includes(callerKey)) {
return pseudonymizer.person(callerValue);
}
if (callerKey === 'email') {
return pseudonymizer.email(callerValue);
}

return pseudonymizer.scrubText(callerValue);
});
// the caller is the request's security context (e.g. `{ EpilotAuth: { email, claims, token } }`):
// apply the embedded-user rules at every depth, then scrub what is left
return scrubStringLeaves(
applyStrategy({ value, strategy: 'user', pseudonymizer }),
pseudonymizer,
);
}
if (key === 'payload') {
// entity payloads are anonymized by the walker; scrub remaining free text
Expand Down Expand Up @@ -227,8 +226,10 @@ const isEntityLike = (record: Record<string, unknown>): boolean =>
const isEntityOperationLike = (record: Record<string, unknown>): boolean =>
typeof record.operation === 'string' && ('payload' in record || 'diff' in record) && 'activity_id' in record;

// `caller` is optional: activities created through `POST /v1/activity` (e.g. by the message API)
// are stored without one, and must still have their payload scrubbed
const isActivityLike = (record: Record<string, unknown>): boolean =>
typeof record.type === 'string' &&
typeof record.title === 'string' &&
typeof record.message === 'string' &&
'caller' in record;
('caller' in record || 'payload' in record);
164 changes: 164 additions & 0 deletions src/stable360-13580.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
import { describe, expect, it } from 'vitest';

import { createAnonymizer } from './index';
import type { Schema, SchemaResolver } from './types';

// STABLE360-13580: PII that passed through anonymized responses (activity feed, thread
// timeline, message/thread titles, cc names, file names)
const anonymizer = createAnonymizer({ orgId: '123', secret: 'test-secret' });

const schemas: Record<string, Schema> = {
opportunity: {
slug: 'opportunity',
attributes: [
{ name: 'delivery_address', type: 'relation_address' },
{ name: 'status', type: 'select' },
],
},
message: { slug: 'message', attributes: [{ name: 'subject', type: 'string' }] },
file: { slug: 'file', attributes: [{ name: 'filename', type: 'string' }] },
};
const schemaResolver: SchemaResolver = async ({ slug }) => schemas[slug] ?? null;

const USER_EMAIL = 'jane.user@epilot.cloud';
const caller = {
EpilotAuth: {
token: 'eyJ.payload.sig',
userId: '10006129',
email: USER_EMAIL,
claims: { sub: 's-1', email: USER_EMAIL, 'cognito:username': USER_EMAIL, 'custom:ivy_user_id': '10006129' },
},
};

const leaks = (value: unknown, ...needles: string[]) => {
const json = JSON.stringify(value);

return needles.filter((needle) => json.includes(needle));
};

describe('STABLE360-13580', () => {
it('masks the nested caller security context of activities', async () => {
const result = await anonymizer.anonymizeResponse(
{ results: [{ _id: 'a1', type: 'EntityUpdated', title: 'Updated', message: '{{caller}} updated', caller, payload: {} }] },
{ schemaResolver },
);

const masked = result.results[0].caller.EpilotAuth;
expect(leaks(result, USER_EMAIL, 'eyJ.payload.sig')).toEqual([]);
expect(masked.email).toMatch(/@anonymized\.invalid$/);
expect(masked.claims['cognito:username']).toBe(masked.email);
expect(masked.token).toBe('[REDACTED]');
// ids stay joinable
expect(masked.userId).toBe('10006129');
expect(masked.claims['custom:ivy_user_id']).toBe('10006129');
});

it('masks relation_address values and non-schema journey data in activity entity snapshots', async () => {
const result = await anonymizer.anonymizeResponse(
{
results: [
{
_id: 'a1',
type: 'EntityUpdated',
title: 'Updated',
message: 'updated',
caller,
payload: {
entity: {
_id: 'o1',
_org: '123',
_schema: 'opportunity',
status: 'open',
journey_data: { first_name: 'Max', last_name: 'Mustermann', street: 'Hauptstr. 1', city: 'Köln', zip: '50667' },
delivery_address: [
{
_id: 'addr-1',
$relation_ref: { entity_id: 'c1', path: 'address' },
street: 'Hauptstr.',
street_number: '1',
city: 'Köln',
postal_code: '50667',
},
],
},
},
},
],
},
{ schemaResolver },
);

const entity = result.results[0].payload.entity;
expect(leaks(entity, 'Max', 'Mustermann', 'Hauptstr')).toEqual([]);
expect(entity.status).toBe('open');
expect(entity.delivery_address[0].$relation_ref).toEqual({ entity_id: 'c1', path: 'address' });
expect(entity.delivery_address[0].street).toMatch(/^street_/);
expect(entity.delivery_address[0].street_number).toBeUndefined();
// quasi-identifiers are kept by policy (ADDRESS_KEEP_KEYS)
expect(entity.delivery_address[0].postal_code).toBe('50667');
});

it('scrubs payloads of activities stored without a caller (thread timeline)', async () => {
const result = await anonymizer.anonymizeResponse({
results: [
{
_id: 'a2',
type: 'ThreadMarkedAsDone',
title: 'Thread marked as Done',
message: '{{caller payload.caller}} marked the Thread as Done',
timestamp: '2026-10-30T10:00:00Z',
payload: { orgId: '123', threadId: 't1', caller: { userId: 'u1', email: USER_EMAIL } },
operations: [],
},
],
});

expect(leaks(result, USER_EMAIL)).toEqual([]);
expect(result.results[0].payload.caller.userId).toBe('u1');
});

it('scrubs identifiers in JSON outside any recognized shape', async () => {
const result = await anonymizer.anonymizeResponse({ data: { note: `contact ${USER_EMAIL}` } });

expect(result.data.note).toMatch(/^contact [0-9a-f]{8}@anonymized\.invalid$/);
});

it('masks message titles and cc/bcc names, also inside thread.latest_message', async () => {
const message = {
_id: 'm1',
_org: '123',
_schema: 'message',
_title: 'Rechnung für Max Mustermann',
subject: 'Rechnung für Max Mustermann',
from: { name: 'Max Mustermann', address: 'max@example.com' },
cc: [{ name: 'Erika Musterfrau', address: 'erika@example.com' }],
bcc: [{ name: 'Hans Meier', address: 'hans@example.com' }],
};
const thread = { _id: 't1', _org: '123', _schema: 'thread', _title: 'Rechnung für Max Mustermann', latest_message: message };

const result = await anonymizer.anonymizeResponse({ results: [message, thread] }, { schemaResolver });

expect(leaks(result, 'Mustermann', 'Musterfrau', 'Meier', '@example.com')).toEqual([]);
expect(result.results[0]._title).toBe('[REDACTED]');
expect(result.results[0].cc[0].name).toMatch(/^person_/);
expect(result.results[1]._title).toBe('[REDACTED]');
expect(result.results[1].latest_message._title).toBe('[REDACTED]');
});

it('masks file names consistently with the file title', async () => {
const file = {
_id: 'f1',
_org: '123',
_schema: 'file',
_title: 'Rechnung_Mustermann.pdf',
filename: 'Rechnung_Mustermann.pdf',
mime_type: 'application/pdf',
};

const result = await anonymizer.anonymizeResponse({ results: [file] }, { schemaResolver });

expect(result.results[0].filename).toMatch(/^value_[0-9a-f]{8}$/);
expect(result.results[0]._title).toBe(result.results[0].filename);
expect(result.results[0].mime_type).toBe('application/pdf');
});
});
Loading