Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 19 additions & 2 deletions packages/translate/src/translate.js
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ export async function translatePost(input) {
},
read: (text) => {
const parsed = readTagged(text, ['title', 'summary', 'source_language']);
return parsed && finishPost(parsed);
return parsed && finishPost(fixSourceLanguage(parsed, input));
},
});
}
Expand Down Expand Up @@ -310,7 +310,7 @@ export async function translateArticle(input) {
read: (text) => {
const parsed = readTagged(text, ['title', 'summary', 'source_language', 'body']);
if (!parsed) return null;
const result = finishArticle({ ...parsed, content_html: parsed.body });
const result = finishArticle({ ...fixSourceLanguage(parsed, input), content_html: parsed.body });
return result && { ...result, truncated: peerCut };
},
});
Expand Down Expand Up @@ -423,6 +423,23 @@ export function readTagged(text, fields) {
return out;
}

/**
* A small model often reports the language it wrote in (the target) as the
* source. When it names the target, trust the feed's declared language
* instead, or record none: "translated from English into English" is wrong
* either way.
*
* @param {Record<string, string>} parsed
* @param {{ targetLang: string, sourceLang?: string|null }} input
*/
export function fixSourceLanguage(parsed, input) {
const base = (/** @type {unknown} */ v) => String(v ?? '').trim().toLowerCase().split(/[-_]/)[0];
if (parsed.source_language && base(parsed.source_language) === base(input.targetLang)) {
return { ...parsed, source_language: input.sourceLang ? String(input.sourceLang) : '' };
}
return parsed;
}

/**
* Cuts HTML at a paragraph-ish boundary at or before `max` characters, so the
* model is not handed half a tag.
Expand Down
7 changes: 7 additions & 0 deletions packages/translate/test/translate.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,10 @@ test('readTagged reads the short fields and a body full of markup', async () =>
assert.equal(readTagged('<title>x</title>', ['title', 'summary']), null);
assert.equal(readTagged('<title>x</title><body><p>cut off', ['title', 'body']), null);
});

test('a peer answer that names the target as its source falls back to the declared language', async () => {
const { fixSourceLanguage } = await import('../src/translate.js');
assert.equal(fixSourceLanguage({ source_language: 'en' }, { targetLang: 'en', sourceLang: 'de' }).source_language, 'de');
assert.equal(fixSourceLanguage({ source_language: 'EN-us' }, { targetLang: 'en' }).source_language, '');
assert.equal(fixSourceLanguage({ source_language: 'fr' }, { targetLang: 'en', sourceLang: 'de' }).source_language, 'fr');
});
Loading