-
Notifications
You must be signed in to change notification settings - Fork 2.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Uses the machine translation API of DeepL to automatically translate any missing values
- Loading branch information
Showing
9 changed files
with
417 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| /******************************************************************************** | ||
| * Copyright (C) 2021 TypeFox and others. | ||
| * | ||
| * This program and the accompanying materials are made available under the | ||
| * terms of the Eclipse Public License v. 2.0 which is available at | ||
| * http://www.eclipse.org/legal/epl-2.0. | ||
| * | ||
| * This Source Code may also be made available under the following Secondary | ||
| * Licenses when the conditions for such availability set forth in the Eclipse | ||
| * Public License v. 2.0 are satisfied: GNU General Public License, version 2 | ||
| * with the GNU Classpath Exception which is available at | ||
| * https://www.gnu.org/software/classpath/license.html. | ||
| * | ||
| * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 | ||
| ********************************************************************************/ | ||
|
|
||
| export interface Localization { | ||
| [key: string]: string | Localization | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| /******************************************************************************** | ||
| * Copyright (C) 2021 TypeFox and others. | ||
| * | ||
| * This program and the accompanying materials are made available under the | ||
| * terms of the Eclipse Public License v. 2.0 which is available at | ||
| * http://www.eclipse.org/legal/epl-2.0. | ||
| * | ||
| * This Source Code may also be made available under the following Secondary | ||
| * Licenses when the conditions for such availability set forth in the Eclipse | ||
| * Public License v. 2.0 are satisfied: GNU General Public License, version 2 | ||
| * with the GNU Classpath Exception which is available at | ||
| * https://www.gnu.org/software/classpath/license.html. | ||
| * | ||
| * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 | ||
| ********************************************************************************/ | ||
|
|
||
| import * as bent from 'bent'; | ||
|
|
||
| const post = bent('POST', 'json', 200); | ||
| // 50 is the maximum amount of translations per request | ||
| const deeplLimit = 50; | ||
|
|
||
| export async function deepl( | ||
| parameters: DeeplParameters | ||
| ): Promise<DeeplResponse> { | ||
| const sub_domain = parameters.free_api ? 'api-free' : 'api'; | ||
| const textChunks: string[][] = []; | ||
| const textArray = [...parameters.text]; | ||
| while (textArray.length > 0) { | ||
| textChunks.push(textArray.splice(0, deeplLimit)); | ||
| } | ||
| const responses: DeeplResponse[] = await Promise.all(textChunks.map(chunk => { | ||
| const parameterCopy: DeeplParameters = { ...parameters, text: chunk }; | ||
| return post(`https://${sub_domain}.deepl.com/v2/translate`, Buffer.from(toFormData(parameterCopy)), { | ||
| 'Content-Type': 'application/x-www-form-urlencoded', | ||
| 'User-Agent': 'Theia-Localization-Manager' | ||
| }); | ||
| })); | ||
| const mergedResponse: DeeplResponse = { translations: [] }; | ||
| for (const response of responses) { | ||
| mergedResponse.translations.push(...response.translations); | ||
| } | ||
| return mergedResponse; | ||
| } | ||
|
|
||
| function toFormData(parameters: DeeplParameters): string { | ||
| const str: string[] = []; | ||
| for (const [key, value] of Object.entries(parameters)) { | ||
| if (typeof value === 'string') { | ||
| str.push(encodeURIComponent(key) + '=' + encodeURIComponent(value.toString())); | ||
| } else if (Array.isArray(value)) { | ||
| for (const item of value) { | ||
| str.push(encodeURIComponent(key) + '=' + encodeURIComponent(item.toString())); | ||
| } | ||
| } | ||
| } | ||
| return str.join('&'); | ||
| } | ||
|
|
||
| export type DeeplLanguage = | ||
| | 'BG' | ||
| | 'CS' | ||
| | 'DA' | ||
| | 'DE' | ||
| | 'EL' | ||
| | 'EN-GB' | ||
| | 'EN-US' | ||
| | 'EN' | ||
| | 'ES' | ||
| | 'ET' | ||
| | 'FI' | ||
| | 'FR' | ||
| | 'HU' | ||
| | 'IT' | ||
| | 'JA' | ||
| | 'LT' | ||
| | 'LV' | ||
| | 'NL' | ||
| | 'PL' | ||
| | 'PT-PT' | ||
| | 'PT-BR' | ||
| | 'PT' | ||
| | 'RO' | ||
| | 'RU' | ||
| | 'SK' | ||
| | 'SL' | ||
| | 'SV' | ||
| | 'ZH'; | ||
|
|
||
| export const supportedLanguages = [ | ||
| 'BG', 'CD', 'DA', 'DE', 'EL', 'EN-GB', 'EN-US', 'EN', 'ES', 'ET', 'FI', 'FR', 'HU', 'IT', | ||
| 'JA', 'LT', 'LV', 'NL', 'PL', 'PT-PT', 'PT-BR', 'PT', 'RO', 'RU', 'SK', 'SL', 'SV', 'ZH' | ||
| ]; | ||
|
|
||
| export function isSupportedLanguage(language: string): language is DeeplLanguage { | ||
| return supportedLanguages.includes(language.toUpperCase()); | ||
| } | ||
|
|
||
| export interface DeeplParameters { | ||
| free_api: Boolean | ||
| auth_key: string | ||
| text: string[] | ||
| source_lang?: DeeplLanguage | ||
| target_lang: DeeplLanguage | ||
| split_sentences?: '0' | '1' | 'nonewlines' | ||
| preserve_formatting?: '0' | '1' | ||
| formality?: 'default' | 'more' | 'less' | ||
| tag_handling?: string[] | ||
| non_splitting_tags?: string[] | ||
| outline_detection?: string | ||
| splitting_tags?: string[] | ||
| ignore_tags?: string[] | ||
| } | ||
|
|
||
| export interface DeeplResponse { | ||
| translations: DeeplTranslation[] | ||
| } | ||
|
|
||
| export interface DeeplTranslation { | ||
| detected_source_language: string | ||
| text: string | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
80 changes: 80 additions & 0 deletions
80
dev-packages/localization-manager/src/localization-manager.spec.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /******************************************************************************** | ||
| * Copyright (C) 2021 TypeFox and others. | ||
| * | ||
| * This program and the accompanying materials are made available under the | ||
| * terms of the Eclipse Public License v. 2.0 which is available at | ||
| * http://www.eclipse.org/legal/epl-2.0. | ||
| * | ||
| * This Source Code may also be made available under the following Secondary | ||
| * Licenses when the conditions for such availability set forth in the Eclipse | ||
| * Public License v. 2.0 are satisfied: GNU General Public License, version 2 | ||
| * with the GNU Classpath Exception which is available at | ||
| * https://www.gnu.org/software/classpath/license.html. | ||
| * | ||
| * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 | ||
| ********************************************************************************/ | ||
|
|
||
| import * as assert from 'assert'; | ||
| import { DeeplParameters, DeeplResponse } from './deepl-api'; | ||
| import { LocalizationManager, LocalizationOptions } from './localization-manager'; | ||
|
|
||
| describe('localization-manager#translateLanguage', () => { | ||
|
|
||
| async function mockLocalization(parameters: DeeplParameters): Promise<DeeplResponse> { | ||
| return { | ||
| translations: parameters.text.map(value => ({ | ||
| detected_source_language: '', | ||
| text: `[${value}]` | ||
| })) | ||
| }; | ||
| } | ||
|
|
||
| const manager = new LocalizationManager(mockLocalization); | ||
| const defaultOptions: LocalizationOptions = { | ||
| authKey: '', | ||
| freeApi: false, | ||
| sourceFile: '', | ||
| targetLanguages: ['EN'] | ||
| }; | ||
|
|
||
| it('should translate a single value', async () => { | ||
| const input = { | ||
| key: 'value' | ||
| }; | ||
| const target = {}; | ||
| await manager.translateLanguage(input, target, 'EN', defaultOptions); | ||
| assert.deepStrictEqual(target, { | ||
| key: '[value]' | ||
| }); | ||
| }); | ||
|
|
||
| it('should translate nested values', async () => { | ||
| const input = { | ||
| a: { | ||
| b: 'b' | ||
| }, | ||
| c: 'c' | ||
| }; | ||
| const target = {}; | ||
| await manager.translateLanguage(input, target, 'EN', defaultOptions); | ||
| assert.deepStrictEqual(target, { | ||
| a: { | ||
| b: '[b]' | ||
| }, | ||
| c: '[c]' | ||
| }); | ||
| }); | ||
|
|
||
| it('should not override existing targets', async () => { | ||
| const input = { | ||
| a: 'a' | ||
| }; | ||
| const target = { | ||
| a: 'b' | ||
| }; | ||
| await manager.translateLanguage(input, target, 'EN', defaultOptions); | ||
| assert.deepStrictEqual(target, { | ||
| a: 'b' | ||
| }); | ||
| }); | ||
| }); |
Oops, something went wrong.