/* eslint-env node */ /** * This script is used for the internal Zendesk translation system, and it creates or updates the source YAML file for translations, * extracting the strings from the source code. * * It searches for i18next calls (`{t("my-key", "My value")}`) and it adds the strings to the specified YAML file * (if not already present) with empty title and screenshot. * ```yaml * - translation: * key: "my-key" * title: "" * screenshot: "" * value: "My value" * ``` * * If a string present in the source YAML file is not found in the source code, it will be marked as obsolete if the * `--mark-obsolete` flag is passed. * * If the value in the YAML file differs from the value in the source code, a warning will be printed in the console, * since the script cannot know which one is correct and cannot write back in the source code files. This can happen for * example after a "reverse string sweep", and can be eventually fixed manually. * * The script uses the i18next-parser library for extracting the strings and it adds a custom transformer for creating * the file in the required YAML format. * * For usage instructions, run `node extract-strings.mjs --help` */ import vfs from "vinyl-fs"; import Vinyl from "vinyl"; import { transform as I18NextTransform } from "i18next-parser"; import { Transform } from "node:stream"; import { readFileSync } from "node:fs"; import { load, dump } from "js-yaml"; import { resolve } from "node:path"; import { glob } from "glob"; import { parseArgs } from "node:util"; const { values: args } = parseArgs({ options: { "mark-obsolete": { type: "boolean", }, module: { type: "string", }, help: { type: "boolean", }, }, }); if (args.help) { const helpMessage = ` Usage: extract-strings.mjs [options] Options: --mark-obsolete Mark removed strings as obsolete in the source YAML file --module Extract strings only for the specified module. The module name should match the folder name in the src/modules folder If not specified, the script will extract strings for all modules --help Display this help message Examples: node extract-strings.mjs node extract-strings.mjs --mark-obsolete node extract-strings.mjs --module=ticket-fields `; console.log(helpMessage); process.exit(0); } const OUTPUT_YML_FILE_NAME = "en-us.yml"; const OUTPUT_BANNER = `# # This file is used for the internal Zendesk translation system, and it is generated from the extract-strings.mjs script. # It contains the English strings to be translated, which are used for generating the JSON files containing the translation strings # for each language. # # If you are building your own theme, you can remove this file and just load the translation JSON files, or provide # your translations with a different method. # `; /** @type {import("i18next-parser").UserConfig} */ const config = { // Our translation system requires that we add all 6 forms (zero, one, two, few, many, other) keys for plurals // i18next-parser extracts plural keys based on the target locale, so we are passing a // locale that need exactly the 6 forms, even if we are extracting English strings locales: ["ar"], keySeparator: false, namespaceSeparator: false, pluralSeparator: ".", // This path is used only as a Virtual FS path by i18next-parser, and it doesn't get written on the FS output: "locales/en.json", }; class SourceYmlTransform extends Transform { #parsedInitialContent; #outputDir; #counters = { added: 0, obsolete: 0, mismatch: 0, }; constructor(outputDir) { super({ objectMode: true }); this.#outputDir = outputDir; this.#parsedInitialContent = this.#getSourceYmlContent(); } _transform(file, encoding, done) { try { const strings = JSON.parse(file.contents.toString(encoding)); const outputContent = { ...this.#parsedInitialContent, parts: this.#parsedInitialContent.parts || [], }; // Find obsolete keys for (const { translation } of outputContent.parts) { if (!(translation.key in strings) && !translation.obsolete) { this.#counters.obsolete++; if (args["mark-obsolete"]) { translation.obsolete = this.#getObsoleteDate(); } } } // Add new keys to the source YAML or log mismatched value for (let [key, value] of Object.entries(strings)) { value = this.#fixPluralValue(key, value, strings); const existingPart = outputContent.parts.find( (part) => part.translation.key === key ); if (existingPart === undefined) { outputContent.parts.push({ translation: { key, title: "", screenshot: "", value, }, }); this.#counters.added++; } else if (value !== existingPart.translation.value) { console.warn( `\nFound a mismatch value for the key "${key}".\n\tSource code value: ${value}\n\tTranslation file value: ${existingPart.translation.value}` ); this.#counters.mismatch++; } } const virtualFile = new Vinyl({ path: OUTPUT_YML_FILE_NAME, contents: Buffer.from( OUTPUT_BANNER + "\n" + dump(outputContent, { quotingType: `"`, forceQuotes: true }) ), }); this.push(virtualFile); this.#printInfo(); done(); } catch (e) { console.error(e); process.exit(1); } } #getSourceYmlContent() { const outputPath = resolve(this.#outputDir, OUTPUT_YML_FILE_NAME); return load(readFileSync(outputPath, "utf-8")); } #getObsoleteDate() { const today = new Date(); const obsolete = new Date(today.setMonth(today.getMonth() + 3)); return obsolete.toISOString().split("T")[0]; } #printInfo() { const message = `Package ${this.#parsedInitialContent.packages[0]} Added strings: ${this.#counters.added} ${this.#getObsoleteInfoMessage()} Strings with mismatched value: ${this.#counters.mismatch} `; console.log(message); } #getObsoleteInfoMessage() { if (args["mark-obsolete"]) { return `Removed strings (marked as obsolete): ${this.#counters.obsolete}`; } let result = `Obsolete strings: ${this.#counters.obsolete}`; if (this.#counters.obsolete > 0) { result += " - Use --mark-obsolete to mark them as obsolete"; } return result; } // if the key ends with .zero, .one, .two, .few, .many, .other and the value is empty // find the same key with the `.other` suffix in strings and return the value #fixPluralValue(key, value, strings) { if (key.endsWith(".zero") && value === "") { return strings[key.replace(".zero", ".other")] || ""; } if (key.endsWith(".one") && value === "") { return strings[key.replace(".one", ".other")] || ""; } if (key.endsWith(".two") && value === "") { return strings[key.replace(".two", ".other")] || ""; } if (key.endsWith(".few") && value === "") { return strings[key.replace(".few", ".other")] || ""; } if (key.endsWith(".many") && value === "") { return strings[key.replace(".many", ".other")] || ""; } return value; } } const sourceFilesGlob = args.module ? `src/modules/${args.module}/translations/en-us.yml` : "src/modules/**/translations/en-us.yml"; const sourceFiles = await glob(sourceFilesGlob); for (const sourceFile of sourceFiles) { const moduleName = sourceFile.split("/")[2]; const inputGlob = `src/modules/${moduleName}/**/*.{ts,tsx}`; const outputDir = resolve( process.cwd(), `src/modules/${moduleName}/translations` ); vfs .src([inputGlob]) .pipe(new I18NextTransform(config)) .pipe(new SourceYmlTransform(outputDir)) .pipe(vfs.dest(outputDir)); }