first load

bin/extract-strings.mjs

@@ -0,0 +1,256 @@
+/* eslint-env node */
+/**
+ * This script is used for the internal Zendesk translation system, and it creates or updates the source YAML file for translations,
+ * extracting the strings from the source code.
+ *
+ * It searches for i18next calls (`{t("my-key", "My value")}`) and it adds the strings to the specified YAML file
+ * (if not already present) with empty title and screenshot.
+ * ```yaml
+ *   - translation:
+ *       key: "my-key"
+ *       title: ""
+ *       screenshot: ""
+ *       value: "My value"
+ * ```
+ *
+ * If a string present in the source YAML file is not found in the source code, it will be marked as obsolete if the
+ * `--mark-obsolete` flag is passed.
+ *
+ * If the value in the YAML file differs from the value in the source code, a warning will be printed in the console,
+ * since the script cannot know which one is correct and cannot write back in the source code files. This can happen for
+ * example after a "reverse string sweep", and can be eventually fixed manually.
+ *
+ * The script uses the i18next-parser library for extracting the strings and it adds a custom transformer for creating
+ * the file in the required YAML format.
+ *
+ * For usage instructions, run `node extract-strings.mjs --help`
+ */
+import vfs from "vinyl-fs";
+import Vinyl from "vinyl";
+import { transform as I18NextTransform } from "i18next-parser";
+import { Transform } from "node:stream";
+import { readFileSync } from "node:fs";
+import { load, dump } from "js-yaml";
+import { resolve } from "node:path";
+import { glob } from "glob";
+import { parseArgs } from "node:util";
+
+const { values: args } = parseArgs({
+  options: {
+    "mark-obsolete": {
+      type: "boolean",
+    },
+    module: {
+      type: "string",
+    },
+    help: {
+      type: "boolean",
+    },
+  },
+});
+
+if (args.help) {
+  const helpMessage = `
+  Usage: extract-strings.mjs [options]
+
+  Options:
+    --mark-obsolete   Mark removed strings as obsolete in the source YAML file
+    --module          Extract strings only for the specified module. The module name should match the folder name in the src/modules folder
+                      If not specified, the script will extract strings for all modules
+    --help            Display this help message
+
+  Examples:
+    node extract-strings.mjs
+    node extract-strings.mjs --mark-obsolete
+    node extract-strings.mjs --module=ticket-fields
+  `;
+
+  console.log(helpMessage);
+  process.exit(0);
+}
+
+const OUTPUT_YML_FILE_NAME = "en-us.yml";
+
+const OUTPUT_BANNER = `#
+# This file is used for the internal Zendesk translation system, and it is generated from the extract-strings.mjs script.
+# It contains the English strings to be translated, which are used for generating the JSON files containing the translation strings
+# for each language.
+#
+# If you are building your own theme, you can remove this file and just load the translation JSON files, or provide
+# your translations with a different method.
+#
+`;
+
+/** @type {import("i18next-parser").UserConfig} */
+const config = {
+  // Our translation system requires that we add all 6 forms (zero, one, two, few, many, other) keys for plurals
+  // i18next-parser extracts plural keys based on the target locale, so we are passing a
+  // locale that need exactly the 6 forms, even if we are extracting English strings
+  locales: ["ar"],
+  keySeparator: false,
+  namespaceSeparator: false,
+  pluralSeparator: ".",
+  // This path is used only as a Virtual FS path by i18next-parser, and it doesn't get written on the FS
+  output: "locales/en.json",
+};
+
+class SourceYmlTransform extends Transform {
+  #parsedInitialContent;
+  #outputDir;
+
+  #counters = {
+    added: 0,
+    obsolete: 0,
+    mismatch: 0,
+  };
+
+  constructor(outputDir) {
+    super({ objectMode: true });
+
+    this.#outputDir = outputDir;
+    this.#parsedInitialContent = this.#getSourceYmlContent();
+  }
+
+  _transform(file, encoding, done) {
+    try {
+      const strings = JSON.parse(file.contents.toString(encoding));
+
+      const outputContent = {
+        ...this.#parsedInitialContent,
+        parts: this.#parsedInitialContent.parts || [],
+      };
+
+      // Find obsolete keys
+      for (const { translation } of outputContent.parts) {
+        if (!(translation.key in strings) && !translation.obsolete) {
+          this.#counters.obsolete++;
+
+          if (args["mark-obsolete"]) {
+            translation.obsolete = this.#getObsoleteDate();
+          }
+        }
+      }
+
+      // Add new keys to the source YAML or log mismatched value
+      for (let [key, value] of Object.entries(strings)) {
+        value = this.#fixPluralValue(key, value, strings);
+
+        const existingPart = outputContent.parts.find(
+          (part) => part.translation.key === key
+        );
+
+        if (existingPart === undefined) {
+          outputContent.parts.push({
+            translation: {
+              key,
+              title: "",
+              screenshot: "",
+              value,
+            },
+          });
+          this.#counters.added++;
+        } else if (value !== existingPart.translation.value) {
+          console.warn(
+            `\nFound a mismatch value for the key "${key}".\n\tSource code value: ${value}\n\tTranslation file value: ${existingPart.translation.value}`
+          );
+          this.#counters.mismatch++;
+        }
+      }
+
+      const virtualFile = new Vinyl({
+        path: OUTPUT_YML_FILE_NAME,
+        contents: Buffer.from(
+          OUTPUT_BANNER +
+            "\n" +
+            dump(outputContent, { quotingType: `"`, forceQuotes: true })
+        ),
+      });
+      this.push(virtualFile);
+      this.#printInfo();
+      done();
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    }
+  }
+
+  #getSourceYmlContent() {
+    const outputPath = resolve(this.#outputDir, OUTPUT_YML_FILE_NAME);
+    return load(readFileSync(outputPath, "utf-8"));
+  }
+
+  #getObsoleteDate() {
+    const today = new Date();
+    const obsolete = new Date(today.setMonth(today.getMonth() + 3));
+    return obsolete.toISOString().split("T")[0];
+  }
+
+  #printInfo() {
+    const message = `Package ${this.#parsedInitialContent.packages[0]}
+    Added strings: ${this.#counters.added}
+    ${this.#getObsoleteInfoMessage()}
+    Strings with mismatched value: ${this.#counters.mismatch}
+    `;
+
+    console.log(message);
+  }
+
+  #getObsoleteInfoMessage() {
+    if (args["mark-obsolete"]) {
+      return `Removed strings (marked as obsolete): ${this.#counters.obsolete}`;
+    }
+
+    let result = `Obsolete strings: ${this.#counters.obsolete}`;
+    if (this.#counters.obsolete > 0) {
+      result += " - Use --mark-obsolete to mark them as obsolete";
+    }
+
+    return result;
+  }
+
+  // if the key ends with .zero, .one, .two, .few, .many, .other and the value is empty
+  // find the same key with the `.other` suffix in strings and return the value
+  #fixPluralValue(key, value, strings) {
+    if (key.endsWith(".zero") && value === "") {
+      return strings[key.replace(".zero", ".other")] || "";
+    }
+
+    if (key.endsWith(".one") && value === "") {
+      return strings[key.replace(".one", ".other")] || "";
+    }
+
+    if (key.endsWith(".two") && value === "") {
+      return strings[key.replace(".two", ".other")] || "";
+    }
+
+    if (key.endsWith(".few") && value === "") {
+      return strings[key.replace(".few", ".other")] || "";
+    }
+
+    if (key.endsWith(".many") && value === "") {
+      return strings[key.replace(".many", ".other")] || "";
+    }
+
+    return value;
+  }
+}
+
+const sourceFilesGlob = args.module
+  ? `src/modules/${args.module}/translations/en-us.yml`
+  : "src/modules/**/translations/en-us.yml";
+
+const sourceFiles = await glob(sourceFilesGlob);
+for (const sourceFile of sourceFiles) {
+  const moduleName = sourceFile.split("/")[2];
+  const inputGlob = `src/modules/${moduleName}/**/*.{ts,tsx}`;
+  const outputDir = resolve(
+    process.cwd(),
+    `src/modules/${moduleName}/translations`
+  );
+
+  vfs
+    .src([inputGlob])
+    .pipe(new I18NextTransform(config))
+    .pipe(new SourceYmlTransform(outputDir))
+    .pipe(vfs.dest(outputDir));
+}