Add Crowdin translations for all mods

.github/crowdin/extract_mod_sources.py

@@ -0,0 +1,210 @@
+import argparse
+import hashlib
+import io
+import json
+import re
+import shutil
+import sys
+import time
+import urllib.request
+import zipfile
+from concurrent.futures import ThreadPoolExecutor
+from pathlib import Path
+
+USER_AGENT = "Polyfrost/OneConfig (crowdin-sync; +https://github.com/Polyfrost/OneConfig)"
+LANG_ENTRY = re.compile(r"^assets/([a-z0-9_.-]+)/lang/([A-Za-z]{2,3}_[A-Za-z]{2,3})\.(json|lang)$")
+NESTED_JAR = re.compile(r"^META-INF/(jars|jarjar)/[^/]+\.jar$")
+SOURCE_LANGUAGE = "en_us"
+EXCLUDED_NAMESPACES = {"minecraft", "realms"}
+MAX_NESTING = 3
+ATTEMPTS = 3
+MAX_FAILED_SHARE = 0.05
+
+
+def fetch(url):
+    request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
+    with urllib.request.urlopen(request, timeout=60) as response:
+        return response.read()
+
+
+def fetch_verified(urls, sha1):
+    for attempt in range(ATTEMPTS):
+        if attempt:
+            time.sleep(5 * attempt)
+        for url in urls:
+            try:
+                data = fetch(url)
+            except Exception as error:
+                print(f"warning: failed to download {url}: {error}", file=sys.stderr)
+                continue
+            if sha1 and hashlib.sha1(data).hexdigest() != sha1.lower():
+                print(f"warning: sha1 mismatch for {url}", file=sys.stderr)
+                continue
+            return data
+    return None
+
+
+def version_key(version):
+    return tuple(int(part) if part.isdigit() else 0 for part in re.split(r"[.\-]", version))
+
+
+def parse_lang(data, kind):
+    text = data.decode("utf-8-sig", errors="replace")
+    if kind == "json":
+        parsed = json.loads(text)
+        if not isinstance(parsed, dict):
+            return {}
+        return {key: value for key, value in parsed.items() if isinstance(key, str) and isinstance(value, str)}
+    entries = {}
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, value = line.split("=", 1)
+        entries[key] = value
+    return entries
+
+
+def read_lang_entry(entry, read, name, strict=False):
+    match = LANG_ENTRY.match(entry)
+    if not match:
+        return None
+    namespace, language, kind = match.groups()
+    if namespace in EXCLUDED_NAMESPACES:
+        return None
+    try:
+        return namespace, language.lower(), parse_lang(read(), kind)
+    except Exception as error:
+        if strict:
+            raise
+        print(f"warning: skipping {name}!{entry}: {error}", file=sys.stderr)
+        return None
+
+
+def scan_jar(data, name, depth=0):
+    found = []
+    try:
+        archive = zipfile.ZipFile(io.BytesIO(data))
+    except Exception as error:
+        print(f"warning: {name} is not a valid jar: {error}", file=sys.stderr)
+        return found
+    with archive:
+        for entry in archive.namelist():
+            lang = read_lang_entry(entry, lambda: archive.read(entry), name)
+            if lang:
+                found.append(lang)
+            elif depth < MAX_NESTING and NESTED_JAR.match(entry):
+                try:
+                    nested = archive.read(entry)
+                except Exception as error:
+                    print(f"warning: skipping {name}!{entry}: {error}", file=sys.stderr)
+                    continue
+                found.extend(scan_jar(nested, f"{name}!{entry}", depth + 1))
+    return found
+
+
+def scan_resources(root):
+    found = []
+    for file in sorted(root.glob("assets/*/lang/*")):
+        lang = read_lang_entry(file.relative_to(root).as_posix(), file.read_bytes, str(root), strict=True)
+        if lang:
+            found.append(lang)
+    return found
+
+
+def collect_jars(base_url):
+    metadata = json.loads(fetch(f"{base_url}/oneclient/bundles/metadata.json"))
+    jars = {}
+    for version in sorted(metadata.get("versions", {}), key=version_key):
+        for loader, bundles in sorted(metadata["versions"][version].items()):
+            for bundle in bundles:
+                pack = fetch_verified([base_url + bundle["path"]], bundle.get("sha1"))
+                if pack is None:
+                    raise RuntimeError(f"failed to download bundle {bundle['path']}")
+                with zipfile.ZipFile(io.BytesIO(pack)) as archive:
+                    index = json.loads(archive.read("modrinth.index.json"))
+                print(f"{version} {loader}: {index.get('name', bundle['path'])} ({len(index.get('files', []))} files)")
+                for file in index.get("files", []):
+                    path = file.get("path", "")
+                    sha1 = file.get("hashes", {}).get("sha1")
+                    if not path.startswith("mods/") or not path.endswith(".jar") or not sha1:
+                        continue
+                    jars.pop(sha1, None)
+                    jars[sha1] = (path, file.get("downloads", []))
+    return jars
+
+
+def locale_directory(language):
+    code, region = language.split("_", 1)
+    return f"{code}_{region.upper()}"
+
+
+def write_json(path, entries):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8", newline="\n") as output:
+        json.dump(dict(sorted(entries.items())), output, ensure_ascii=False, indent=2)
+        output.write("\n")
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--output", type=Path, default=Path("build/crowdin"))
+    parser.add_argument("--meta-url", default="https://data-v2.polyfrost.org")
+    parser.add_argument("--threads", type=int, default=8)
+    parser.add_argument("--local-resources", type=Path, action="append", default=[])
+    args = parser.parse_args()
+
+    jars = collect_jars(args.meta_url.rstrip("/"))
+    print(f"scanning {len(jars)} unique mod jars")
+
+    failed = []
+
+    def scan(item):
+        sha1, (path, urls) = item
+        data = fetch_verified(urls, sha1)
+        if data is None:
+            failed.append(path)
+            return []
+        return scan_jar(data, path)
+
+    sources = {}
+    translations = {}
+
+    def merge(found):
+        for namespace, language, entries in found:
+            if language == SOURCE_LANGUAGE:
+                sources.setdefault(namespace, {}).update(entries)
+            else:
+                translations.setdefault(namespace, {}).setdefault(language, {}).update(entries)
+
+    with ThreadPoolExecutor(max_workers=args.threads) as pool:
+        for found in pool.map(scan, jars.items()):
+            merge(found)
+    if failed:
+        print(f"warning: skipped {len(failed)} jars that could not be downloaded: {', '.join(sorted(failed))}", file=sys.stderr)
+    if len(failed) > len(jars) * MAX_FAILED_SHARE:
+        raise RuntimeError(f"failed to download {len(failed)} of {len(jars)} jars")
+    for root in args.local_resources:
+        merge(scan_resources(root))
+
+    sources = {namespace: entries for namespace, entries in sources.items() if entries}
+    if not sources:
+        raise RuntimeError("no source strings found")
+
+    shutil.rmtree(args.output / "sources", ignore_errors=True)
+    shutil.rmtree(args.output / "translations", ignore_errors=True)
+    translated_files = 0
+    for namespace, entries in sources.items():
+        write_json(args.output / "sources" / f"{namespace}.json", entries)
+        for language, values in translations.get(namespace, {}).items():
+            values = {key: value for key, value in values.items() if key in entries and value != entries[key]}
+            if values:
+                write_json(args.output / "translations" / locale_directory(language) / f"{namespace}.json", values)
+                translated_files += 1
+
+    strings = sum(len(entries) for entries in sources.values())
+    print(f"wrote {len(sources)} namespaces, {strings} source strings, {translated_files} existing translation files")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/crowdin.yml

@@ -0,0 +1,99 @@
+name: Crowdin
+
+on:
+  push:
+    branches:
+      - v1
+    paths:
+      - 'minecraft/src/main/resources/assets/oneconfig/lang/en_us.json'
+      - 'crowdin.yml'
+      - '.github/crowdin/**'
+  schedule:
+    - cron: '0 4 * * 1'
+    - cron: '0 5 * * *'
+  workflow_dispatch:
+    inputs:
+      task:
+        description: Task to run
+        type: choice
+        default: all
+        options:
+          - all
+          - sync
+          - release
+
+concurrency:
+  group: crowdin
+  cancel-in-progress: false
+
+permissions:
+  contents: read
+
+jobs:
+  sync:
+    name: Sync sources
+
+    if: >-
+      github.repository == 'Polyfrost/OneConfig' && (github.event_name == 'push' ||
+      github.event.schedule == '0 4 * * 1' ||
+      (github.event_name == 'workflow_dispatch' && contains(fromJSON('["all", "sync"]'), inputs.task)))
+
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+
+      - name: Setup Python
+        uses: actions/setup-python@v7
+        with:
+          python-version: '3.13'
+
+      - name: Extract sources
+        run: >-
+          python .github/crowdin/extract_mod_sources.py
+          --output build/crowdin
+          --local-resources minecraft/src/main/resources
+
+      - name: Upload sources and existing translations
+        uses: crowdin/github-action@v3
+        env:
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+        with:
+          upload_sources: true
+          upload_translations: true
+          auto_approve_imported: false
+          import_eq_suggestions: false
+          download_translations: false
+
+  release:
+    name: Release OTA distribution
+
+    needs: sync
+
+    if: >-
+      !cancelled() && needs.sync.result != 'failure' && github.repository == 'Polyfrost/OneConfig' &&
+      (github.event_name == 'schedule' ||
+      (github.event_name == 'workflow_dispatch' && contains(fromJSON('["all", "release"]'), inputs.task)))
+
+    runs-on: ubuntu-26.04
+
+    steps:
+      - name: Release distributions
+        shell: bash
+        env:
+          CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }}
+          CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }}
+        run: |
+          api="https://api.crowdin.com/api/v2/projects/$CROWDIN_PROJECT_ID/distributions"
+          auth="Authorization: Bearer $CROWDIN_PERSONAL_TOKEN"
+          hashes=$(curl -fsS -H "$auth" "$api?limit=500" | jq -r '.data[].data.hash')
+          if [ -z "$hashes" ]; then
+            echo "::warning::No OTA distributions to release"
+            exit 0
+          fi
+          for hash in $hashes; do
+            curl -fsS -X POST -H "$auth" "$api/$hash/release" > /dev/null
+            echo "Released distribution $hash"
+          done

crowdin.yml

@@ -0,0 +1,13 @@
+project_id_env: CROWDIN_PROJECT_ID
+api_token_env: CROWDIN_PERSONAL_TOKEN
+base_path: build/crowdin
+preserve_hierarchy: true
+
+files:
+  - source: /sources/*.json
+    dest: /mods/%original_file_name%
+    translation: /translations/%locale_with_underscore%/%original_file_name%
+    update_option: update_as_unapproved
+    languages_mapping:
+      locale_with_underscore:
+        nb: no_NO

minecraft/src/main/java/org/polyfrost/oneconfig/internal/OneConfigMixinInit.java

@@ -109,6 +109,10 @@ public List<String> getMixins() {
         mixins.add("Mixin_MainMenuFpsUncap");
         //? if > 1.8.9
         mixins.add("Mixin_VersionedResourcePacks");
+        //? if > 1.8.9
+        mixins.add("Mixin_CommunityTranslations");
+        //? if = 1.8.9
+        //mixins.add("Mixin_LegacyCommunityTranslations");
         //? yacl_compat
         mixins.add("compat.yacl.Mixin_YetAnotherConfigLib_Builder");
 

minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java

@@ -0,0 +1,28 @@
+package org.polyfrost.oneconfig.internal.mixin;
+
+//? if > 1.8.9 {
+import com.llamalad7.mixinextras.sugar.Local;
+import java.util.List;
+import java.util.Map;
+import net.minecraft.client.resources.language.ClientLanguage;
+import net.minecraft.server.packs.resources.ResourceManager;
+import org.polyfrost.oneconfig.internal.l10n.CommunityTranslations;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.injection.At;
+import org.spongepowered.asm.mixin.injection.Inject;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable;
+
+@Mixin(ClientLanguage.class)
+public class Mixin_CommunityTranslations {
+    @Inject(
+        method = "loadFrom",
+        //? if >= 1.21.4 {
+        at = @At(value = "INVOKE", target = "Ljava/util/Map;copyOf(Ljava/util/Map;)Ljava/util/Map;")
+        //?} else
+        //at = @At(value = "INVOKE", target = "Lcom/google/common/collect/ImmutableMap;copyOf(Ljava/util/Map;)Lcom/google/common/collect/ImmutableMap;")
+    )
+    private static void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List<String> languages, boolean defaultRightToLeft, CallbackInfoReturnable<ClientLanguage> cir, @Local Map<String, String> translations) {
+        CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::get, translations::put);
+    }
+}
+//?}

minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java

@@ -0,0 +1,33 @@
+package org.polyfrost.oneconfig.internal.mixin;
+
+//? if = 1.8.9 {
+/*import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+import net.minecraft.client.resource.language.Locale;
+import net.minecraft.client.resource.manager.ResourceManager;
+import org.polyfrost.oneconfig.internal.l10n.CommunityTranslations;
+import org.spongepowered.asm.mixin.Mixin;
+import org.spongepowered.asm.mixin.Shadow;
+import org.spongepowered.asm.mixin.Unique;
+import org.spongepowered.asm.mixin.injection.At;
+import org.spongepowered.asm.mixin.injection.Inject;
+import org.spongepowered.asm.mixin.injection.callback.CallbackInfo;
+
+@Mixin(Locale.class)
+public class Mixin_LegacyCommunityTranslations {
+    @Unique
+    private static final Pattern oneconfig$NUMERIC_FORMAT = Pattern.compile("%(\\d+\\$)?[\\d.]*[df]");
+
+    @Shadow
+    Map<String, String> translations;
+
+    @Inject(
+        method = "reload",
+        at = @At(value = "INVOKE", target = "Lnet/minecraft/client/resource/language/Locale;checkUnicodeUsage()V")
+    )
+    private void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List<String> languages, CallbackInfo ci) {
+        CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::get, (key, value) -> translations.put(key, oneconfig$NUMERIC_FORMAT.matcher(value).replaceAll("%$1s")));
+    }
+}
+*///?}