From f35e0e92f4d262b3211f5f344eba560951b27100 Mon Sep 17 00:00:00 2001 From: Deftu Date: Sun, 18 Oct 2026 15:04:45 +0200 Subject: [PATCH 1/2] Add Crowdin translations for all mods --- .github/crowdin/extract_mod_sources.py | 202 ++++++++++++++++++ .github/workflows/crowdin.yml | 99 +++++++++ crowdin.yml | 10 + .../internal/OneConfigMixinInit.java | 4 + .../mixin/Mixin_CommunityTranslations.java | 28 +++ .../Mixin_LegacyCommunityTranslations.java | 33 +++ .../internal/l10n/CommunityTranslations.kt | 183 ++++++++++++++++ 7 files changed, 559 insertions(+) create mode 100644 .github/crowdin/extract_mod_sources.py create mode 100644 .github/workflows/crowdin.yml create mode 100644 crowdin.yml create mode 100644 minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java create mode 100644 minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java create mode 100644 minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt diff --git a/.github/crowdin/extract_mod_sources.py b/.github/crowdin/extract_mod_sources.py new file mode 100644 index 000000000..190fd7c11 --- /dev/null +++ b/.github/crowdin/extract_mod_sources.py @@ -0,0 +1,202 @@ +import argparse +import hashlib +import io +import json +import re +import shutil +import sys +import time +import urllib.request +import zipfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +USER_AGENT = "Polyfrost/OneConfig (crowdin-sync; +https://github.com/Polyfrost/OneConfig)" +LANG_ENTRY = re.compile(r"^assets/([a-z0-9_.-]+)/lang/([A-Za-z]{2,3}_[A-Za-z]{2,3})\.(json|lang)$") +NESTED_JAR = re.compile(r"^META-INF/(jars|jarjar)/[^/]+\.jar$") +SOURCE_LANGUAGE = "en_us" +EXCLUDED_NAMESPACES = {"minecraft", "realms"} +MAX_NESTING = 3 +ATTEMPTS = 3 + + +def fetch(url): + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=60) as response: + return response.read() + + +def fetch_verified(urls, sha1): + for attempt in range(ATTEMPTS): + if attempt: + time.sleep(5 * attempt) + for url in urls: + try: + data = fetch(url) + except Exception as error: + print(f"warning: failed to download {url}: {error}", file=sys.stderr) + continue + if sha1 and hashlib.sha1(data).hexdigest() != sha1.lower(): + print(f"warning: sha1 mismatch for {url}", file=sys.stderr) + continue + return data + return None + + +def version_key(version): + return tuple(int(part) if part.isdigit() else 0 for part in re.split(r"[.\-]", version)) + + +def parse_lang(data, kind): + text = data.decode("utf-8-sig", errors="replace") + if kind == "json": + parsed = json.loads(text) + if not isinstance(parsed, dict): + return {} + return {key: value for key, value in parsed.items() if isinstance(key, str) and isinstance(value, str)} + entries = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + entries[key] = value + return entries + + +def read_lang_entry(entry, read, name, strict=False): + match = LANG_ENTRY.match(entry) + if not match: + return None + namespace, language, kind = match.groups() + if namespace in EXCLUDED_NAMESPACES: + return None + try: + return namespace, language.lower(), parse_lang(read(), kind) + except Exception as error: + if strict: + raise + print(f"warning: skipping {name}!{entry}: {error}", file=sys.stderr) + return None + + +def scan_jar(data, name, depth=0): + found = [] + try: + archive = zipfile.ZipFile(io.BytesIO(data)) + except Exception as error: + print(f"warning: {name} is not a valid jar: {error}", file=sys.stderr) + return found + with archive: + for entry in archive.namelist(): + lang = read_lang_entry(entry, lambda: archive.read(entry), name) + if lang: + found.append(lang) + elif depth < MAX_NESTING and NESTED_JAR.match(entry): + try: + nested = archive.read(entry) + except Exception as error: + print(f"warning: skipping {name}!{entry}: {error}", file=sys.stderr) + continue + found.extend(scan_jar(nested, f"{name}!{entry}", depth + 1)) + return found + + +def scan_resources(root): + found = [] + for file in sorted(root.glob("assets/*/lang/*")): + lang = read_lang_entry(file.relative_to(root).as_posix(), file.read_bytes, str(root), strict=True) + if lang: + found.append(lang) + return found + + +def collect_jars(base_url): + metadata = json.loads(fetch(f"{base_url}/oneclient/bundles/metadata.json")) + jars = {} + for version in sorted(metadata.get("versions", {}), key=version_key): + for loader, bundles in sorted(metadata["versions"][version].items()): + for bundle in bundles: + pack = fetch_verified([base_url + bundle["path"]], bundle.get("sha1")) + if pack is None: + raise RuntimeError(f"failed to download bundle {bundle['path']}") + with zipfile.ZipFile(io.BytesIO(pack)) as archive: + index = json.loads(archive.read("modrinth.index.json")) + print(f"{version} {loader}: {index.get('name', bundle['path'])} ({len(index.get('files', []))} files)") + for file in index.get("files", []): + path = file.get("path", "") + sha1 = file.get("hashes", {}).get("sha1") + if not path.startswith("mods/") or not path.endswith(".jar") or not sha1: + continue + jars.pop(sha1, None) + jars[sha1] = (path, file.get("downloads", [])) + return jars + + +def locale_directory(language): + code, region = language.split("_", 1) + return f"{code}_{region.upper()}" + + +def write_json(path, entries): + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as output: + json.dump(dict(sorted(entries.items())), output, ensure_ascii=False, indent=2) + output.write("\n") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", type=Path, default=Path("build/crowdin")) + parser.add_argument("--meta-url", default="https://data-v2.polyfrost.org") + parser.add_argument("--threads", type=int, default=8) + parser.add_argument("--local-resources", type=Path, action="append", default=[]) + args = parser.parse_args() + + jars = collect_jars(args.meta_url.rstrip("/")) + print(f"scanning {len(jars)} unique mod jars") + + def scan(item): + sha1, (path, urls) = item + data = fetch_verified(urls, sha1) + if data is None: + raise RuntimeError(f"failed to download {path}") + return scan_jar(data, path) + + sources = {} + translations = {} + + def merge(found): + for namespace, language, entries in found: + if language == SOURCE_LANGUAGE: + sources.setdefault(namespace, {}).update(entries) + else: + translations.setdefault(namespace, {}).setdefault(language, {}).update(entries) + + with ThreadPoolExecutor(max_workers=args.threads) as pool: + for found in pool.map(scan, jars.items()): + merge(found) + for root in args.local_resources: + merge(scan_resources(root)) + + sources = {namespace: entries for namespace, entries in sources.items() if entries} + if not sources: + raise RuntimeError("no source strings found") + + shutil.rmtree(args.output / "sources", ignore_errors=True) + shutil.rmtree(args.output / "translations", ignore_errors=True) + translated_files = 0 + for namespace, entries in sources.items(): + write_json(args.output / "sources" / f"{namespace}.json", entries) + for language, values in translations.get(namespace, {}).items(): + values = {key: value for key, value in values.items() if key in entries and value != entries[key]} + if values: + write_json(args.output / "translations" / locale_directory(language) / f"{namespace}.json", values) + translated_files += 1 + + strings = sum(len(entries) for entries in sources.values()) + print(f"wrote {len(sources)} namespaces, {strings} source strings, {translated_files} existing translation files") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/crowdin.yml b/.github/workflows/crowdin.yml new file mode 100644 index 000000000..99738cbe3 --- /dev/null +++ b/.github/workflows/crowdin.yml @@ -0,0 +1,99 @@ +name: Crowdin + +on: + push: + branches: + - v1 + paths: + - 'minecraft/src/main/resources/assets/oneconfig/lang/en_us.json' + - 'crowdin.yml' + - '.github/crowdin/**' + schedule: + - cron: '0 4 * * 1' + - cron: '0 5 * * *' + workflow_dispatch: + inputs: + task: + description: Task to run + type: choice + default: all + options: + - all + - sync + - release + +concurrency: + group: crowdin + cancel-in-progress: false + +permissions: + contents: read + +jobs: + sync: + name: Sync sources + + if: >- + github.repository == 'Polyfrost/OneConfig' && (github.event_name == 'push' || + github.event.schedule == '0 4 * * 1' || + (github.event_name == 'workflow_dispatch' && contains(fromJSON('["all", "sync"]'), inputs.task))) + + runs-on: ubuntu-26.04 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.13' + + - name: Extract sources + run: >- + python .github/crowdin/extract_mod_sources.py + --output build/crowdin + --local-resources minecraft/src/main/resources + + - name: Upload sources and existing translations + uses: crowdin/github-action@v3 + env: + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} + with: + upload_sources: true + upload_translations: true + auto_approve_imported: false + import_eq_suggestions: false + download_translations: false + + release: + name: Release OTA distribution + + needs: sync + + if: >- + !cancelled() && needs.sync.result != 'failure' && github.repository == 'Polyfrost/OneConfig' && + (github.event_name == 'schedule' || + (github.event_name == 'workflow_dispatch' && contains(fromJSON('["all", "release"]'), inputs.task))) + + runs-on: ubuntu-26.04 + + steps: + - name: Release distributions + shell: bash + env: + CROWDIN_PROJECT_ID: ${{ secrets.CROWDIN_PROJECT_ID }} + CROWDIN_PERSONAL_TOKEN: ${{ secrets.CROWDIN_PERSONAL_TOKEN }} + run: | + api="https://api.crowdin.com/api/v2/projects/$CROWDIN_PROJECT_ID/distributions" + auth="Authorization: Bearer $CROWDIN_PERSONAL_TOKEN" + hashes=$(curl -fsS -H "$auth" "$api?limit=500" | jq -r '.data[].data.hash') + if [ -z "$hashes" ]; then + echo "::warning::No OTA distributions to release" + exit 0 + fi + for hash in $hashes; do + curl -fsS -X POST -H "$auth" "$api/$hash/release" > /dev/null + echo "Released distribution $hash" + done diff --git a/crowdin.yml b/crowdin.yml new file mode 100644 index 000000000..901072715 --- /dev/null +++ b/crowdin.yml @@ -0,0 +1,10 @@ +project_id_env: CROWDIN_PROJECT_ID +api_token_env: CROWDIN_PERSONAL_TOKEN +base_path: build/crowdin +preserve_hierarchy: true + +files: + - source: /sources/*.json + dest: /mods/%original_file_name% + translation: /translations/%locale_with_underscore%/%original_file_name% + update_option: update_as_unapproved diff --git a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/OneConfigMixinInit.java b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/OneConfigMixinInit.java index 0d2260ae1..a454d1bb5 100644 --- a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/OneConfigMixinInit.java +++ b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/OneConfigMixinInit.java @@ -109,6 +109,10 @@ public List getMixins() { mixins.add("Mixin_MainMenuFpsUncap"); //? if > 1.8.9 mixins.add("Mixin_VersionedResourcePacks"); + //? if > 1.8.9 + mixins.add("Mixin_CommunityTranslations"); + //? if = 1.8.9 + //mixins.add("Mixin_LegacyCommunityTranslations"); //? yacl_compat mixins.add("compat.yacl.Mixin_YetAnotherConfigLib_Builder"); diff --git a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java new file mode 100644 index 000000000..57ab7068e --- /dev/null +++ b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java @@ -0,0 +1,28 @@ +package org.polyfrost.oneconfig.internal.mixin; + +//? if > 1.8.9 { +import com.llamalad7.mixinextras.sugar.Local; +import java.util.List; +import java.util.Map; +import net.minecraft.client.resources.language.ClientLanguage; +import net.minecraft.server.packs.resources.ResourceManager; +import org.polyfrost.oneconfig.internal.l10n.CommunityTranslations; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.injection.At; +import org.spongepowered.asm.mixin.injection.Inject; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfoReturnable; + +@Mixin(ClientLanguage.class) +public class Mixin_CommunityTranslations { + @Inject( + method = "loadFrom", + //? if >= 1.21.4 { + at = @At(value = "INVOKE", target = "Ljava/util/Map;copyOf(Ljava/util/Map;)Ljava/util/Map;") + //?} else + //at = @At(value = "INVOKE", target = "Lcom/google/common/collect/ImmutableMap;copyOf(Ljava/util/Map;)Lcom/google/common/collect/ImmutableMap;") + ) + private static void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List languages, boolean defaultRightToLeft, CallbackInfoReturnable cir, @Local Map translations) { + CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::put); + } +} +//?} diff --git a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java new file mode 100644 index 000000000..dfa8bfea0 --- /dev/null +++ b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java @@ -0,0 +1,33 @@ +package org.polyfrost.oneconfig.internal.mixin; + +//? if = 1.8.9 { +/*import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import net.minecraft.client.resource.language.Locale; +import net.minecraft.client.resource.manager.ResourceManager; +import org.polyfrost.oneconfig.internal.l10n.CommunityTranslations; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.Shadow; +import org.spongepowered.asm.mixin.Unique; +import org.spongepowered.asm.mixin.injection.At; +import org.spongepowered.asm.mixin.injection.Inject; +import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; + +@Mixin(Locale.class) +public class Mixin_LegacyCommunityTranslations { + @Unique + private static final Pattern oneconfig$NUMERIC_FORMAT = Pattern.compile("%(\\d+\\$)?[\\d.]*[df]"); + + @Shadow + Map translations; + + @Inject( + method = "reload", + at = @At(value = "INVOKE", target = "Lnet/minecraft/client/resource/language/Locale;checkUnicodeUsage()V") + ) + private void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List languages, CallbackInfo ci) { + CommunityTranslations.apply(languages, resourceManager.getNamespaces(), (key, value) -> translations.put(key, oneconfig$NUMERIC_FORMAT.matcher(value).replaceAll("%$1s"))); + } +} +*///?} diff --git a/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt b/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt new file mode 100644 index 000000000..f27f5c8fd --- /dev/null +++ b/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt @@ -0,0 +1,183 @@ +package org.polyfrost.oneconfig.internal.l10n + +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import net.minecraft.client.Minecraft +import org.apache.logging.log4j.LogManager +import org.polyfrost.oneconfig.utils.v1.JsonUtils +import org.polyfrost.oneconfig.utils.v1.Multithreading +import org.polyfrost.oneconfig.utils.v1.NetworkUtils +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption +import java.util.Locale +import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer + +object CommunityTranslations { + private const val DISTRIBUTION_HASH = "" + private const val DISTRIBUTION_URL = "https://distributions.crowdin.net/" + private const val SOURCE_LANGUAGE = "en_us" + private val MANIFEST_TTL = TimeUnit.HOURS.toMillis(6) + private val RETRY_DELAY = TimeUnit.MINUTES.toMillis(10) + private val LOGGER = LogManager.getLogger("OneConfig/Translations") + private val SAFE_NAME = Regex("[a-z0-9_.-]+") + private val LANGUAGE_NAME = Regex("[a-z0-9_]+") + + private val cacheDir: Path = Paths.get("oneconfig", "translations") + private val manifestFile: Path = cacheDir.resolve("manifest.json") + private val refreshLock = Any() + + @Volatile + private var lastFailure = 0L + + private class Manifest(val timestamp: Long, val files: Map>) + + @JvmStatic + fun apply(languages: List, namespaces: Collection, sink: BiConsumer) { + if (DISTRIBUTION_HASH.isEmpty()) return + runCatching { + val wanted = languages.map(::normalize).filter { it != SOURCE_LANGUAGE }.distinct() + if (wanted.isEmpty()) return + val present = namespaces.toSet() + val manifest = readManifest() + if (manifest != null) { + for (language in wanted) { + for (namespace in manifest.files[language]?.keys.orEmpty()) { + if (namespace in present) readTranslations(cachedFile(manifest, language, namespace))?.forEach(sink) + } + } + } + if (!isUpToDate(manifest, wanted, present) && System.currentTimeMillis() - lastFailure >= RETRY_DELAY) { + Multithreading.submit { + if (synchronized(refreshLock) { refresh(wanted, present) }) reloadLanguages() + } + } + }.onFailure { LOGGER.warn("Failed to apply community translations", it) } + } + + private fun isUpToDate(manifest: Manifest?, languages: List, namespaces: Set): Boolean { + if (manifest == null || manifestAge() >= MANIFEST_TTL) return false + return languages.all { language -> + manifest.files[language].orEmpty().keys.all { it !in namespaces || Files.isRegularFile(cachedFile(manifest, language, it)) } + } + } + + private fun refresh(languages: List, namespaces: Set): Boolean { + val previous = readManifest() + if (isUpToDate(previous, languages, namespaces)) return false + var manifest = previous + var manifestText: String? = null + if (manifest == null || manifestAge() >= MANIFEST_TTL) { + manifestText = NetworkUtils.getString("$DISTRIBUTION_URL$DISTRIBUTION_HASH/manifest.json") + manifest = manifestText?.let(::parseManifest) + if (manifest == null) { + lastFailure = System.currentTimeMillis() + return false + } + } + var changed = false + var failed = false + for (language in languages) { + for ((namespace, path) in manifest.files[language].orEmpty()) { + if (namespace !in namespaces) continue + val target = cachedFile(manifest, language, namespace) + if (Files.isRegularFile(target)) continue + val text = NetworkUtils.getString("$DISTRIBUTION_URL$DISTRIBUTION_HASH$path?timestamp=${manifest.timestamp}") + if (text != null && JsonUtils.parseOrNull(text) is JsonObject && write(target, text)) { + changed = true + } else { + failed = true + } + } + } + val released = previous != null && previous.timestamp != manifest.timestamp + if (failed) { + lastFailure = System.currentTimeMillis() + if (released) return false + } + if (manifestText != null && !write(manifestFile, manifestText)) { + lastFailure = System.currentTimeMillis() + return false + } + if (!failed) pruneReleases(manifest.timestamp) + return changed || released + } + + private fun reloadLanguages() { + val mc = Minecraft.getInstance() + //? if > 1.8.9 { + mc.execute { mc.languageManager.onResourceManagerReload(mc.resourceManager) } + //?} else + //mc.executeTask(Runnable { mc.languageManager.reload(mc.resourceManager) }) + } + + private fun parseManifest(text: String): Manifest? { + val json = JsonUtils.parseOrNull(text) as? JsonObject ?: return null + val timestamp = json.get("timestamp")?.takeIf { it.isJsonPrimitive }?.asLong ?: return null + val content = json.get("content") as? JsonObject ?: return null + val files = HashMap>() + for ((key, value) in content.entrySet()) { + for (element in value as? JsonArray ?: continue) { + val path = element.takeIf { it.isJsonPrimitive }?.asString ?: continue + if (!path.startsWith("/") || !path.endsWith(".json")) continue + val segments = path.split('/') + val namespace = segments.last().removeSuffix(".json") + if (!SAFE_NAME.matches(namespace)) continue + val languages = listOfNotNull(segments.getOrNull(segments.size - 2), key).map(::normalize) + for (language in languages.distinct()) { + if (LANGUAGE_NAME.matches(language)) files.getOrPut(language) { HashMap() }.putIfAbsent(namespace, path) + } + } + } + return Manifest(timestamp, files) + } + + private fun readManifest(): Manifest? { + if (!Files.isRegularFile(manifestFile)) return null + return runCatching { parseManifest(String(Files.readAllBytes(manifestFile), StandardCharsets.UTF_8)) }.getOrNull() + } + + private fun manifestAge(): Long = + runCatching { System.currentTimeMillis() - Files.getLastModifiedTime(manifestFile).toMillis() }.getOrDefault(Long.MAX_VALUE) + + private fun readTranslations(file: Path): Map? { + if (!Files.isRegularFile(file)) return null + return runCatching { + val json = JsonUtils.parseOrNull(String(Files.readAllBytes(file), StandardCharsets.UTF_8)) as? JsonObject ?: return null + json.entrySet().mapNotNull { (key, value) -> + value.takeIf { it.isJsonPrimitive && it.asJsonPrimitive.isString }?.let { key to it.asString } + }.toMap() + }.onFailure { LOGGER.warn("Failed to read translations from {}", file, it) }.getOrNull() + } + + private fun cachedFile(manifest: Manifest, language: String, namespace: String): Path = + cacheDir.resolve(manifest.timestamp.toString()).resolve(language).resolve("$namespace.json") + + private fun pruneReleases(current: Long) { + runCatching { + Files.list(cacheDir).use { entries -> + entries.filter { Files.isDirectory(it) && it.fileName.toString() != current.toString() }.forEach { release -> + Files.walk(release).use { walk -> walk.sorted(reverseOrder()).forEach(Files::deleteIfExists) } + } + } + }.onFailure { LOGGER.warn("Failed to prune old translation releases", it) } + } + + private fun write(target: Path, text: String): Boolean { + var temp: Path? = null + return runCatching { + Files.createDirectories(target.parent) + val file = Files.createTempFile(target.parent, target.fileName.toString(), ".tmp").also { temp = it } + Files.write(file, text.toByteArray(StandardCharsets.UTF_8)) + Files.move(file, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE) + }.onFailure { + LOGGER.warn("Failed to write {}", target, it) + temp?.let { file -> runCatching { Files.deleteIfExists(file) } } + }.isSuccess + } + + private fun normalize(language: String): String = language.lowercase(Locale.ROOT).replace('-', '_') +} From b14e690e9c946c615bc2e2a5c57f6aa5bb081745 Mon Sep 17 00:00:00 2001 From: Deftu Date: Sun, 18 Oct 2026 15:53:26 +0200 Subject: [PATCH 2/2] Address review feedback --- .github/crowdin/extract_mod_sources.py | 10 ++- crowdin.yml | 3 + .../mixin/Mixin_CommunityTranslations.java | 2 +- .../Mixin_LegacyCommunityTranslations.java | 2 +- .../internal/l10n/CommunityTranslations.kt | 84 ++++++++++++------- 5 files changed, 66 insertions(+), 35 deletions(-) diff --git a/.github/crowdin/extract_mod_sources.py b/.github/crowdin/extract_mod_sources.py index 190fd7c11..2749bf9e0 100644 --- a/.github/crowdin/extract_mod_sources.py +++ b/.github/crowdin/extract_mod_sources.py @@ -18,6 +18,7 @@ EXCLUDED_NAMESPACES = {"minecraft", "realms"} MAX_NESTING = 3 ATTEMPTS = 3 +MAX_FAILED_SHARE = 0.05 def fetch(url): @@ -156,11 +157,14 @@ def main(): jars = collect_jars(args.meta_url.rstrip("/")) print(f"scanning {len(jars)} unique mod jars") + failed = [] + def scan(item): sha1, (path, urls) = item data = fetch_verified(urls, sha1) if data is None: - raise RuntimeError(f"failed to download {path}") + failed.append(path) + return [] return scan_jar(data, path) sources = {} @@ -176,6 +180,10 @@ def merge(found): with ThreadPoolExecutor(max_workers=args.threads) as pool: for found in pool.map(scan, jars.items()): merge(found) + if failed: + print(f"warning: skipped {len(failed)} jars that could not be downloaded: {', '.join(sorted(failed))}", file=sys.stderr) + if len(failed) > len(jars) * MAX_FAILED_SHARE: + raise RuntimeError(f"failed to download {len(failed)} of {len(jars)} jars") for root in args.local_resources: merge(scan_resources(root)) diff --git a/crowdin.yml b/crowdin.yml index 901072715..7b5b6bba9 100644 --- a/crowdin.yml +++ b/crowdin.yml @@ -8,3 +8,6 @@ files: dest: /mods/%original_file_name% translation: /translations/%locale_with_underscore%/%original_file_name% update_option: update_as_unapproved + languages_mapping: + locale_with_underscore: + nb: no_NO diff --git a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java index 57ab7068e..c06dec76d 100644 --- a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java +++ b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_CommunityTranslations.java @@ -22,7 +22,7 @@ public class Mixin_CommunityTranslations { //at = @At(value = "INVOKE", target = "Lcom/google/common/collect/ImmutableMap;copyOf(Ljava/util/Map;)Lcom/google/common/collect/ImmutableMap;") ) private static void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List languages, boolean defaultRightToLeft, CallbackInfoReturnable cir, @Local Map translations) { - CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::put); + CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::get, translations::put); } } //?} diff --git a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java index dfa8bfea0..576e33a5d 100644 --- a/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java +++ b/minecraft/src/main/java/org/polyfrost/oneconfig/internal/mixin/Mixin_LegacyCommunityTranslations.java @@ -27,7 +27,7 @@ public class Mixin_LegacyCommunityTranslations { at = @At(value = "INVOKE", target = "Lnet/minecraft/client/resource/language/Locale;checkUnicodeUsage()V") ) private void oneconfig$applyCommunityTranslations(ResourceManager resourceManager, List languages, CallbackInfo ci) { - CommunityTranslations.apply(languages, resourceManager.getNamespaces(), (key, value) -> translations.put(key, oneconfig$NUMERIC_FORMAT.matcher(value).replaceAll("%$1s"))); + CommunityTranslations.apply(languages, resourceManager.getNamespaces(), translations::get, (key, value) -> translations.put(key, oneconfig$NUMERIC_FORMAT.matcher(value).replaceAll("%$1s"))); } } *///?} diff --git a/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt b/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt index f27f5c8fd..468109253 100644 --- a/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt +++ b/minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/l10n/CommunityTranslations.kt @@ -15,6 +15,7 @@ import java.nio.file.StandardCopyOption import java.util.Locale import java.util.concurrent.TimeUnit import java.util.function.BiConsumer +import java.util.function.Function object CommunityTranslations { private const val DISTRIBUTION_HASH = "" @@ -23,8 +24,10 @@ object CommunityTranslations { private val MANIFEST_TTL = TimeUnit.HOURS.toMillis(6) private val RETRY_DELAY = TimeUnit.MINUTES.toMillis(10) private val LOGGER = LogManager.getLogger("OneConfig/Translations") - private val SAFE_NAME = Regex("[a-z0-9_.-]+") + private val SAFE_NAME = Regex("[a-z0-9_-][a-z0-9_.-]*") private val LANGUAGE_NAME = Regex("[a-z0-9_]+") + private val FORMAT_SPECIFIER = Regex("%(?:(\\d+)\\$)?[\\d.]*([A-Za-z%])") + private val LANGUAGE_ALIASES = mapOf("nb_no" to "no_no") private val cacheDir: Path = Paths.get("oneconfig", "translations") private val manifestFile: Path = cacheDir.resolve("manifest.json") @@ -36,7 +39,7 @@ object CommunityTranslations { private class Manifest(val timestamp: Long, val files: Map>) @JvmStatic - fun apply(languages: List, namespaces: Collection, sink: BiConsumer) { + fun apply(languages: List, namespaces: Collection, existing: Function, sink: BiConsumer) { if (DISTRIBUTION_HASH.isEmpty()) return runCatching { val wanted = languages.map(::normalize).filter { it != SOURCE_LANGUAGE }.distinct() @@ -46,7 +49,11 @@ object CommunityTranslations { if (manifest != null) { for (language in wanted) { for (namespace in manifest.files[language]?.keys.orEmpty()) { - if (namespace in present) readTranslations(cachedFile(manifest, language, namespace))?.forEach(sink) + if (namespace !in present) continue + readTranslations(language, namespace, manifest.timestamp)?.forEach { (key, value) -> + val current = existing.apply(key) + if (current == null || arity(current) == arity(value)) sink.accept(key, value) + } } } } @@ -61,7 +68,7 @@ object CommunityTranslations { private fun isUpToDate(manifest: Manifest?, languages: List, namespaces: Set): Boolean { if (manifest == null || manifestAge() >= MANIFEST_TTL) return false return languages.all { language -> - manifest.files[language].orEmpty().keys.all { it !in namespaces || Files.isRegularFile(cachedFile(manifest, language, it)) } + manifest.files[language].orEmpty().keys.all { it !in namespaces || Files.isRegularFile(cachedFile(language, it, manifest.timestamp)) } } } @@ -69,41 +76,33 @@ object CommunityTranslations { val previous = readManifest() if (isUpToDate(previous, languages, namespaces)) return false var manifest = previous - var manifestText: String? = null if (manifest == null || manifestAge() >= MANIFEST_TTL) { - manifestText = NetworkUtils.getString("$DISTRIBUTION_URL$DISTRIBUTION_HASH/manifest.json") - manifest = manifestText?.let(::parseManifest) - if (manifest == null) { + val text = NetworkUtils.getString("$DISTRIBUTION_URL$DISTRIBUTION_HASH/manifest.json") + val fetched = text?.let(::parseManifest) + if (text == null || fetched == null || !write(manifestFile, text)) { lastFailure = System.currentTimeMillis() return false } + manifest = fetched } - var changed = false + var changed = previous != null && previous.timestamp != manifest.timestamp var failed = false for (language in languages) { for ((namespace, path) in manifest.files[language].orEmpty()) { if (namespace !in namespaces) continue - val target = cachedFile(manifest, language, namespace) + val target = cachedFile(language, namespace, manifest.timestamp) if (Files.isRegularFile(target)) continue val text = NetworkUtils.getString("$DISTRIBUTION_URL$DISTRIBUTION_HASH$path?timestamp=${manifest.timestamp}") if (text != null && JsonUtils.parseOrNull(text) is JsonObject && write(target, text)) { + pruneOlder(target) changed = true } else { failed = true } } } - val released = previous != null && previous.timestamp != manifest.timestamp - if (failed) { - lastFailure = System.currentTimeMillis() - if (released) return false - } - if (manifestText != null && !write(manifestFile, manifestText)) { - lastFailure = System.currentTimeMillis() - return false - } - if (!failed) pruneReleases(manifest.timestamp) - return changed || released + if (failed) lastFailure = System.currentTimeMillis() + return changed } private fun reloadLanguages() { @@ -143,8 +142,10 @@ object CommunityTranslations { private fun manifestAge(): Long = runCatching { System.currentTimeMillis() - Files.getLastModifiedTime(manifestFile).toMillis() }.getOrDefault(Long.MAX_VALUE) - private fun readTranslations(file: Path): Map? { - if (!Files.isRegularFile(file)) return null + private fun readTranslations(language: String, namespace: String, timestamp: Long): Map? { + val file = cachedFile(language, namespace, timestamp).takeIf { Files.isRegularFile(it) } + ?: newestCached(language, namespace) + ?: return null return runCatching { val json = JsonUtils.parseOrNull(String(Files.readAllBytes(file), StandardCharsets.UTF_8)) as? JsonObject ?: return null json.entrySet().mapNotNull { (key, value) -> @@ -153,17 +154,35 @@ object CommunityTranslations { }.onFailure { LOGGER.warn("Failed to read translations from {}", file, it) }.getOrNull() } - private fun cachedFile(manifest: Manifest, language: String, namespace: String): Path = - cacheDir.resolve(manifest.timestamp.toString()).resolve(language).resolve("$namespace.json") + private fun arity(text: String): Int { + var sequential = 0 + var highest = 0 + for (match in FORMAT_SPECIFIER.findAll(text)) { + val conversion = match.groupValues[2] + if (conversion == "%" || conversion == "n") continue + val index = match.groupValues[1].toIntOrNull() + if (index != null) highest = maxOf(highest, index) else sequential++ + } + return maxOf(highest, sequential) + } + + private fun cachedFile(language: String, namespace: String, timestamp: Long): Path = + cacheDir.resolve(language).resolve(namespace).resolve("$timestamp.json") - private fun pruneReleases(current: Long) { + private fun releaseOf(file: Path): Long? = file.fileName.toString().removeSuffix(".json").toLongOrNull() + + private fun newestCached(language: String, namespace: String): Path? = runCatching { + Files.list(cacheDir.resolve(language).resolve(namespace)).use { files -> + files.filter { releaseOf(it) != null }.max(compareBy { releaseOf(it) }).orElse(null) + } + }.getOrNull() + + private fun pruneOlder(current: Path) { runCatching { - Files.list(cacheDir).use { entries -> - entries.filter { Files.isDirectory(it) && it.fileName.toString() != current.toString() }.forEach { release -> - Files.walk(release).use { walk -> walk.sorted(reverseOrder()).forEach(Files::deleteIfExists) } - } + Files.list(current.parent).use { files -> + files.filter { it != current && releaseOf(it) != null }.forEach(Files::deleteIfExists) } - }.onFailure { LOGGER.warn("Failed to prune old translation releases", it) } + }.onFailure { LOGGER.warn("Failed to prune old translations in {}", current.parent, it) } } private fun write(target: Path, text: String): Boolean { @@ -179,5 +198,6 @@ object CommunityTranslations { }.isSuccess } - private fun normalize(language: String): String = language.lowercase(Locale.ROOT).replace('-', '_') + private fun normalize(language: String): String = + language.lowercase(Locale.ROOT).replace('-', '_').let { LANGUAGE_ALIASES[it] ?: it } }