diff --git a/.github/workflows/fix-encoding-errors.yml b/.github/workflows/fix-encoding-errors.yml new file mode 100644 index 00000000..214b057e --- /dev/null +++ b/.github/workflows/fix-encoding-errors.yml @@ -0,0 +1,196 @@ +name: Build FixEncodingErrors (SE5) + +on: + push: + branches: [main] + paths: + - 'se5/FixEncodingErrors/**' + - 'se5/FixEncodingErrors.Tests/**' + - 'se5/Plugin-Shared/**' + - '.github/workflows/fix-encoding-errors.yml' + pull_request: + paths: + - 'se5/FixEncodingErrors/**' + - 'se5/FixEncodingErrors.Tests/**' + - 'se5/Plugin-Shared/**' + - '.github/workflows/fix-encoding-errors.yml' + workflow_dispatch: + inputs: + release: + description: 'Publish a GitHub release. Bumps the minor in plugin.json and commits it back.' + type: boolean + default: false + version: + description: 'Optional explicit version (e.g. 1.2.0). Overrides the auto-minor-bump.' + required: false + type: string + +permissions: + contents: write + +jobs: + prepare: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.set.outputs.version }} + tag: ${{ steps.set.outputs.tag }} + ref: ${{ steps.set.outputs.ref }} + release: ${{ steps.set.outputs.release }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Resolve version (and bump on release) + id: set + run: | + set -e + current=$(jq -r .version se5/FixEncodingErrors/plugin.json) + release_flag="false" + ref="${{ github.sha }}" + version="$current" + tag="" + + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.release }}" = "true" ]; then + release_flag="true" + if [ -n "${{ inputs.version }}" ]; then + version="${{ inputs.version }}" + else + IFS=. read -r major minor patch <<< "$current" + version="$major.$((minor + 1)).0" + fi + + tmp=$(mktemp) + jq --arg v "$version" '.version = $v' se5/FixEncodingErrors/plugin.json > "$tmp" + mv "$tmp" se5/FixEncodingErrors/plugin.json + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add se5/FixEncodingErrors/plugin.json + git commit -m "Bump FixEncodingErrors to $version [skip ci]" + + # Retry-with-rebase: if a concurrent release dispatched for a different + # plugin won the push first, pull --rebase and try again. + attempts=0 + while ! git push 2>/tmp/push.err; do + attempts=$((attempts + 1)) + if [ "$attempts" -ge 5 ]; then + echo "Failed to push after $attempts attempts:" >&2 + cat /tmp/push.err >&2 + exit 1 + fi + echo "Push rejected (attempt $attempts), pulling --rebase and retrying..." + git pull --rebase --no-edit + done + ref=$(git rev-parse HEAD) + + IFS=. read -r major minor patch <<< "$version" + tag="se5-fix-encoding-errors-v$major.$minor" + fi + + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "release=$release_flag" >> "$GITHUB_OUTPUT" + + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup .NET 8 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Test + run: dotnet test se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj -c Release + + build: + needs: [prepare, test] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - rid: win-x64 + os: windows + exe: FixEncodingErrors.exe + - rid: win-arm64 + os: windows + exe: FixEncodingErrors.exe + - rid: linux-x64 + os: linux + exe: FixEncodingErrors + - rid: linux-arm64 + os: linux + exe: FixEncodingErrors + - rid: osx-x64 + os: macos + exe: FixEncodingErrors + - rid: osx-arm64 + os: macos + exe: FixEncodingErrors + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ needs.prepare.outputs.ref }} + + - name: Setup .NET 8 + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '8.0.x' + + - name: Publish self-contained (${{ matrix.rid }}) + working-directory: se5/FixEncodingErrors + run: | + dotnet publish FixEncodingErrors.csproj \ + -c Release \ + -r ${{ matrix.rid }} \ + --self-contained true \ + -p:DebugType=None \ + -p:DebugSymbols=false \ + -o staging/FixEncodingErrors + + - name: Rewrite plugin.json for ${{ matrix.os }} + working-directory: se5/FixEncodingErrors + run: | + jq ' + del(.runtime, .entry) | + .executables = { "${{ matrix.os }}": "${{ matrix.exe }}" } + ' plugin.json > staging/FixEncodingErrors/plugin.json + + - name: Package zip + working-directory: se5/FixEncodingErrors/staging + run: zip -r "$GITHUB_WORKSPACE/FixEncodingErrors-${{ matrix.rid }}.zip" FixEncodingErrors + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: FixEncodingErrors-${{ matrix.rid }} + path: FixEncodingErrors-${{ matrix.rid }}.zip + + release: + name: Publish GitHub Release + needs: [prepare, build] + if: needs.prepare.outputs.release == 'true' + runs-on: ubuntu-latest + steps: + - name: Download all zips + uses: actions/download-artifact@v4 + with: + path: dist + pattern: FixEncodingErrors-* + merge-multiple: true + + - name: Create release and upload zips + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${{ needs.prepare.outputs.tag }}" dist/FixEncodingErrors-*.zip \ + --repo "${{ github.repository }}" \ + --target "${{ needs.prepare.outputs.ref }}" \ + --title "FixEncodingErrors v${{ needs.prepare.outputs.version }}" \ + --notes "Self-contained FixEncodingErrors plugin builds for win/linux/osx (x64 + arm64)." diff --git a/README.md b/README.md index 8e11d54b..e99b97d5 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ index [`se5-plugins.json`](se5-plugins.json). | [ARTE check](se5/ArteCheck) | Checks and fixes EBU STL teletext subtitles for ARTE delivery. | | [American to British](se5/AmericanToBritish) | Converts American English spellings to British English. | | [British to American](se5/BritishToAmerican) | Converts British English spellings to American English. | +| [Fix encoding errors](se5/FixEncodingErrors) | Repairs UTF-8 text read with the wrong code page: résumé → résumé, don’t → don’t. | | [Haxor](se5/Haxor) | Translates text to "haxor" - the minimal reference plugin. | | [Italic each line](se5/ItalicEachLine) | Switches italic tags between one tag per subtitle and a tag on each line. | | [Persian Subtitle Fixes](se5/PersianErrors) | Fixes common errors in Persian (Farsi) subtitles - 1,500+ rules in 12 groups. | diff --git a/se5/FixEncodingErrors.Tests/EncodingFixerTests.cs b/se5/FixEncodingErrors.Tests/EncodingFixerTests.cs new file mode 100644 index 00000000..9d482704 --- /dev/null +++ b/se5/FixEncodingErrors.Tests/EncodingFixerTests.cs @@ -0,0 +1,140 @@ +using SubtitleEdit.Plugins.FixEncodingErrors; +using System.Text; + +namespace FixEncodingErrors.Tests; + +public class EncodingFixerTests +{ + static EncodingFixerTests() => Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + /// What a UTF-8 text looks like after being read with . + private static string Break(string text, int codePage = 1252) => + Encoding.GetEncoding(codePage).GetString(Encoding.UTF8.GetBytes(text)); + + [Theory] + [InlineData("résumé", "résumé")] + [InlineData("don’t", "don’t")] + [InlineData("“Quoteâ€\u009D", "“Quote”")] + [InlineData("“Quoteâ€", "“Quote”")] // 0x9D lost + [InlineData("25°C", "25°C")] + [InlineData("Hello", "Hello")] + [InlineData("Wait à bientôt", "Wait à bientôt")] // A0 turned into a space + [InlineData("😀 hi", "😀 hi")] + public void FixesWestern(string broken, string expected) + { + Assert.Equal(expected, EncodingFixer.Fix(broken, 1252, confirmed: true)); + } + + [Theory] + [InlineData("cache of such communiquã©s,", "cache of such communiqués,")] + [InlineData("and dust off my rã©sumã©.", "and dust off my résumé.")] + [InlineData("madame la secrã©taire.", "madame la secrétaire.")] + public void FixesTextLowercasedAfterItBroke(string broken, string expected) + { + // Ed's Madam Secretary subtitles: © alone tells the subtitle is broken. + Assert.Equal(expected, EncodingFixer.Fix(broken, 1252, confirmed: false)); + } + + [Theory] + [InlineData("Ça va? Où êtes-vous? À bientôt, Œdipe!")] + [InlineData("Größe, Übermaß, naïve, Señor, Ñandú, Ærø")] + [InlineData("“Smart quotes” – dashes — and… ellipsis ‘ok’ 😀")] + public void RoundTripsWestern(string text) + { + Assert.Equal(text, EncodingFixer.Fix(Break(text), 1252, confirmed: true)); + } + + [Theory] + [InlineData("Привет, как дела? Всё хорошо.", 1251)] + [InlineData("Příliš žluťoučký kůň úpěl ďábelské ódy.", 1250)] + [InlineData("Καλημέρα κόσμε", 1253)] + [InlineData("Günaydın, nasılsın? Şimdi değil.", 1254)] + public void RoundTripsOtherCodePages(string text, int codePage) + { + // Detection may pick a code page that reads the same bytes the same way (1252/1254 for Turkish). + var broken = Break(text, codePage); + var detected = EncodingFixer.Detect(new[] { broken }).CodePage; + Assert.Equal(text, EncodingFixer.Fix(broken, detected, confirmed: true)); + Assert.Equal(text, EncodingFixer.Fix(broken, codePage, confirmed: true)); + } + + [Fact] + public void DetectsCyrillic() + { + Assert.Equal(1251, EncodingFixer.Detect(new[] { Break("Привет, как дела?", 1251) }).CodePage); + } + + [Theory] + [InlineData("Привет, как дела?")] + [InlineData("Καλημέρα κόσμε")] + [InlineData("שלום עולם")] + [InlineData("مرحبا بالعالم")] + public void FixesOtherScriptsMisreadAsWestern(string text) + { + var broken = Break(text); + Assert.Equal(1252, EncodingFixer.Detect(new[] { broken }).CodePage); + Assert.Equal(text, EncodingFixer.Fix(broken, 1252, EncodingFixer.IsBroken(new[] { broken }, 1252))); + } + + [Fact] + public void FixesDoubleBroken() + { + Assert.Equal("café", EncodingFixer.Fix(Break(Break("café")), 1252, confirmed: true)); + } + + [Theory] + [InlineData("Não”")] + [InlineData("A minha irmã… e a mãe")] + [InlineData("«Não»")] + [InlineData("Irmã Lúcia")] + [InlineData("Ça coûte 25 €, café»")] + [InlineData("Plain ASCII text")] + public void LeavesRealTextAlone(string text) + { + Assert.Equal(0, EncodingFixer.CountTelling(new[] { text }, 1252)); + Assert.Equal(text, EncodingFixer.Fix(text, 1252, confirmed: false)); + } + + [Theory] + [InlineData("Привет, Сёма! В доме.")] + [InlineData("Příliš žluťoučký kůň")] + public void LeavesRealTextAloneInItsOwnCodePage(string text) + { + Assert.Equal(1252, EncodingFixer.Detect(new[] { text }).CodePage); + Assert.Equal(text, EncodingFixer.Fix(text, 1252, confirmed: false)); + } + + [Fact] + public void AmbiguousMatchNeedsConfirmation() + { + // "Ö" (Ö) at a word end could be real text; fixed once the subtitle is known to be broken. + Assert.Equal("BÖ", EncodingFixer.Fix("BÖ", 1252, confirmed: false)); + Assert.Equal("BÖ", EncodingFixer.Fix("BÖ", 1252, confirmed: true)); + Assert.True(EncodingFixer.IsBroken(new[] { "Österreich", "café" }, 1252)); + Assert.False(EncodingFixer.IsBroken(new[] { "Österreich" }, 1252)); + } + + [Fact] + public void DetectsWesternForEd() + { + var lines = new[] { "and dust off my rã©sumã©.", "I'll send you his rã©sumã©." }; + Assert.Equal(1252, EncodingFixer.Detect(lines).CodePage); + Assert.Equal(4, EncodingFixer.CountTelling(lines, 1252)); + } + + [Fact] + public void RealMacedonianIsNotBroken() + { + // ј, џ, ќ, ѓ are continuation bytes in Windows-1251: "ој" must not become Greek "μ", "рџ" not "п". + var lines = new[] { "Тој е мојот брат.", "Џорџ веќе учење, која боја." }; + Assert.False(EncodingFixer.IsBroken(lines, 1251)); + Assert.Equal(1252, EncodingFixer.Detect(lines).CodePage); + Assert.All(lines, l => Assert.Equal(l, EncodingFixer.Fix(l, 1251, confirmed: false))); + } + + [Fact] + public void KeepsTagsAndLineBreaks() + { + Assert.Equal("résumé\r\n{\\an8}café", EncodingFixer.Fix("résumé\r\n{\\an8}café", 1252, confirmed: false)); + } +} diff --git a/se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj b/se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj new file mode 100644 index 00000000..4bb38f49 --- /dev/null +++ b/se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj @@ -0,0 +1,27 @@ + + + + net8.0 + enable + enable + false + + + + + + + + + + + + + + + + + + + + diff --git a/se5/FixEncodingErrors.Tests/MainViewModelTests.cs b/se5/FixEncodingErrors.Tests/MainViewModelTests.cs new file mode 100644 index 00000000..48ea9ec8 --- /dev/null +++ b/se5/FixEncodingErrors.Tests/MainViewModelTests.cs @@ -0,0 +1,101 @@ +using SubtitleEdit.Plugins.FixEncodingErrors; +using SubtitleEdit.Plugins.Shared; + +namespace FixEncodingErrors.Tests; + +public class MainViewModelTests +{ + private static PluginRequest Request(params string[] texts) => new() + { + Subtitle = new PluginSubtitle + { + Format = "Advanced Sub Station Alpha", + Paragraphs = texts.Select((t, i) => new PluginParagraph { StartMs = i * 1000, EndMs = i * 1000 + 900, Text = t, Style = "Top", Actor = "A" }).ToList(), + }, + }; + + [Fact] + public void ListsBrokenLines() + { + var vm = new MainViewModel(Request("and dust off my rã©sumã©.", "Plain", "don’t")); + + Assert.Equal(0, vm.SelectedEncoding.CodePage); + Assert.Equal(1252, vm.EffectiveEncoding.CodePage); + Assert.Equal(new[] { 0, 2 }, vm.Changes.Select(c => c.Index)); + Assert.Equal("and dust off my résumé.", vm.Changes[0].After); + } + + [Fact] + public void ConfirmedSubtitleFixesAmbiguousMatches() + { + // "BÖ" alone is left; with a clear "é" elsewhere in the subtitle it is fixed too. + Assert.Empty(new MainViewModel(Request("BÖ")).Changes); + Assert.Equal(2, new MainViewModel(Request("BÖ", "café")).Changes.Count); + } + + [Fact] + public void ResponseKeepsParagraphFieldsAndSkipsUnchecked() + { + var vm = new MainViewModel(Request("café", "résumé")); + vm.Changes[1].IsSelected = false; + + var response = vm.BuildResponse(); + + Assert.Equal(PluginStatus.Ok, response.Status); + var paragraphs = response.Subtitle!.Paragraphs!; + Assert.Equal("café", paragraphs[0].Text); + Assert.Equal("résumé", paragraphs[1].Text); + Assert.All(paragraphs, p => Assert.Equal("Top", p.Style)); + } + + [Fact] + public void SelectedLinesScope() + { + var request = Request("café", "résumé"); + request.SelectedIndices = new List { 1 }; + var vm = new MainViewModel(request); + + Assert.True(vm.SelectedLinesOnly); + Assert.Equal(1, Assert.Single(vm.Changes).Index); + + vm.AllLines = true; + Assert.Equal(2, vm.Changes.Count); + } + + [Fact] + public void PickedCodePageIsRemembered() + { + var vm = new MainViewModel(Request("café")); + vm.SelectedEncoding = vm.EncodingOptions.Single(e => e.CodePage == 1251); + Assert.Empty(vm.Changes); + + var cancel = vm.BuildCancelResponse(); + var request = Request("café"); + request.Settings = cancel.Settings; + request.SettingsVersion = cancel.SettingsVersion; + Assert.Equal(1251, new MainViewModel(request).SelectedEncoding.CodePage); + } + + [Fact] + public void NothingSelectedCancels() + { + var vm = new MainViewModel(Request("café")); + vm.SelectNoneCommand.Execute(null); + + Assert.Equal(PluginStatus.Cancelled, vm.BuildResponse().Status); + } + + [Fact] + public void LegacyRequestUsesSubRip() + { + var vm = new MainViewModel(new PluginRequest + { + Subtitle = new PluginSubtitle { SubRip = "1\n00:00:01,000 --> 00:00:02,000\nrésumé\n" }, + }); + + var response = vm.BuildResponse(); + + Assert.Equal("SubRip", response.Subtitle!.Format); + Assert.Contains("résumé", response.Subtitle.Native); + } +} diff --git a/se5/FixEncodingErrors/App.axaml b/se5/FixEncodingErrors/App.axaml new file mode 100644 index 00000000..8e90aac7 --- /dev/null +++ b/se5/FixEncodingErrors/App.axaml @@ -0,0 +1,8 @@ + + + + + diff --git a/se5/FixEncodingErrors/App.axaml.cs b/se5/FixEncodingErrors/App.axaml.cs new file mode 100644 index 00000000..259a12a6 --- /dev/null +++ b/se5/FixEncodingErrors/App.axaml.cs @@ -0,0 +1,12 @@ +using Avalonia.Controls; +using Avalonia.Markup.Xaml; +using SubtitleEdit.Plugins.Shared; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +public partial class App : PluginApp +{ + public override void Initialize() => AvaloniaXamlLoader.Load(this); + + protected override Window CreateMainWindow(PluginRequest request) => new MainWindow(request); +} diff --git a/se5/FixEncodingErrors/ChangeItem.cs b/se5/FixEncodingErrors/ChangeItem.cs new file mode 100644 index 00000000..24421fe1 --- /dev/null +++ b/se5/FixEncodingErrors/ChangeItem.cs @@ -0,0 +1,26 @@ +using CommunityToolkit.Mvvm.ComponentModel; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +/// One subtitle line with broken characters. +public sealed partial class ChangeItem : ObservableObject +{ + [ObservableProperty] private bool _isSelected = true; + + public ChangeItem(int index, string before, string after, string timeRange) + { + Index = index; + Before = before; + After = after; + TimeRange = timeRange; + (BeforeSegments, AfterSegments) = TextDiff.Compute(before.Replace("\r\n", "\n"), after.Replace("\r\n", "\n")); + } + + public int Index { get; } + public string Before { get; } + public string After { get; } + public string Number => "#" + (Index + 1); + public string TimeRange { get; } + public IReadOnlyList BeforeSegments { get; } + public IReadOnlyList AfterSegments { get; } +} diff --git a/se5/FixEncodingErrors/EncodingFixer.cs b/se5/FixEncodingErrors/EncodingFixer.cs new file mode 100644 index 00000000..0b162835 --- /dev/null +++ b/se5/FixEncodingErrors/EncodingFixer.cs @@ -0,0 +1,349 @@ +using System.Text; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +/// A single-byte code page that UTF-8 text may have been misread with. +public sealed record SourceEncoding(int CodePage, string Name) +{ + public override string ToString() => Name; +} + +/// +/// Repairs UTF-8 text that was decoded with a single-byte code page and saved again ("mojibake"): +/// résumé → résumé, don’t → don’t. Each character is mapped back to its byte +/// in the code page and runs of bytes that form valid UTF-8 are decoded. Also handles text that was lowercased +/// after it broke (rã©sumã©), a lost 0x9D byte (†→ ”) and a byte order mark +/// read as text (). +/// +public static class EncodingFixer +{ + public static readonly IReadOnlyList Encodings = new[] + { + new SourceEncoding(1252, "Western (Windows-1252)"), + new SourceEncoding(1250, "Central European (Windows-1250)"), + new SourceEncoding(1251, "Cyrillic (Windows-1251)"), + new SourceEncoding(1253, "Greek (Windows-1253)"), + new SourceEncoding(1254, "Turkish (Windows-1254)"), + new SourceEncoding(1257, "Baltic (Windows-1257)"), + }; + + private const int MaxPasses = 3; + + // Punctuation that also follows a real accented letter at the end of a word ("Não”", "irmã…"): + // a two-character match ending in one of these is only fixed when there is other evidence. + private const string AmbiguousFollowers = "…‘’“”–—«»‹›„‚•· "; + + // Letters (to char.IsLetter) that are still telling as the second character: ú, õ, ö in Western. + private const string TellingLetters = "ªµº"; + + // Share of the code page's non-ASCII characters that must be part of matches before a subtitle counts as + // broken. Broken text is nearly all matches; real Macedonian has a few look-alikes (ој, ње) among normal letters. + private const double MinCoverage = 0.3; + + /// + /// What a two-byte character decoded from a code page may be: its own script plus Latin-1 symbols + /// (°, «, », ...), and the lead bytes a lowercased letter is tried as. + /// + private sealed record Script((int From, int To)[] TwoByteRanges, byte[] LowercasedLeads); + + private static readonly Script Latin = new(new[] { (0xA0, 0x24F) }, new byte[] { 0xC3, 0xC4, 0xC5 }); + + // Western PCs misread everything, so Windows-1252 also decodes to Greek, Cyrillic, Hebrew and Arabic. + private static readonly Script Western = new(new[] { (0xA0, 0x24F), (0x370, 0x52F), (0x590, 0x6FF) }, new byte[] { 0xC3, 0xC4, 0xC5 }); + private static readonly Script Cyrillic = new(new[] { (0xA0, 0xBF), (0x400, 0x52F) }, new byte[] { 0xD0, 0xD1 }); + private static readonly Script Greek = new(new[] { (0xA0, 0xBF), (0x370, 0x3FF) }, new byte[] { 0xCE, 0xCF }); + + private static Script ScriptOf(int codePage) => codePage switch + { + 1252 => Western, + 1251 => Cyrillic, + 1253 => Greek, + _ => Latin, + }; + + private static readonly Dictionary> Maps = new(); + private static readonly object MapsLock = new(); + + static EncodingFixer() => Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + /// + /// The code page with the most unambiguous matches among those the texts look broken for, or Windows-1252. + /// + public static SourceEncoding Detect(IEnumerable texts) + { + var list = texts as IReadOnlyCollection ?? texts.ToList(); + var best = Encodings[0]; + var bestCount = 0; + foreach (var encoding in Encodings) + { + var analysis = Analyze(list, encoding.CodePage); + if (analysis.IsBroken && analysis.Telling > bestCount) + { + best = encoding; + bestCount = analysis.Telling; + } + } + + return best; + } + + /// + /// Whether the texts look broken for the code page: unambiguous matches (like é or ’), and + /// matches make up a good share of the non-ASCII characters. Then ambiguous matches are fixed too. + /// + public static bool IsBroken(IEnumerable texts, int codePage) => Analyze(texts, codePage).IsBroken; + + /// Number of unambiguous matches in the texts. + public static int CountTelling(IEnumerable texts, int codePage) => Analyze(texts, codePage).Telling; + + private readonly record struct Analysis(int Telling, int MatchedChars, int NonAsciiChars) + { + public bool IsBroken => Telling > 0 && MatchedChars >= NonAsciiChars * MinCoverage; + } + + private static Analysis Analyze(IEnumerable texts, int codePage) + { + var map = GetMap(codePage); + var script = ScriptOf(codePage); + int telling = 0, matched = 0, nonAscii = 0; + foreach (var text in texts) + { + if (string.IsNullOrEmpty(text)) + { + continue; + } + + for (var i = 0; i < text.Length; i++) + { + if (text[i] >= 0x80 && map.ContainsKey(text[i])) + { + nonAscii++; + } + + if (TryMatch(text, i, map, script, out var match)) + { + telling += match.Telling ? 1 : 0; + matched += match.Length; + for (var k = 1; k < match.Length; k++) + { + nonAscii += text[i + k] >= 0x80 && map.ContainsKey(text[i + k]) ? 1 : 0; + } + + i += match.Length - 1; + } + } + } + + return new Analysis(telling, matched, nonAscii); + } + + /// + /// Fixes one text. With (the subtitle ) ambiguous matches are + /// fixed as well; without, only the unambiguous ones. + /// Repeats for text that was broken more than once. + /// + public static string Fix(string text, int codePage, bool confirmed) + { + if (string.IsNullOrEmpty(text)) + { + return text; + } + + var map = GetMap(codePage); + var script = ScriptOf(codePage); + for (var pass = 0; pass < MaxPasses; pass++) + { + // Text broken twice is broken once after the first pass; go again only when that is clear, + // so correct text ("Всё") is never decoded a second time. + if (pass > 0 && CountTelling(new[] { text }, codePage) == 0) + { + break; + } + + var fixedText = FixOnce(text, map, script, confirmed); + if (fixedText == text) + { + break; + } + + text = fixedText; + } + + return text; + } + + private static string FixOnce(string text, Dictionary map, Script script, bool confirmed) + { + StringBuilder? sb = null; + var copied = 0; + for (var i = 0; i < text.Length; i++) + { + if (!TryMatch(text, i, map, script, out var match)) + { + continue; + } + + if (!match.Telling && !confirmed) + { + continue; + } + + sb ??= new StringBuilder(text.Length); + sb.Append(text, copied, i - copied).Append(match.Replacement); + i += match.Length - 1; + copied = i + 1; + } + + if (sb == null) + { + return text; + } + + sb.Append(text, copied, text.Length - copied); + return sb.ToString(); + } + + private readonly record struct Match(int Length, string Replacement, bool Telling); + + private static bool TryMatch(string text, int i, Dictionary map, Script script, out Match match) + { + match = default; + var c = text[i]; + if (c < 0x80 || !map.TryGetValue(c, out var lead)) + { + return false; + } + + if (lead >= 0xC2 && lead <= 0xF4 && TryDecode(text, i, lead, map, script, lowercased: false, out match)) + { + return true; + } + + // Lowercased after it broke: "é" → "ã©". Only the leads of the code page's letters are tried, and the + // result must be a letter. + var upper = char.ToUpperInvariant(c); + if (upper != c && map.TryGetValue(upper, out var upperLead) && Array.IndexOf(script.LowercasedLeads, upperLead) >= 0 && + TryDecode(text, i, upperLead, map, script, lowercased: true, out match) && char.IsLetter(match.Replacement, 0)) + { + return true; + } + + // "â€" with the third byte (0x9D, undefined in Windows-1252) lost: the closing quote ”. + if (lead == 0xE2 && i + 1 < text.Length && map.TryGetValue(text[i + 1], out var second) && second == 0x80 && + (i + 2 >= text.Length || !map.TryGetValue(text[i + 2], out var third) || third < 0x80 || third > 0xBF)) + { + match = new Match(2, "”", true); + return true; + } + + // "à " / " " where the second byte (A0, a no-break space) became a normal space: "à", " ". + if (c is 'Â' or 'Ã' && lead is 0xC2 or 0xC3 && i + 1 < text.Length && text[i + 1] == ' ') + { + match = new Match(2, c == 'Â' ? " " : "à", false); + return true; + } + + return false; + } + + private static bool TryDecode(string text, int i, byte lead, Dictionary map, Script script, bool lowercased, out Match match) + { + match = default; + var continuationCount = lead >= 0xF0 ? 3 : lead >= 0xE0 ? 2 : 1; + if (i + continuationCount >= text.Length) + { + return false; + } + + var codePoint = lead & (0x3F >> continuationCount); + for (var k = 1; k <= continuationCount; k++) + { + if (!map.TryGetValue(text[i + k], out var b) || b < 0x80 || b > 0xBF) + { + return false; + } + + codePoint = (codePoint << 6) | (b & 0x3F); + } + + string replacement; + switch (continuationCount) + { + case 1: + if (!script.TwoByteRanges.Any(r => codePoint >= r.From && codePoint <= r.To)) + { + return false; // C1 controls, or another script: real "ој" is not Greek "μ" + } + + replacement = ((char)codePoint).ToString(); + break; + case 2: + if (codePoint == 0xFEFF) + { + replacement = string.Empty; // byte order mark + } + else if (codePoint < 0x800 || codePoint >= 0x2E80 || codePoint is >= 0xD800 and <= 0xDFFF) + { + // Not valid, or CJK and beyond: a real "é…" is far likelier than a lone CJK character. + return false; + } + else + { + replacement = ((char)codePoint).ToString(); + } + + break; + default: + if (codePoint is < 0x1F000 or > 0x1FAFF) + { + return false; // only emoji + } + + replacement = char.ConvertFromUtf32(codePoint); + break; + } + + if (lowercased) + { + replacement = replacement.ToLowerInvariant(); + } + + var follower = text[i + 1]; + var telling = continuationCount > 1 || + (!char.IsLetter(follower) || TellingLetters.Contains(follower)) && + !AmbiguousFollowers.Contains(follower) && !char.IsWhiteSpace(follower); + match = new Match(continuationCount + 1, replacement, telling); + return true; + } + + private static Dictionary GetMap(int codePage) + { + lock (MapsLock) + { + if (Maps.TryGetValue(codePage, out var cached)) + { + return cached; + } + + var encoding = Encoding.GetEncoding(codePage, EncoderFallback.ExceptionFallback, DecoderFallback.ReplacementFallback); + var map = new Dictionary(); + for (var b = 0x80; b <= 0xFF; b++) + { + var decoded = encoding.GetString(new[] { (byte)b }); + if (decoded.Length == 1 && decoded[0] != '�') + { + map.TryAdd(decoded[0], (byte)b); + } + } + + // Bytes the code page leaves undefined are often decoded as Latin-1 C1 controls (Windows-1252 0x9D "”"). + for (var b = 0x80; b <= 0x9F; b++) + { + map.TryAdd((char)b, (byte)b); + } + + Maps[codePage] = map; + return map; + } + } +} diff --git a/se5/FixEncodingErrors/FixEncodingErrors.csproj b/se5/FixEncodingErrors/FixEncodingErrors.csproj new file mode 100644 index 00000000..c5de7eb0 --- /dev/null +++ b/se5/FixEncodingErrors/FixEncodingErrors.csproj @@ -0,0 +1,27 @@ + + + + WinExe + net8.0 + enable + enable + FixEncodingErrors + SubtitleEdit.Plugins.FixEncodingErrors + true + app.manifest + + + + + + + + + + + + + + + + diff --git a/se5/FixEncodingErrors/MainViewModel.cs b/se5/FixEncodingErrors/MainViewModel.cs new file mode 100644 index 00000000..f8232a4b --- /dev/null +++ b/se5/FixEncodingErrors/MainViewModel.cs @@ -0,0 +1,253 @@ +using CommunityToolkit.Mvvm.ComponentModel; +using CommunityToolkit.Mvvm.Input; +using SubtitleEdit.Plugins.Shared; +using System.Collections.ObjectModel; +using System.Globalization; +using System.Text.Json; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +public sealed partial class MainViewModel : ObservableObject +{ + private const int SettingsVersion = 1; + + private readonly PluginRequest _request; + private readonly bool _legacyRequest; + private readonly List _paragraphs; + private readonly long _videoOffsetMs; + private readonly SourceEncoding _detected; + private bool _loading; + + [ObservableProperty] private SourceEncoding _selectedEncoding; + [ObservableProperty] private bool _selectedLinesOnly; + [ObservableProperty] private int _selectedCount; + + public MainViewModel(PluginRequest request) + { + _request = request; + _videoOffsetMs = request.Rules?.VideoOffsetMs ?? 0; + _legacyRequest = request.Subtitle.Paragraphs == null; + _paragraphs = _legacyRequest + ? SubRipParser.Parse(request.Subtitle.SubRip) + .Select(b => new PluginParagraph { StartMs = b.StartMs, EndMs = b.EndMs, Text = b.Text }).ToList() + : request.Subtitle.Paragraphs!.Select(p => p.Clone()).ToList(); + SelectedIndices = request.SelectedIndices.Where(i => i >= 0 && i < _paragraphs.Count).Distinct().Order().ToList(); + + var detected = EncodingFixer.Detect(_paragraphs.Select(p => p.Text ?? string.Empty)); + AutoEncoding = new SourceEncoding(0, "Auto-detect - " + detected.Name); + _detected = detected; + EncodingOptions = new[] { AutoEncoding }.Concat(EncodingFixer.Encodings).ToList(); + _selectedEncoding = AutoEncoding; + + _loading = true; + LoadSettings(); + _loading = false; + Analyze(); + } + + public ObservableCollection Changes { get; } = new(); + public IReadOnlyList EncodingOptions { get; } + public SourceEncoding AutoEncoding { get; } + + /// The code page used: the detected one when Auto is picked. + public SourceEncoding EffectiveEncoding => SelectedEncoding.CodePage == 0 ? _detected : SelectedEncoding; + public IReadOnlyList SelectedIndices { get; } + + public string SubtitleInfo + { + get + { + var name = string.IsNullOrEmpty(_request.Subtitle.FileName) ? "Untitled" : Path.GetFileName(_request.Subtitle.FileName); + var format = string.IsNullOrEmpty(_request.Subtitle.Format) ? string.Empty : $" · {_request.Subtitle.Format}"; + return $"{name}{format} · {_paragraphs.Count:N0} lines"; + } + } + + public bool HasSelectedLines => SelectedIndices.Count > 0; + public string SelectedLinesText => SelectedIndices.Count == 1 ? "Selected line (1)" : $"Selected lines ({SelectedIndices.Count:N0})"; + public bool AllLines { get => !SelectedLinesOnly; set => SelectedLinesOnly = !value; } + public int ChangeCount => Changes.Count; + public bool HasChanges => Changes.Count > 0; + public bool IsClean => Changes.Count == 0; + public bool CanApply => SelectedCount > 0; + public string CleanText => $"No broken characters found for {EffectiveEncoding.Name}."; + + public string StatusText => Changes.Count == 0 + ? $"{LinesInScope:N0} lines checked" + : $"{SelectedCount:N0} of {Changes.Count:N0} lines selected"; + + private int LinesInScope => SelectedLinesOnly && HasSelectedLines ? SelectedIndices.Count : _paragraphs.Count; + + partial void OnSelectedEncodingChanged(SourceEncoding value) + { + OnPropertyChanged(nameof(EffectiveEncoding)); + OnPropertyChanged(nameof(CleanText)); + Analyze(); + } + + partial void OnSelectedLinesOnlyChanged(bool value) + { + OnPropertyChanged(nameof(AllLines)); + Analyze(); + } + + + partial void OnSelectedCountChanged(int value) + { + OnPropertyChanged(nameof(CanApply)); + OnPropertyChanged(nameof(StatusText)); + } + + private IEnumerable IndicesInScope => + SelectedLinesOnly && HasSelectedLines ? SelectedIndices : Enumerable.Range(0, _paragraphs.Count); + + private void Analyze() + { + if (_loading) + { + return; + } + + // The whole subtitle (not only the lines in scope) decides whether it is broken. + var codePage = EffectiveEncoding.CodePage; + var confirmed = EncodingFixer.IsBroken(_paragraphs.Select(p => p.Text ?? string.Empty), codePage); + Changes.Clear(); + foreach (var index in IndicesInScope) + { + var text = _paragraphs[index].Text ?? string.Empty; + var after = EncodingFixer.Fix(text, codePage, confirmed); + if (after != text) + { + var item = new ChangeItem(index, text, after, FormatTimeRange(_paragraphs[index])); + item.PropertyChanged += (_, e) => + { + if (e.PropertyName == nameof(ChangeItem.IsSelected)) + { + SelectedCount = Changes.Count(c => c.IsSelected); + } + }; + Changes.Add(item); + } + } + + SelectedCount = Changes.Count; + OnPropertyChanged(nameof(ChangeCount)); + OnPropertyChanged(nameof(HasChanges)); + OnPropertyChanged(nameof(IsClean)); + OnPropertyChanged(nameof(StatusText)); + } + + [RelayCommand] + private void SelectAll() => SetSelection(_ => true); + + [RelayCommand] + private void SelectNone() => SetSelection(_ => false); + + [RelayCommand] + private void InvertSelection() => SetSelection(c => !c.IsSelected); + + private void SetSelection(Func selected) + { + foreach (var change in Changes) + { + change.IsSelected = selected(change); + } + } + + /// The response for OK: the selected lines, fixed. + public PluginResponse BuildResponse() + { + var selected = Changes.Where(c => c.IsSelected).ToList(); + if (selected.Count == 0) + { + return BuildCancelResponse(); + } + + var texts = _paragraphs.Select(p => p.Text ?? string.Empty).ToArray(); + foreach (var change in selected) + { + texts[change.Index] = change.After; + } + + PluginSubtitle subtitle; + if (_legacyRequest) + { + subtitle = new PluginSubtitle + { + Format = "SubRip", + Native = SubRipParser.Serialize(_paragraphs.Select((p, i) => new SrtBlock + { + StartMs = (long)Math.Round(p.StartMs), + EndMs = (long)Math.Round(p.EndMs), + Text = texts[i], + }).ToList()), + }; + } + else + { + subtitle = new PluginSubtitle + { + Format = _request.Subtitle.Format, + Paragraphs = _paragraphs.Select((p, i) => + { + var copy = p.Clone(); + copy.Text = texts[i]; + return copy; + }).ToList(), + }; + } + + return new PluginResponse + { + Status = PluginStatus.Ok, + Message = selected.Count == 1 ? "Fix encoding errors: 1 line fixed." : $"Fix encoding errors: {selected.Count:N0} lines fixed.", + UndoDescription = "Fix encoding errors", + Subtitle = subtitle, + Settings = BuildSettings(), + SettingsVersion = SettingsVersion, + }; + } + + public PluginResponse BuildCancelResponse() => + new() { Status = PluginStatus.Cancelled, Settings = BuildSettings(), SettingsVersion = SettingsVersion }; + + private JsonElement BuildSettings() + { + var payload = new Dictionary + { + ["codePage"] = SelectedEncoding.CodePage, + ["selectedLinesOnly"] = SelectedLinesOnly, + }; + return JsonSerializer.SerializeToElement(payload); + } + + private void LoadSettings() + { + // Lines selected in Subtitle Edit mean "work on these" - default to them. + SelectedLinesOnly = HasSelectedLines; + if (_request.SettingsVersion != SettingsVersion || _request.Settings is not { ValueKind: JsonValueKind.Object } settings) + { + return; + } + + if (settings.TryGetProperty("codePage", out var codePage) && codePage.TryGetInt32(out var value) && + EncodingOptions.FirstOrDefault(e => e.CodePage == value) is { } encoding) + { + SelectedEncoding = encoding; + } + + if (settings.TryGetProperty("selectedLinesOnly", out var selectedOnly) && selectedOnly.ValueKind is JsonValueKind.True or JsonValueKind.False) + { + SelectedLinesOnly = selectedOnly.GetBoolean() && HasSelectedLines; + } + } + + private string FormatTimeRange(PluginParagraph p) => + $"{FormatTime(p.StartMs + _videoOffsetMs)} → {FormatTime(p.EndMs + _videoOffsetMs)}"; + + private static string FormatTime(double ms) + { + var time = TimeSpan.FromMilliseconds(Math.Max(0, ms)); + return string.Format(CultureInfo.InvariantCulture, "{0:00}:{1:00}:{2:00},{3:000}", (int)time.TotalHours, time.Minutes, time.Seconds, time.Milliseconds); + } +} diff --git a/se5/FixEncodingErrors/MainWindow.axaml b/se5/FixEncodingErrors/MainWindow.axaml new file mode 100644 index 00000000..acbf304b --- /dev/null +++ b/se5/FixEncodingErrors/MainWindow.axaml @@ -0,0 +1,323 @@ + + + + + + + + + + + + + + + + + + + + Cascadia Mono, Consolas, Menlo, DejaVu Sans Mono, monospace + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/se5/FixEncodingErrors/MainWindow.axaml.cs b/se5/FixEncodingErrors/MainWindow.axaml.cs new file mode 100644 index 00000000..6453982f --- /dev/null +++ b/se5/FixEncodingErrors/MainWindow.axaml.cs @@ -0,0 +1,58 @@ +using Avalonia.Controls; +using Avalonia.Input; +using Avalonia.Interactivity; +using Avalonia.Markup.Xaml; +using SubtitleEdit.Plugins.Shared; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +public partial class MainWindow : Window +{ + private readonly MainViewModel _vm; + + public MainWindow() : this(new PluginRequest()) { } + + public MainWindow(PluginRequest request) + { + AvaloniaXamlLoader.Load(this); + _vm = new MainViewModel(request); + DataContext = _vm; + App.Response = _vm.BuildCancelResponse(); + KeyDown += OnKeyDown; + } + + protected override void OnOpened(EventArgs e) + { + base.OnOpened(e); + this.BringToForeground(); + } + + private void OnKeyDown(object? sender, KeyEventArgs e) + { + if (e.Key == Key.Escape) + { + e.Handled = true; + OnCancel(sender, e); + } + } + + private void OnCancel(object? sender, RoutedEventArgs e) + { + App.Response = _vm.BuildCancelResponse(); + Close(); + } + + private void OnOk(object? sender, RoutedEventArgs e) + { + try + { + App.Response = _vm.BuildResponse(); + } + catch (Exception exception) + { + App.Response = new PluginResponse { Status = PluginStatus.Error, Message = "Fix encoding errors failed: " + exception.Message }; + } + + Close(); + } +} diff --git a/se5/FixEncodingErrors/Program.cs b/se5/FixEncodingErrors/Program.cs new file mode 100644 index 00000000..e7c28d11 --- /dev/null +++ b/se5/FixEncodingErrors/Program.cs @@ -0,0 +1,16 @@ +using Projektanker.Icons.Avalonia; +using Projektanker.Icons.Avalonia.MaterialDesign; +using SubtitleEdit.Plugins.Shared; +using System; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +public static class Program +{ + [STAThread] + public static int Main(string[] args) + { + IconProvider.Current.Register(); + return PluginBootstrap.Run(args); + } +} diff --git a/se5/FixEncodingErrors/README.md b/se5/FixEncodingErrors/README.md new file mode 100644 index 00000000..ae7a7896 --- /dev/null +++ b/se5/FixEncodingErrors/README.md @@ -0,0 +1,71 @@ +# Fix encoding errors (Subtitle Edit 5 plugin) + +Repairs UTF-8 text that was read with the wrong code page and saved again +("mojibake"): + +| Broken | Fixed | +|---|---| +| `résumé`, `communiqués` | `résumé`, `communiqués` | +| `don’t`, `“Hiâ€` | `don’t`, `“Hi”` | +| `25°`, `😀` | `25°`, `😀` | +| `rã©sumã©` (lowercased after it broke) | `résumé` | +| `Привет` (Western) / `Привет` (Cyrillic) | `Привет` | + +Often seen in subtitles extracted from MKV/MP4 files: Matroska text tracks are +UTF-8, so a track muxed from an SRT that was misread somewhere along the way +keeps the broken characters, and Subtitle Edit shows exactly what is stored. + +![Fix encoding errors](docs/screenshot.png) + +Built on Avalonia and the shared [`Plugin-Shared`](../Plugin-Shared/) library. + +## How it works + +Each character is mapped back to its byte in the code page the text was misread +with, and runs of bytes that form valid UTF-8 are decoded. Code pages: +Western (1252), Central European (1250), Cyrillic (1251), Greek (1253), +Turkish (1254) and Baltic (1257). *Auto-detect* picks the one with the most +clear matches. + +Also handled: + +- Text lowercased after it broke - `é` became `ã©`, so the lead byte is + tried as its uppercase letter. +- A lost `0x9D` byte (undefined in Windows-1252): `â€` → `”`. +- A no-break space turned into a normal space: `à ` → `à`. +- A byte order mark read as text: `` is removed. +- Text broken twice (`é`) is fixed in more passes. + +### Avoiding false fixes + +- A two-character match must decode to a script the text could be in: Latin, + plus Greek, Cyrillic, Hebrew and Arabic for Western (a Western PC misreads + everything); Cyrillic for 1251 and Greek for 1253. So real Macedonian `ој` + never turns into Greek `μ`. +- Matches ending in punctuation that can follow a real accented letter + (`Não”`, `irmã…`, `«Não»`) are *ambiguous*. They are fixed only when the + subtitle is clearly broken: it has unambiguous matches like `é` or `’`, and + matches make up at least 30% of its non-ASCII characters. +- Three-byte matches that would decode to CJK are skipped; four-byte matches + must be emoji. + +Checked against all Subtitle Edit UI translations and word lists (about 360,000 +lines in 35 languages): with *Auto-detect*, the only change was a line that +really is broken. + +## UI + +- *Text was misread as* - Auto-detect or a fixed code page; remembered. +- *All lines* / *Selected lines* - lines selected in Subtitle Edit are the + default scope. +- Every changed line is a checkable before/after card with the changed + characters highlighted. *OK* applies the checked lines as one + "Fix encoding errors" undo step. + +Works on the lines exactly as Subtitle Edit holds them, so styles, actors and +other fields of formats like ASSA are kept (falls back to SubRip on older SE +versions). + +## Build + +See `.github/workflows/fix-encoding-errors.yml`. diff --git a/se5/FixEncodingErrors/SubtitleTextBlock.cs b/se5/FixEncodingErrors/SubtitleTextBlock.cs new file mode 100644 index 00000000..b9dd6df7 --- /dev/null +++ b/se5/FixEncodingErrors/SubtitleTextBlock.cs @@ -0,0 +1,201 @@ +using Avalonia; +using Avalonia.Controls; +using Avalonia.Controls.Documents; +using Avalonia.Media; +using System.Text; + +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +/// +/// Shows one side of a . With off the raw text is shown with +/// tags colored and changed text tinted; with it on the tags are hidden and italic text is drawn in italics. +/// +public sealed class SubtitleTextBlock : TextBlock +{ + public static readonly StyledProperty?> SegmentsProperty = + AvaloniaProperty.Register?>(nameof(Segments)); + + public static readonly StyledProperty RenderedProperty = + AvaloniaProperty.Register(nameof(Rendered)); + + private static readonly IBrush RemovedBrush = new SolidColorBrush(Color.Parse("#55E5484D")); + private static readonly IBrush AddedBrush = new SolidColorBrush(Color.Parse("#552FA36B")); + private static readonly IBrush TagBrush = new SolidColorBrush(Color.Parse("#EA580C")); + private static readonly IBrush ItalicBrush = new SolidColorBrush(Color.Parse("#22EA580C")); + + static SubtitleTextBlock() + { + SegmentsProperty.Changed.AddClassHandler((block, _) => block.Rebuild()); + RenderedProperty.Changed.AddClassHandler((block, _) => block.Rebuild()); + } + + public IReadOnlyList? Segments + { + get => GetValue(SegmentsProperty); + set => SetValue(SegmentsProperty, value); + } + + public bool Rendered + { + get => GetValue(RenderedProperty); + set => SetValue(RenderedProperty, value); + } + + protected override Type StyleKeyOverride => typeof(TextBlock); + + private void Rebuild() + { + var segments = Segments ?? Array.Empty(); + Inlines = Rendered ? BuildRendered(string.Concat(segments.Select(s => s.Text))) : BuildRaw(segments); + } + + /// Raw text: tags in the tag color, removed/added text on a tint. + private static InlineCollection BuildRaw(IReadOnlyList segments) + { + var inlines = new InlineCollection(); + var inTag = false; + foreach (var segment in segments) + { + var background = segment.Kind switch + { + DiffKind.Removed => RemovedBrush, + DiffKind.Added => AddedBrush, + _ => null, + }; + + var run = new StringBuilder(); + var runIsTag = inTag; + void Flush() + { + if (run.Length > 0) + { + var inline = new Run(run.ToString()) { Background = background }; + if (runIsTag) + { + inline.Foreground = TagBrush; + inline.FontWeight = FontWeight.SemiBold; + } + + inlines.Add(inline); + run.Clear(); + } + } + + foreach (var c in segment.Text) + { + if (c == '\r') + { + continue; + } + + if (c == '\n') + { + Flush(); + if (background != null) + { + // A changed line break would otherwise be invisible. + inlines.Add(new Run("↵") { Background = background, Foreground = TagBrush }); + } + + inlines.Add(new LineBreak()); + continue; + } + + var isTag = inTag || c == '<'; + if (isTag != runIsTag) + { + Flush(); + runIsTag = isTag; + } + + run.Append(c); + if (c == '<') + { + inTag = true; + } + else if (c == '>') + { + inTag = false; + Flush(); + runIsTag = false; + } + } + + Flush(); + } + + return inlines; + } + + /// Rendered text: tags hidden, <i> drawn in italics on a soft tint, <b>/<u> applied too. + private static InlineCollection BuildRendered(string text) + { + var inlines = new InlineCollection(); + var italic = 0; + var bold = 0; + var underline = 0; + var run = new StringBuilder(); + void Flush() + { + if (run.Length > 0) + { + var inline = new Run(run.ToString()) + { + FontStyle = italic > 0 ? FontStyle.Italic : FontStyle.Normal, + FontWeight = bold > 0 ? FontWeight.Bold : FontWeight.Normal, + }; + if (underline > 0) + { + inline.TextDecorations = Avalonia.Media.TextDecorations.Underline; + } + + if (italic > 0) + { + inline.Background = ItalicBrush; + } + + inlines.Add(inline); + run.Clear(); + } + } + + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + if (c == '\r') + { + continue; + } + + if (c == '\n') + { + Flush(); + inlines.Add(new LineBreak()); + continue; + } + + var end = c is '<' or '{' ? text.IndexOf(c == '<' ? '>' : '}', i) : -1; + if (end < 0) + { + run.Append(c); + continue; + } + + Flush(); + var tag = text.Substring(i, end - i + 1).ToLowerInvariant(); + i = end; + switch (tag) + { + case "": italic++; break; + case "": italic = Math.Max(0, italic - 1); break; + case "": bold++; break; + case "": bold = Math.Max(0, bold - 1); break; + case "": underline++; break; + case "": underline = Math.Max(0, underline - 1); break; + } + } + + Flush(); + return inlines; + } +} diff --git a/se5/FixEncodingErrors/TextDiff.cs b/se5/FixEncodingErrors/TextDiff.cs new file mode 100644 index 00000000..6c6a2f96 --- /dev/null +++ b/se5/FixEncodingErrors/TextDiff.cs @@ -0,0 +1,138 @@ +namespace SubtitleEdit.Plugins.FixEncodingErrors; + +public enum DiffKind +{ + Same, + Removed, + Added, +} + +public sealed record DiffSegment(string Text, DiffKind Kind); + +/// Character diff (LCS) used to highlight what a fix changed. +public static class TextDiff +{ + private const int MaxCells = 1_000_000; + + /// Returns the segments of the old text (same/removed) and of the new text (same/added). + public static (List Before, List After) Compute(string before, string after) + { + // Same number of lines: diff line by line, so line breaks never pair up with the wrong line. + var beforeLines = before.Split('\n'); + var afterLines = after.Split('\n'); + if (beforeLines.Length > 1 && beforeLines.Length == afterLines.Length) + { + var beforeSegments = new List(); + var afterSegments = new List(); + for (var i = 0; i < beforeLines.Length; i++) + { + if (i > 0) + { + Add(beforeSegments, "\n", DiffKind.Same); + Add(afterSegments, "\n", DiffKind.Same); + } + + var (b, a) = ComputeLine(beforeLines[i], afterLines[i]); + b.ForEach(segment => Add(beforeSegments, segment.Text, segment.Kind)); + a.ForEach(segment => Add(afterSegments, segment.Text, segment.Kind)); + } + + return (beforeSegments, afterSegments); + } + + return ComputeLine(before, after); + } + + private static (List Before, List After) ComputeLine(string before, string after) + { + var prefix = 0; + while (prefix < before.Length && prefix < after.Length && before[prefix] == after[prefix]) + { + prefix++; + } + + var suffix = 0; + while (suffix < before.Length - prefix && suffix < after.Length - prefix && + before[before.Length - 1 - suffix] == after[after.Length - 1 - suffix]) + { + suffix++; + } + + var a = before.Substring(prefix, before.Length - prefix - suffix); + var b = after.Substring(prefix, after.Length - prefix - suffix); + var beforeSegments = new List(); + var afterSegments = new List(); + Add(beforeSegments, before[..prefix], DiffKind.Same); + Add(afterSegments, after[..prefix], DiffKind.Same); + + if ((long)a.Length * b.Length > MaxCells) + { + Add(beforeSegments, a, DiffKind.Removed); + Add(afterSegments, b, DiffKind.Added); + } + else + { + Lcs(a, b, beforeSegments, afterSegments); + } + + Add(beforeSegments, before[(before.Length - suffix)..], DiffKind.Same); + Add(afterSegments, after[(after.Length - suffix)..], DiffKind.Same); + return (beforeSegments, afterSegments); + } + + private static void Lcs(string a, string b, List beforeSegments, List afterSegments) + { + var lengths = new int[a.Length + 1, b.Length + 1]; + for (var i = a.Length - 1; i >= 0; i--) + { + for (var j = b.Length - 1; j >= 0; j--) + { + lengths[i, j] = a[i] == b[j] + ? lengths[i + 1, j + 1] + 1 + : Math.Max(lengths[i + 1, j], lengths[i, j + 1]); + } + } + + int x = 0, y = 0; + while (x < a.Length && y < b.Length) + { + if (a[x] == b[y]) + { + Add(beforeSegments, a[x].ToString(), DiffKind.Same); + Add(afterSegments, b[y].ToString(), DiffKind.Same); + x++; + y++; + } + else if (lengths[x + 1, y] >= lengths[x, y + 1]) + { + Add(beforeSegments, a[x].ToString(), DiffKind.Removed); + x++; + } + else + { + Add(afterSegments, b[y].ToString(), DiffKind.Added); + y++; + } + } + + Add(beforeSegments, a[x..], DiffKind.Removed); + Add(afterSegments, b[y..], DiffKind.Added); + } + + private static void Add(List segments, string text, DiffKind kind) + { + if (text.Length == 0) + { + return; + } + + if (segments.Count > 0 && segments[^1].Kind == kind) + { + segments[^1] = segments[^1] with { Text = segments[^1].Text + text }; + } + else + { + segments.Add(new DiffSegment(text, kind)); + } + } +} diff --git a/se5/FixEncodingErrors/app.manifest b/se5/FixEncodingErrors/app.manifest new file mode 100644 index 00000000..e68fabca --- /dev/null +++ b/se5/FixEncodingErrors/app.manifest @@ -0,0 +1,10 @@ + + + + + + true/pm + PerMonitorV2 + + + diff --git a/se5/FixEncodingErrors/docs/screenshot.png b/se5/FixEncodingErrors/docs/screenshot.png new file mode 100644 index 00000000..2bac684c Binary files /dev/null and b/se5/FixEncodingErrors/docs/screenshot.png differ diff --git a/se5/FixEncodingErrors/plugin.json b/se5/FixEncodingErrors/plugin.json new file mode 100644 index 00000000..c40849d7 --- /dev/null +++ b/se5/FixEncodingErrors/plugin.json @@ -0,0 +1,12 @@ +{ + "apiVersion": 1, + "name": "Fix encoding errors", + "description": "Repairs UTF-8 text that was read with the wrong code page (\"mojibake\"): résumé → résumé, don’t → don’t. Also handles text lowercased afterwards (rã©sumã©). Common in subtitles extracted from MKV files. Shows a checkable before/after preview.", + "version": "1.0.0", + "author": "Subtitle Edit", + "url": "https://github.com/SubtitleEdit/plugins/tree/main/se5/FixEncodingErrors", + "menu": "Tools", + "minSeVersion": "5.0.0", + "runtime": "dotnet", + "entry": "FixEncodingErrors.dll" +}