Add SE5 Fix encoding errors plugin

.github/workflows/fix-encoding-errors.yml

@@ -0,0 +1,196 @@
+name: Build FixEncodingErrors (SE5)
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'se5/FixEncodingErrors/**'
+      - 'se5/FixEncodingErrors.Tests/**'
+      - 'se5/Plugin-Shared/**'
+      - '.github/workflows/fix-encoding-errors.yml'
+  pull_request:
+    paths:
+      - 'se5/FixEncodingErrors/**'
+      - 'se5/FixEncodingErrors.Tests/**'
+      - 'se5/Plugin-Shared/**'
+      - '.github/workflows/fix-encoding-errors.yml'
+  workflow_dispatch:
+    inputs:
+      release:
+        description: 'Publish a GitHub release. Bumps the minor in plugin.json and commits it back.'
+        type: boolean
+        default: false
+      version:
+        description: 'Optional explicit version (e.g. 1.2.0). Overrides the auto-minor-bump.'
+        required: false
+        type: string
+
+permissions:
+  contents: write
+
+jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.set.outputs.version }}
+      tag:     ${{ steps.set.outputs.tag }}
+      ref:     ${{ steps.set.outputs.ref }}
+      release: ${{ steps.set.outputs.release }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Resolve version (and bump on release)
+        id: set
+        run: |
+          set -e
+          current=$(jq -r .version se5/FixEncodingErrors/plugin.json)
+          release_flag="false"
+          ref="${{ github.sha }}"
+          version="$current"
+          tag=""
+
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.release }}" = "true" ]; then
+            release_flag="true"
+            if [ -n "${{ inputs.version }}" ]; then
+              version="${{ inputs.version }}"
+            else
+              IFS=. read -r major minor patch <<< "$current"
+              version="$major.$((minor + 1)).0"
+            fi
+
+            tmp=$(mktemp)
+            jq --arg v "$version" '.version = $v' se5/FixEncodingErrors/plugin.json > "$tmp"
+            mv "$tmp" se5/FixEncodingErrors/plugin.json
+
+            git config user.name  "github-actions[bot]"
+            git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+            git add se5/FixEncodingErrors/plugin.json
+            git commit -m "Bump FixEncodingErrors to $version [skip ci]"
+
+            # Retry-with-rebase: if a concurrent release dispatched for a different
+            # plugin won the push first, pull --rebase and try again.
+            attempts=0
+            while ! git push 2>/tmp/push.err; do
+              attempts=$((attempts + 1))
+              if [ "$attempts" -ge 5 ]; then
+                echo "Failed to push after $attempts attempts:" >&2
+                cat /tmp/push.err >&2
+                exit 1
+              fi
+              echo "Push rejected (attempt $attempts), pulling --rebase and retrying..."
+              git pull --rebase --no-edit
+            done
+            ref=$(git rev-parse HEAD)
+
+            IFS=. read -r major minor patch <<< "$version"
+            tag="se5-fix-encoding-errors-v$major.$minor"
+          fi
+
+          echo "version=$version"  >> "$GITHUB_OUTPUT"
+          echo "tag=$tag"          >> "$GITHUB_OUTPUT"
+          echo "ref=$ref"          >> "$GITHUB_OUTPUT"
+          echo "release=$release_flag" >> "$GITHUB_OUTPUT"
+
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup .NET 8
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+
+      - name: Test
+        run: dotnet test se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj -c Release
+
+  build:
+    needs: [prepare, test]
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - rid: win-x64
+            os: windows
+            exe: FixEncodingErrors.exe
+          - rid: win-arm64
+            os: windows
+            exe: FixEncodingErrors.exe
+          - rid: linux-x64
+            os: linux
+            exe: FixEncodingErrors
+          - rid: linux-arm64
+            os: linux
+            exe: FixEncodingErrors
+          - rid: osx-x64
+            os: macos
+            exe: FixEncodingErrors
+          - rid: osx-arm64
+            os: macos
+            exe: FixEncodingErrors
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.prepare.outputs.ref }}
+
+      - name: Setup .NET 8
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: '8.0.x'
+
+      - name: Publish self-contained (${{ matrix.rid }})
+        working-directory: se5/FixEncodingErrors
+        run: |
+          dotnet publish FixEncodingErrors.csproj \
+            -c Release \
+            -r ${{ matrix.rid }} \
+            --self-contained true \
+            -p:DebugType=None \
+            -p:DebugSymbols=false \
+            -o staging/FixEncodingErrors
+
+      - name: Rewrite plugin.json for ${{ matrix.os }}
+        working-directory: se5/FixEncodingErrors
+        run: |
+          jq '
+            del(.runtime, .entry) |
+            .executables = { "${{ matrix.os }}": "${{ matrix.exe }}" }
+          ' plugin.json > staging/FixEncodingErrors/plugin.json
+
+      - name: Package zip
+        working-directory: se5/FixEncodingErrors/staging
+        run: zip -r "$GITHUB_WORKSPACE/FixEncodingErrors-${{ matrix.rid }}.zip" FixEncodingErrors
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: FixEncodingErrors-${{ matrix.rid }}
+          path: FixEncodingErrors-${{ matrix.rid }}.zip
+
+  release:
+    name: Publish GitHub Release
+    needs: [prepare, build]
+    if: needs.prepare.outputs.release == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download all zips
+        uses: actions/download-artifact@v4
+        with:
+          path: dist
+          pattern: FixEncodingErrors-*
+          merge-multiple: true
+
+      - name: Create release and upload zips
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh release create "${{ needs.prepare.outputs.tag }}" dist/FixEncodingErrors-*.zip \
+            --repo "${{ github.repository }}" \
+            --target "${{ needs.prepare.outputs.ref }}" \
+            --title "FixEncodingErrors v${{ needs.prepare.outputs.version }}" \
+            --notes "Self-contained FixEncodingErrors plugin builds for win/linux/osx (x64 + arm64)."

README.md

@@ -18,6 +18,7 @@ index [`se5-plugins.json`](se5-plugins.json).
 | [ARTE check](se5/ArteCheck) | Checks and fixes EBU STL teletext subtitles for ARTE delivery. |
 | [American to British](se5/AmericanToBritish) | Converts American English spellings to British English. |
 | [British to American](se5/BritishToAmerican) | Converts British English spellings to American English. |
+| [Fix encoding errors](se5/FixEncodingErrors) | Repairs UTF-8 text read with the wrong code page: résumé → résumé, don’t → don’t. |
 | [Haxor](se5/Haxor) | Translates text to "haxor" - the minimal reference plugin. |
 | [Italic each line](se5/ItalicEachLine) | Switches italic tags between one tag per subtitle and a tag on each line. |
 | [Persian Subtitle Fixes](se5/PersianErrors) | Fixes common errors in Persian (Farsi) subtitles - 1,500+ rules in 12 groups. |

se5/FixEncodingErrors.Tests/EncodingFixerTests.cs

@@ -0,0 +1,140 @@
+using SubtitleEdit.Plugins.FixEncodingErrors;
+using System.Text;
+
+namespace FixEncodingErrors.Tests;
+
+public class EncodingFixerTests
+{
+    static EncodingFixerTests() => Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
+
+    /// <summary>What a UTF-8 text looks like after being read with <paramref name="codePage"/>.</summary>
+    private static string Break(string text, int codePage = 1252) =>
+        Encoding.GetEncoding(codePage).GetString(Encoding.UTF8.GetBytes(text));
+
+    [Theory]
+    [InlineData("résumé", "résumé")]
+    [InlineData("don’t", "don’t")]
+    [InlineData("“Quoteâ€\u009D", "“Quote”")]
+    [InlineData("“Quoteâ€", "“Quote”")] // 0x9D lost
+    [InlineData("25°C", "25°C")]
+    [InlineData("Hello", "Hello")]
+    [InlineData("Wait à  bientôt", "Wait à bientôt")] // A0 turned into a space
+    [InlineData("😀 hi", "😀 hi")]
+    public void FixesWestern(string broken, string expected)
+    {
+        Assert.Equal(expected, EncodingFixer.Fix(broken, 1252, confirmed: true));
+    }
+
+    [Theory]
+    [InlineData("cache of such communiquã©s,", "cache of such communiqués,")]
+    [InlineData("and dust off my rã©sumã©.", "and dust off my résumé.")]
+    [InlineData("madame la secrã©taire.", "madame la secrétaire.")]
+    public void FixesTextLowercasedAfterItBroke(string broken, string expected)
+    {
+        // Ed's Madam Secretary subtitles: © alone tells the subtitle is broken.
+        Assert.Equal(expected, EncodingFixer.Fix(broken, 1252, confirmed: false));
+    }
+
+    [Theory]
+    [InlineData("Ça va? Où êtes-vous? À bientôt, Œdipe!")]
+    [InlineData("Größe, Übermaß, naïve, Señor, Ñandú, Ærø")]
+    [InlineData("“Smart quotes” – dashes — and… ellipsis ‘ok’ 😀")]
+    public void RoundTripsWestern(string text)
+    {
+        Assert.Equal(text, EncodingFixer.Fix(Break(text), 1252, confirmed: true));
+    }
+
+    [Theory]
+    [InlineData("Привет, как дела? Всё хорошо.", 1251)]
+    [InlineData("Příliš žluťoučký kůň úpěl ďábelské ódy.", 1250)]
+    [InlineData("Καλημέρα κόσμε", 1253)]
+    [InlineData("Günaydın, nasılsın? Şimdi değil.", 1254)]
+    public void RoundTripsOtherCodePages(string text, int codePage)
+    {
+        // Detection may pick a code page that reads the same bytes the same way (1252/1254 for Turkish).
+        var broken = Break(text, codePage);
+        var detected = EncodingFixer.Detect(new[] { broken }).CodePage;
+        Assert.Equal(text, EncodingFixer.Fix(broken, detected, confirmed: true));
+        Assert.Equal(text, EncodingFixer.Fix(broken, codePage, confirmed: true));
+    }
+
+    [Fact]
+    public void DetectsCyrillic()
+    {
+        Assert.Equal(1251, EncodingFixer.Detect(new[] { Break("Привет, как дела?", 1251) }).CodePage);
+    }
+
+    [Theory]
+    [InlineData("Привет, как дела?")]
+    [InlineData("Καλημέρα κόσμε")]
+    [InlineData("שלום עולם")]
+    [InlineData("مرحبا بالعالم")]
+    public void FixesOtherScriptsMisreadAsWestern(string text)
+    {
+        var broken = Break(text);
+        Assert.Equal(1252, EncodingFixer.Detect(new[] { broken }).CodePage);
+        Assert.Equal(text, EncodingFixer.Fix(broken, 1252, EncodingFixer.IsBroken(new[] { broken }, 1252)));
+    }
+
+    [Fact]
+    public void FixesDoubleBroken()
+    {
+        Assert.Equal("café", EncodingFixer.Fix(Break(Break("café")), 1252, confirmed: true));
+    }
+
+    [Theory]
+    [InlineData("Não”")]
+    [InlineData("A minha irmã… e a mãe")]
+    [InlineData("«Não»")]
+    [InlineData("Irmã Lúcia")]
+    [InlineData("Ça coûte 25 €, café»")]
+    [InlineData("Plain ASCII text")]
+    public void LeavesRealTextAlone(string text)
+    {
+        Assert.Equal(0, EncodingFixer.CountTelling(new[] { text }, 1252));
+        Assert.Equal(text, EncodingFixer.Fix(text, 1252, confirmed: false));
+    }
+
+    [Theory]
+    [InlineData("Привет, Сёма! В доме.")]
+    [InlineData("Příliš žluťoučký kůň")]
+    public void LeavesRealTextAloneInItsOwnCodePage(string text)
+    {
+        Assert.Equal(1252, EncodingFixer.Detect(new[] { text }).CodePage);
+        Assert.Equal(text, EncodingFixer.Fix(text, 1252, confirmed: false));
+    }
+
+    [Fact]
+    public void AmbiguousMatchNeedsConfirmation()
+    {
+        // "Ö" (Ö) at a word end could be real text; fixed once the subtitle is known to be broken.
+        Assert.Equal("BÖ", EncodingFixer.Fix("BÖ", 1252, confirmed: false));
+        Assert.Equal("BÖ", EncodingFixer.Fix("BÖ", 1252, confirmed: true));
+        Assert.True(EncodingFixer.IsBroken(new[] { "Österreich", "café" }, 1252));
+        Assert.False(EncodingFixer.IsBroken(new[] { "Österreich" }, 1252));
+    }
+
+    [Fact]
+    public void DetectsWesternForEd()
+    {
+        var lines = new[] { "and dust off my rã©sumã©.", "I'll send you his rã©sumã©." };
+        Assert.Equal(1252, EncodingFixer.Detect(lines).CodePage);
+        Assert.Equal(4, EncodingFixer.CountTelling(lines, 1252));
+    }
+
+    [Fact]
+    public void RealMacedonianIsNotBroken()
+    {
+        // ј, џ, ќ, ѓ are continuation bytes in Windows-1251: "ој" must not become Greek "μ", "рџ" not "п".
+        var lines = new[] { "Тој е мојот брат.", "Џорџ веќе учење, која боја." };
+        Assert.False(EncodingFixer.IsBroken(lines, 1251));
+        Assert.Equal(1252, EncodingFixer.Detect(lines).CodePage);
+        Assert.All(lines, l => Assert.Equal(l, EncodingFixer.Fix(l, 1251, confirmed: false)));
+    }
+
+    [Fact]
+    public void KeepsTagsAndLineBreaks()
+    {
+        Assert.Equal("<i>résumé</i>\r\n{\\an8}café", EncodingFixer.Fix("<i>résumé</i>\r\n{\\an8}café", 1252, confirmed: false));
+    }
+}

se5/FixEncodingErrors.Tests/FixEncodingErrors.Tests.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
+  <PropertyGroup>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageReference Include="xunit" Version="2.9.2" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\FixEncodingErrors\FixEncodingErrors.csproj" />
+  </ItemGroup>
+
+</Project>