Marking conflicting messages

* Update _prepare function in ConcatenateCatalog to check conflicting messages and to not parse po-files twice

* Add _conflicts field in Catalog to mark conflicts

* Update tests

Author: soft-suroleb

babel/messages/catalog.py

@@ -12,13 +12,14 @@
 
 import datetime
 import re
+import os
 from collections.abc import Iterable, Iterator
 from copy import copy
 from difflib import SequenceMatcher
 from email import message_from_string
 from heapq import nlargest
 from string import Formatter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypedDict
 
 from babel import __version__ as VERSION
 from babel.core import Locale, UnknownLocaleError
@@ -357,6 +358,13 @@ def _force_text(s: str | bytes, encoding: str = 'utf-8', errors: str = 'strict')
     return str(s)
 
 
+class ConflictInfo(TypedDict):
+    message: Message
+    file_name: str
+    project: str
+    version: str
+
+
 class Catalog:
     """Representation of a message catalog."""
 
@@ -400,6 +408,7 @@ def __init__(
         self.locale = locale
         self._header_comment = header_comment
         self._messages: dict[str | tuple[str, str], Message] = {}
+        self._conflicts: dict[str | tuple[str, str], list[ConflictInfo]] = {}
 
         self.project = project or 'PROJECT'
         self.version = version or 'VERSION'
@@ -780,6 +789,19 @@ def __setitem__(self, id: _MessageID, message: Message) -> None:
                 )
             self._messages[key] = message
 
+    def add_conflict(self, message: Message, file_name: str, project: str, version: str):
+        key = message.id
+        if key not in self._conflicts:
+            self._conflicts[key] = []
+
+        self._conflicts[key].append({
+            'message': message,
+            'file_name': file_name,
+            'project': project,
+            'version': version,
+        })
+        message.flags |= {'fuzzy'}
+
     def add(
         self,
         id: _MessageID,

babel/messages/frontend.py

@@ -21,15 +21,15 @@
 import sys
 import tempfile
 import warnings
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
 from configparser import RawConfigParser
 from io import StringIO
 from typing import Any, BinaryIO, Iterable, Literal
 
 from babel import Locale, localedata
 from babel import __version__ as VERSION
 from babel.core import UnknownLocaleError
-from babel.messages.catalog import DEFAULT_HEADER, Catalog
+from babel.messages.catalog import DEFAULT_HEADER, Catalog, ConflictInfo
 from babel.messages.extract import (
     DEFAULT_KEYWORDS,
     DEFAULT_MAPPING,
@@ -960,7 +960,7 @@ def initialize_options(self):
         self.stringtable_input = None
         self.to_code = None
         # the first translation is always used temporarily
-        self.use_first = True #~
+        self.use_first = False #~
         self.lang = None
         self.color = None
         self.style = None
@@ -1000,36 +1000,49 @@ def finalize_options(self):
             self.less_than = 2
 
     def _prepare(self):
-        self.message_count = defaultdict(int)
+        templates: list[tuple[str, Catalog]] = []
+        message_info = {}
 
         for filename in self.input_files:
             with open(filename, 'r') as pofile:
                 template = read_po(pofile)
             for message in template:
-                self.message_count[message.id] += 1
+                if message.id not in message_info:
+                    message_info[message.id] = {
+                        'count': 0,
+                        'strings': set(),
+                    }
+                message_info[message.id]['count'] += 1
+                message_info[message.id]['strings'].add(message.string if isinstance(message.string, str) else tuple(message.string))
+            templates.append((filename, template, ))
+
+        return templates, message_info
 
     def run(self):
         catalog = Catalog(fuzzy=False)
-        self._prepare()
+        templates, message_info = self._prepare()
 
-        for filename in self.input_files:
-            with open(filename, 'r') as pofile:
-                template = read_po(pofile)
-                if catalog.locale is None:
-                    catalog.locale = template.locale
+        for path, template in templates:
+            if catalog.locale is None:
+                catalog.locale = template.locale
 
-                for message in template:
-                    if not message.id:
-                        continue
+            for message in template:
+                if not message.id:
+                    continue
+
+                count = message_info[message.id]['count']
+                diff_string_count = len(message_info[message.id]['strings'])
+                if count <= self.more_than or (self.less_than is not None and count >= self.less_than):
+                    continue
 
-                    if message.id in catalog and catalog[message.id].string != message.string and not self.use_first:
-                        raise NotImplementedError()
+                if count > 1 and not self.use_first and diff_string_count > 1:
+                    file_name = os.path.basename(path)
+                    catalog.add_conflict(message, file_name, template.project, template.version)
 
-                    message_count = self.message_count[message.id]
-                    if message_count > self.more_than and (self.less_than is None or message_count < self.less_than):
-                        catalog[message.id] = message
+                catalog[message.id] = message
 
         catalog.fuzzy = any(message.fuzzy for message in catalog)
+
         with open(self.output_file, 'wb') as outfile:
             write_po(
                 outfile,

babel/messages/pofile.py

@@ -17,8 +17,8 @@
 from typing import TYPE_CHECKING, Literal
 
 from babel.core import Locale
-from babel.messages.catalog import Catalog, Message
-from babel.util import TextWrapper
+from babel.messages.catalog import Catalog, Message, ConflictInfo
+from babel.util import TextWrapper, _cmp
 
 if TYPE_CHECKING:
     from typing import IO, AnyStr
@@ -351,8 +351,11 @@ def parse(self, fileobj: IO[AnyStr] | Iterable[AnyStr]) -> None:
                 continue
             if needs_decode:
                 line = line.decode(self.catalog.charset)
-            if line[0] == '#':
-                if line[:2] == '#~':
+            if line.startswith('#'):
+                if line[1:].startswith('-'):
+                    self._invalid_pofile(line, lineno, 'cannot parse po file with conflicts')
+
+                if line[1:].startswith('~'):
                     self._process_message_line(lineno, line[2:].lstrip(), obsolete=True)
                 else:
                     try:
@@ -646,6 +649,37 @@ def _format_comment(comment, prefix=''):
         for line in comment_wrapper.wrap(comment):
             yield f"#{prefix} {line.strip()}\n"
 
+    def _format_conflict_comment(file, project, version, prefix=''):
+        comment = f"#-#-#-#-#  {file} ({project} {version})  #-#-#-#-#"
+        yield f"{normalize(comment, prefix=prefix, width=width)}\n"
+
+    def _format_conflict(key: str | tuple[str, str], conflicts: list[ConflictInfo], prefix=''):
+        for conflict in conflicts:
+            message = conflict['message']
+            if message.context:
+                yield from _format_conflict_comment(conflict['file_name'], conflict['project'], conflict['version'], prefix=prefix)
+                yield f"{prefix}msgctxt {normalize(message.context, prefix=prefix, width=width)}\n"
+
+        if isinstance(key, (list, tuple)):
+            yield f"{prefix}msgid {normalize(key[0], prefix=prefix, width=width)}\n"
+            yield f"{prefix}msgid_plural {normalize(key[1], prefix=prefix, width=width)}\n"
+        else:
+            yield f"{prefix}msgid {normalize(key, prefix=prefix, width=width)}\n"
+        yield f"{prefix}msgstr {normalize('', prefix=prefix, width=width)}\n"
+
+        for conflict in conflicts:
+            message = conflict['message']
+            yield from _format_conflict_comment(conflict['file_name'], conflict['project'], conflict['version'], prefix=prefix)
+            if isinstance(key, (list, tuple)):
+                for idx in range(catalog.num_plurals):
+                    try:
+                        string = message.string[idx]
+                    except IndexError:
+                        string = ''
+                    yield f"{prefix}msgstr[{idx:d}] {normalize(string, prefix=prefix, width=width)}\n"
+            else:
+                yield f"{normalize(message.string, prefix=prefix, width=width)}\n"
+
     def _format_message(message, prefix=''):
         if isinstance(message.id, (list, tuple)):
             if message.context:
@@ -717,7 +751,10 @@ def _format_message(message, prefix=''):
                 norm_previous_id = normalize(message.previous_id[1], width=width)
                 yield from _format_comment(f'msgid_plural {norm_previous_id}', prefix='|')
 
-        yield from _format_message(message)
+        if len(conflicts := catalog._conflicts.get(message.id, [])) > 0:
+            yield from _format_conflict(message.id, conflicts)
+        else:
+            yield from _format_message(message)
         yield '\n'
 
     if not ignore_obsolete: