Marking conflicting messages

* Update _prepare function in ConcatenateCatalog to check conflicting messages and to not parse po-files twice

* Add _conflicts field in Catalog to mark conflicts

* Update tests

Author: soft-suroleb

babel/messages/catalog.py

@@ -11,13 +11,14 @@
 
 import datetime
 import re
+import os
 from collections.abc import Iterable, Iterator
 from copy import copy
 from difflib import SequenceMatcher
 from email import message_from_string
 from heapq import nlargest
 from string import Formatter
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, TypedDict
 
 from babel import __version__ as VERSION
 from babel.core import Locale, UnknownLocaleError
@@ -338,6 +339,13 @@ def _force_text(s: str | bytes, encoding: str = 'utf-8', errors: str = 'strict')
     return str(s)
 
 
+class ConflictInfo(TypedDict):
+    message: Message
+    file_name: str
+    project: str
+    version: str
+
+
 class Catalog:
     """Representation of a message catalog."""
 
@@ -381,6 +389,7 @@ def __init__(
         self.locale = locale
         self._header_comment = header_comment
         self._messages: dict[str | tuple[str, str], Message] = {}
+        self._conflicts: dict[str | tuple[str, str], list[ConflictInfo]] = {}
 
         self.project = project or 'PROJECT'
         self.version = version or 'VERSION'
@@ -747,6 +756,19 @@ def __setitem__(self, id: _MessageID, message: Message) -> None:
                     f"Expected sequence but got {type(message.string)}"
             self._messages[key] = message
 
+    def add_conflict(self, message: Message, file_name: str, project: str, version: str):
+        key = message.id
+        if key not in self._conflicts:
+            self._conflicts[key] = []
+
+        self._conflicts[key].append({
+            'message': message,
+            'file_name': file_name,
+            'project': project,
+            'version': version,
+        })
+        message.flags |= {'fuzzy'}
+
     def add(
         self,
         id: _MessageID,

babel/messages/frontend.py

@@ -20,15 +20,15 @@
 import sys
 import tempfile
 import warnings
-from collections import OrderedDict, defaultdict
+from collections import defaultdict
 from configparser import RawConfigParser
 from io import StringIO
 from typing import BinaryIO, Iterable, Literal
 
 from babel import Locale, localedata
 from babel import __version__ as VERSION
 from babel.core import UnknownLocaleError
-from babel.messages.catalog import DEFAULT_HEADER, Catalog
+from babel.messages.catalog import DEFAULT_HEADER, Catalog, ConflictInfo
 from babel.messages.extract import (
     DEFAULT_KEYWORDS,
     DEFAULT_MAPPING,
@@ -925,7 +925,7 @@ def initialize_options(self):
         self.stringtable_input = None
         self.to_code = None
         # the first translation is always used temporarily
-        self.use_first = True #~
+        self.use_first = False #~
         self.lang = None
         self.color = None
         self.style = None
@@ -965,36 +965,49 @@ def finalize_options(self):
             self.less_than = 2
 
     def _prepare(self):
-        self.message_count = defaultdict(int)
+        templates: list[tuple[str, Catalog]] = []
+        message_info = {}
 
         for filename in self.input_files:
             with open(filename, 'r') as pofile:
                 template = read_po(pofile)
             for message in template:
-                self.message_count[message.id] += 1
+                if message.id not in message_info:
+                    message_info[message.id] = {
+                        'count': 0,
+                        'strings': set(),
+                    }
+                message_info[message.id]['count'] += 1
+                message_info[message.id]['strings'].add(message.string if isinstance(message.string, str) else tuple(message.string))
+            templates.append((filename, template, ))
+
+        return templates, message_info
 
     def run(self):
         catalog = Catalog(fuzzy=False)
-        self._prepare()
+        templates, message_info = self._prepare()
 
-        for filename in self.input_files:
-            with open(filename, 'r') as pofile:
-                template = read_po(pofile)
-                if catalog.locale is None:
-                    catalog.locale = template.locale
+        for path, template in templates:
+            if catalog.locale is None:
+                catalog.locale = template.locale
 
-                for message in template:
-                    if not message.id:
-                        continue
+            for message in template:
+                if not message.id:
+                    continue
+
+                count = message_info[message.id]['count']
+                diff_string_count = len(message_info[message.id]['strings'])
+                if count <= self.more_than or (self.less_than is not None and count >= self.less_than):
+                    continue
 
-                    if message.id in catalog and catalog[message.id].string != message.string and not self.use_first:
-                        raise NotImplementedError()
+                if count > 1 and not self.use_first and diff_string_count > 1:
+                    file_name = os.path.basename(path)
+                    catalog.add_conflict(message, file_name, template.project, template.version)
 
-                    message_count = self.message_count[message.id]
-                    if message_count > self.more_than and (self.less_than is None or message_count < self.less_than):
-                        catalog[message.id] = message
+                catalog[message.id] = message
 
         catalog.fuzzy = any(message.fuzzy for message in catalog)
+
         with open(self.output_file, 'wb') as outfile:
             write_po(
                 outfile,

babel/messages/pofile.py

@@ -16,7 +16,7 @@
 from typing import TYPE_CHECKING, Literal
 
 from babel.core import Locale
-from babel.messages.catalog import Catalog, Message
+from babel.messages.catalog import Catalog, Message, ConflictInfo
 from babel.util import TextWrapper, _cmp
 
 if TYPE_CHECKING:
@@ -349,6 +349,9 @@ def parse(self, fileobj: IO[AnyStr] | Iterable[AnyStr]) -> None:
             if not line:
                 continue
             if line.startswith('#'):
+                if line[1:].startswith('-'):
+                    self._invalid_pofile(line, lineno, 'cannot parse po file with conflicts')
+
                 if line[1:].startswith('~'):
                     self._process_message_line(lineno, line[2:].lstrip(), obsolete=True)
                 else:
@@ -642,6 +645,37 @@ def _format_comment(comment, prefix=''):
         for line in comment_wrapper.wrap(comment):
             yield f"#{prefix} {line.strip()}\n"
 
+    def _format_conflict_comment(file, project, version, prefix=''):
+        comment = f"#-#-#-#-#  {file} ({project} {version})  #-#-#-#-#"
+        yield f"{normalize(comment, prefix=prefix, width=width)}\n"
+
+    def _format_conflict(key: str | tuple[str, str], conflicts: list[ConflictInfo], prefix=''):
+        for conflict in conflicts:
+            message = conflict['message']
+            if message.context:
+                yield from _format_conflict_comment(conflict['file_name'], conflict['project'], conflict['version'], prefix=prefix)
+                yield f"{prefix}msgctxt {normalize(message.context, prefix=prefix, width=width)}\n"
+
+        if isinstance(key, (list, tuple)):
+            yield f"{prefix}msgid {normalize(key[0], prefix=prefix, width=width)}\n"
+            yield f"{prefix}msgid_plural {normalize(key[1], prefix=prefix, width=width)}\n"
+        else:
+            yield f"{prefix}msgid {normalize(key, prefix=prefix, width=width)}\n"
+        yield f"{prefix}msgstr {normalize('', prefix=prefix, width=width)}\n"
+
+        for conflict in conflicts:
+            message = conflict['message']
+            yield from _format_conflict_comment(conflict['file_name'], conflict['project'], conflict['version'], prefix=prefix)
+            if isinstance(key, (list, tuple)):
+                for idx in range(catalog.num_plurals):
+                    try:
+                        string = message.string[idx]
+                    except IndexError:
+                        string = ''
+                    yield f"{prefix}msgstr[{idx:d}] {normalize(string, prefix=prefix, width=width)}\n"
+            else:
+                yield f"{normalize(message.string, prefix=prefix, width=width)}\n"
+
     def _format_message(message, prefix=''):
         if isinstance(message.id, (list, tuple)):
             if message.context:
@@ -711,7 +745,10 @@ def _format_message(message, prefix=''):
                 norm_previous_id = normalize(message.previous_id[1], width=width)
                 yield from _format_comment(f'msgid_plural {norm_previous_id}', prefix='|')
 
-        yield from _format_message(message)
+        if len(conflicts := catalog._conflicts.get(message.id, [])) > 0:
+            yield from _format_conflict(message.id, conflicts)
+        else:
+            yield from _format_message(message)
         yield '\n'
 
     if not ignore_obsolete:

tests/messages/test_frontend.py

@@ -715,14 +715,14 @@ def test_supports_width(self):
         assert expected_content == actual_content
 
 
-class ConcatanationCatalogTestCase(unittest.TestCase):
+class ConcatanateCatalogTestCase(unittest.TestCase):
 
     def setUp(self):
         self.olddir = os.getcwd()
         os.chdir(data_dir)
 
         self.dist = Distribution(TEST_PROJECT_DISTRIBUTION_DATA)
-        self.cmd = frontend.ConcatenationCatalog(self.dist)
+        self.cmd = frontend.ConcatenateCatalog(self.dist)
         self.cmd.initialize_options()
 
         self.temp1 = f'{i18n_dir}/msgcat_temp1.po'
@@ -752,13 +752,13 @@ def tearDown(self):
             if os.path.isfile(file):
                     os.unlink(file)
 
-    def _get_expected(self, messages):
+    def _get_expected(self, messages, fuzzy=False):
         date = format_datetime(datetime(1994, 11, 11, 00, 00), 'yyyy-MM-dd HH:mmZ', tzinfo=LOCALTZ, locale='en')
         return fr"""# Translations template for PROJECT.
 # Copyright (C) 1994 ORGANIZATION
 # This file is distributed under the same license as the PROJECT project.
 # FIRST AUTHOR <EMAIL@ADDRESS>, 1994.
-#
+#{'\n#, fuzzy' if fuzzy else ''}
 msgid ""
 msgstr ""
 "Project-Id-Version: PROJECT VERSION\n"
@@ -805,6 +805,64 @@ def test_default(self):
 msgid "same"
 msgstr "Same"
 
+#: hard.py:1000 simple.py:1000
+#, flag2, flag3, fuzzy
+msgid "almost_same"
+msgstr ""
+"#-#-#-#-#  msgcat_temp1.po (PROJECT VERSION)  #-#-#-#-#"
+"Almost same"
+"#-#-#-#-#  msgcat_temp2.po (PROJECT VERSION)  #-#-#-#-#"
+"A bit same"
+
+#: hard.py:2000 simple.py:2000
+#, fuzzy
+msgid "plural"
+msgid_plural "plurals"
+msgstr ""
+"#-#-#-#-#  msgcat_temp1.po (PROJECT VERSION)  #-#-#-#-#"
+msgstr[0] "Plural"
+msgstr[1] "Plurals"
+"#-#-#-#-#  msgcat_temp2.po (PROJECT VERSION)  #-#-#-#-#"
+msgstr[0] "Plural"
+msgstr[1] "Plurals other"
+
+#: hard.py:1
+msgid "other3"
+msgstr "Other 3"
+
+#: hard.py:10
+msgid "other4"
+msgstr "Other 4"
+
+""", fuzzy=True)
+
+        with open(self.output_file, 'r') as f:
+            actual_content = f.read()
+        assert expected_content == actual_content
+
+    @freeze_time("1994-11-11")
+    def test_use_first(self):
+        self.cmd.input_files = [self.temp1, self.temp2]
+        self.cmd.output_file = self.output_file
+        self.cmd.use_first = True
+
+        self.cmd.finalize_options()
+        self.cmd.run()
+
+        expected_content = self._get_expected(fr"""#: simple.py:1
+#, flag1000
+msgid "other1"
+msgstr "Other 1"
+
+#: simple.py:10
+msgid "other2"
+msgstr "Other 2"
+
+#: hard.py:100 simple.py:100
+#, flag1, flag1.2, flag4
+msgid "same"
+msgstr "Same"
+
 #: hard.py:1000 simple.py:1000
 #, flag2, flag3
 msgid "almost_same"
@@ -885,17 +943,27 @@ def test_more_than(self):
 msgstr "Same"
 
 #: hard.py:1000 simple.py:1000
-#, flag2, flag3
+#, flag2, flag3, fuzzy
 msgid "almost_same"
-msgstr "Almost same"
+msgstr ""
+"#-#-#-#-#  msgcat_temp1.po (PROJECT VERSION)  #-#-#-#-#"
+"Almost same"
+"#-#-#-#-#  msgcat_temp2.po (PROJECT VERSION)  #-#-#-#-#"
+"A bit same"
 
 #: hard.py:2000 simple.py:2000
+#, fuzzy
 msgid "plural"
 msgid_plural "plurals"
+msgstr ""
+"#-#-#-#-#  msgcat_temp1.po (PROJECT VERSION)  #-#-#-#-#"
 msgstr[0] "Plural"
 msgstr[1] "Plurals"
+"#-#-#-#-#  msgcat_temp2.po (PROJECT VERSION)  #-#-#-#-#"
+msgstr[0] "Plural"
+msgstr[1] "Plurals other"
 
-""")
+""", fuzzy=True)
 
         with open(self.output_file, 'r') as f:
             actual_content = f.read()
@@ -1029,7 +1097,7 @@ def test_default(self):
     def test_compenidum(self):
         self.cmd.input_files = [self.temp_def, self.temp_ref]
         self.cmd.output_file = self.output_file
-        self.cmd.compendium = self.compendium
+        self.cmd.compendium = [self.compendium,]
         self.cmd.no_fuzzy_matching = True
         self.cmd.no_compendium_comment = True
         self.cmd.finalize_options()
@@ -1057,7 +1125,7 @@ def test_compenidum(self):
     def test_compenidum_overwrite(self):
         self.cmd.input_files = [self.temp_def, self.temp_ref]
         self.cmd.output_file = self.output_file
-        self.cmd.compendium = self.compendium
+        self.cmd.compendium = [self.compendium,]
         self.cmd.no_fuzzy_matching = True
         self.cmd.no_compendium_comment = True
         self.cmd.compendium_overwrite = True