Merge pull request #419 from derek73/fix/413-harvest-backticked-names

fix(differential): harvest names written in backticks

Author: derek73

docs/release_log.rst

@@ -13,11 +13,14 @@ Release Log
     Nothing moved between vocabularies. The rename itself changes no
     parse at all; the parsing changes below are separate fixes, and
     only one of them reaches the default name order -- a title no
-    longer changing how the name behind it is read, which moves one of
-    the 751 names of the differential corpora. Both family-first
-    orders change too, below -- the same eight names under each, seven
-    of them from the family-first fix and the eighth the one name the
-    title fix moves in every order.
+    longer changing how the name behind it is read, which moves four
+    names of the differential corpora. Both family-first orders change too,
+    below -- seven of those from the family-first fix, plus the names
+    the title fix moves in every order. The corpora have roughly
+    doubled across this cycle (#414 added the rules doc's examples,
+    #413 taught the issue harvester to read backticks), so the counts
+    in the bullets below are the classified summary at the time each
+    was written; the gate output is the source of record.
     What breaks is code that *writes* to a default word list, and code
     that imports one by its 1.x name has until 3.0.
 

tests/v2/test_differential.py

@@ -1057,3 +1057,67 @@ def test_validate_rules_rejects_two_rules_sharing_an_issue() -> None:
             [{"issue": "dup", "name_regex": "Smith", "fields": ["given"]},
              {"issue": "dup", "name_regex": "Jones", "fields": ["given"]}],
             "expected_since_1.4.0.toml")
+
+
+harvester = load_tool("build_issues_corpus")
+
+
+def test_a_backticked_name_is_harvested() -> None:
+    """This tracker writes names in backticks, because issues are
+    markdown. Matching only quoted names left the corpus whose whole
+    purpose is "what users reported" blind to how this project reports
+    (#413).
+    """
+    assert harvester._harvest(
+        "the tussenvoegsel in `Beethoven, Ludwig van` is filed behind "
+        "the given name") == {"Beethoven, Ludwig van"}
+
+
+def test_a_call_inside_backticks_yields_the_name_not_the_call() -> None:
+    """The reason backticks are a SECOND PASS rather than a third
+    alternative in one pattern.
+
+    A reporter writes the failing input as a call inside a code span.
+    Regex scanning takes the leftmost match, so a backtick alternative
+    starts one character before the HumanName branch can and swallows
+    the whole span -- harvesting the call and losing the name inside
+    it. This is the bug the two-pass structure exists to prevent, and
+    it is reachable from a single string.
+    """
+    got = harvester._harvest('reported as `HumanName("John  Q  Doe")`')
+    assert got == {"John  Q  Doe"}
+
+    # ... and a call anywhere in the span, not only at its start
+    assert not any("HumanName(" in n for n in
+                   harvester._harvest('see `Foo HumanName("Jane Roe")`'))
+
+
+def test_a_sentence_is_not_a_name() -> None:
+    """Prose is quoted and backticked as often as names are, and a
+    capitalized phrase is well-formed enough that the character screen
+    cannot see it.
+    """
+    for text in ("`What this gate does not cover` is the section",
+                 "the note said `Takes everything` about it",
+                 "`C is CONSTANTS` in that example",
+                 "wrote `NO Latin twins on purpose` there",
+                 "raised `TypeError: a bytes-like object is required`"):
+        assert harvester._harvest(text) == set(), text
+
+
+def test_the_sentence_screen_keeps_its_hands_off_names() -> None:
+    """The screen matches a function word only where it is written
+    LOWERCASE, which is what keeps it off names: a name capitalizes its
+    words and a sentence's function words do not.
+
+    Every name here contains a word that is in the stop list, and every
+    one is a real naming convention -- Burmese, Norwegian, Estonian,
+    Cantonese, Luhya, Dutch, Lakota. Without the case test the screen
+    drops all of them, silently, from the one corpus meant to catch
+    what users report.
+    """
+    for name in ("U Than Shwe", "Sven Are Olsen", "Kertu Must",
+                 "Chan On Kei", "Wafula Were", "John Does",
+                 "Marie Takes War Bonnet", "Rob And Beth Edmunds",
+                 "Duke of Wellington", "Anh Do", "Will De Groot"):
+        assert harvester._harvest(f"reported `{name}` today") == {name}, name

tests/v2/test_ledger_guards.py

@@ -189,7 +189,7 @@ that stops being run. Pass `--corpus PATH` (repeatable) to narrow it.
 They are deliberately separate rather than merged: `corpus.jsonl` is
 reproducible forever from an immutable git ref, while the issue
 tracker is mutable, so regenerating `corpus_issues.jsonl` is an
-explicit, reviewable act that can only add names — and the CJK corpus
+explicit, reviewable act that can only add names while the harvester is unchanged (changing its screens, as #413 did, can remove one) — and the CJK corpus
 exists because BOTH of those are structurally blind to unspaced CJK (v1's
 banks never tested it; `build_issues_corpus.py` requires an internal
 space, which unspaced names never have). It regenerates from the case
@@ -211,6 +211,29 @@ intended-but-unclassified 2.0 behaviors (#273 typographic delimiters,
 #269 non-Latin vocabulary) plus one shape no test had considered: a
 **leading** `"Ph. D."`, which v1 split into title `Ph.` + given `D.`.
 
+It earned it again at #413. The harvester matched names in quotes
+only, while this tracker writes them in backticks, so the corpus whose
+purpose is "what users reported" was blind to how this project
+reports — 200 names became 381. Among the arrivals were the headline
+example of nearly every issue that fixed them, including
+`Ursula von der Leyen geb. Albrecht`, which sat in #399's own title
+while that issue shipped noting the class was invisible to this
+harness. Classifying the arrivals widened `fix(#367)`, which had
+asked in writing to be widened, added six rules, gave
+`fix(leading-credential)` the `family` role its own shape always moved,
+and narrowed the trailing-`Ph. D.` exclusion as its comment instructed
+the moment a `<suffix> Ph. D.` name appeared.
+
+Backticks prompted a second look at prose, and the same change added
+two screens neither branch had: `:` joins the structural characters (it
+appears in no name across all four corpora, and accounts for three
+error messages and a PyPI trove classifier), and a short list of English function words
+rejects capitalized sentences the character screen cannot see —
+`What this gate does not cover` is well-formed as a phrase. That list
+is narrow on purpose: `and`, `the`, `of`, `will`, `can` and `do` all
+stay out of it, because `Rob And Beth Edmunds`, `The Hon`, `Duke of
+Wellington` and `Anh Do` are real corpus names.
+
 The rules corpus earned its place the same way — 113 of its 155 names
 were in no other corpus, and on its first run 12 of them turned out to
 have moved during the 2.2 cycle with nothing observing it. Classifying

tools/differential/README.md

@@ -33,24 +33,110 @@
 import sys
 
 # Either an explicit HumanName("...") call -- the reporter showing the
-# failing input -- or a quoted, capitalized phrase, which is how names
-# appear in prose ("parsing 'Hans "Hansi" Müller' gives ...").
+# failing input -- or a capitalized phrase in quotes or BACKTICKS,
+# which is how names appear in prose ("parsing 'Hans "Hansi" Müller'
+# gives ...", or `Beethoven, Ludwig van` in a markdown issue).
+#
+# The backtick branch was missing until #413, and the cost was not
+# marginal: this tracker writes names in backticks because issues are
+# markdown, so the corpus whose whole purpose is "what users reported"
+# was blind to the way this project reports. Under the screens this
+# file ships the backtick pass adds 94 names that the quoted branch
+# never saw, none of them in the previous corpus, and they include the
+# headline example of the issue that fixed them -- `Beethoven, Ludwig
+# van` (#379), `Berg, Jan vd` (#380), `Ursula von der Leyen geb.
+# Albrecht` (#399). That last one sat in its issue's TITLE while #399
+# shipped noting the class was invisible to this harness.
 _CANDIDATE = re.compile(
     r"""HumanName\(\s*["']([^"']{3,60})["']"""
     r"""|["']([A-Z][^"'\n]{4,60})["']"""
 )
+# Backticks are scanned SEPARATELY, not as a third alternative, and the
+# reason is a name this cost before it was split out. A reporter writes
+# the failing input as `HumanName("John  Q  Doe")` -- a call inside a
+# code span. Regex scanning takes the leftmost match, so a backtick
+# alternative starts one character before the HumanName branch can and
+# swallows the whole span, harvesting the CALL and losing the name
+# inside it. Two passes over the same text cannot shadow each other.
+_BACKTICKED = re.compile(r"""`([A-Z][^`\n]{4,60})`""")
+# ... and the same code span is dropped from the backtick pass, or the
+# call itself is harvested as a name. The HumanName branch above has
+# already taken the argument out of it, which is the part anyone meant.
+#
+# Searched, not matched: a call is code wherever it sits in the span,
+# and anchoring at offset 0 let `Foo HumanName("x y")` through with the
+# call still attached.
+_A_CALL = re.compile(r"""[A-Za-z_]\w*\(""")
 # Structural characters that mean we caught code or markup, not a name.
-_NOT_A_NAME = set('{}<>=/\\|')
+#
+# Extended when the backtick branch landed (#413): backticks wrap code
+# far more often than quotes do, so the branch admits code-shaped
+# strings a quoted phrase rarely produces -- 'Constants.__init__(self,
+# **state)', 'DEVIATION #364', 'SUFFIX_ACRONYMS ∩ SUFFIX_NOT_ACRONYMS',
+# 'TypeError: a bytes-like object is required'. None of '*#~;_:'
+# appears in a name; the colon alone accounts for three error messages
+# and a PyPI trove classifier, and none of the six characters occurs in
+# any name across the four corpora. The one existing entry this drops,
+# 'St. ___', is a placeholder, not a name.
+_NOT_A_NAME = set('{}<>=/\\|*#~;_:')
+
+#: Words that make a string a SENTENCE rather than a name. The
+#: character screen above cannot reach prose that is merely prose --
+#: 'What this gate does not cover', 'Takes everything', 'C is
+#: CONSTANTS' are all well-formed capitalized phrases.
+#:
+#: NOT a backtick problem, though the backtick branch is what prompted
+#: looking: five of the six sentences this drops come from the QUOTED
+#: branch, which has been running since the harvester was written.
+#: They are new to the corpus because the tracker grew, not because of
+#: the new pass -- re-harvesting with backticks disabled still adds 88
+#: names over the previous snapshot. Scoping this screen to the
+#: backtick pass would look like a tightening and would silently
+#: readmit five of them.
+#:
+#: Matched only against a word written in LOWERCASE, which is what
+#: keeps the screen off names. A name capitalizes its words and a
+#: sentence's function words do not, so 'Chan On Kei', 'U Than Shwe',
+#: 'Sven Are Olsen', 'Kertu Must', 'Wafula Were' and 'John Does' all
+#: survive while 'NO Latin twins on purpose' does not. Without the
+#: case test this list silently drops real names, and the corpus it
+#: drops them from is the one whose whole purpose is catching what
+#: users report.
+#:
+#: The exclusions are the load-bearing part, and they divide into two
+#: kinds. 'and', 'the', 'of', 'do' and 'will' are excluded on corpus
+#: evidence -- 'Rob And Beth Edmunds', 'The Hon', 'Duke of Wellington',
+#: 'Anh Do' and 'Will De Groot' are all in the corpora. 'can' and 'no'
+#: are excluded on no corpus evidence at all: they are Turkish Can and
+#: Korean No, names this harvest has not reached yet. The distinction
+#: matters because the corpus cannot supply evidence for a word any
+#: screen already rejects, so absence from it is not a licence. A name is allowed to
+#: be odd -- 'der, y van' is real -- it is not allowed to be a
+#: sentence.
+_PROSE = frozenset({
+    "is", "are", "was", "were", "be", "been", "this", "that", "these",
+    "those", "what", "which", "how", "why", "when", "not", "does",
+    "did", "without", "everything", "takes", "reads", "sure", "about",
+    "because", "than", "then", "into", "over", "under", "should",
+    "would", "could", "must", "on",
+})
 
 
 def _harvest(text: str) -> set[str]:
     found = set()
-    for match in _CANDIDATE.finditer(text):
-        value = (match.group(1) or match.group(2) or "").strip()
+    values = [(m.group(1) or m.group(2) or "").strip()
+              for m in _CANDIDATE.finditer(text)]
+    values += [m.group(1).strip() for m in _BACKTICKED.finditer(text)
+               if not _A_CALL.search(m.group(1))]
+    for value in values:
         # Require an internal space: a single word is a surname at best
         # and carries no structure to disagree about.
-        if value and " " in value and not (_NOT_A_NAME & set(value)):
-            found.add(value)
+        if not value or " " not in value or (_NOT_A_NAME & set(value)):
+            continue
+        if any(s and s == s.lower() and s in _PROSE
+               for s in (w.strip(".,'\"") for w in value.split())):
+            continue
+        found.add(value)
     return found
 
 

tools/differential/build_issues_corpus.py

@@ -357,7 +357,7 @@ def _is_latin_only(name: str) -> bool:
 _CORPUS_FLOORS = {
     "corpus.jsonl": 480,        # 486 today, from v1's banks at a pinned ref
     "corpus_cjk.jsonl": 95,     # 97 today, generated from the case table
-    "corpus_issues.jsonl": 190,  # 200 today, harvested and append-only
+    "corpus_issues.jsonl": 370,  # 381 today, harvested and append-only
     "corpus_rules.jsonl": 150,  # 155 today, generated from rules.md
 }