From 260b93ba1a3e10b18c2e2c7189836009fb758bb9 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:22:51 -0800 Subject: [PATCH 1/9] Record the choice of languages on the Lexicon (#754, 4i-c) Lexicon gains _languages/_strict (read as languages/strict, with Parser.languages/strict reading through); a choice filters the base's readings and the caller's log applies after it. parser_for unions a base's choice with its packs', joins the packs' fragments under the caller's lexicon, and refuses a clash naming the edit. _apply_choice and its delta retire. Co-Authored-By: Claude Opus 5.5 --- nameparser/_lexicon.py | 315 +++++++++++++++++++++++++++++++-------- nameparser/_locale.py | 31 +--- nameparser/_parser.py | 176 ++++++++-------------- tests/v2/test_lexicon.py | 79 +++++++++- tests/v2/test_parser.py | 289 +++++++++++++++++++++++------------ 5 files changed, 594 insertions(+), 296 deletions(-) diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 1a2ed53c..48cf6cdd 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -527,18 +527,6 @@ def _normpairs( _KIND_FIELDS = ("generation_words", "credential_words", "patronymic_particles") -#: Each marker set over the base set it qualifies. A compile from -#: readings holds every pair by construction, the marker existing only -#: beside its base reading; `_parser._apply_choice` reads the pairs to -#: drop a word's marks where a choice drops it from the base. -_MARKER_BASES = ( - ("particles_ambiguous", "particles"), - ("suffix_acronyms_ambiguous", "suffix_acronyms"), - ("honorific_tails", "suffix_words"), - ("suffix_words_ambiguous", "suffix_words"), -) - - @dataclass(frozen=True, slots=True, eq=False, repr=False) class _LexiconSets: """The vocabulary sets a parse reads: the pipeline, `_render`, @@ -933,16 +921,54 @@ def _check_pool(pool: Pool, words: Iterable[str]) -> None: f".remove({w!r}, kind='bound_given')") +@functools.lru_cache(maxsize=32) +def _chosen_pool(base: object, chosen: frozenset[str], strict: bool) -> Pool: + """A base's readings under the choice of the file tags `chosen`, + before the caller's edits (decisions/vocabulary-model, 2026-11-02 + #754, item 4): `choose` over the base's readings, under `strict` + only the chosen files' and native-script ones; a reading an + unchosen file leaves off stays in the pool off, so a caller's add + of its kind still turns it on, as it does with no choice.""" + pool = _base_pool(base) + if not chosen: + return pool + ri = _index_module() + readings = [r for rs in pool.values() for r in rs] + if strict: + readings = [r for r in readings + if r.tag in chosen or ri.is_native_script(r.text)] + out: dict[str, list[Any]] = {} + for r in ri.choose(readings, chosen): + out.setdefault(r.text, []).append(r) + for r in readings: + if not r.on_by_default and r.tag not in chosen: + out.setdefault(r.text, []).append(r) + return MappingProxyType({t: tuple(rs) for t, rs in out.items()}) + + @functools.lru_cache(maxsize=128) -def _built(base: object, edits: tuple[_Edit, ...]) -> tuple[ - Pool, Mapping[str, frozenset[str]]]: +def _built(base: object, edits: tuple[_Edit, ...], chosen: frozenset[str], + strict: bool) -> tuple[Pool, Mapping[str, frozenset[str]]]: """A lexicon's readings by word and the sets they compile to, kept - per base and log: a compile costs milliseconds, paid once per - configuration (decisions/vocabulary-model, 2026-11-01 #754).""" - pool = dict(_base_pool(base)) + per base, log and choice: a compile costs milliseconds, paid once + per configuration (decisions/vocabulary-model, 2026-11-01 #754). + The choice filters the base's readings and the log applies after + it, so a caller's reading survives any choice; one that clashes + with a reading the choice turns on is refused, naming the edit + (2026-11-02 #754, item 3).""" + pool = dict(_chosen_pool(base, chosen, strict)) for edit in edits: _apply_edit(pool, edit) - _check_pool(pool, (w for e in edits for w in e.words)) + for w in dict.fromkeys(w for e in edits for w in e.words): + try: + _check_pool(pool, (w,)) + except ValueError as exc: + if not chosen or _clashes_unchosen(base, edits): + raise + mine = " and ".join(_rendered(e for e in edits if w in e.words)) + raise ValueError( + f"the lexicon's {mine} clashes with a reading the choice " + f"turns on: {exc}") from None ri = _index_module() try: compiled = ri.compile_readings( @@ -952,14 +978,95 @@ def _built(base: object, edits: tuple[_Edit, ...]) -> tuple[ return MappingProxyType(pool), MappingProxyType(_vocab_fields(compiled)) -def _compiled_for(base: object, - edits: tuple[_Edit, ...]) -> Mapping[str, frozenset[str]]: +def _clashes_unchosen(base: object, edits: tuple[_Edit, ...]) -> bool: + """Whether `edits` clash on `base` with no choice, so that a clash + under one is the log's own rather than the choice's.""" + try: + _built(base, edits, frozenset(), False) + except ValueError: + return True + return False + + +def _compiled_for(base: object, edits: tuple[_Edit, ...], + chosen: frozenset[str] = frozenset(), + strict: bool = False) -> Mapping[str, frozenset[str]]: if not edits: - if base == "default": - return _default_compiled() - if base == "empty": - return _NO_SETS - return _built(base, edits)[1] + if not chosen: + if base == "default": + return _default_compiled() + if base == "empty": + return _NO_SETS + elif base == "default": + # the shipped index under the choice, compiled once per + # choice by the reader every pack shares + return _choice_compiled(chosen, strict) + return _built(base, edits, chosen, strict)[1] + + +@functools.lru_cache(maxsize=32) +def _choice_compiled(chosen: frozenset[str], + strict: bool) -> Mapping[str, frozenset[str]]: + ri = _index_module() + return MappingProxyType(_vocab_fields(ri.compile_choice(chosen, strict))) + + +def _check_choice(languages: object, strict: object) -> tuple[ + frozenset[str], frozenset[str]]: + """A recorded choice validated as `parser_for` validates one: its + languages in canonical case and the file tags they select, the + selection raising for a tag no file serves (decisions/ + vocabulary-model, 2026-11-02 #754, item 3).""" + if not isinstance(strict, bool): + raise TypeError(f"_strict must be True or False, got {strict!r}") + if (isinstance(languages, (str, bytes, bytearray, Mapping)) + or not isinstance(languages, Iterable)): + raise TypeError(f"_languages must be a set of BCP 47 tags, e.g. " + f"frozenset({{'pt'}}), got {languages!r}") + tags = frozenset(cast("Iterable[object]", languages)) + for tag in tags: + if not isinstance(tag, str): + raise TypeError(f"_languages entries must be str BCP 47 tags, " + f"got {tag!r}") + if not tags: + if strict: + raise ValueError( + "a strict choice keeps only the chosen languages' " + "vocabulary, and _languages chooses none; choose through " + "parser_for(..., strict=True) with a pack that chooses one") + return frozenset(), frozenset() + canon = frozenset(_canonical(cast(str, t)) for t in tags) + return canon, _index_module().selected(canon, strict=strict) + + +def _canonical(tag: str) -> str: + """`tag` in BCP 47's canonical case: the language subtag lowercase, + a four-letter script subtag title case, a two-letter region + uppercase, every other subtag lowercase (RFC 5646, 2.1.1). Equal + choices then compare and hash equal, whatever case they were + written in.""" + first, *rest = tag.split("-") + out = [first.lower()] + for sub in rest: + if len(sub) == 4 and sub.isalpha(): + out.append(sub.title()) + elif len(sub) == 2 and sub.isalpha(): + out.append(sub.upper()) + else: + out.append(sub.lower()) + return "-".join(out) + + +#: The most tags a repr lists (a `Lexicon`'s choice, a `Locale`'s); a +#: longer choice is shown by its size, so the repr stays bounded however +#: many languages are chosen. +_REPR_TAGS = 3 + + +def _shown_choice(languages: frozenset[str]) -> str: + if len(languages) > _REPR_TAGS: + return f"languages: {len(languages)}" + return "languages " + "+".join(sorted(languages)) def _check_base(base: object) -> object: @@ -1014,6 +1121,45 @@ def _check_base(base: object) -> object: return base +@functools.cache +def _choice_masks(chosen: frozenset[str], strict: bool = False) -> tuple[ + frozenset[str], tuple[tuple[str, str], ...]]: + """The case masks the choice of the file tags `chosen` moves against + the default's compile: the ones that leave and the ones set. A mask + is no reading (a `Lexicon` keeps its masks in a field of their own), + so `parser_for` moves them in that field, as 3b's delta did.""" + ri = _index_module() + old = ri.compile_choice(frozenset())["CAPITALIZATION_EXCEPTIONS"] + new = ri.compile_choice(chosen, strict)["CAPITALIZATION_EXCEPTIONS"] + return (frozenset(old.keys() - new.keys()), + tuple(sorted((k, v) for k, v in new.items() if old.get(k) != v))) + + +def _rechosen_masks(lexicon: Lexicon, chosen: frozenset[str], + strict: bool) -> tuple[tuple[str, str], ...]: + """`lexicon`'s masks moved from its own choice's to `chosen`'s: the + base's choice's moves undone, then the new choice's applied, so a + chained choice moves them as the union does.""" + default = _index_module().compile_choice( + frozenset())["CAPITALIZATION_EXCEPTIONS"] + cap = dict(lexicon.capitalization_exceptions) + if lexicon.languages: + gone, joined = _choice_masks(lexicon._chosen, lexicon.strict) + for k in gone: + cap.setdefault(k, default[k]) + for k, v in joined: + if cap.get(k) == v: + if k in default: + cap[k] = default[k] + else: + del cap[k] + gone, joined = _choice_masks(chosen, strict) + for k in gone: + cap.pop(k, None) + cap.update(joined) + return tuple(sorted(cap.items())) + + # -- the 2.x set fields, translated -------------------------------------- #: The fields whose membership puts a word in a category. A marker set @@ -1226,18 +1372,17 @@ def _inert_dropped(sets: Mapping[str, frozenset[str]], def _edits_toward(pool: Pool, current: Mapping[str, frozenset[str]], - target: Mapping[str, frozenset[str]], - hints: Pool | None = None) -> tuple[_Edit, ...]: + target: Mapping[str, frozenset[str]]) -> tuple[_Edit, ...]: """The edits that take the readings of `pool`, which compile to the sets `current`, toward the sets `target` (by `_LexiconSets` name; a set it leaves out stays as `current` has it), word by word: for each word whose memberships differ, a `remove` of each kind it no longer holds, then an `add` of each kind it lacks or whose properties change. The one translation from sets to readings, which - the refusals of a 2.x spelling, the load of a 2.x pickle and - `_parser._apply_choice` share; a best effort, so each caller - compiles the result where it must be exact. `hints` are readings a - word's new kinds are taken from before its held ones (a choice's).""" + the refusals of a 2.x spelling and the load of a 2.x pickle share; a + best effort, so each caller compiles the result where it must be + exact. A new kind is taken from a reading the word holds off where + it holds one.""" ri = _index_module() kinds_known = all(k in target for k in _KIND_FIELDS) fields = _SET_FIELDS + (_KIND_FIELDS if kinds_known else ()) @@ -1260,18 +1405,13 @@ def heads(phrases: frozenset[str], word: str) -> frozenset[str]: for w in sorted(changed): held = pool.get(w, ()) on = _by_kind(r for r in held if r.on_by_default) - everything = _by_kind(held) - # a new kind is taken from the hints first, then from the - # readings the word holds turned off - hinted = _by_kind((hints or {}).get(w, ())) - for k, p in everything.items(): - hinted.setdefault(k, p) + every = _by_kind(held) member = frozenset(f for f in fields if f != "title_phrases" and w in full[f]) new_heads = heads(full["title_phrases"], w) moved = new_heads != heads(current["title_phrases"], w) want = _target_readings(w, member, new_heads if moved else None, - on, hinted, kinds_known) + on, every, kinds_known) dropped = [k for k in on if k not in want] gone_categories = {k for k in dropped if ri.CATEGORY[k] == k} for k in dropped: @@ -1360,7 +1500,8 @@ def _translation(lexicon: Lexicon, changes: Mapping[str, frozenset[str]], try: edits = _edits_toward(lexicon._pool(), current, target) result = _inert_dropped( - _built(lexicon._base, lexicon._edits + edits)[1]) + _built(lexicon._base, lexicon._edits + edits, lexicon._chosen, + lexicon._strict)[1]) exact = all(result[f] == wanted[f] for f in _SET_FIELDS) except (TypeError, ValueError): edits, exact = (), False @@ -1488,7 +1629,8 @@ def distance(compiled: Mapping[str, frozenset[str]]) -> int: return {"surnames": sets["surnames"], "given_names": sets["given_names"], "capitalization_exceptions": state["capitalization_exceptions"], - "_base": base, "_edits": edits} + "_base": base, "_edits": edits, + "_languages": frozenset(), "_strict": False} # -- the public types ---------------------------------------------------- @@ -1577,9 +1719,10 @@ class Lexicon: ``surnames``, ``given_names`` and ``capitalization_exceptions``. State is a base (the shipped vocabulary, none, or a pickled - release's), the ordered log of :meth:`add` and :meth:`remove` calls - applied after it, and the three fields; two lexicons are equal when - all of them are, the log compared call by call as written, so two + release's), the choice of languages :func:`~nameparser.parser_for` + records (:attr:`languages`, :attr:`strict`), the ordered log of + :meth:`add` and :meth:`remove` calls applied after the choice, and + the three fields; two lexicons are equal when all of them are, the log compared call by call as written, so two logs that read alike can still differ (decisions/vocabulary-model, 2026-11-02 #754, item 3).""" @@ -1621,12 +1764,18 @@ class Lexicon: # canonicalizes here; this closes the caller-aliasing hole and keeps # Lexicon hashable. Read via capitalization_exceptions_map. capitalization_exceptions: tuple[tuple[str, str], ...] = () - # The base and the log: constructor parameters under underscore - # names, so that dataclasses.replace and copy.replace carry them, - # and no public API (decisions/vocabulary-model, 2026-11-02 #754, - # item 3). 4i-c adds the choice beside them. + # The base, the log and the choice: constructor parameters under + # underscore names, so that dataclasses.replace and copy.replace + # carry them, and no public API (decisions/vocabulary-model, + # 2026-11-02 #754, item 3); a choice is set publicly by parser_for + # alone, and read through `languages` and `strict`. _base: Any = "empty" _edits: tuple[_Edit, ...] = () + _languages: frozenset[str] = frozenset() + _strict: bool = False + # the file tags the choice selects, derived from it + _chosen: frozenset[str] = field( + init=False, repr=False, compare=False, hash=False) _cap_map: Mapping[str, str] = field( init=False, repr=False, compare=False, hash=False) _sets: _LexiconSets = field( @@ -1640,6 +1789,8 @@ def __init__( | Iterable[tuple[str, str]]) = (), _base: object = "empty", _edits: Iterable[_Edit] = (), + _languages: Iterable[str] = frozenset(), + _strict: bool = False, **removed: object, ) -> None: put = functools.partial(object.__setattr__, self) @@ -1664,7 +1815,11 @@ def __init__( "_edits holds an edit not in canonical form; build a " "Lexicon's log with add() and remove()") put("_edits", canon) - compiled = _compiled_for(self._base, canon) + languages, chosen = _check_choice(_languages, _strict) + put("_languages", languages) + put("_strict", _strict) + put("_chosen", chosen) + compiled = _compiled_for(self._base, canon, chosen, _strict) put("_sets", _LexiconSets( **compiled, surnames=self.surnames, given_names=self.given_names)) if removed: @@ -1700,13 +1855,23 @@ def __repr__(self) -> str: # Bounded: counts readings by kind against the nearer of the two # named lexicons, never naming a word (the design rule in # nameparser._types' module docstring) - if self == Lexicon.default(): - return "Lexicon(default)" - if self == Lexicon.empty(): - return "Lexicon(empty)" + # A choice is shown by its tags, and the readings counted against + # the named lexicon under the same choice + choice = "" + named = [("default", Lexicon.default()), ("empty", Lexicon.empty())] + if self._languages: + choice = ", " + _shown_choice(self._languages) + ( + ", strict" if self._strict else "") + named = [(label, dataclasses.replace( + other, _languages=self._languages, _strict=self._strict, + capitalization_exceptions=_rechosen_masks( + other, self._chosen, self._strict))) + for label, other in named] + for label, other in named: + if self == other: + return f"Lexicon({label}{choice})" candidates = [(label, self._deltas_from(other)) - for label, other in (("default", Lexicon.default()), - ("empty", Lexicon.empty()))] + for label, other in named] label, (kinds, lists) = min( candidates, key=lambda c: sum(a + r for _, a, r in c[1][0] + c[1][1])) @@ -1718,7 +1883,8 @@ def counts(deltas: list[tuple[str, int, int]]) -> str: for name, a, r in deltas) rendered = "; ".join(filter(None, (counts(kinds), counts(lists)))) - return (f"Lexicon({label} + {rendered or f'edits: {len(self._edits)}'})") + return (f"Lexicon({label} + {rendered or f'edits: {len(self._edits)}'}" + f"{choice})") def __getstate__(self) -> dict[str, object]: # the base as a snapshot of its rows, so a pickle parses as it @@ -1730,7 +1896,8 @@ def __getstate__(self) -> dict[str, object]: base = READINGS return {"surnames": self.surnames, "given_names": self.given_names, "capitalization_exceptions": self.capitalization_exceptions, - "_base": base, "_edits": tuple(tuple(e) for e in self._edits)} + "_base": base, "_edits": tuple(tuple(e) for e in self._edits), + "_languages": self._languages, "_strict": self._strict} def __setstate__(self, state: dict[str, object]) -> None: # A 2.x state holds the compiled sets, translated onto a base @@ -1791,9 +1958,18 @@ def __or__(self, other: Lexicon) -> Lexicon: right-biased, and the logs concatenated, this one's first, so a later lexicon's edits apply after an earlier one's (decisions/ vocabulary-model, 2026-11-02 #754, item 3); the base is the one - that is not empty, and two different ones raise.""" + that is not empty and the choice the one that is made, and two + different bases, or two different choices, raise.""" if not isinstance(other, Lexicon): return NotImplemented + if (self._languages and other._languages + and (self._languages, self._strict) + != (other._languages, other._strict)): + raise ValueError( + "cannot join two lexicons compiled under different choices " + "of languages; combine choices with parser_for, which " + "unions them: parser_for(*packs, base=parser)") + mine = self if self._languages else other if "empty" in (self._base, other._base) or self._base == other._base: base = other._base if self._base == "empty" else self._base else: @@ -1806,7 +1982,8 @@ def __or__(self, other: Lexicon) -> Lexicon: surnames=self.surnames | other.surnames, given_names=self.given_names | other.given_names, capitalization_exceptions=tuple(sorted(merged.items())), - _base=base, _edits=self._edits + other._edits) + _base=base, _edits=self._edits + other._edits, + _languages=mine._languages, _strict=mine._strict) # -- properties ------------------------------------------------------- @@ -1814,6 +1991,23 @@ def __or__(self, other: Lexicon) -> Lexicon: def capitalization_exceptions_map(self) -> Mapping[str, str]: return self._cap_map + @property + def languages(self) -> frozenset[str]: + """The languages this lexicon's readings are chosen under, as + BCP 47 tags; empty where none was chosen. Set by + :func:`~nameparser.parser_for`, which unions a base's choice + with its packs' (decisions/vocabulary-model, 2026-11-02 #754, + item 4): the shipped readings are filtered by the choice, and + the lexicon's own :meth:`add` and :meth:`remove` calls apply + after it.""" + return self._languages + + @property + def strict(self) -> bool: + """Whether the choice is strict: only the chosen languages' + vocabulary files and native-script vocabulary are read.""" + return self._strict + # -- access ----------------------------------------------------------- def readings_of(self, word: str) -> tuple[LexiconReading, ...]: @@ -1917,8 +2111,9 @@ def remove(self, *words: str, kind: str | None = None, def _pool(self) -> Pool: if not self._edits: - return _base_pool(self._base) - return _built(self._base, self._edits)[0] + return _chosen_pool(self._base, self._chosen, self._strict) + return _built(self._base, self._edits, self._chosen, + self._strict)[0] def _changed(self, op: str, entries: Mapping[str, object]) -> dict[str, object]: diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 0dabccbf..5c11d787 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -21,7 +21,7 @@ from dataclasses import dataclass, field from typing import Any -from nameparser._lexicon import Lexicon +from nameparser._lexicon import Lexicon, _canonical, _shown_choice from nameparser._policy import ( _ORDER_CONSTANT_NAMES, UNSET, PatronymicRule, PolicyPatch, _reject_str_and_mapping, @@ -107,37 +107,12 @@ def __repr__(self) -> str: if getattr(self.policy, f.name) is not UNSET] # a choice is shown by its tags while it is short, as one is # (a pack chooses one or two), and by its size past that - if len(self.languages) > _REPR_TAGS: - patched.insert(0, f"languages: {len(self.languages)}") - elif self.languages: - patched.insert(0, "languages " + "+".join(sorted(self.languages))) + if self.languages: + patched.insert(0, _shown_choice(self.languages)) suffix = f": {', '.join(patched)}" if patched else "" return f"Locale({self.code!r}{suffix})" -#: The most tags Locale's repr lists; a longer choice is shown by its -#: size, so the repr stays bounded however many languages are chosen. -_REPR_TAGS = 3 - - -def _canonical(tag: str) -> str: - """`tag` in BCP 47's canonical case: the language subtag lowercase, - a four-letter script subtag title case, a two-letter region - uppercase, every other subtag lowercase (RFC 5646, 2.1.1). Equal - choices then compare and hash equal, whatever case they were - written in.""" - first, *rest = tag.split("-") - out = [first.lower()] - for sub in rest: - if len(sub) == 4 and sub.isalpha(): - out.append(sub.title()) - elif len(sub) == 2 and sub.isalpha(): - out.append(sub.upper()) - else: - out.append(sub.lower()) - return "-".join(out) - - def _tags(value: object) -> frozenset[str]: """`value` as Locale.languages stores it, each tag in canonical case, or the TypeError or ValueError naming what is wrong with it.""" diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 9d10af85..ee635886 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -11,11 +11,9 @@ import dataclasses import functools import warnings -from collections.abc import Mapping from dataclasses import dataclass, field -from types import MappingProxyType -from typing import Any +from nameparser._lexicon import _rechosen_masks from nameparser._lexicon import Lexicon from nameparser._locale import Locale from nameparser._pipeline import run @@ -141,6 +139,21 @@ def __repr__(self) -> str: seg = f", segmenter={name}" return f"Parser({self.lexicon!r}, {self.policy!r}{seg})" + # -- properties ------------------------------------------------------ + + @property + def languages(self) -> frozenset[str]: + """The languages this parser's vocabulary is chosen under, + :attr:`Lexicon.languages` read through: set by + :func:`parser_for`, empty where none was chosen.""" + return self.lexicon.languages + + @property + def strict(self) -> bool: + """Whether the choice is strict, :attr:`Lexicon.strict` read + through.""" + return self.lexicon.strict + def parse(self, text: str) -> ParsedName: """Parse one name string into a :class:`ParsedName`. Never raises on string content (unparseable input yields empty @@ -297,95 +310,12 @@ def _reading_index_selected(languages: frozenset[str], return selected(languages, strict=strict) -@functools.cache -def _choice_delta(chosen: frozenset[str], strict: bool = False) -> tuple[ - dict[str, tuple[frozenset[str], frozenset[str]]], - tuple[frozenset[str], tuple[tuple[str, str], ...]]]: - """What the choice of the file tags `chosen` changes against the - default's compile: per Lexicon field, the entries that leave and - the ones that join, and the case masks that leave and the ones set. - A delta, so it applies to a caller's own Lexicon as well as to the - default (the RFC's "Where the provenance lives at runtime"); a - `strict` choice's delta is the same comparison, its compile holding - only the chosen files and native-script vocabulary.""" - from nameparser._lexicon import _vocab_fields - from nameparser._reading_index import compile_choice - default = compile_choice(frozenset()) - choice = compile_choice(chosen, strict) - was, now = _vocab_fields(default), _vocab_fields(choice) - fields = {name: (was[name] - now[name], now[name] - was[name]) - for name in was if was[name] != now[name]} - old, new = (default["CAPITALIZATION_EXCEPTIONS"], - choice["CAPITALIZATION_EXCEPTIONS"]) - masks = (frozenset(old.keys() - new.keys()), - tuple(sorted((k, v) for k, v in new.items() if old.get(k) != v))) - return fields, masks - - -@functools.cache -def _choice_readings(chosen: frozenset[str], - strict: bool = False) -> Mapping[str, tuple[Any, ...]]: - """The readings the choice of the file tags `chosen` compiles, by - word: the kinds `_apply_choice` gives a word the choice changes.""" - from nameparser._reading_index import choose, index, strict_pool - by_text: dict[str, list[Any]] = {} - for r in choose(strict_pool(chosen) if strict else index(), chosen): - by_text.setdefault(r.text, []).append(r) - return MappingProxyType({t: tuple(rs) for t, rs in by_text.items()}) - - -def _apply_choice(lexicon: Lexicon, languages: frozenset[str], - strict: bool = False) -> Lexicon: - """`lexicon` under the choice of `languages`, a delta against the - default's compile: what the choice drops leaves `lexicon` whoever - put it there, a caller included, what it turns on joins, and every - other entry stays as the caller had it. Written as readings, the - edits that take each changed word's readings to its sets under the - choice, appended to the lexicon's log (#754, 4i-b), until 4i-c - records the choice on the `Lexicon` and retires this delta.""" - from nameparser._lexicon import ( - _MARKER_BASES, _compiled_for, _edits_toward, _fields_of) - chosen = _reading_index_selected(languages, strict) - fields, (unmasked, masks) = _choice_delta(chosen, strict) - current = _fields_of(lexicon._sets) - target = {name: (current[name] - gone) | joined - for name, (gone, joined) in fields.items()} - # a word the choice drops from a base set leaves that set's markers - # too, a caller's own mark of it included: no reading compiles a - # marker without its base - for marker, base in _MARKER_BASES: - if base in fields and fields[base][0]: - target[marker] = target.get(marker, current[marker]) - fields[base][0] - updates: dict[str, object] = {} - if target: - wanted = current | target - edits = _edits_toward(lexicon._pool(), current, wanted, - _choice_readings(chosen, strict)) - if edits: - updates["_edits"] = lexicon._edits + edits - # a reading the choice turns on can clash with one the - # caller wrote (`st`'s cs generation beside a caller's - # credential: one suffix kind per word), and no readings - # give the delta's sets back; refused, as construction - # refuses the same clash (decisions/vocabulary-model, - # 2026-11-02 #754, item 3), where 3b's delta raised the - # set guard it broke - got = _fields_of(_compiled_for(lexicon._base, - lexicon._edits + edits)) - clash = sorted({w for f in wanted for w in got[f] ^ wanted[f]}) - if clash: - raise ValueError( - f"the readings the choice turns on clash with the " - f"lexicon's own for {', '.join(map(repr, clash[:8]))}; " - f"remove the lexicon's reading of each before choosing") - if unmasked or masks: - cap = {k: v for k, v in lexicon.capitalization_exceptions - if k not in unmasked} - updates["capitalization_exceptions"] = tuple( - sorted((cap | dict(masks)).items())) - if not updates: - return lexicon - return dataclasses.replace(lexicon, **updates) # type: ignore[arg-type] +def _rewrapped(exc: Exception, message: str) -> Exception: + """`exc`'s message with `message` before it, as a plain TypeError or + ValueError: a subclass may take other constructor arguments + (_lexicon._MaskValueError takes three).""" + return (TypeError if isinstance(exc, TypeError) else ValueError)( + f"{message}: {exc}") def parser_for(*locales: Locale, base: Parser | None = None, @@ -459,10 +389,17 @@ def parser_for(*locales: Locale, base: Parser | None = None, # segmenter it was asked to drop. if segmenter is UNSET: segmenter = base.segmenter if base is not None else None - # one choice for all the packs, so applying them is order-free; each - # pack's tags are checked on their own first, so a tag no file holds - # is reported with the pack that named it (rules/D2) - languages: frozenset[str] = frozenset() + # one choice for all the packs and the base's, so applying them is + # order-free; each pack's tags are checked on their own first, so a + # tag no file holds is reported with the pack that named it + # (rules/D2) + if lexicon.languages and lexicon.strict != strict: + raise ValueError( + f"base's lexicon was chosen with strict={lexicon.strict}, and " + f"a chained choice keeps it: pass strict={lexicon.strict}, or " + f"build the parser from Lexicon.default() with every pack, " + f"parser_for(*packs, strict={strict})") + languages = lexicon.languages for loc in locales: if loc.languages: try: @@ -473,18 +410,38 @@ def parser_for(*locales: Locale, base: Parser | None = None, languages |= loc.languages if strict and not languages: raise ValueError(_no_strict_language(locales)) - if languages: + # the packs' fragments join in pack order, and the caller's lexicon + # after them all, so its own readings outrank every pack's + # (decisions/vocabulary-model, 2026-11-02 #754, item 3) + fragments = Lexicon.empty() + for loc in locales: try: - lexicon = _apply_choice(lexicon, languages, strict) + fragments = fragments | loc.lexicon except (TypeError, ValueError) as exc: - # rules/D2: "applying a locale pack wraps any such error with - # the locale's code, so a stacked configuration names which - # layer broke" -- the choice is every pack's at once, so it - # names the languages (history: decisions/vocabulary-model, - # 2026-10-29 3b) - raise type(exc)( - f"while applying the languages chosen, " - f"{', '.join(sorted(languages))}: {exc}") from exc + raise _rewrapped(exc, f"while applying locale {loc.code!r}") from exc + try: + lexicon = fragments | lexicon + except (TypeError, ValueError) as exc: + raise _rewrapped(exc, "while joining the packs' lexicons to " + "base's") from exc + try: + if languages: + # the compile and its refusals, here: a reading of the + # lexicon's own that clashes with one the choice turns on + # raises, naming the edit + lexicon = dataclasses.replace( + lexicon, _languages=languages, _strict=strict, + capitalization_exceptions=_rechosen_masks( + lexicon, _reading_index_selected(languages, strict), + strict)) + except (TypeError, ValueError) as exc: + # rules/D2: "applying a locale pack wraps any such error with + # the locale's code, so a stacked configuration names which + # layer broke" -- the choice is every pack's at once, so it + # names the languages (history: decisions/vocabulary-model, + # 2026-10-29 3b) + raise _rewrapped(exc, "while applying the languages chosen, " + + ", ".join(sorted(languages))) from exc # each scalar field's setter so far and the value it left in the # policy: a later pack leaving another value overrides it, which is # warned; one leaving the same value (cs and sk's two_family_names) @@ -496,14 +453,9 @@ def parser_for(*locales: Locale, base: Parser | None = None, if f.metadata.get("compose") != "union"] for loc in locales: try: - lexicon = lexicon | loc.lexicon policy = apply_patch(policy, loc.policy) except (TypeError, ValueError) as exc: - # safe: every raise in the apply path (Policy.__post_init__, - # Lexicon.__or__, apply_patch) is a PLAIN TypeError/ValueError -- - # a subclass with extra mandatory args would break this rewrap - raise type(exc)( - f"while applying locale {loc.code!r}: {exc}") from exc + raise _rewrapped(exc, f"while applying locale {loc.code!r}") from exc for name in scalars: if getattr(loc.policy, name) is UNSET: continue diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index b46f855b..4dbad744 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -12,7 +12,7 @@ import pytest -from nameparser import LexiconReading, Parser +from nameparser import Locale, LexiconReading, Parser, parser_for from nameparser._lexicon import ( Lexicon, _ADDED_IN, _KIND_FIELDS, _LIST_FIELDS, _MaskValueError, _SET_FIELDS, _VOCAB_FIELDS, _default_lexicon, _fields_of, _normalize, @@ -1025,6 +1025,83 @@ def test_union_refuses_two_different_bases() -> None: Lexicon.default() | older +# -- the choice (decisions/vocabulary-model, 2026-11-02 #754, items 3 +# and 4) -------------------------------------------------------------------- + +def test_the_choice_travels_with_replace_union_and_pickle() -> None: + chosen = dataclasses.replace(Lexicon.default(), _languages=frozenset({"pt-br"})) + assert chosen.languages == {"pt-BR"} and chosen.strict is False + assert chosen != Lexicon.default() + added = chosen.add("zzq", kind="title") + assert added.languages == {"pt-BR"} + assert (added | Lexicon(surnames={"김"})).languages == {"pt-BR"} + assert (Lexicon.empty() | added).languages == {"pt-BR"} + strict = dataclasses.replace(Lexicon.default(), _languages=frozenset({"vi"}), + _strict=True) + for lex in (chosen, added, strict): + loaded = pickle.loads(pickle.dumps(lex)) + assert loaded == lex and hash(loaded) == hash(lex) + assert (loaded.languages, loaded.strict) == (lex.languages, lex.strict) + + +@pytest.mark.parametrize("left,right", [ + ({"_languages": {"pt"}}, {"_languages": {"en"}}), + ({"_languages": {"ko"}}, {"_languages": {"ko"}, "_strict": True}), +]) +def test_union_refuses_two_different_choices_naming_parser_for( + left: dict[str, object], right: dict[str, object]) -> None: + a = dataclasses.replace(Lexicon.default(), **left) # type: ignore[arg-type] + b = dataclasses.replace(Lexicon.default(), **right) # type: ignore[arg-type] + with pytest.raises(ValueError, match="different choices.*parser_for"): + a | b + assert (a | a).languages == a.languages + + +@pytest.mark.parametrize("kwargs,error,match", [ + ({"_languages": "pt"}, TypeError, "_languages must be a set"), + ({"_languages": {"pt": 1}}, TypeError, "_languages must be a set"), + ({"_languages": {5}}, TypeError, "entries must be str"), + ({"_languages": {"pt"}, "_strict": 1}, TypeError, + "_strict must be True or False"), + ({"_strict": True}, ValueError, "chooses none"), + ({"_languages": {"xx"}}, ValueError, "no vocabulary file holds"), + ({"_languages": {"mul"}}, ValueError, "not a language to choose"), + # a file holding no reading is a strict choice only, as at parser_for + ({"_languages": {"vi"}}, ValueError, "strict=True"), +]) +def test_the_choice_is_validated_as_parser_for_validates_one( + kwargs: dict[str, object], error: type[Exception], match: str) -> None: + with pytest.raises(error, match=match): + Lexicon(**kwargs) # type: ignore[arg-type] + + +def test_a_choice_filters_the_base_and_the_log_applies_after_it() -> None: + # en prefers its credential of `do` over pt's particle; a removal + # written before the choice was made still applies after it + # (the RFC's item 4 example) + removed = Lexicon.default().remove("do", kind="credential") + chosen = dataclasses.replace(removed, _languages=frozenset({"en"})) + assert [r.category for r in chosen.readings_of("do")] == ["name", "name"] + assert [r.kind for r in dataclasses.replace( + Lexicon.default(), _languages=frozenset({"pt"})).readings_of("do") + if r.category != "name"] == ["preposition"] + + +def test_the_repr_shows_the_choice_and_counts_against_it() -> None: + chosen = dataclasses.replace(Lexicon.default(), _languages=frozenset({"pt"})) + assert repr(chosen) == "Lexicon(default, languages pt)" + assert (repr(chosen.add("zzq", kind="title")) + == "Lexicon(default + title: +1, languages pt)") + # the masks a strict choice drops are parser_for's to move, and the + # repr counts against the named lexicon with them moved + vi = Locale(code="vi", lexicon=Lexicon.empty(), languages=frozenset({"vi"})) + strict = parser_for(vi, strict=True).lexicon + assert repr(strict) == "Lexicon(default, languages vi, strict)" + many = dataclasses.replace(Lexicon.default(), + _languages=frozenset({"pt", "en", "es", "de"})) + assert repr(many) == "Lexicon(default, languages: 4)" + + def test_the_internal_sets_hold_each_field_under_its_own_name() -> None: # the roster pinned so that a field added to one and not the other # fails at once; the kinds the views read follow the 2.x sets (4i-b) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index b6712f66..464450b2 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2472,7 +2472,7 @@ def test_the_packs_choices_are_one_choice_in_any_order() -> None: assert parser_for(pt, en).lexicon == parser_for(en, pt).lexicon == both -def test_a_choice_is_a_delta_that_keeps_a_callers_vocabulary() -> None: +def test_a_choice_keeps_a_callers_vocabulary() -> None: base = Parser(lexicon=Lexicon.default().add("zzq", kind="title")) got = parser_for(_choosing("pt"), base=base) assert "zzq" in got.lexicon._sets.titles @@ -2511,96 +2511,178 @@ def test_a_word_a_choice_drops_leaves_the_callers_markers_of_it_too() -> None: assert "vd" not in got.particles and "vd" not in got.particles_ambiguous +@pytest.mark.parametrize("mask,error", [ + # a value not a string: a TypeError + (5, TypeError), + # a mask spelling other letters: _lexicon._MaskValueError, whose + # constructor takes three arguments, rewrapped as a plain ValueError + ("Abc", ValueError), +]) def test_an_error_applying_a_choice_names_the_choice( - monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch: pytest.MonkeyPatch, mask: object, + error: type[Exception]) -> None: # an error the Lexicon raises while the choice is applied names the - # languages chosen, as a patch error names its pack. No shipped - # choice raises, so a delta joining a mask that is not a string - # stands in for one (an orphan marker did until 4i-b, which left a - # choice no way to write one) + # languages chosen, as a patch error names its pack. A shipped + # choice raises only on a caller's clashing reading (the `st` test + # below), so a choice moving a mask the Lexicon refuses stands in import nameparser._parser as parser_mod - bad_mask: object = ({}, (frozenset(), (("zzq", 5),))) - monkeypatch.setattr(parser_mod, "_choice_delta", - lambda chosen, strict=False: bad_mask) - with pytest.raises(TypeError, - match="while applying the languages chosen, en, pt: " - "capitalization_exceptions entries must be"): + monkeypatch.setattr(parser_mod, "_rechosen_masks", + lambda lexicon, chosen, strict: (("zzq", mask),)) + with pytest.raises(error, match="while applying the languages chosen, " + "en, pt: capitalization_exceptions"): parser_for(_choosing("pt", code="aa"), _choosing("en", code="bb")) -def test_a_choice_moves_case_masks_as_it_moves_words( - monkeypatch: pytest.MonkeyPatch) -> None: - # no shipped choice changes a mask (no reading off by default or - # preferred away carries `cased`), so a stand-in compile does: under - # it `phd`'s mask leaves and `zzq`'s joins, and the delta applies the - # same to a caller's base, keeping the caller's own masks - import nameparser._parser as parser_mod - import nameparser._reading_index as ri - real = ri.compile_choice - default = real(frozenset()) - - def stand_in(chosen: frozenset[str], strict: bool = False) -> object: - if not chosen: - return default - masks = {k: v for k, v in default["CAPITALIZATION_EXCEPTIONS"].items() - if k != "phd"} | {"zzq": "ZzQ"} - return {**default, "CAPITALIZATION_EXCEPTIONS": masks} - monkeypatch.setattr(ri, "compile_choice", stand_in) - delta = parser_mod._choice_delta.__wrapped__(frozenset({"pt"})) # type: ignore[attr-defined] - assert delta == ({}, (frozenset({"phd"}), (("zzq", "ZzQ"),))) - monkeypatch.setattr(parser_mod, "_choice_delta", - lambda chosen, strict=False: delta) - base = dataclasses.replace(Lexicon.default(), capitalization_exceptions=( +def test_a_choice_moves_case_masks_as_it_moves_words() -> None: + # cs's credentials carry masks the default leaves off (`judr` -> + # JUDr), a strict choice drops the masks of every file it drops, + # and a caller's own mask the choice does not touch is kept + from nameparser import locales + mine = dataclasses.replace(Lexicon.default(), capitalization_exceptions=( *Lexicon.default().capitalization_exceptions, ("abcq", "AbcQ"))) - got = parser_mod._apply_choice(base, frozenset({"pt"})) - cap = got.capitalization_exceptions_map - assert "phd" not in cap and cap["zzq"] == "ZzQ" and cap["abcq"] == "AbcQ" - - -def test_a_choice_chained_through_base_applies_after_the_bases() -> None: - # decided 2026-10-29 (decisions/vocabulary-model, chained choices): - # a choice is one call's, and a Parser records none, so en chained on - # pt's parser drops pt's particle after pt dropped en's credential. - # A change to Parser recording its choice (option b) changes this - # test on purpose + cs = parser_for(locales.CS, base=Parser(lexicon=mine)).lexicon + assert cs.capitalization_exceptions_map["judr"] == "JUDr" + assert cs.capitalization_exceptions_map["abcq"] == "AbcQ" + ko = parser_for(_choosing("ko"), strict=True).lexicon + assert "phd" not in ko.capitalization_exceptions_map + # chained, the masks move as the union's do: what ko's strict choice + # dropped and en's keeps comes back + en, ko_pack = _choosing("en", code="en"), _choosing("ko", code="ko") + chained = parser_for(en, base=parser_for(ko_pack, strict=True), + strict=True) + assert "phd" in chained.lexicon.capitalization_exceptions_map + assert chained.lexicon == parser_for(ko_pack, en, strict=True).lexicon + + +def test_a_choice_chained_through_base_unions_with_the_bases() -> None: + # decisions/vocabulary-model, 2026-11-02 #754, item 4: the lexicon + # records its choice, so en chained on pt's parser chooses both, as + # one call choosing both does (superseding 2026-10-29's chained + # choices, where en dropped pt's particle after pt dropped en's + # credential: family `Carmo`, family `DO`) pt, en = _choosing("pt", code="pt"), _choosing("en", code="en") chained = parser_for(en, base=parser_for(pt)) - assert chained.parse("Maria do Carmo").family == "Carmo" - assert chained.parse("John Smith DO").family == "DO" + assert chained.parse("Maria do Carmo").family == "do Carmo" + assert chained.parse("John Smith DO").suffix == "DO" together = parser_for(pt, en) - assert together.parse("Maria do Carmo").family == "do Carmo" - assert together.parse("John Smith DO").suffix == "DO" - assert chained.lexicon != together.lexicon + assert chained.lexicon == together.lexicon + assert chained.languages == together.languages == {"en", "pt"} def test_a_choice_whose_reading_clashes_with_the_callers_is_refused() -> None: # cs turns on `st`'s generation beside a caller's credential, and a - # word holds one suffix kind: 3b's delta broke a set guard here, and - # 4i-b's readings refuse it too, rather than drop the caller's - # reading in silence (found by review of 4i-b) + # word holds one suffix kind: refused at parser_for, naming the + # languages, the edit, and the remove that resolves it + # (decisions/vocabulary-model, 2026-11-02 #754, item 3) from nameparser import locales mine = Parser(lexicon=Lexicon.default().add("st", kind="credential")) with pytest.raises(ValueError, match=( - r"while applying the languages chosen, cs: the readings the " - r"choice turns on clash with the lexicon's own for 'st'")): + r"while applying the languages chosen, cs: the lexicon's " + r"\.add\('st', kind='credential'\) clashes with a reading the " + r"choice turns on: 'st' would hold two suffix kinds.*" + r"\.remove\('st', kind='generation'\)")) as raised: parser_for(locales.CS, base=mine) - - -def test_a_choice_drops_a_callers_own_entry_it_removes() -> None: - # decided with the chained choices: the delta removes what the - # choice drops whoever listed it, and keeps every untouched entry + # the remove it offers applies after the choice, so it runs + offered = re.search(r"(\.remove\(.*\))$", str(raised.value)) + assert offered is not None + fixed = eval("Lexicon.default().add('st', kind='credential')" # noqa: S307 + + offered.group(1), {"Lexicon": Lexicon}) + got = parser_for(locales.CS, base=Parser(lexicon=fixed)) + assert got.parse("Jiří Dienstbier st.").suffix == "st." + + +def test_a_clash_the_log_makes_without_the_choice_is_not_the_choices( + ) -> None: + # an edit made on a chosen lexicon that clashes with the base alone + # is refused as it is with no choice, the choice not named + chosen = parser_for(_choosing("pt")).lexicon + with pytest.raises(ValueError, match="would hold two suffix kinds") as e: + chosen.add("jr", kind="credential") + assert "the choice" not in str(e.value) + + +def test_a_callers_reading_survives_a_choice_whoever_wrote_first() -> None: + # superseding 2026-10-29's chained choices item 2, where the delta + # dropped what the choice removed whoever listed it: a caller's + # reading applies after the choice mine = Lexicon(surnames=Lexicon.default().surnames).add( - "do", "van", kind="particle") + "do", "van", kind="preposition") got = parser_for(_choosing("en"), base=Parser(lexicon=mine)).lexicon - assert got._sets.particles == {"van"} + assert got._sets.particles == {"do", "van"} + # and a caller's removal too: with en's credential of `do` taken out + # after en preferred it over pt's particle, `do` reads as neither + removed = Lexicon.default().remove("do", kind="credential") + en = parser_for(_choosing("en"), base=Parser(lexicon=removed)) + assert en.parse("Maria do Carmo").middle == "do" + + +def test_a_choice_selects_nothing_in_an_empty_base() -> None: + # a base built from Lexicon.empty() holds no shipped reading, so a + # choice turns none on (the 2026-11-02 #754 entry's item 4) + # (the Korean surnames kept, so the default policy's hangul + # segmentation has its list) + empty = Lexicon(surnames=Lexicon.default().surnames) + got = parser_for(_choosing("cs"), base=Parser(lexicon=empty)) + assert got.languages == {"cs"} + from nameparser._lexicon import _fields_of + assert _fields_of(got.lexicon._sets) == _fields_of(empty._sets) + + +def test_the_choice_is_recorded_and_read_through_the_parser() -> None: + plain = parser_for(_choosing("pt-br")) + assert plain.languages == plain.lexicon.languages == {"pt-BR"} + assert plain.strict is plain.lexicon.strict is False + strict = parser_for(_choosing("ko"), strict=True) + assert strict.strict is strict.lexicon.strict is True + assert Parser().languages == frozenset() and Parser().strict is False + # a pack choosing nothing chained on a choice keeps it + assert parser_for(Locale(code="xx", lexicon=Lexicon.empty()), + base=plain).lexicon == plain.lexicon + + +@pytest.mark.parametrize("base_strict", [False, True]) +def test_a_chained_call_of_another_strictness_is_refused( + base_strict: bool) -> None: + base = parser_for(_choosing("ko"), strict=base_strict) + with pytest.raises(ValueError, match=( + f"chosen with strict={base_strict}.*pass " + f"strict={base_strict}.*parser_for\\(\\*packs, " + f"strict={not base_strict}\\)")): + parser_for(_choosing("en"), base=base, strict=not base_strict) + + +def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: + # the caller's lexicon goes last, so its own statement outranks a + # pack's: a pack's credential `do` against a caller's removal of it, + # and a pack's mask against the caller's (decisions/vocabulary-model, + # 2026-11-02 #754, item 3) + fragment = dataclasses.replace( + Lexicon.empty().add("do", kind="credential"), + capitalization_exceptions=(("zzq", "ZZq"),)) + pack = _choosing("en", lexicon=fragment) + mine = dataclasses.replace( + Lexicon.default().remove("do", kind="credential"), + capitalization_exceptions=( + *Lexicon.default().capitalization_exceptions, ("zzq", "ZzQ"))) + got = parser_for(pack, base=Parser(lexicon=mine)) + assert got.parse("John Smith DO").suffix == "" + assert got.lexicon.capitalization_exceptions_map["zzq"] == "ZzQ" + # and a later pack's fragment after an earlier one's + first = _choosing("en", code="aa", + lexicon=Lexicon.empty().add("zzq", kind="title")) + second = _choosing("en", code="bb", + lexicon=Lexicon.empty().remove("zzq", kind="title")) + assert parser_for(first, second).parse("Zzq Smith").title == "" + assert parser_for(second, first).parse("Zzq Smith").title == "Zzq" def test_reading_languages_name_the_default_files_under_a_choice() -> None: # decisions/vocabulary-model, 2026-10-29 phase 3b: the field names the # files holding the reading on by default whatever choice built the - # parser, a Parser recording none. Under pt, `it`'s dropped reading of - # `e` is still named; under cs, the generation `st` it turned on names - # no file. A Parser.languages (option b) would change both on purpose + # parser. Under pt, `it`'s dropped reading of `e` is still named; + # under cs, the generation `st` it turned on names no file. Its + # following the choice the lexicon now records is 4i-d's, optional, + # and would change both on purpose def languages(parser: Parser, name: str, text: str) -> tuple[str, ...]: token = next(t for t in parser.parse(name).tokens if t.text == text) assert token.reading is not None @@ -2613,14 +2695,17 @@ def languages(parser: Parser, name: str, text: str) -> tuple[str, ...]: assert languages(pt, "Maria do Carmo", "do") == ("pt",) -def test_a_choice_that_changes_nothing_leaves_the_lexicon_itself() -> None: +def test_a_choice_that_changes_nothing_leaves_the_sets_as_they_are() -> None: # sv.toml holds no reading the default leaves off or another file's - # reading clashes with, so choosing it changes no compiled set - import nameparser._parser as parser_mod + # reading clashes with, so choosing it changes no compiled set; the + # lexicon records the choice all the same from nameparser._reading_index import changed_words, selected assert changed_words(selected(["sv"])) == frozenset() mine = Lexicon.default().add("zzq", kind="title") - assert parser_mod._apply_choice(mine, frozenset({"sv"})) is mine + got = parser_for(_choosing("sv"), base=Parser(lexicon=mine)).lexicon + from nameparser._lexicon import _fields_of + assert _fields_of(got._sets) == _fields_of(mine._sets) + assert got.languages == {"sv"} # -- strict (#753, 3j: file-strict, decisions/vocabulary-model's @@ -2707,12 +2792,16 @@ def test_strict_is_a_bool() -> None: parser_for(_choosing("ko"), strict=1) # type: ignore[arg-type] -def test_a_strict_choice_is_a_delta_over_the_union_in_any_order() -> None: - # a caller's listing of a dropped word leaves, a word no file holds - # stays (measured in the decisions entry); the union, then strict +def test_a_strict_choice_keeps_a_callers_readings_over_the_union( + ) -> None: + # a caller's reading of a word strict drops stays, as one of a word + # no file holds does (superseding 3j's clause that the delta dropped + # it, decisions/vocabulary-model, 2026-11-02 #754, item 4); the + # union, then strict base = Parser(lexicon=Lexicon.default().add("dr", "zzq", kind="title")) got = parser_for(_choosing("vi"), base=base, strict=True).lexicon._sets - assert "zzq" in got.titles and "dr" not in got.titles + assert {"zzq", "dr"} <= got.titles + assert "dr" not in parser_for(_choosing("vi"), strict=True).lexicon._sets.titles ko, en = _choosing("ko", code="aa"), _choosing("en", code="bb") assert (parser_for(ko, en, strict=True).lexicon == parser_for(en, ko, strict=True).lexicon @@ -2730,7 +2819,7 @@ def _choice_tags() -> list[tuple[str, bool]]: + [(t, True) for t in sorted(files() - UNCHOOSABLE)]) -#: The lexicons a choice is written into: the default, the empty one, +#: The lexicons a choice is filtered under: the default, the empty one, #: and one a caller configured (a removed gate, an added title, two #: added particles, one of them a word a choice turns a suffix on for) _CHOICE_BASES = { @@ -2743,26 +2832,36 @@ def _choice_tags() -> list[tuple[str, bool]]: @pytest.mark.parametrize("tag,strict", _choice_tags(), ids=lambda v: str(v)) -def test_a_choice_written_as_readings_gives_the_deltas_sets( +def test_a_choice_moves_only_the_words_the_callers_log_leaves_alone( tag: str, strict: bool) -> None: - # 4i-b writes 3b's delta as readings (`_apply_choice` through - # `_lexicon._edits_toward`): for every single file tag a choice may - # name, plain and strict, on each of _CHOICE_BASES, the compiled sets - # are the delta applied to the base's sets, field by field, the kind - # sets the views read included (decisions/vocabulary-model's - # 2026-11-02 #754, sub-phase 4i-b entry counts this population) - from nameparser._lexicon import _MARKER_BASES, _fields_of - from nameparser._parser import ( - _apply_choice, _choice_delta, _reading_index_selected) - chosen = _reading_index_selected(frozenset({tag}), strict) - fields, _ = _choice_delta(chosen, strict) + # 4i-c filters the base's readings by the choice and applies the + # caller's log after it (decisions/vocabulary-model, 2026-11-02 + # #754, item 4): for every single file tag a choice may name, plain + # and strict, the default under the choice compiles as the shipped + # index does under it, the empty lexicon stays empty, and a + # configured lexicon differs from the choice alone only in entries + # holding a word its log names + from nameparser._lexicon import _NO_SETS, _built, _fields_of, _vocab_fields + from nameparser._reading_index import compile_choice, selected + chosen = selected(frozenset({tag}), strict=strict) + want = _vocab_fields(compile_choice(chosen, strict)) for name, build in _CHOICE_BASES.items(): lexicon = build() - current = _fields_of(lexicon._sets) - want = {f: (current[f] - gone) | joined - for f, (gone, joined) in fields.items()} - for marker, base in _MARKER_BASES: - if base in fields and fields[base][0]: - want[marker] = want.get(marker, current[marker]) - fields[base][0] - got = _fields_of(_apply_choice(lexicon, frozenset({tag}), strict)._sets) - assert got == current | want, name + got = dataclasses.replace(lexicon, _languages=frozenset({tag}), + _strict=strict) + assert got.languages == {tag} and got.strict is strict + sets = _fields_of(got._sets) + if name == "empty": + assert sets == _fields_of(_NO_SETS), name + continue + if name == "default": + # the shortcut every pack takes, and the compile it stands for + assert sets == _fields_of(want), name + assert _fields_of(_built("default", (), chosen, strict)[1]) == sets + continue + touched = {w for e in lexicon._edits for w in e.words} + for field, words in sets.items(): + def kept(entries: frozenset[str]) -> frozenset[str]: + return frozenset(e for e in entries + if not touched & set(e.split())) + assert kept(words) == kept(want[field]), (name, field) From 6c390f7d39c7eb4852b7a6b35e0ffefcac399249 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:26:36 -0800 Subject: [PATCH 2/9] Document the recorded choice; mark the superseded entries in effect (#754, 4i-c) Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 +-- docs/design/decisions/vocabulary-model.md | 20 ++++--- docs/design/mechanisms.md | 6 +-- docs/design/vocabulary-model-rfc.md | 6 +-- docs/locales.rst | 64 +++++++++++++++-------- docs/release_log.rst | 1 + nameparser/_locale.py | 19 ++++--- nameparser/_parser.py | 37 +++++++------ 8 files changed, 95 insertions(+), 64 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 61dc629b..7a66951a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -212,7 +212,7 @@ Latin-script input is never touched by any of this: "Kim Min-jun" is genuinely o ### Vocabulary (`nameparser/vocab/`) and the configuration layer (`nameparser/config/`) -**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows, and every file's tag (`FILES`, a file holding only exclusions included, for a strict choice to name), as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and applies as a delta against the default's compile, `choose` there selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped; under `parser_for(..., strict=True)`, 3j, only the chosen files and native-script vocabulary compiled, through `strict_pool`), `changed_words` deriving a choice's deviation surface which tests/v2/test_locales.py's gate reads, and `strict_ungated` the words a strict choice ungates against the plain one, which tests/v2/test_reading_index.py holds to a reviewed table (decisions/vocabulary-model, 2026-10-29 3b; 2026-10-31 #753, phase 3j); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. +**The default vocabulary is written per word, in TOML, and compiled.** `nameparser/vocab/.toml` holds one file per BCP 47 language tag, each `[[word]]` entry a `text` and its readings keyed by kind. What a file means, how an entry is built, the kinds and properties with the set each compiles into, exclusions, bearer counts and the steps for adding a word are in `nameparser/vocab/README.md`, the reference for anyone editing the TOML; `tests/v2/test_vocab.py` holds its kind, property, derived-marker and reserved-name tables to the names `build.py` accepts and the sets it compiles, so read it rather than restating it here. The parser never reads the TOML: `tools/vocab/build.py` runs the build checks that replaced the modules' import-time asserts (its docstring numbers them), turns every reading into an index row with its file's tag, and writes the rows, and every file's tag (`FILES`, a file holding only exclusions included, for a strict choice to name), as the committed reading index `nameparser/config/_index.py` (#753, phase 3a) and, compiled, as the committed `nameparser/config/_compiled.py`, frozenset literals one entry per line; the build never reads `_index.py` back. A file a shipped pack reads may also hold a `[rules]` table, each key a `Policy` field (`patronymic_rules`, `name_order`, `two_family_names`, `two_given_names`), which the build checks against its `RULE_KEYS` and `PACK_RULES` and writes as `nameparser/config/_rules.py`; `_locale.rules_patch(tag)` turns a file's table into the `PolicyPatch` a pack carries, RU's and TR_AZ's from `ru.toml` and `tr.toml`, and the `two_family_names` of ES, CS and SK from `es.toml`, `cs.toml` and `sk.toml` (#754, 4f; decisions/vocabulary-model's 2026-10-29 4f entry). The compiler is `nameparser/_reading_index.py`'s `compile_readings`, with the kind and property tables it reads (`CATEGORY`, `BEHAVIORAL`, `GATED_WORDS`), so the build and a choice of languages compile alike: `Locale.languages` (3.0, phase 3b) is a choice of BCP 47 tags, which `parser_for` unions across its packs and base's recorded choice and records on the `Lexicon` (`Lexicon.languages`, 3.0's 4i-c), the lexicon compiling its base's readings through `choose` and its own edits after them, `choose` selecting the readings (a chosen file's `default = false` ones on; on a word a chosen file holds a vocabulary reading of, other files' vocabulary readings off, save `mul.toml`'s and `und.toml`'s; a chosen conjunction's `may_be_initial` dropped; under `parser_for(..., strict=True)`, 3j, only the chosen files and native-script vocabulary compiled, through `strict_pool`), `changed_words` deriving a choice's deviation surface which tests/v2/test_locales.py's gate reads, and `strict_ungated` the words a strict choice ungates against the plain one, which tests/v2/test_reading_index.py holds to a reviewed table (decisions/vocabulary-model, 2026-10-29 3b; 2026-10-31 #753, phase 3j); the build keeps the checks and loads that module by path, as it loads `_lexicon.py`, and `tests/v2/test_reading_index.py` holds that the index compiled with no choice is `_compiled.py` exactly. `readings_of(text)` there is the run-time reader of the index, every reading of a word with its file's tag, loaded on first call; `languages_of(text, categories)` is the reader a parse calls, from assemble, for `Reading.languages` (3.0), so the first parse that reads a word from its `Lexicon`'s vocabulary loads the index. Each `config/` module re-exports its constants from `_compiled.py` (`TITLES = _compiled.TITLES`) under its autodoc docstring, so the public names and the docs did not move. `tests/v2/test_vocab.py` fails when `_compiled.py` is stale, and `tests/v2/test_reading_index.py` when the index is, each naming the refresh command, and `test_vocab.py` gives every build check a mutation. The model, the phasing and what was declined: decisions/vocabulary-model. The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` on any of them, so a default word list is changed at runtime by configuring a `Lexicon`, never by editing the constant, and the shipped default is changed by editing the TOML and rebuilding. **Neither dict was frozen, and `CAPITALIZATION_EXCEPTIONS` is not covered by anything else either.** `REGEXES` is a compiled-pattern table rather than vocabulary and was never in #293's scope; `CAPITALIZATION_EXCEPTIONS` is vocabulary-shaped and is a decided, in-scope exemption. So the hazard the freeze closes is still live for it: `CAPITALIZATION_EXCEPTIONS['dclinpsy'] = 'DClinPsy'` after the first parse misses the cached `Lexicon.default()`. Same advice — configure the object (`dataclasses.replace(lexicon, capitalization_exceptions=...)`). `tests/v2/test_contracts.py::test_every_vocabulary_constant_is_frozen` names both dicts as explicit exemptions rather than letting its `isinstance` filter drop them. @@ -243,14 +243,14 @@ The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2 - **A kind is worth adding only if a reader would hesitate too**: the test is not "does the code take a branch" but whether a person reading that input would genuinely be unsure. "Smith, John V" reads as a middle initial to anyone -- the comma settles it -- so reporting it would be noise that teaches callers to ignore the field, which costs more than the missing report. Reachability of the second branch is necessary, not sufficient. Prefer leaving a fork silent and documenting the omission over emitting on input nobody finds ambiguous. - **Parser owns config-dependent conveniences**: `Parser.matches`/`Parser.capitalized`/`Parser.revise` exist because the `ParsedName` equivalents fall back to DEFAULT config for str/omitted arguments (documented loudly in both docstrings). `revise` harvests tokens from a full sub-parse of each replacement value (tags kept minus `FOLDED_TAG`, roles forced, the R1 entry pass `suffix_entries` re-run over the forced state so a suffix value's entries follow its own commas, ambiguities discarded); the merge tail is shared with `replace()` via `ParsedName._with_field_tokens`. `Parser.capitalized` delegates through `name.capitalized(self.lexicon)` specifically so `_parser` never imports `_render` — keep it that way. - **A reading is of one word, save a birth marker and a title addressing the given name** (the build's check 8, which `_lexicon._make_edit` runs on every `add` and RAISES on, since a reading is now a caller's statement the compile can refuse as the build does: 3.0's 4i-b). The two lists, `surnames` and `given_names`, and `capitalization_exceptions` keys still WARN on a multi-word entry (`_normset`/`_normpairs` via `_warn_at_caller`, UserWarning, never a raise — an inert entry produces nothing). The default vocabulary and every locale pack must stay warning-free (`test_default_lexicon_builds_warning_free`, `test_pack_vocabulary_entries_are_single_words`). -- **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. Since 3.0's 4i-b a `Lexicon` compiles from readings, which hold the 2.x marker-over-base relations by construction; the checks a caller's edit reaches are the build's checks 3 (a kind and its properties, `may_be_initial` on a single cased letter alone), 4 (one suffix kind per word), 8 (a phrase), 9 (a bound given name that is a particle has a `given` reading) and 11's agnomen clause (an agnomen has a name reading), in `_lexicon._make_edit` and `_check_pool`, and again in `_parser._apply_choice`, which refuses a choice whose readings clash with the caller's, each raising with the call that fixes it, because each is a wrong parse. A given-addressing title run the parse can never build is not checked, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). +- **Invariants guard harm, not no-ops**: add a constructor check when violating it produces a *wrong parse*, not when it produces *nothing*. A false positive costs a working configuration; a true positive on an inert condition costs the user nothing, so that trade is never worth taking. Since 3.0's 4i-b a `Lexicon` compiles from readings, which hold the 2.x marker-over-base relations by construction; the checks a caller's edit reaches are the build's checks 3 (a kind and its properties, `may_be_initial` on a single cased letter alone), 4 (one suffix kind per word), 8 (a phrase), 9 (a bound given name that is a particle has a `given` reading) and 11's agnomen clause (an agnomen has a name reading), in `_lexicon._make_edit` and `_check_pool`, which `_lexicon._built` runs again under a choice of languages, refusing a caller's reading that clashes with one the choice turns on and naming the edit, each raising with the call that fixes it, because each is a wrong parse. A given-addressing title run the parse can never build is not checked, because an unreachable entry is simply never consulted (see Gotchas). Before adding one, construct the config it forbids and check what actually breaks. **Between raise and silence sits the construction-time `UserWarning`**, for a gap that is statically decidable, harmless to SOME deliberate caller, and indistinguishable-from-working for everyone else: the segmenterless activation (#337 — `parser_for(locales.JA)` without a segmenter behaved exactly like a working parser minus the feature) warns rather than raises because the inert JA registration is itself a pinned property, and a warning is filterable by the caller who wants exactly that. The message must carry every applicable remedy and no inapplicable one (the `ja_segmenter` hint fires only when a Japanese script is among the dead ones). Test fuzzers that legitimately construct such configs suppress the warning by MESSAGE, never by category — a blanket `UserWarning` ignore would mask the next construction diagnostic (`_quiet_parser` in `tests/v2/test_properties.py` is the pattern). - **Reprs are bounded**: render which fields deviate from a named baseline and by how much, never contents (`Lexicon(default + titles: +2)`). `PolicyPatch`'s repr shows only set (non-UNSET) fields; `_order_repr` must never raise even on an unvalidated patch's garbage `name_order` (PolicyPatch defers validation to apply time); the sweep test in `tests/v2/test_reprs.py` pins that no config repr leaks the UNSET sentinel. - **Every pipeline stage is one module holding one public function of the same name** (`_tokenize.py`/`tokenize`), with the exceptions named where they stand — `_group.py`'s `marker_run_length`, a shared predicate (mechanisms.md#ONE-PREDICATE-PER-QUESTION), and since #511 `_post_rules.py`'s `suffix_entries`, the R1 entry pass as a function, because `Parser.revise` runs that one pass over a forced-role sub-parse and a stage's tail cannot be called on its own — **and its module docstring declares the contract in three labelled lines**: `Consumes:` what it takes from `ParseState`, `Produces:` what it hands back, `Reads:` which `Policy`/`Lexicon` fields it consults. `Reads:` is the load-bearing one — it makes "which stage do I touch for this feature?" a grep rather than a read-through. A `Reads:` line names a vocabulary set as `Lexicon.`, and every library module but `_lexicon` reads it as `lexicon._sets.` (`_lexicon._LexiconSets`, #754's 4i-a), never off the public field, `surnames` and `given_names` included though they stay public, which let 4i-b make the other twenty internal without touching a reader; `tests/v2/test_layering.py` refuses a direct read, field by field, in the library and in every tool. The authority for the stage set is `_pipeline/__init__.py`'s `STAGES` (eight, and not public API) with the field-ownership map in `ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`; NOT the rules, which are implementation-free by their own preamble and whose `implemented:` names modules honoring a rule rather than stages. A `_pipeline/` module that is not a stage says so in its first line instead (`_assemble.py`: "Not a stage: …", with a `Reads:` line all the same, since it names a reported field by `Policy.middle_as_family` and a token's reading by three `Lexicon` sets), so absence of the three lines is a claim about the module rather than an oversight. Provenance: §5 of the 2026-07-11 conventions spec, recorded here 2026-08-16. - **Frames are a budget for hot and scaling paths; everywhere else, prefer the shared predicate.** The parser's cost is counted in Python frames (`sys.setprofile` "call" events — deterministic and load-immune, which is why #475 chose them over wall-clock timing), but one frame is small: measured 2026-09-28 on py3.11, a trivial call costs about 13 ns against 40–80 µs for a whole parse, so one frame per parse is roughly 0.03% of it. What the budget exists to catch is how a cost SCALES and how it ACCUMULATES, so weigh an extra frame by four questions, in order. (1) Does it multiply with the input — per token, per member, per pair? Then it matters, and a hand-inlined fast path can earn its place (the inline tag tests ahead of `_pieces.starts_a_credential_run` in its loops, each a necessary half of the predicate so nothing can drift; `_vocab.run_word_fold` was the measured example until #613 deleted the run test it served). Folding a copy into a shared helper is where such a pre-check gets lost: #617 replaced `_comma._title_chains`, whose loop tested the conjunction tag inline, with a call into group's loop, which asks `is_conj_piece` per piece, and every family-comma parse paid two frames per word of the part (`Smith, John` +2) while the reference row fell -- review caught it; the fix gates the call on the tag walk the caller already makes, and `test_a_comma_part_with_no_free_connective_skips_the_join` holds it. Count a comma name, not only the reference row, when a fold touches the comma path. (2) How many names pay it — every parse, where plain names like `Smith, John` must keep their cost, or only a rare shape? (3) Does it move an enforced guard — the ±2% reference band in `test_benchmark.py` (`_CALL_BASELINE`, decisions/parse-cost) or a scaling ratio? (4) What does the inline alternative cost to maintain? A hand copy of a predicate that can drift is the defect class mechanisms.md#ONE-PREDICATE-PER-QUESTION names, and it compounds; a NARROWER question stated exactly is not a copy and may stay inline on that ground alone. A constant frame on a rare shape that no guard sees loses to readability: #544's cleanup declined `is_title_piece` for one frame on comma credential runs (24 ns a parse, no guard moved), a stricter call than this rule asks for; by this rule the inline test is still right, but because it asks the narrower question (a lone word in both vocabularies), not because of the frame. The converse holds too: the frame counter cannot see C-level work (`x in some_list`, slices, string concatenation), so "no new frames" is not "no new cost" — #553's quadratic was a list membership test inside a loop, invisible to every frame guard. - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in the 2.x `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`, deleted with the facade in 3.0), which cannot be wrong the way a sentence can. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. -- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` and `LexiconReading` keep their own copy of the guard (layering); a new unpicklable slot type needs the same treatment plus a round-trip test. Since 3.0's 4i-b a `Lexicon` pickles five things: its lists, its masks, its edit log and its base as a SNAPSHOT of the reading index's rows, so a pickle parses as it did where it was written, a snapshot equal to the running release's index loading as the by-reference default again (decisions/vocabulary-model, 2026-11-02 #754, item 5); its private `_sets` and `_cap_map` fields are built at construction and never pickled. `Lexicon`'s copy of the guard admits the `ADDED_IN` marker, read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older pickles. A 2.x state holds the compiled sets instead, and `_lexicon._translated_2x` translates it onto the nearer named base and a per-word log, loading where the compile gives the sets back and refusing otherwise, naming the fields, so the 2.8 `Lexicon`, `Parser` and locale-pack pickles in `tests/v2/data/lexicon_v28.pickle` load as the 3.0 calls that build them. And unlike the shared guard it DOES re-validate: a load goes through the constructor, which checks the base, refuses an edit not in canonical form, and refuses a list or mask entry that is not already normalized, because an old blob would otherwise load a different lexicon in silence (`particles_ambiguous` changed meaning between v1 and v2 without changing its name). +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` and `LexiconReading` keep their own copy of the guard (layering); a new unpicklable slot type needs the same treatment plus a round-trip test. Since 3.0's 4i-b a `Lexicon` pickles its lists, its masks, its edit log, its choice of languages (`_languages` and `_strict`, 4i-c, unmarked because no release pickled a `Lexicon` without them, a 2.x state translating to no choice) and its base as a SNAPSHOT of the reading index's rows, so a pickle parses as it did where it was written, a snapshot equal to the running release's index loading as the by-reference default again (decisions/vocabulary-model, 2026-11-02 #754, item 5); its private `_sets`, `_cap_map` and `_chosen` fields are built at construction and never pickled. `Lexicon`'s copy of the guard admits the `ADDED_IN` marker, read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older pickles. A 2.x state holds the compiled sets instead, and `_lexicon._translated_2x` translates it onto the nearer named base and a per-word log, loading where the compile gives the sets back and refusing otherwise, naming the fields, so the 2.8 `Lexicon`, `Parser` and locale-pack pickles in `tests/v2/data/lexicon_v28.pickle` load as the 3.0 calls that build them. And unlike the shared guard it DOES re-validate: a load goes through the constructor, which checks the base, refuses an edit not in canonical form, and refuses a list or mask entry that is not already normalized, because an old blob would otherwise load a different lexicon in silence (`particles_ambiguous` changed meaning between v1 and v2 without changing its name). - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package, one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, `test_regex_sync.py`, which pins the pipeline's hand-copied patterns against their sources, and `test_snapshot.py`, which holds every corpus name's reading to `tools/differential/snapshot.jsonl` — sharing `_differential_fixtures.py` with `test_differential.py` and `test_corpora.py` for the handles on `tools/differential/`), names stating behavior; the 1.x suite's parsing tests stand ported to `parse()` in `tests/*.py`. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index d828102c..de2439fa 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -14,12 +14,12 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - `es` turns on the courtesy, academic and professional titles that lead a Spanish name with no complement (`lic`, `licenciado`, `licenciada`, `lcdo`, `lcda`, `ingeniero`, `ingeniera`, `arq`, `mtro`, `mtra`, `doctora`, `profesor`, `profesora`, `doña`) and `pt` `doutor`, `doutora`, `professora` and `profa`, `pt-BR` with it; `don`, `dona` and `dom` are excluded, the packs included, names opening with each as a name more often than as the title; `maestro`, a family name whose form of address is written `Mtro.`, is excluded too, and so is `h`, the hijo an initial is written as; `sr` keeps a generation reading beside its salutation in both files, so a choice leaves it the dual position decides; no connective continues a title run, the shape unwritten in name data, and the office titles that take a complement stay out — 2026-10-30 #753, phase 3g. - Of the romanized titles a pack's phase lists, one whose native spelling is a default title turns on unless, counted in persons and pooled across every population measured, names open with it more often than it opens one as the title; a spelling outside the phase's list (`Shreemati`, `Srimati`) stays open until a phase writes it as a reading: `hi` turns on `smt`, `shrimati` and `shriyut`, `bn` `smt`, `shrimati` and `adhyapak` (`shriyut` with no bearer anywhere, `adhyapak` with family-name bearers only, every one where the title claim, acting at the head, does not reach), while `shri`, opening given names in every population measured, stays excluded, and `sushri`, an Odia given name, waits with no reading written for data strict can read since 3j, a `default = false` salutation in hi.toml with the Odia name declared in or.toml (2026-10-31 #753, phase 3j), a plain choice being made for data that holds those names — 2026-10-30 #753, phase 3e. - A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict is file-strict, the chosen files and native-script vocabulary only, `mul.toml` and `und.toml` not among them, so it needs no 3i; third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek); 2026-10-29 #753, phase 3b; 2026-10-30 #753, file-strict. -- A strict choice is `parser_for(..., strict=True)` over the whole choice, applied as a delta as a plain choice is; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j. -- A choice is matched without regard to case, a tag whose file does not exist selects its ancestors' (`pt-PT` selects `pt`), `mul` and `und` are no language to choose, and a reading turned on also prefers, so `de`'s generation `i` drops `ca`'s and `pl`'s conjunction under a `de` choice; the `Lexicon` is to record its choice, so that a choice chained through `base=` unions with the base's, and a caller's own edits are to apply after the choice, plain or strict, in the order written (4i-c; until it lands a choice is one call's and drops the caller's base entries it removes) — 2026-10-29 #753, phase 3b; 2026-11-02 #754, the `Lexicon` surface (Derek). +- A strict choice is `parser_for(..., strict=True)` over the whole choice, filtering the base's readings as a plain choice does, the caller's own edits applying after it, and a chained call of the other strictness raises; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j; 2026-11-02 #754, sub-phase 4i-c. +- A choice is matched without regard to case, a tag whose file does not exist selects its ancestors' (`pt-PT` selects `pt`), `mul` and `und` are no language to choose, and a reading turned on also prefers, so `de`'s generation `i` drops `ca`'s and `pl`'s conjunction under a `de` choice; the `Lexicon` records its choice, so that a choice chained through `base=` unions with the base's, and a caller's own edits apply after the choice, plain or strict, in the order written, a pack's fragment joining before the caller's lexicon — 2026-10-29 #753, phase 3b; 2026-11-02 #754, the `Lexicon` surface (Derek); 2026-11-02 #754, sub-phase 4i-c. - A reading right in its own language and wrong only as a default, because another language written in the same script collides with it, is written `default = false` in its file, never excluded, so that after the packs an exclusion means wrong everywhere: cs `st` since phase 2h, and cs `ml`, hr, sk and sl `ml` and `st` and fr `père` since 3c, the rest in their packs' PRs — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3c. - A name reading written `default = false` derives no marker in the default and counts for no check guarding the default's compile; a particle whose word's only `given` readings are off is refused until a pack needs one — 2026-10-29 #753, phase 3a. - A caller configures vocabulary by reading, `Lexicon` deriving the sets and gates the build derives, and the compiled sets become internal in 3.0 — 2026-10-28 #754, the `Lexicon` surface (Derek). -- The configure-by-reading API: `add(*words, kind, **properties)` and `remove(*words, kind=None)`, the views reading a word's kind from the lexicon, `readings_of` returning a public `LexiconReading` and a repr counting readings by kind, a five-part state (base, ordered edit log, choice, lists, masks) compared structurally, a clash a later choice exposes raised at `parser_for` naming the choice, the choice recorded on the `Lexicon` with chained choices unioning, the default's base lazy, a 2.x pickle translated onto the nearer named base and a per-word log, old spellings refused with a per-word translation, built in 4i-a to 4i-d with no default parse moving — 2026-11-02 #754, the `Lexicon` surface (Derek). +- The configure-by-reading API: `add(*words, kind, **properties)` and `remove(*words, kind=None)`, the views reading a word's kind from the lexicon, `readings_of` returning a public `LexiconReading` and a repr counting readings by kind, a five-part state (base, ordered edit log, choice, lists, masks) compared structurally, a clash a later choice exposes raised at `parser_for` naming the choice, the choice recorded on the `Lexicon` with chained choices unioning, the default's base lazy, a 2.x pickle translated onto the nearer named base and a per-word log, old spellings refused with a per-word translation, built in 4i-a to 4i-d with no default parse moving, 4i-a to 4i-c built — 2026-11-02 #754, the `Lexicon` surface (Derek); 2026-11-02 #754, sub-phase 4i-a; 2026-11-02 #754, sub-phase 4i-b; 2026-11-02 #754, sub-phase 4i-c. - Configured by reading, an `add` of a title reading that a `given` reading keeps a title only inside a phrase is to warn, naming `remove(w, kind="given")`, save one that only updates a held title reading's properties — 2026-11-01 #754, the title warning (Derek). - A 2.x call or pickle that removed a word from its base set but left it in a marker subset reads as though the marker went with the word, for every marker set, the two readings 2.x gave such an orphan accepted as moving — 2026-11-02 #754, a marker left behind. - A `Locale` may carry CLDR-shaped render patterns, display and sorting, a later addition whose shape is agreed; how a call reaches one is open — 2026-10-28 #754, render patterns. @@ -314,10 +314,10 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **Measured** 2026-10-29 on `bd02d68f` plus this change, Python 3.11.17, each tag chosen alone against the default's readings with `snapshot.py --languages TAG`. Committed corpora, entries moved (names held only by `corpus_cldr.jsonl` among them): `pt` 38 (1), `pt-BR` 38 (1), `en` 54 (7), `de` 49 (0), `it` 14 (0), `cs` 11 (2), `es` 3 (1), `ar` 2 (1), `fr` 0. `changed_words` gives the words behind them: `pt` `do`, `e`, `sr`; `pt-BR` the same, pt-BR.toml holding no word that moves; `en` `abd`, `do`, `ra`, `sr`, `vd`; `de` `i`, `ra`; `it` `e`; `cs` `st`; `es` `h`, `sr`; `ar` `abd`; `fr` none. Every move read is one of these words: `pt` reads the one-case `e` as the connective in every comma shape and under all three orders (`ANA SILVA E SOUZA` → family `SILVA E SOUZA`; `SILVA E SOUZA ANA` under `FAMILY_FIRST` → family `SILVA E SOUZA`, given `ANA`), exactly as a lowercase `e` in mixed case reads in the default (`Ana Silva e Souza`), its report gone where no role moves (`SILVA E SOUZA, ANA`), and a mixed-case capital `E` stays the initial the writing makes it; `pt` and `es` read `Sr.` as the salutation, so `J. Smith Sr.` → title `Sr.`; `cs` turns on its generation `st` and the Saint readings leave, so `Jiří Dienstbier st.` → suffix `st.`, reported suffix-or-name since en's `given` reading still gates it, and `Jill St. John` → family `John`; `ar` reads `Haddad, Abd` given `Abd`. Local corpora, entries moved against the same tree's default readings (`snapshot.py --local SOURCE --base . --languages TAG`): `tse-donors` under `pt` 4,824 of 854,531, 4,751 of them names holding a bare `e` (`ABILIO ROMUALDO LIMA E SOUZA` → family `LIMA E SOUZA`), 72 more a `do` (a trailing `DO` read as a name word, `ALESSANDRA ALVES DOS SANTOS DO` → family `DO`) and one an `SR`, counted by the bare word each name holds; `tse-donors` under `en` 14,559, all but one holding `do`, which loses its particle (`ABADIA DO CARMO CARVALHO` → family `CARVALHO`); `ctgov-officials` under `en` 327 of 536,981, 223 of them names holding `Abd`, whose bound given reading leaves for the credential (`A M Abd El Baky` → family `M`), 43 `Sr`, 39 `do`, 18 `vd`, one `Ra`, and three a `do` inside a hyphenated word, which changes only its case repair (`Giao DO-PHAM` → `Do-Pham`); `psc-individuals` under `en` 209 (its 1,040,708 filed names), 131 holding `do` (`Alexandre Martelliti Do Amaral` → family `Amaral`), 38 `Abd`, 35 `Sr`, three `Ra`, de's academic title leaving it (`Mr Ra Jojo Cole` → given `Ra`), one `vd` and one a hyphenated `do` (`Costa-Do-Amaral`, case repair only); `ukec-individuals` under `en` and `nepal-candidates` under `ne` none, ne.toml holding no reading the default leaves off or that clashes. So choosing a language alone costs the readings of every word it holds a vocabulary reading of: `en`, which holds the credentials, is the costliest choice, and a pack is a language whose clashes are few. Each local count is by the first of the changed words a moved name holds as a bare word, a word inside a hyphenated one counted apart; every moved name holds one, so the local corpora show no break of non-interference either. Recompute: the commands above, with `uv run python -c "from nameparser import _reading_index as ri; print(sorted(ri.changed_words(ri.selected(['pt']))))"` for each tag's words. - **The no-choice path does not move.** tests/v2/test_snapshot.py passes unchanged; `parser_for` with packs choosing no language returns the base lexicon's value; and `tools/differential/measure.py --base` a checkout of `bd02d68f` reports no move in any committed or local corpus check. - **The deviation surface is derived (item 7).** tests/v2/test_locales.py's `test_non_interference_each_choice` runs every corpus name under `pt`, `en`, `cs`, `de` and `pt` with `en`, requiring the default's reading for every name holding no word of `changed_words` (matched over-wide: each piece whole, periods removed, cut at non-word characters, and a non-ASCII word anywhere), about 1 s a choice on one worker; its negative control removes `e` from `pt`'s surface and fails on the names it moves. tests/v2/test_reading_index.py holds, for every file tag chosen alone, that a choice changes only words its selected files hold. decisions/deviates-registry records the registry half: a registered vocabulary pack's `DEVIATES` is the derived surface from 3d. - - **A choice chained through `base=` is applied on top of the base's, not unioned with it**, decided in the next entry (option (a) of three weighed there). A `Parser` records no choice, so `parser_for(EN, base=parser_for(PT))` applies `en`'s delta to a lexicon `pt` already changed: `Maria do Carmo` reads family `Carmo` and `John Smith DO` family `DO`, where `parser_for(PT, EN)` keeps both readings (`do Carmo`, `Smith`). Item 1's order-independence holds within one call. + - (Superseded by 2026-11-02 #754, the `Lexicon` surface, effective with 4i-c on 2026-11-02: a chained choice now unions with the base's.) **A choice chained through `base=` is applied on top of the base's, not unioned with it**, decided in the next entry (option (a) of three weighed there). A `Parser` records no choice, so `parser_for(EN, base=parser_for(PT))` applies `en`'s delta to a lexicon `pt` already changed: `Maria do Carmo` reads family `Carmo` and `John Smith DO` family `DO`, where `parser_for(PT, EN)` keeps both readings (`do Carmo`, `Smith`). Item 1's order-independence holds within one call. - **`Reading.languages` does not follow a choice: it names the files holding the reading on by default, whatever choice built the parser.** SUPERSEDES: the 4a follow-up entry's expectation, in its item 1, that 3b changes `languages_of`'s filter on `on_by_default`. Measured on this branch merged with that follow-up: under `pt`, the one-case `E` of `ANA SILVA E SOUZA` reads `conjunction` with languages `('it', 'pt')`, though prefer dropped `it`'s reading from the pool; under `cs`, the `st.` of `Jiří Dienstbier st.` reads `generation` with languages `()`, though `cs.toml` holds the reading the choice turned on. Kept, for two reasons: the follow-up decided the field names the default's files whatever `Lexicon` the parse used, a caller's own `St` in `suffix_words` naming none for the same `cs` reading; and a parse cannot know its choice, a `Parser` recording none (the chained-choices entry below), so a field that followed the choice needs the `Parser.languages` left open there. The cost is that a reading a choice turns on names no file and a language a choice dropped is still named; tests/v2/test_parser.py's `test_reading_languages_name_the_default_files_under_a_choice` pins both, so a change with `Parser.languages` moves them deliberately. - Not touched: `Lexicon` and `Parser` gain no field; `_locale.py` imports nothing new beyond `_types.ADDED_IN` and `_policy`'s string-and-mapping guard; `_parser.py` imports `_reading_index` at call time, and tests/v2/test_layering.py's table says so. 4f's `[rules]` compiled into a `Locale`'s policy patch needs no change to this shape: a pack's policy patch and its `languages` are separate fields. Strict (item 4) is not built. -- **2026-10-29 #753, phase 3b, chained choices (Derek) — SUPERSEDED by 2026-11-02 #754, the `Lexicon` surface (to take effect with 4i-c) — ONLY ONE CALL'S CHOICE COUNTS: A CHOICE CHAINED THROUGH `base=` APPLIES ON TOP OF THE BASE'S, AND A CHOICE DROPS THE CALLER'S OWN BASE ENTRIES THAT IT REMOVES.** Derek decided in chat on 2026-10-29, choosing option (a) of the three the phase 3b entry above weighed. `parser_for` unions the `languages` of the packs passed to it and applies that one choice as a delta against the default's compile; a `Parser` records no choice, so a base built by an earlier `parser_for` carries its choice only as the lexicon it changed. Two consequences, both pinned by tests in tests/v2/test_parser.py so that a change to them is deliberate: +- **2026-10-29 #753, phase 3b, chained choices (Derek) — SUPERSEDED by 2026-11-02 #754, the `Lexicon` surface (in effect since 4i-c, 2026-11-02 #754, sub-phase 4i-c) — ONLY ONE CALL'S CHOICE COUNTS: A CHOICE CHAINED THROUGH `base=` APPLIES ON TOP OF THE BASE'S, AND A CHOICE DROPS THE CALLER'S OWN BASE ENTRIES THAT IT REMOVES.** Derek decided in chat on 2026-10-29, choosing option (a) of the three the phase 3b entry above weighed. `parser_for` unions the `languages` of the packs passed to it and applies that one choice as a delta against the default's compile; a `Parser` records no choice, so a base built by an earlier `parser_for` carries its choice only as the lexicon it changed. Two consequences, both pinned by tests in tests/v2/test_parser.py so that a change to them is deliberate: 1. **Chained choices compose one after the other.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `Carmo` and `John Smith DO` family `DO`: `en` drops `pt`'s particle and `pt` had dropped `en`'s credential. The way to choose both is one call, `parser_for(PT, EN)`, which keeps both readings (`do Carmo`, suffix `DO`). 2. **A choice drops what it removes from the base, whoever put it there.** `parser_for(EN, base=Parser(lexicon=Lexicon.empty().add(particles={'do', 'van'})))` keeps only `van`: the delta removes `do` from `particles`, the caller's listing included, and leaves every entry it does not touch. Weighed against: (b) a `Parser.languages` field recording the choice, so a base's choice unions with the packs' (a new public name, safe for pickles, `Parser`'s guard honoring `ADDED_IN`), deferred rather than declined; (c) a `Lexicon` field recording it, which every older `Lexicon`, `Parser` and pack pickle would refuse, `Lexicon`'s guard not honoring `ADDED_IN` (an objection removed on 2026-10-30, when that guard learned the marker, #960, decisions/A1). Open: (b), with the configure-by-reading method names (the 2026-10-28 #754 `Lexicon` surface entry), and with it whether `Reading.languages` follows the choice (the phase 3b entry above). @@ -450,7 +450,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **3i's evidence, kept** from the RFC paragraph this entry retires, should 3i be wanted again: all measured 2026-10-28 on master `c8357e74`, not on this entry's tree: of the 227 name readings, 165 words carried one; `do`'s family box led by Brazil in ORCID (none of Brazil's 7,227 `do` alone, 2,629 opening with it and 4,598 holding it inside) but by share Vietnam (1.90% against 0.81%; PatentsView 1.78% and 0.64%), Vietnam having 618 boxes of `do` alone and Korea 151; `kim`'s family box 17.67% of ORCID's Korean holders and 13.30% of PatentsView's, its given box 0.93% of PatentsView's Danish inventors. Recompute: `uv run python tools/name_usage.py do kim --no-wikidata --boxes`, and the whole-box split from the cached ORCID extract's `family` and `country` keys. - **Candidate next steps, not taken here:** Vietnamese degree prefixes as vi.toml readings, each checked for collisions and bearers per nameparser/vocab/AGENTS.md first; `ông` and `bà` stay excluded, Ông being a Vietnamese surname, a clash within Vietnamese strict does not settle; 3h's `nl` prepositions and 3e's `sushri` as data with the clashing name readings declared in ko.toml, vi.toml and or.toml, gated under the plain choice and not under strict, though the plain `nl` choice still reads `Park In Hwan` family `In Hwan` and `Nguyen To Nga` family `To Nga`, item 4's recorded cost, the marker acting at the head only (re-run 2026-10-31), which a follow-up must settle before shipping them; `_reading_index`'s `UNCHOOSABLE` comment and `selected`'s refusal text, which say no choice removes a reading of `mul` or `und`, updated with 3j; the 16 ungated words read one by one. - **Recompute:** a strict choice of the tags `T` is `compile_readings(choose(rows, T))` from `nameparser._reading_index`, `rows` the `index()` rows whose tag is in `T` or whose text has a letter and no letter whose `unicodedata.name` contains `LATIN`, applied to `Lexicon.default()` as `_parser._apply_choice` applies `_choice_delta` (the delta taken against `compile_choice(frozenset())`, the markers of a dropped base entry removed), and a pack's own patch and fragment applied by `parser_for(dataclasses.replace(pack, languages=frozenset()), base=Parser(lexicon=that))`. Count a word's share of the moves as the moves whose name, split at spaces and commas with edge periods dropped and casefolded, holds it. Score ORCID with `local_agreement.read(record, local_agreement.FIELDED["orcid-researchers"], parser, local_agreement._title_words(parser.lexicon))` over the cache's `raw.jsonl` records of each country, and PSC the same with `"psc-individuals"` and the `nationality` key; the title-word filter is the given box's first word, split at spaces and periods and casefolded, in {dr, prof, professor, mr, mrs, ms, miss, dra, ing, mgr, assoc, assist, er, engr, drs, ir}. The ungated words: for each choosable tag, the words in a marker (`particles_ambiguous`, `suffix_acronyms_ambiguous`, `suffix_words_ambiguous`, `titles_ambiguous`) of the plain choice of that tag (`parser_for(Locale(code="x", lexicon=Lexicon.empty(), languages={tag})).lexicon`) that stay in the base set of the strict lexicon and leave the marker, or join `titles`; compared with `Lexicon.default()`'s markers instead, it misses the readings a choice turns on (`st` under cs, hr, sk and sl, `nic` under gd) and finds 15 words. The scripts are in no tree: they are a few dozen lines each from this recipe. -- **2026-10-31 #753, phase 3j (its built clause that a caller's listing of a dropped word is dropped by the delta superseded by 2026-11-02 #754, the `Lexicon` surface, to take effect with 4i-c) — FILE-STRICT IS BUILT: `parser_for(..., strict=True)` COMPILES ONLY THE CHOSEN FILES AND NATIVE-SCRIPT VOCABULARY; A STRICT CALL CHOOSING NO LANGUAGE RAISES; NO `locales.choose`; THE WORDS STRICT UNGATES ARE DERIVED AND HELD BY A TEST; A FILE HOLDING NO READING IS A STRICT CHOICE ONLY; THE DEFAULT DOES NOT MOVE.** SUPERSEDES: in the 2026-10-30 #753, file-strict entry, its verdict's "the rest of its API is proposed, not decided, and nothing is built"; its "Proposed, not decided" bullet, every item decided as proposed save `locales.choose(*tags)`, which is left out; and the files-with-no-readings bullet's "strict `vi` needs vi.toml readings first". Carries out 3j of item 11 of the 2026-10-27 #753 entry, with the definition of the 2026-10-30 entry. The four API points below were decided on 2026-10-31; the rest of the 2026-10-30 entry's proposed bullet is built as proposed (no `Locale.strict` field; the chosen files' `default = false` readings turned on and no other file's; patches and fragments applied after the strict compile, untouched; `Lexicon.given_names` and `Lexicon.surnames` outside it; packs stacked as a union; a caller's listing of a dropped word dropped by the delta). Built on master `8438ccbb`. +- **2026-10-31 #753, phase 3j (its built clause that a caller's listing of a dropped word is dropped by the delta superseded by 2026-11-02 #754, the `Lexicon` surface, in effect since 4i-c, 2026-11-02 #754, sub-phase 4i-c) — FILE-STRICT IS BUILT: `parser_for(..., strict=True)` COMPILES ONLY THE CHOSEN FILES AND NATIVE-SCRIPT VOCABULARY; A STRICT CALL CHOOSING NO LANGUAGE RAISES; NO `locales.choose`; THE WORDS STRICT UNGATES ARE DERIVED AND HELD BY A TEST; A FILE HOLDING NO READING IS A STRICT CHOICE ONLY; THE DEFAULT DOES NOT MOVE.** SUPERSEDES: in the 2026-10-30 #753, file-strict entry, its verdict's "the rest of its API is proposed, not decided, and nothing is built"; its "Proposed, not decided" bullet, every item decided as proposed save `locales.choose(*tags)`, which is left out; and the files-with-no-readings bullet's "strict `vi` needs vi.toml readings first". Carries out 3j of item 11 of the 2026-10-27 #753 entry, with the definition of the 2026-10-30 entry. The four API points below were decided on 2026-10-31; the rest of the 2026-10-30 entry's proposed bullet is built as proposed (no `Locale.strict` field; the chosen files' `default = false` readings turned on and no other file's; patches and fragments applied after the strict compile, untouched; `Lexicon.given_names` and `Lexicon.surnames` outside it; packs stacked as a union; a caller's listing of a dropped word dropped by the delta). Built on master `8438ccbb`. 1. **A strict call that chooses no language raises `ValueError`.** The message names the packs given and offers a pack that chooses a language, the first pack's code where a strict choice accepts it as one (`parser_for(locales.ZH, strict=True)` offers `Locale(code='zh', lexicon=Lexicon.empty(), languages=frozenset({'zh'}))`), else `en`; tests/v2/test_parser.py runs the offered code. 2. **No public `locales.choose`.** A bare choice is a `Locale` with an empty lexicon and the tags, the spelling docs/locales.rst teaches. 3. **The words strict ungates are derived**, `_reading_index.strict_ungated`, each strict choice against the plain choice of the same tags and never against the default's markers, which miss a reading the choice turns on; tests/v2/test_reading_index.py holds every one-tag strict choice to a per-tag table, which is the 2026-10-30 entry's 16 words and 27 word-and-tag pairs over 13 tags exactly, so a word that joins or leaves is a reviewed diff. @@ -478,7 +478,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **Revised after the third independent review** (2026-11-01, of `911d97a0..f5b4e014`): the phrase quoted as 3j's, "a bare choice has no constructor", is the Standing line's restatement, not the 2026-10-31 #753, phase 3j entry's text, whose item 2 is now quoted; that item binds the public spelling of a bare choice, so the underscore parameters are a private route beside it and no exception to it, and item 4 says the choice is set publicly only by `parser_for`; `parser_for` joins the packs' fragments in pack order, a later pack winning as the policy fold has it, before the caller's lexicon; edits are stored frozen; and the clash at `parser_for` is offered as raise (recommended) or caller-wins. - Not touched: no code, no test, no rule; `Lexicon`, `Parser` and `Locale` keep their 2.x shape until 4i-b. - **2026-11-01 #754, the title warning (Derek) — AN `add` OF A TITLE READING THAT A `given` READING KEEPS A TITLE ONLY INSIDE A PHRASE WARNS, NAMING `remove(w, kind="given")`; DECIDED AHEAD OF THE REST OF THE `Lexicon` SURFACE, WHICH STAYS PROPOSED.** Decided on 2026-11-01, on item 1 of the 2026-11-01 #754, the `Lexicon` surface proposed entry. Under the configure-by-reading API, when it is built (4i-b), an `add` of a title reading warns at construction (`UserWarning`) where, in the compile under no choice, the word's `given` reading makes it a title only inside a phrase (rules/H9), the warning naming `remove(w, kind="given")`: the trap of the 1.4 recipe `add(titles={"dean"})`, whose literal translations `add("dean", kind="title")` and `add("dean", kind="office")` move no parse (0 of 45 shapes, against 27 of 45 for the 2.x spelling, measured by an independent review on 2026-11-01). It fires where the `add` creates a title reading or re-adds a held one with no property changed, and not where it only updates the properties of a title reading the word holds (`add("prince", kind="nobility", attaches=[...])`), the scope the review proposed with the decision. The reference is the compile under no choice, stated because a caller's edits apply after any later choice; nothing else of the proposal is decided by this entry. -- **2026-11-02 #754, the `Lexicon` surface (Derek) — THE CONFIGURE-BY-READING API IS DECIDED AS RECOMMENDED; CALLER-WINS AT `parser_for`, AN EAGER DEFAULT BASE AND REFUSING EVERY 2.x PICKLE ARE DECLINED; NOTHING IS BUILT.** Decides the 2026-11-01 #754, the `Lexicon` surface proposed entry, each item as it recommends; its arguments are the RFC's Phase 4 section "The `Lexicon` surface: configuring by reading", and its measurements that entry. SUPERSEDES: both items of the 2026-10-29 #753, phase 3b, chained choices entry, and the 2026-10-31 #753, phase 3j entry's built clause that a caller's listing of a dropped word is dropped by the delta (each effective when 4i-c lands). +- **2026-11-02 #754, the `Lexicon` surface (Derek) — THE CONFIGURE-BY-READING API IS DECIDED AS RECOMMENDED; CALLER-WINS AT `parser_for`, AN EAGER DEFAULT BASE AND REFUSING EVERY 2.x PICKLE ARE DECLINED; NOTHING IS BUILT.** Decides the 2026-11-01 #754, the `Lexicon` surface proposed entry, each item as it recommends; its arguments are the RFC's Phase 4 section "The `Lexicon` surface: configuring by reading", and its measurements that entry. SUPERSEDES: both items of the 2026-10-29 #753, phase 3b, chained choices entry, and the 2026-10-31 #753, phase 3j entry's built clause that a caller's listing of a dropped word is dropped by the delta (each in effect since 4i-c landed, 2026-11-02 #754, sub-phase 4i-c). 1. **Configuring.** `Lexicon.add(*words, kind, **properties)` and `Lexicon.remove(*words, kind=None)`, kinds and properties as the language files spell them; derived markers through `compile_readings`; the `generation`, `credentials` and `patronymic` views reading a listed word's kind from the lexicon, rules/R8 and rules/R9 amended when 4i-b builds it. The title warning was decided on 2026-11-01 (2026-11-01 #754, the title warning). 2. **Queries.** `Lexicon.readings_of(word)` returns a public `LexiconReading`; the repr counts readings by kind against the nearer named lexicon. 3. **State.** Five parts, a base, an ordered log of the caller's edits, the choice, the lists and the masks, compared structurally with the log as written and each edit stored frozen; the base, log and choice constructor parameters under underscore names, validated as `parser_for` validates a choice; `|` later-wins, `parser_for` joining the packs' fragments in pack order and the caller's lexicon last. An edit that clashes with a reading a later choice turns on raises at `parser_for`, naming the choice and the edit. @@ -504,4 +504,8 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-11-02 #754, a marker left behind — A 2.x CALL OR PICKLE THAT REMOVED A WORD FROM ITS BASE SET BUT LEFT IT IN A MARKER SUBSET READS AS THOUGH THE MARKER WENT WITH THE WORD, FOR EVERY MARKER SET; THE TWO READINGS 2.x GAVE SUCH AN ORPHAN ARE ACCEPTED AS MOVING.** Decided in chat on 2026-11-02, on the review of 4i-b's fix commit. In 3.0 a marker is a property of its reading (`addresses="given"` on the title, `may_be_initial` on the conjunction, the `modifier` and `style` kinds, a name reading beside a suffix word) or a kind, so taking the reading out takes the marker: `remove(titles={"sir"})` removes `sir` as a title, and being a given-name title goes with it. 4i-b's `_lexicon._inert_dropped` drops each marker entry its base sets do not hold (`_MARKER_READ_WITH`), on both sides of every compare, so the refusal's translation of such a call and the load of such a pickle agree with it. - **Accepted: what 2.x read of an orphan, measured on master `3804b1a1` against this tree by the orchestrating review (2026-11-02).** (1) An orphan in `given_name_titles` still made rules/H2's period-shape title address the given name: with `remove(titles={"sir"})`, `Sir. Smith`, `Sir. Elton` and `Smith, Sir.` read given `Smith` and `Elton` on master and family on the tree; `add(given_name_titles={"zork"})` and `Zork. Smith` the same. (2) Any orphan counted as a listed word in the capitals shape's "unlisted" test (`_vocab.in_any_wordlist`): with `remove(titles={"air"})`, `John Smith, AIR` reads given `AIR`, family `John Smith` on master and suffix `AIR` on the tree, unlisted as any word a caller removed. tests/v2/test_lexicon.py's `_LEFT_BEHIND` pins the tree's readings, under the translated call and the loaded pickle alike; recompute by building the 2.x state with the word taken out of its base set alone, as that test's `_state_2x` does, and parsing each text on both trees. - Weighed against: keeping the orphan's two readings, which would need a reading that marks without its base, the very state the per-word model was built not to express. - +- **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. + - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. + - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **The strictness mismatch is read off base's recorded choice,** so a chained call choosing no language (`parser_for(locales.RU, base=strict_parser)`) raises too unless it passes `strict=True`; the message offers both spellings. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. + - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. + - **Measured.** On this branch over `db7e8fc4`, Python 3.11.17: tests/v2/test_snapshot.py passes unchanged; `uv run python tools/differential/cldr_agreement.py` reports the CLDR locale view and the Wikidata native locale view matching their records (JA not measured, namedivider absent); frames, `uv run python tools/perf/call_count.py --against db7e8fc4`, the reference row 311.48 on both, `_CALL_BASELINE` unchanged and the serial benchmark passing. diff --git a/docs/design/mechanisms.md b/docs/design/mechanisms.md index 8c6f2fe9..168d1ce5 100644 --- a/docs/design/mechanisms.md +++ b/docs/design/mechanisms.md @@ -332,11 +332,11 @@ a fork for all ambiguous particles on "Dr. Van Jr.". **Problem shape.** Language-specific behavior needs a home that cannot drift from the core. -**Contract statement.** A locale pack is pure data — a choice of languages, a Lexicon fragment of ADDITIONS and a Policy patch — applied by parser_for, a shipped pack's patch compiled from its language file's `[rules]` table where one is written, each key the `Policy` field of its name: the packs' languages union into one choice, which selects among the default vocabulary's readings by the file each sits in, the fragments union onto the result, and the patches fold in order; packs contain no code paths of their own. A vocabulary pack's deviations are derived from the reading index, the words whose compiled readings its choice changes; a policy or segmentation pack declares its own in its docstring and `DEVIATES`. +**Contract statement.** A locale pack is pure data — a choice of languages, a Lexicon fragment of ADDITIONS and a Policy patch — applied by parser_for, a shipped pack's patch compiled from its language file's `[rules]` table where one is written, each key the `Policy` field of its name: the packs' languages union with the base's recorded choice into one choice, recorded on the `Lexicon`, which selects among the base's shipped readings by the file each sits in, the fragments join in pack order and the caller's lexicon after them, its own readings applying after the choice, and the patches fold in order; packs contain no code paths of their own. A vocabulary pack's deviations are derived from the reading index, the words whose compiled readings its choice changes; a policy or segmentation pack declares its own in its docstring and `DEVIATES`. -**How it works.** What is policy for every language (an order constant) is policy, not pack data; packs are lowercase modules exposing uppercase constants; a pack error is wrapped with the pack's code (rule D2). The fragment never removes a base entry; a choice may, and only by selection, never by a list of removals: a chosen language's `default = false` readings join; on a word a chosen language holds a vocabulary reading of, other languages' vocabulary readings leave, save `mul.toml`'s; and a chosen language's conjunction loses `may_be_initial`, its clash with the initial being a shape reading a preference outranks, which is how a `pt` choice drops `e` from `conjunctions_ambiguous` (decisions/vocabulary-model, 2026-10-27 #753, items 1, 2 and 7; built 2026-10-29, phase 3b). A strict choice, `parser_for(..., strict=True)`, selects by file alone: only the chosen files and native-script vocabulary compile, `mul.toml`'s and `und.toml`'s readings leaving with every other file's, and it too is applied as a delta (decisions/vocabulary-model, 2026-10-30 #753, file-strict; built 2026-10-31, phase 3j). So a bad default is routed around only by choosing the language whose reading is right, which is why #342-class fixes still change shipped vocabulary. The choice is applied as a delta against the default's compile: it adds to the base lexicon what the choice turns on and removes from it what the choice drops, whichever of the default and the caller put a removed entry there, and leaves every other entry, a caller's own included; a pack choosing no language leaves the base's exactly as it was. The choice is one call's: a `Parser` records none, so `parser_for(EN, base=parser_for(PT))` applies `en`'s after `pt`'s and keeps neither language's `do`, where `parser_for(PT, EN)` keeps both (decisions/vocabulary-model, 2026-10-29, chained choices). Pure data means a pack can be audited: a vocabulary pack's whole content is its tag, but what it changes reaches other files too, prefer dropping their readings of its words (a `de` choice drops `ca.toml`'s conjunction `i`), so the audit is the derived surface, `changed_words`, not the pack's file alone. +**How it works.** What is policy for every language (an order constant) is policy, not pack data; packs are lowercase modules exposing uppercase constants; a pack error is wrapped with the pack's code (rule D2). A choice changes the base's readings only by selection, never by a list of removals: a chosen language's `default = false` readings join; on a word a chosen language holds a vocabulary reading of, other languages' vocabulary readings leave, save `mul.toml`'s; and a chosen language's conjunction loses `may_be_initial`, its clash with the initial being a shape reading a preference outranks, which is how a `pt` choice drops `e` from `conjunctions_ambiguous` (decisions/vocabulary-model, 2026-10-27 #753, items 1, 2 and 7; built 2026-10-29, phase 3b). A strict choice, `parser_for(..., strict=True)`, selects by file alone: only the chosen files and native-script vocabulary compile, `mul.toml`'s and `und.toml`'s readings leaving with every other file's (decisions/vocabulary-model, 2026-10-30 #753, file-strict; built 2026-10-31, phase 3j). So a bad default is routed around only by choosing the language whose reading is right, which is why #342-class fixes still change shipped vocabulary. The choice filters the base's shipped readings and the caller's own `add` and `remove` calls apply after it, so a choice never drops a caller's word, and one that clashes with a reading the choice turns on is refused at `parser_for`, naming the edit; a lexicon built from `Lexicon.empty()` holds no shipped reading, so a choice turns none on in it; a pack choosing no language leaves the base's exactly as it was. The choice is recorded on the `Lexicon`, so `parser_for(EN, base=parser_for(PT))` chooses both, as `parser_for(PT, EN)` does (decisions/vocabulary-model, 2026-11-02 #754, the `Lexicon` surface, items 3 and 4; built 2026-11-02, sub-phase 4i-c). Pure data means a pack can be audited: a vocabulary pack's whole content is its tag, but what it changes reaches other files too, prefer dropping their readings of its words (a `de` choice drops `ca.toml`'s conjunction `i`), so the audit is the derived surface, `changed_words`, not the pack's file alone. -**Lives in.** nameparser/locales/ (packs), nameparser/_parser.py (parser_for, the one applier), nameparser/_reading_index.py (the selection, `choose`, and the derived surface, `changed_words`), nameparser/_locale.py (`rules_patch`, a file's compiled `[rules]` as a patch) and tools/vocab/build.py (`RULE_KEYS`, the rules a file may turn on, and `PACK_RULES`, the files the shipped packs read; decisions/vocabulary-model, 2026-10-29 4f). A rule a file turns on is a `Policy` field; a new rule kind is a core change, never code in a pack: a new `Policy` field (with `ADDED_IN`), a row in build.py's `RULE_KEYS` and in `_locale._RULE_VALUES`, and a mechanisms.md entry of its own for the rule kind. +**Lives in.** nameparser/locales/ (packs), nameparser/_parser.py (parser_for, the one applier), nameparser/_lexicon.py (the compile under a recorded choice, `_chosen_pool` and `_built`), nameparser/_reading_index.py (the selection, `choose`, and the derived surface, `changed_words`), nameparser/_locale.py (`rules_patch`, a file's compiled `[rules]` as a patch) and tools/vocab/build.py (`RULE_KEYS`, the rules a file may turn on, and `PACK_RULES`, the files the shipped packs read; decisions/vocabulary-model, 2026-10-29 4f). A rule a file turns on is a `Policy` field; a new rule kind is a core change, never code in a pack: a new `Policy` field (with `ADDED_IN`), a row in build.py's `RULE_KEYS` and in `_locale._RULE_VALUES`, and a mechanisms.md entry of its own for the rule kind. **Reach for it when.** A language fix wants an if-statement — make it vocabulary or policy in a pack instead. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index 6a56d711..c365f1a8 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -463,7 +463,7 @@ How a call reaches a pack's pattern is open, and each answer costs something: `P ### The `Lexicon` surface: configuring by reading -Status: **decided 2026-11-02 as recommended below (decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry), item 1's title warning on 2026-11-01 (decisions/vocabulary-model's 2026-11-01 #754, the title warning entry); 4i-a and 4i-b are built (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-a and sub-phase 4i-b entries), 4i-c and 4i-d are not.** What is decided is the direction (2026-10-28): a caller configures vocabulary by reading, `Lexicon` derives the sets and gates the build derives, and the compiled sets become internal (on 2026-11-01, 20 of `Lexicon`'s 23 public fields: the 19 of `_lexicon._COMPILED_FIELDS` and the derived `particles_ambiguous`; item 3 keeps the other three), with no 2.x warning (2026-10-29). This section proposes the API that direction leaves open, each question with the options weighed and one recommendation; decisions/vocabulary-model's 2026-11-01 #754, the `Lexicon` surface proposed entry records the proposal and its measurements, and decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry the decision, each recommendation accepted and the alternatives below declined. Where this section says "proposed" or "recommended", read "decided"; the wording is kept as the record of what was weighed. Every parse below was run on master `a3b1f3e1` (Python 3.11.17) through a throwaway prototype that compiles a lexicon's readings with the shipped `compile_readings` over `choose(index(), chosen)`, the whole index with its readings off by default, as item 4 has the default hold them, the caller's edits applied after the choice, and folds the result into today's `Lexicon`, so the real parser reads it; the prototype is not committed. +Status: **decided 2026-11-02 as recommended below (decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry), item 1's title warning on 2026-11-01 (decisions/vocabulary-model's 2026-11-01 #754, the title warning entry); 4i-a, 4i-b and 4i-c are built (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-a, sub-phase 4i-b and sub-phase 4i-c entries), the optional 4i-d is not.** What is decided is the direction (2026-10-28): a caller configures vocabulary by reading, `Lexicon` derives the sets and gates the build derives, and the compiled sets become internal (on 2026-11-01, 20 of `Lexicon`'s 23 public fields: the 19 of `_lexicon._COMPILED_FIELDS` and the derived `particles_ambiguous`; item 3 keeps the other three), with no 2.x warning (2026-10-29). This section proposes the API that direction leaves open, each question with the options weighed and one recommendation; decisions/vocabulary-model's 2026-11-01 #754, the `Lexicon` surface proposed entry records the proposal and its measurements, and decisions/vocabulary-model's 2026-11-02 #754, the `Lexicon` surface entry the decision, each recommendation accepted and the alternatives below declined. Where this section says "proposed" or "recommended", read "decided"; the wording is kept as the record of what was weighed. Every parse below was run on master `a3b1f3e1` (Python 3.11.17) through a throwaway prototype that compiles a lexicon's readings with the shipped `compile_readings` over `choose(index(), chosen)`, the whole index with its readings off by default, as item 4 has the default hold them, the caller's edits applied after the choice, and folds the result into today's `Lexicon`, so the real parser reads it; the prototype is not committed. **1. Configuring: `add` and `remove` take words and a reading kind.** Proposed: @@ -561,7 +561,7 @@ All three were checked on a frozen, slotted throwaway dataclass on 2026-11-01. T |---|---|---| | 4i-a | The pipeline reads the compiled sets through one internal object, not `Lexicon`'s public fields; the three tools move to it or to the index. No public change; a frozenset test stays a C-level attribute read, so the frame count holds (`_CALL_BASELINE`). Landed 2026-11-02 (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-a entry) | — | | 4i-b | The break: `Lexicon` holds readings, `add`/`remove` by kind, `readings_of` and `LexiconReading`, the narrowed constructor, the three refusals with their translation, the per-kind repr, the warning on a title reading made phrase-only by a `given` reading, the views reading the lexicon's kinds with rules/R8 and rules/R9 amended, the lazy base, the 2.x pickle translation; the tests and docs moved, `docs/migrate.rst`'s table, a release-log bullet. Landed 2026-11-02 (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-b entry) | 4i-a | -| 4i-c | The choice recorded on the `Lexicon`: chained choices union, a caller's readings applied after the choice, the strict mismatch raised, `|` refusing two choices, a pack's fragment joined under the caller's lexicon, the compile and its refusals at `parser_for`, the underscore choice parameters validated, `_apply_choice` retired; the chained-choices entry and 3j's dropped-listing clause superseded | 4i-b | +| 4i-c | The choice recorded on the `Lexicon`: chained choices union, a caller's readings applied after the choice, the strict mismatch raised, `|` refusing two choices, a pack's fragment joined under the caller's lexicon, the compile and its refusals at `parser_for`, the underscore choice parameters validated, `_apply_choice` retired; the chained-choices entry and 3j's dropped-listing clause superseded. Landed 2026-11-02 (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry) | 4i-b | | 4i-d | Optional: `Reading.languages` follows the choice | 4i-c | ### The order of work @@ -578,7 +578,7 @@ Each sub-phase is one PR, measured on its own base as phase 2's and phase 3's ar | 4f | `[rules]` compiled into a `Locale`'s policy patch; RU and TR_AZ rebuilt from their files. Landed (decisions/vocabulary-model's 2026-10-29 sub-phase 4f entry) | 3b | | 4g | the compound given name, a rule kind (#704): built 2026-10-30, `Policy.two_given_names` (rules/O7) | 4f | | 4h | the shape inventory per order (#469), starting with the title between a family and a given name under `FAMILY_FIRST`. Landed as an inventory, no reading moved: the arrangements each order is written in, listed with what each reads (decisions/vocabulary-model's 2026-10-30 #754, sub-phase 4h entry); the title between, and bare after, under a family-first order stays a name word, #938 declined on a survey of real names, which write the title in front (decisions/H5's 2026-10-30 #938 entry) | — | -| 4i | the `Lexicon` surface: configuring by reading, the compiled sets internal, and the choice recorded on the `Lexicon`. Decided 2026-11-02, to be built in four PRs, 4i-a and 4i-b landed (4i-a to 4i-d, "The `Lexicon` surface: configuring by reading" above) | 3b, 3j | +| 4i | the `Lexicon` surface: configuring by reading, the compiled sets internal, and the choice recorded on the `Lexicon`. Decided 2026-11-02, to be built in four PRs, 4i-a, 4i-b and 4i-c landed (4i-a to 4i-d, "The `Lexicon` surface: configuring by reading" above) | 3b, 3j | **Interleaved with phase 3** (its sub-phases 3a to 3j, above): 4a can start beside 3a, its `languages` waiting for 3a's index; 4d needs nothing of phase 3, and 4b and 4c need 3a's index only for a listed word's kind, a suffix word's and a particle's; they unblock its packs, `cs`'s generations (3f) and `pt-BR`'s agnomens (3h) on 4b, `es` and `pt` (3g) on 4d and 4g; 4f lands with or after 3b, since both decide what a shipped `Locale` holds; an `is` pack, which no phase 3 sub-phase lists, needs 3b, 4c and 4f. 3j, file-strict, needs 3b alone (3i, which name-strict would have read, is set aside), and 4e and 4h stand apart from both. So the order is 3a and 4a, then 4b to 4d with 3b and 3c (4b after 3a), then 4f, then the packs with 4g, with 4e at any point and 3j at any point after 3b. diff --git a/docs/locales.rst b/docs/locales.rst index 7c7342c5..f4bf8a11 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -200,32 +200,37 @@ checked in one of four ways: A pack that chooses no language leaves the base parser's vocabulary exactly as it was. -Choosing in one call -^^^^^^^^^^^^^^^^^^^^ +Chaining choices +^^^^^^^^^^^^^^^^ -A choice belongs to one :func:`~nameparser.parser_for` call: the -languages of the packs passed together are one choice. A base parser -records no choice of its own, so a choice made on top of a base built -by an earlier call applies after the base's, and settles the base's -words its own way. To choose two languages, pass both packs in one -call: +A parser records the languages it was chosen under, as +:attr:`Parser.languages ` (read from its +:attr:`Lexicon.languages `), so a choice +made on top of a base built by an earlier call joins the base's: the +two read as one call choosing both. .. doctest:: >>> chained = parser_for(en, base=parser_for(pt)) - >>> chained.parse("Maria do Carmo").family - 'Carmo' - >>> chained.parse("John Smith DO").family - 'DO' - >>> together = parser_for(pt, en) - >>> together.parse("Maria do Carmo").family, together.parse("John Smith DO").suffix + >>> sorted(chained.languages) + ['en', 'pt'] + >>> chained.parse("Maria do Carmo").family, chained.parse("John Smith DO").suffix ('do Carmo', 'DO') + >>> chained.lexicon == parser_for(pt, en).lexicon + True + +A strict choice (`Reading one language only`_) chains only with another +strict one: a chained call whose ``strict`` differs from its base's +raises ``ValueError``. -A choice applied to a base also removes what it drops from the base's -vocabulary, even where you added the word yourself, and keeps every -word it does not touch. Under ``en``, a base listing ``do`` and ``van`` -as particles keeps only ``van``, English holding ``do`` as a -credential: +Your own words outrank a choice +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A choice selects among the shipped readings; what you added or removed +with :meth:`Lexicon.add ` and +:meth:`Lexicon.remove ` applies after it, +whichever was written first. Under ``en``, which reads ``do`` as a +credential, a base listing ``do`` and ``van`` as particles keeps both: .. doctest:: @@ -235,7 +240,20 @@ credential: >>> chosen = parser_for(en, base=Parser(lexicon=mine)).lexicon >>> [w for w in ("do", "van") ... if any(r.category == "particle" for r in chosen.readings_of(w))] - ['van'] + ['do', 'van'] + +A pack's own lexicon is joined before yours, so where the two disagree +yours wins, and of two packs the later one wins. A word of yours that +cannot stand beside a reading the choice turns on raises +``ValueError`` at :func:`~nameparser.parser_for`, naming your call and +the ``remove`` that resolves it: the Czech pack reads ``st.`` as a +generation (the elder), and a word holds one kind of suffix, so a +lexicon that added it as a credential is refused under ``locales.CS``. + +A lexicon built from :meth:`Lexicon.empty +` holds no shipped reading, so a choice turns +none on in it: to read only the chosen languages' vocabulary, choose +strictly. .. _strict-choice: @@ -305,9 +323,9 @@ drops from every name: strict one reads the other scripts' words and nothing in Latin letters, which unchains Dutch ``van`` from a Vietnamese middle name (``Do Van Minh`` reads middle ``Van``, family ``Minh``). -- **It is a delta, like a choice.** A word a strict choice drops leaves - a base parser's vocabulary even where you added it yourself; add it - again in the pack's own lexicon, which is applied after. +- **Your own words stay.** A word a strict choice drops is kept where + you added it yourself, your lexicon's readings applying after the + choice (`Your own words outrank a choice`_). .. _west-south-slavic-packs: diff --git a/docs/release_log.rst b/docs/release_log.rst index 8beb2c6a..0d906c26 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -20,6 +20,7 @@ Release Log - **Add Czech, Slovak, Slovenian and Croatian packs, locales.CS, SK, SL and HR.** ``parser_for(locales.CS).parse("Jan Novák, MUDr")`` gives title ``MUDr``, given ``Jan``, family ``Novák``, where 1.4.0 through 2.8.0 gave given ``MUDr``, family ``Jan Novák``. Each pack chooses its one language and turns on words the default does not read: ``ml.`` and ``st.`` after the name in all four (``Jiří Dienstbier ml.``), the Czech and Slovak academic titles and ``DrSc.``, and Czech ``Pan`` and ``roz.``. Choosing the language drops ``st``'s Saint readings (``St Clair Smith`` reads given ``St``), which choosing ``fr`` as well keeps. ``CS`` and ``SK`` also read two family names, so ``Alena Pešová Řehořová`` gives family ``Pešová Řehořová`` and two given names read as a given name and a family name (``Petr Pavel Novák`` gives family ``Pavel Novák``); a family comma keeps both. :issue:`753` - **Add Hindi and Bengali packs, locales.HI and locales.BN, for the romanized Indic titles.** ``parser_for(locales.HI).parse("Smt Nirmala Sitharaman")`` gives title ``Smt``, given ``Nirmala``, where 1.4.0 through 2.8.0 gave given ``Smt``, middle ``Nirmala``. ``HI`` reads ``Smt``, ``Shrimati`` and ``Shriyut`` as titles and ``BN`` ``Smt``, ``Shrimati`` and ``Adhyapak``; ``Shri`` and ``Sushri`` stay name words under both, since each opens given names (``Shri Kant``, ``Sushri Sangita``). See :doc:`locales`. :issue:`345`, :issue:`753` - **Add parser_for(..., strict=True), to read only the chosen languages' words.** ``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), languages=frozenset({"ko"})), strict=True).parse("Kim Do Hyun")`` gives middle ``Do``, family ``Hyun``, where the default, as in 1.4.0 through 2.8.0, and a plain ``ko`` choice give family ``Do Hyun``, ``Do`` read as the Portuguese particle. Only the chosen languages' vocabulary files are read, with the words written in other scripts; every other language's words leave, the shared ones (``Dr``, ``Prof``) and English credentials included, so ``Kim Min Jun, MD`` gives given ``MD`` unless ``en`` is chosen too. A strict call that chooses no language raises ``ValueError``. ``python -m nameparser`` gains ``--strict``, and ``--locale`` takes a language tag no pack has (``--locale ko``). See :doc:`locales`. :issue:`753` + - **Add Parser.languages and Lexicon.languages, the choice of languages a parser was built under.** ``parser_for(en, base=parser_for(pt)).languages`` is ``{'en', 'pt'}``, and the chained parser reads ``Maria do Carmo`` and ``John Smith DO`` as ``parser_for(pt, en)`` does, the base's choice joined with the call's; ``strict`` is recorded too, and chaining a strict choice with a plain one raises ``ValueError``. A word you add or remove with ``Lexicon.add`` and ``Lexicon.remove`` applies after the choice, so no choice drops it, and one that cannot stand beside a reading the choice turns on raises at ``parser_for``, naming your call. See :doc:`locales`. :issue:`754` - **Add --order to python -m nameparser, to read a name in another order.** ``python -m nameparser --order family-first "Kovács János Péter"`` gives family ``Kovács``, given ``János``, where the command, in 2.0.0 through 2.8.0, read only the default order and gave family ``Péter``. It takes ``given-first``, ``family-first`` or ``family-first-given-last``; a name in Han, Hangul or kana keeps its script's order. :issue:`983` - **Divide Japanese names under python -m nameparser --locale ja when nameparser[ja] is installed.** ``python -m nameparser --locale ja "山田太郎"`` gives family ``山田``, given ``太郎``, where 2.1.0 through 2.8.0 left family ``山田太郎`` and warned. Without the extra it reads as before and prints how to install it, in place of the warning; ``parser_for(locales.JA)`` still needs ``segmenter=locales.ja_segmenter()``. :issue:`984` diff --git a/nameparser/_locale.py b/nameparser/_locale.py index 5c11d787..0533046a 100644 --- a/nameparser/_locale.py +++ b/nameparser/_locale.py @@ -2,9 +2,11 @@ choice of languages. A Locale dissolves at parser construction (parser_for): the packs' -languages are unioned into one choice, which selects among the default -vocabulary's readings by the file each sits in, the lexicon fragments -union onto the result, and the PolicyPatch folds via apply_patch. Packs +languages are unioned with the base's recorded choice into one choice, +recorded on the built Lexicon, which selects among the base's shipped +readings by the file each sits in; the lexicon fragments join in pack +order and the base's lexicon after them, their readings applying after +the choice; and the PolicyPatch folds via apply_patch. Packs are pure data; they have no privileged capabilities (mechanisms.md#LOCALE-PACKS-PURE-DATA). @@ -46,15 +48,18 @@ class Locale: #: Identifier, lowercase ``[a-z0-9_]+`` (e.g. "ru", "tr_az"). code: str - #: Vocabulary ADDED to the base parser's lexicon (unioned after the - #: choice in ``languages``; the fragment never removes base - #: vocabulary, where a choice selects among the default's readings). + #: Vocabulary joined to the base parser's lexicon, before it, so the + #: base's own readings win where the two disagree, and applied after + #: the choice in ``languages``, which selects among the base's + #: shipped readings (decisions/vocabulary-model, 2026-11-02 #754, + #: item 3). lexicon: Lexicon #: Behavior changes folded onto the base policy (set-valued fields #: union; scalars override, later pack wins). policy: PolicyPatch = PolicyPatch() #: The languages chosen, as BCP 47 tags (``{"pt"}``, ``{"pt-BR"}``): - #: ``parser_for`` unions the packs' languages into one choice, which + #: ``parser_for`` unions the packs' languages, and the base's recorded + #: choice, into one choice, which #: turns on the chosen languages' readings the default leaves off #: and, on a word a chosen language holds a vocabulary reading of, #: drops other languages' vocabulary readings. A tag selects its own diff --git a/nameparser/_parser.py b/nameparser/_parser.py index ee635886..ad58696f 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -321,19 +321,22 @@ def _rewrapped(exc: Exception, message: str) -> Exception: def parser_for(*locales: Locale, base: Parser | None = None, segmenter: Segmenter | None | _Unset = UNSET, strict: bool = False) -> Parser: - """The packs' ``languages`` unioned into one choice, applied to - base's lexicon first (so the order of the packs does not change - it); then lexicon fragments unioned left-to-right onto the result; - policy patches applied left-to-right (later wins; set-valued fields - union per the patch metadata). A pack that chooses no language - leaves the base vocabulary exactly as it is; a choice removes from - base's lexicon every entry it drops, a caller's own listing of the - word included, and keeps every entry it does not touch. The choice - is this call's: a base built by an earlier ``parser_for`` records - none, so ``parser_for(EN, base=parser_for(PT))`` applies ``en``'s - choice after ``pt``'s and keeps neither language's reading of - ``do``, where ``parser_for(PT, EN)`` chooses both and keeps both - (decisions/vocabulary-model, 2026-10-29, chained choices). + """The packs' ``languages`` and base's recorded choice unioned into + one choice, recorded on the built parser's lexicon + (:attr:`Parser.languages`), so the order of the packs, and whether + they were chained through ``base``, does not change it: + ``parser_for(EN, base=parser_for(PT))`` reads as ``parser_for(PT, + EN)``. The packs' lexicon fragments are joined left-to-right and + base's lexicon after them, so a caller's own readings outrank every + pack's and a later pack's an earlier one's; the choice filters the + shipped readings and every lexicon's ``add`` and ``remove`` apply + after it, so a choice never drops a caller's word. A caller's + reading that cannot stand beside one the choice turns on raises + ValueError here, naming the call (decisions/vocabulary-model, + 2026-11-02 #754, items 3 and 4). Policy patches apply left-to-right + (later wins; set-valued fields union per the patch metadata). A + pack that chooses no language leaves the base vocabulary exactly + as it is. ``strict=True`` makes the choice strict: only the chosen languages' vocabulary files are compiled, each tag selecting its own and its @@ -343,10 +346,10 @@ def parser_for(*locales: Locale, base: Parser | None = None, them the markers they derive. It is for single-origin data (``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), languages=frozenset({"ko"})), strict=True)`` reads ``Kim Do Hyun`` - with no Portuguese particle ``do``). It applies as a delta, as a - plain choice does, so a caller's own listing of a dropped word - leaves too; the packs' fragments and policy patches apply after it - unchanged. A strict choice accepts a language whose file holds no + with no Portuguese particle ``do``). A caller's own readings and the + packs' fragments apply after it, as after a plain choice, and the + policy patches unchanged; a chained call whose ``strict`` differs + from base's recorded one raises ValueError. A strict choice accepts a language whose file holds no reading yet (``vi``), which a plain one refuses as changing nothing, and raises ValueError when the packs choose no language, since it would then strip every Latin-script reading From cf3f442b4957264cc6f642f532c6a5b48a7bdb87 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:37:30 -0800 Subject: [PATCH 3/9] Keep a strict base's choice through a call choosing nothing; let a pack's mask beat a shipped one (#754, 4i-c review) A chained call choosing no language now keeps base's choice, its strictness included, where the first draft raised on parser_for(locales.RU, base=strict_parser). A shipped mask the caller left unchanged yields to a pack's, as shipped readings do. The kind-set branch of _edits_toward went with the retired delta. The RFC's 3b and 3j passages are put in the past tense. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 3 +- docs/design/vocabulary-model-rfc.md | 6 ++-- docs/locales.rst | 9 +++--- docs/release_log.rst | 2 +- nameparser/_lexicon.py | 19 +++--------- nameparser/_parser.py | 35 +++++++++++++++++------ tests/v2/test_layering.py | 4 +-- tests/v2/test_lexicon.py | 26 ----------------- tests/v2/test_parser.py | 28 ++++++++++++++++-- 9 files changed, 69 insertions(+), 63 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index de2439fa..8dd817f1 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -506,6 +506,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - Weighed against: keeping the orphan's two readings, which would need a reading that marks without its base, the very state the per-word model was built not to express. - **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. - - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **The strictness mismatch is read off base's recorded choice,** so a chained call choosing no language (`parser_for(locales.RU, base=strict_parser)`) raises too unless it passes `strict=True`; the message offers both spellings. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. + - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **The strictness mismatch is read off base's recorded choice** and refused where the chained call chooses a language or asks for strict, the message offering both spellings; a call choosing no language keeps base's choice, its strictness included, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` build as before (the first draft raised on both, found by the code review of this PR). (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. + - **Accepted: a mask the caller deleted comes back when a strict choice is chained.** Undoing base's choice's mask moves restores the shipped value of each mask that choice dropped, and cannot tell one the caller deleted first: with `bsc` deleted from the default's masks, `parser_for(EN, base=parser_for(KO, base=Parser(lexicon=mine), strict=True), strict=True)` holds `bsc` and `parser_for(KO, EN, base=Parser(lexicon=mine), strict=True)` does not (found by the code review of this PR). Telling them apart would need the masks to be a log as the readings are, which item 3 declined to make them. - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. - **Measured.** On this branch over `db7e8fc4`, Python 3.11.17: tests/v2/test_snapshot.py passes unchanged; `uv run python tools/differential/cldr_agreement.py` reports the CLDR locale view and the Wikidata native locale view matching their records (JA not measured, namedivider absent); frames, `uv run python tools/perf/call_count.py --against db7e8fc4`, the reference row 311.48 on both, `_CALL_BASELINE` unchanged and the serial benchmark passing. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index c365f1a8..ade624e9 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -281,13 +281,13 @@ Nine of the ten rows lose a bare `dr` and three `prof` (`Dr.` stays a title by i **The API (the `strict=True` argument decided 2026-10-28; the rest decided and built 2026-10-31).** -- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** No `Locale.strict` field: the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary, so it raises `ValueError`, naming the packs and offering a pack that chooses a language, rather than strip a caller's Latin vocabulary in silence (decided 2026-10-31). A strict choice is one call's, as a choice is (decisions/vocabulary-model's chained-choices entry): as a base it records nothing, so a later plain choice applies its delta to an already stripped lexicon. +- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** No `Locale.strict` field: the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary, so it raises `ValueError`, naming the packs and offering a pack that chooses a language, rather than strip a caller's Latin vocabulary in silence (decided 2026-10-31). As built in 3j a strict choice was one call's, as a choice was (decisions/vocabulary-model's chained-choices entry): as a base it recorded nothing, so a later plain choice applied its delta to an already stripped lexicon. Since 4i-c the `Lexicon` records it (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry): a chained call that chooses a language, or asks for strict, must match the base's strictness or raises `ValueError`, and one choosing no language keeps the base's choice as it is. - **Shipped `Locale` constants: keep them for packs that carry something, and no constructor for the rest (decided 2026-10-31).** A constant per language file would be 78 constants, one per tag a file holds a reading of beside `mul` and `und` (2026-10-31), most of them a tag and nothing else. A caller writes `Locale(code="de", lexicon=Lexicon.empty(), languages=frozenset({"de"}))`, verbose for a choice and nothing more; a `locales.choose("vi", "en")` returning that Locale was proposed and is left out, the bare spelling being the one the docs teach. The constants stay for the language-choice packs with a policy patch, a fragment, or a measured docstring (NE, CS, SK, SL, HR, GA, GD, PT_BR, HI, BN, ES, PT), and ZH, JA, RU and TR_AZ, which choose no language and carry a fragment or a policy patch, stay as they are. Folding a bare tag into `locales.get` was weighed and is not recommended: `get` returns a shipped pack, and a choice it built would be one no test has measured. - **Turned-on readings:** a strict choice turns on the chosen files' `default = false` readings, as a choice does; another file's off reading stays off, a native-script one included. - **A pack's `[rules]` patch and its fragment** are untouched: the patch is policy, and the fragment is unioned after the strict compile, so `locales.ES`'s given-name list and `locales.ZH`'s surnames stay (measured: the 1,000 entries of ES's list under strict `es`). A fragment is the pack's own data, which strict keeps by definition. - **`Lexicon.given_names` and `Lexicon.surnames`** are not compiled from the files, so no choice touches them, strict included: the Korean surname list, outside the per-word model, stays under every strict choice (measured). - **Stacking packs:** the union of their languages, then strict over the union; commutative, as a choice is. -- **A caller's own `Lexicon.add`:** strict applies as a delta, as 3b's choice does, so it removes from the base every entry it drops, a caller's own listing of a dropped word included, and keeps words no file holds: under strict `vi`, a base that added `dr` and `zzz` keeps `zzz` and loses `dr` (measured). A caller who wants `dr` adds it after, in a fragment or to the finished parser's lexicon. Keeping a caller's additions by subtracting the default was weighed and is not recommended: a caller's listing of a word the default holds cannot be told from the default's. +- **A caller's own `Lexicon.add`:** (superseded by decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry: a caller's readings apply after the choice, so under strict `vi` a base that added `dr` and `zzz` keeps both, measured on that branch.) As built in 3j, strict applied as a delta, as 3b's choice did, so it removes from the base every entry it drops, a caller's own listing of a dropped word included, and keeps words no file holds: under strict `vi`, a base that added `dr` and `zzz` keeps `zzz` and loses `dr` (measured). A caller who wants `dr` adds it after, in a fragment or to the finished parser's lexicon. Keeping a caller's additions by subtracting the default was weighed and is not recommended: a caller's listing of a word the default holds cannot be told from the default's. **Candidate next steps, not in this design's PR:** @@ -306,7 +306,7 @@ Measured 2026-10-27 on `cfe8bf92`: the pool holds 2,140 readings over 1,797 word Two cheaper-looking forms do not work. Per-language deltas compiled at build time do not compose: under prefer `{pt, en}`, `do` keeps both its particle and its credential, but Portuguese's delta removes the credential and English's removes the particle. And precompiling every combination of languages is combinatorial. -A choice is applied as a delta against the default's compile, so it composes with a caller's own `Lexicon`: `parser_for(locales.NE, base=Parser(lexicon=custom))` keeps a title the caller added, and a `Locale`'s own `lexicon` fragment is still unioned on after the choice. The delta removes what the choice drops whoever put it in the base: as built in 3b, `parser_for(EN, base=Parser(lexicon=Lexicon.empty().add(particles={'do', 'van'})))` keeps only `van`, since `en` drops `pt`'s particle `do`, the caller's listing of it included. And a choice is one call's (decided 2026-10-29, decisions/vocabulary-model's chained-choices entry): a `Parser` records no choice, so `parser_for(EN, base=parser_for(PT))` applies `en`'s after `pt`'s, reading `Maria do Carmo` family `Carmo` and `John Smith DO` family `DO`, and `parser_for(PT, EN)` is how both are chosen, keeping both readings. Whether a `Parser` records its choice, so that chained choices union, is decided with the configure-by-reading method names. Decided on 2026-11-02, to be built in 4i-c: the `Lexicon` records it, so chained choices union and a caller's own readings apply after the choice (Phase 4's "The `Lexicon` surface: configuring by reading", item 4). +As built in 3b, a choice was applied as a delta against the default's compile, so it composed with a caller's own `Lexicon`: `parser_for(locales.NE, base=Parser(lexicon=custom))` kept a title the caller added, and a `Locale`'s own `lexicon` fragment was unioned on after the choice. The delta removed what the choice drops whoever put it in the base: as built in 3b, `parser_for(EN, base=Parser(lexicon=Lexicon.empty().add(particles={'do', 'van'})))` keeps only `van`, since `en` drops `pt`'s particle `do`, the caller's listing of it included. And a choice was one call's (decided 2026-10-29, decisions/vocabulary-model's chained-choices entry): a `Parser` recorded no choice, so `parser_for(EN, base=parser_for(PT))` applied `en`'s after `pt`'s, reading `Maria do Carmo` family `Carmo` and `John Smith DO` family `DO`, and `parser_for(PT, EN)` is how both are chosen, keeping both readings. Whether a `Parser` records its choice, so that chained choices union, is decided with the configure-by-reading method names. Decided on 2026-11-02 and built in 4i-c (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry): the `Lexicon` records it, so chained choices union and a caller's own readings apply after the choice (Phase 4's "The `Lexicon` surface: configuring by reading", item 4). ### The API and tags diff --git a/docs/locales.rst b/docs/locales.rst index f4bf8a11..f844b765 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -220,8 +220,9 @@ two read as one call choosing both. True A strict choice (`Reading one language only`_) chains only with another -strict one: a chained call whose ``strict`` differs from its base's -raises ``ValueError``. +strict one: a chained call that chooses a language with a ``strict`` +other than its base's raises ``ValueError``, while a pack choosing no +language keeps the base's choice as it is. Your own words outrank a choice ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -252,8 +253,8 @@ lexicon that added it as a credential is refused under ``locales.CS``. A lexicon built from :meth:`Lexicon.empty ` holds no shipped reading, so a choice turns -none on in it: to read only the chosen languages' vocabulary, choose -strictly. +none on in it, strict or not: to read only the chosen languages' +vocabulary, choose strictly over the default lexicon. .. _strict-choice: diff --git a/docs/release_log.rst b/docs/release_log.rst index 0d906c26..13596790 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -20,7 +20,7 @@ Release Log - **Add Czech, Slovak, Slovenian and Croatian packs, locales.CS, SK, SL and HR.** ``parser_for(locales.CS).parse("Jan Novák, MUDr")`` gives title ``MUDr``, given ``Jan``, family ``Novák``, where 1.4.0 through 2.8.0 gave given ``MUDr``, family ``Jan Novák``. Each pack chooses its one language and turns on words the default does not read: ``ml.`` and ``st.`` after the name in all four (``Jiří Dienstbier ml.``), the Czech and Slovak academic titles and ``DrSc.``, and Czech ``Pan`` and ``roz.``. Choosing the language drops ``st``'s Saint readings (``St Clair Smith`` reads given ``St``), which choosing ``fr`` as well keeps. ``CS`` and ``SK`` also read two family names, so ``Alena Pešová Řehořová`` gives family ``Pešová Řehořová`` and two given names read as a given name and a family name (``Petr Pavel Novák`` gives family ``Pavel Novák``); a family comma keeps both. :issue:`753` - **Add Hindi and Bengali packs, locales.HI and locales.BN, for the romanized Indic titles.** ``parser_for(locales.HI).parse("Smt Nirmala Sitharaman")`` gives title ``Smt``, given ``Nirmala``, where 1.4.0 through 2.8.0 gave given ``Smt``, middle ``Nirmala``. ``HI`` reads ``Smt``, ``Shrimati`` and ``Shriyut`` as titles and ``BN`` ``Smt``, ``Shrimati`` and ``Adhyapak``; ``Shri`` and ``Sushri`` stay name words under both, since each opens given names (``Shri Kant``, ``Sushri Sangita``). See :doc:`locales`. :issue:`345`, :issue:`753` - **Add parser_for(..., strict=True), to read only the chosen languages' words.** ``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), languages=frozenset({"ko"})), strict=True).parse("Kim Do Hyun")`` gives middle ``Do``, family ``Hyun``, where the default, as in 1.4.0 through 2.8.0, and a plain ``ko`` choice give family ``Do Hyun``, ``Do`` read as the Portuguese particle. Only the chosen languages' vocabulary files are read, with the words written in other scripts; every other language's words leave, the shared ones (``Dr``, ``Prof``) and English credentials included, so ``Kim Min Jun, MD`` gives given ``MD`` unless ``en`` is chosen too. A strict call that chooses no language raises ``ValueError``. ``python -m nameparser`` gains ``--strict``, and ``--locale`` takes a language tag no pack has (``--locale ko``). See :doc:`locales`. :issue:`753` - - **Add Parser.languages and Lexicon.languages, the choice of languages a parser was built under.** ``parser_for(en, base=parser_for(pt)).languages`` is ``{'en', 'pt'}``, and the chained parser reads ``Maria do Carmo`` and ``John Smith DO`` as ``parser_for(pt, en)`` does, the base's choice joined with the call's; ``strict`` is recorded too, and chaining a strict choice with a plain one raises ``ValueError``. A word you add or remove with ``Lexicon.add`` and ``Lexicon.remove`` applies after the choice, so no choice drops it, and one that cannot stand beside a reading the choice turns on raises at ``parser_for``, naming your call. See :doc:`locales`. :issue:`754` + - **Add Parser.languages and Lexicon.languages, the choice of languages a parser was built under.** With ``pt`` and ``en`` each a ``Locale`` choosing its language, ``parser_for(en, base=parser_for(pt))`` holds both languages in ``.languages`` and reads ``Maria do Carmo`` family ``do Carmo``, as ``parser_for(pt, en)`` does, the base's choice joined with the call's; ``strict`` is recorded too, and a chained call choosing a language with the other ``strict`` raises ``ValueError``. A word you add or remove with ``Lexicon.add`` and ``Lexicon.remove`` applies after the choice, so no choice drops it, and one that cannot stand beside a reading the choice turns on raises at ``parser_for``, naming your call. See :doc:`locales`. :issue:`754` - **Add --order to python -m nameparser, to read a name in another order.** ``python -m nameparser --order family-first "Kovács János Péter"`` gives family ``Kovács``, given ``János``, where the command, in 2.0.0 through 2.8.0, read only the default order and gave family ``Péter``. It takes ``given-first``, ``family-first`` or ``family-first-given-last``; a name in Han, Hangul or kana keeps its script's order. :issue:`983` - **Divide Japanese names under python -m nameparser --locale ja when nameparser[ja] is installed.** ``python -m nameparser --locale ja "山田太郎"`` gives family ``山田``, given ``太郎``, where 2.1.0 through 2.8.0 left family ``山田太郎`` and warned. Without the extra it reads as before and prints how to install it, in place of the warning; ``parser_for(locales.JA)`` still needs ``segmenter=locales.ja_segmenter()``. :issue:`984` diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 48cf6cdd..8845c700 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -1194,7 +1194,7 @@ def _target_readings(word: str, member: frozenset[str], heads: frozenset[str] | None, on: dict[str, dict[str, Any]], hints: dict[str, dict[str, Any]], - kinds_known: bool) -> dict[str, dict[str, Any]]: + ) -> dict[str, dict[str, Any]]: """The readings, kind -> properties, that compile `word` into the sets `member` names, keeping each reading `word` holds (`on`) where it fits and taking a kind it lacks from `hints`, then from the @@ -1237,11 +1237,6 @@ def props(kind: str) -> dict[str, Any]: particled = bool(member & _PARTICLE_FIELDS) if particled: ks = kinds("particle") or ["particle"] - if kinds_known: - if "patronymic_particles" in member: - ks = ["patronymic"] - else: - ks = [k for k in ks if k != "patronymic"] or ["particle"] for k in ks: out[k] = props(k) if member & {"conjunctions", "conjunctions_ambiguous"}: @@ -1266,10 +1261,6 @@ def props(kind: str) -> dict[str, Any]: elif worded: if "agnomens" in member: suffix = "agnomen" - elif kinds_known and "generation_words" in member: - suffix = "generation" - elif kinds_known and "credential_words" in member: - suffix = "credential" elif "suffix_words_ambiguous" in member: # an agnomen is its own kind, so a word that stops being one # is the generation the gated class otherwise holds @@ -1277,8 +1268,7 @@ def props(kind: str) -> dict[str, Any]: else "generation") else: suffix = held or "suffix" - if suffix == "agnomen" or (kinds_known and suffix in ( - "generation", "credential")) or ( + if suffix == "agnomen" or ( suffix in ri.GATED_WORDS and given_needed): suffix = "suffix" # the name readings, which gate the vocabulary readings beside them: @@ -1384,8 +1374,7 @@ def _edits_toward(pool: Pool, current: Mapping[str, frozenset[str]], exact. A new kind is taken from a reading the word holds off where it holds one.""" ri = _index_module() - kinds_known = all(k in target for k in _KIND_FIELDS) - fields = _SET_FIELDS + (_KIND_FIELDS if kinds_known else ()) + fields = _SET_FIELDS full = _inert_dropped({f: target.get(f, current[f]) for f in fields}) current = _inert_dropped({f: current[f] for f in fields}) changed: set[str] = set() @@ -1411,7 +1400,7 @@ def heads(phrases: frozenset[str], word: str) -> frozenset[str]: new_heads = heads(full["title_phrases"], w) moved = new_heads != heads(current["title_phrases"], w) want = _target_readings(w, member, new_heads if moved else None, - on, every, kinds_known) + on, every) dropped = [k for k in on if k not in want] gone_categories = {k for k in dropped if ri.CATEGORY[k] == k} for k in dropped: diff --git a/nameparser/_parser.py b/nameparser/_parser.py index ad58696f..65eec3f7 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -348,8 +348,10 @@ def parser_for(*locales: Locale, base: Parser | None = None, languages=frozenset({"ko"})), strict=True)`` reads ``Kim Do Hyun`` with no Portuguese particle ``do``). A caller's own readings and the packs' fragments apply after it, as after a plain choice, and the - policy patches unchanged; a chained call whose ``strict`` differs - from base's recorded one raises ValueError. A strict choice accepts a language whose file holds no + policy patches unchanged; a chained call that chooses a language, + or asks for strict, with a ``strict`` other than base's recorded one + raises ValueError, and one choosing no language keeps base's + choice, its strictness included. A strict choice accepts a language whose file holds no reading yet (``vi``), which a plain one refuses as changing nothing, and raises ValueError when the packs choose no language, since it would then strip every Latin-script reading @@ -397,11 +399,17 @@ def parser_for(*locales: Locale, base: Parser | None = None, # tag no file holds is reported with the pack that named it # (rules/D2) if lexicon.languages and lexicon.strict != strict: - raise ValueError( - f"base's lexicon was chosen with strict={lexicon.strict}, and " - f"a chained choice keeps it: pass strict={lexicon.strict}, or " - f"build the parser from Lexicon.default() with every pack, " - f"parser_for(*packs, strict={strict})") + # a call choosing no language keeps base's choice as it is, its + # strictness included; one that chooses or asks for strict must + # match it + if strict or any(loc.languages for loc in locales): + raise ValueError( + f"base's lexicon was chosen with strict={lexicon.strict}, " + f"and a chained choice keeps it: pass " + f"strict={lexicon.strict}, or choose every language in one " + f"call from the default lexicon, base's packs included, " + f"parser_for(*packs, strict={strict})") + strict = lexicon.strict languages = lexicon.languages for loc in locales: if loc.languages: @@ -423,10 +431,21 @@ def parser_for(*locales: Locale, base: Parser | None = None, except (TypeError, ValueError) as exc: raise _rewrapped(exc, f"while applying locale {loc.code!r}") from exc try: - lexicon = fragments | lexicon + joined = fragments | lexicon except (TypeError, ValueError) as exc: raise _rewrapped(exc, "while joining the packs' lexicons to " "base's") from exc + # a shipped mask base's lexicon holds unchanged is no statement of + # the caller's, as a shipped reading is none: a pack's mask of the + # word wins over it, as the pack's readings win over the shipped ones + shipped = Lexicon.default().capitalization_exceptions_map + over = {k: v for k, v in fragments.capitalization_exceptions_map.items() + if k in shipped + and lexicon.capitalization_exceptions_map.get(k) == shipped[k]} + if over: + joined = dataclasses.replace(joined, capitalization_exceptions=tuple( + sorted((dict(joined.capitalization_exceptions) | over).items()))) + lexicon = joined try: if languages: # the compile and its refusals, here: a reading of the diff --git a/tests/v2/test_layering.py b/tests/v2/test_layering.py index b0d91976..12e25f07 100644 --- a/tests/v2/test_layering.py +++ b/tests/v2/test_layering.py @@ -613,8 +613,8 @@ def test_the_check_finds_a_tool_read_through_an_alias( #: `getattr`s, all through `sets`, which the undone `sets = lexicon` #: makes a lexicon, and the 3 `sets =` bindings; the tools' 16 are #: attribute loads, every tool scanned. 4i-a's count was 114, with two -#: more `getattr`s in `_parser._apply_choice`, which reads the sets -#: through `_lexicon._fields_of` since 4i-b. +#: more `getattr`s in `_parser._apply_choice`, which read the sets +#: through `_lexicon._fields_of` from 4i-b until 4i-c retired it. #: A reader added through `_sets` moves these, and they are re-taken #: then: they count what the check would refuse with the move undone. _UNGUARDED_READS = {"library": 112, "tools": 16} diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 4dbad744..57437d59 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1469,32 +1469,6 @@ def test_the_translation_keeps_a_held_suffix_kind( assert got == kinds -def test_a_choice_translation_reads_the_kinds_it_is_given() -> None: - # the kind sets name a word's kind where a choice's delta moves it - # (`_apply_choice`): a patronymic, a credential written as a word, - # and a gated generation that a possibly-given particle reading - # beside it turns into a plain suffix word - from nameparser._lexicon import _edits_toward - lex = Lexicon.empty().add("zza", "zzb", kind="particle").add( - "zzc", kind="generation") - current = _fields_of(lex._sets) - target = dict(current) - target["patronymic_particles"] = frozenset({"zza"}) - target["suffix_words"] = current["suffix_words"] | {"zzb"} - target["credential_words"] = frozenset({"zzb"}) - target["particles"] = current["particles"] - {"zzb"} - target["generation_words"] = frozenset() - target["particles"] = target["particles"] | {"zzc"} - target["particles_ambiguous"] = frozenset({"zzc"}) - built = dataclasses.replace(lex, _edits=_edits_toward( - lex._pool(), current, target)) - assert {r.kind for r in built.readings_of("zza")} == {"patronymic"} - assert [(r.kind, r.properties) for r in built.readings_of("zzb")] == [ - ("credential", (("acronym", False),))] - assert {r.kind for r in built.readings_of("zzc")} == { - "suffix", "particle", "given"} - - def test_a_2x_multi_word_title_is_refused_as_a_pickle_3_0_cannot_express() -> None: # 2.x warned on a multi-word entry in a per-word set and kept it; a # reading of a phrase is a birth marker or a given-addressing title diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 464450b2..a3e07e31 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2635,15 +2635,24 @@ def test_the_choice_is_recorded_and_read_through_the_parser() -> None: strict = parser_for(_choosing("ko"), strict=True) assert strict.strict is strict.lexicon.strict is True assert Parser().languages == frozenset() and Parser().strict is False - # a pack choosing nothing chained on a choice keeps it - assert parser_for(Locale(code="xx", lexicon=Lexicon.empty()), - base=plain).lexicon == plain.lexicon + # a pack choosing nothing chained on a choice keeps it, a strict one + # included, whatever strict the call leaves at its default (found by + # review of 4i-c: a policy-only pack on a strict parser raised) + nothing = Locale(code="xx", lexicon=Lexicon.empty()) + assert parser_for(nothing, base=plain).lexicon == plain.lexicon + assert parser_for(nothing, base=strict).lexicon == strict.lexicon + assert parser_for(base=strict, segmenter=None).strict is True @pytest.mark.parametrize("base_strict", [False, True]) def test_a_chained_call_of_another_strictness_is_refused( base_strict: bool) -> None: + # refused where the call chooses a language; a plain base asked for + # strict with none chosen is refused too, strict being asked for base = parser_for(_choosing("ko"), strict=base_strict) + if not base_strict: + with pytest.raises(ValueError, match="chosen with strict=False"): + parser_for(base=base, strict=True) with pytest.raises(ValueError, match=( f"chosen with strict={base_strict}.*pass " f"strict={base_strict}.*parser_for\\(\\*packs, " @@ -2667,6 +2676,19 @@ def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: got = parser_for(pack, base=Parser(lexicon=mine)) assert got.parse("John Smith DO").suffix == "" assert got.lexicon.capitalization_exceptions_map["zzq"] == "ZzQ" + # a shipped mask the caller left as shipped is no statement of the + # caller's, so a pack's mask of the word wins, as its readings win + # over shipped ones; one the caller changed outranks the pack's + # (found by review of 4i-c) + phd = _choosing("en", lexicon=dataclasses.replace( + Lexicon.empty(), capitalization_exceptions=(("phd", "PHD"),))) + assert (parser_for(phd).lexicon.capitalization_exceptions_map["phd"] + == "PHD") + changed = dataclasses.replace(Lexicon.default(), capitalization_exceptions=( + *(p for p in Lexicon.default().capitalization_exceptions + if p[0] != "phd"), ("phd", "Phd"))) + assert (parser_for(phd, base=Parser(lexicon=changed)) + .lexicon.capitalization_exceptions_map["phd"] == "Phd") # and a later pack's fragment after an earlier one's first = _choosing("en", code="aa", lexicon=Lexicon.empty().add("zzq", kind="title")) From 0abf7f1b7977aa69945b138ce4c19cb542660e82 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:44:08 -0800 Subject: [PATCH 4/9] Give parser_for's strict three states, as segmenter's (#754, 4i-c) Omitted (UNSET) keeps base's recorded strictness; strict=False on a strict base raises again, as decided; strict=True on a plain base with a choice makes the union strict. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 2 +- docs/design/vocabulary-model-rfc.md | 2 +- docs/locales.rst | 9 +++--- docs/release_log.rst | 2 +- nameparser/_parser.py | 36 +++++++++++----------- tests/v2/test_parser.py | 37 ++++++++++++++--------- 6 files changed, 50 insertions(+), 38 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index 8dd817f1..c9971af9 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -506,7 +506,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - Weighed against: keeping the orphan's two readings, which would need a reading that marks without its base, the very state the per-word model was built not to express. - **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. - - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **The strictness mismatch is read off base's recorded choice** and refused where the chained call chooses a language or asks for strict, the message offering both spellings; a call choosing no language keeps base's choice, its strictness included, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` build as before (the first draft raised on both, found by the code review of this PR). (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. + - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **`parser_for`'s `strict` has three states, as `segmenter` does (decided 2026-11-02 in chat, on the review of this PR).** Omitted (`UNSET`, the default) keeps base's recorded strictness, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` stay strict; an explicit `False` on a strict base raises `ValueError`, offering both spellings, a strict choice not being undone by a later call; an explicit `True` on a plain base with a choice makes the union strict; and `True` with no language anywhere raises as 3j's does. The first draft refused any mismatch, a policy-only pack and a segmenter swap on a strict parser included (found by the code review); its fix inherited the strictness whenever the call chose no language, which a `bool` default made indistinguishable from an explicit `strict=False`, so that call returned a strict parser in silence (found by the second review), and the `UNSET` default answers both. The annotation is `bool | _Unset`, as `segmenter`'s is. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. - **Accepted: a mask the caller deleted comes back when a strict choice is chained.** Undoing base's choice's mask moves restores the shipped value of each mask that choice dropped, and cannot tell one the caller deleted first: with `bsc` deleted from the default's masks, `parser_for(EN, base=parser_for(KO, base=Parser(lexicon=mine), strict=True), strict=True)` holds `bsc` and `parser_for(KO, EN, base=Parser(lexicon=mine), strict=True)` does not (found by the code review of this PR). Telling them apart would need the masks to be a log as the readings are, which item 3 declined to make them. - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. - **Measured.** On this branch over `db7e8fc4`, Python 3.11.17: tests/v2/test_snapshot.py passes unchanged; `uv run python tools/differential/cldr_agreement.py` reports the CLDR locale view and the Wikidata native locale view matching their records (JA not measured, namedivider absent); frames, `uv run python tools/perf/call_count.py --against db7e8fc4`, the reference row 311.48 on both, `_CALL_BASELINE` unchanged and the serial benchmark passing. diff --git a/docs/design/vocabulary-model-rfc.md b/docs/design/vocabulary-model-rfc.md index ade624e9..fbc29a2d 100644 --- a/docs/design/vocabulary-model-rfc.md +++ b/docs/design/vocabulary-model-rfc.md @@ -281,7 +281,7 @@ Nine of the ten rows lose a bare `dr` and three `prof` (`Dr.` stays a title by i **The API (the `strict=True` argument decided 2026-10-28; the rest decided and built 2026-10-31).** -- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** No `Locale.strict` field: the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary, so it raises `ValueError`, naming the packs and offering a pack that chooses a language, rather than strip a caller's Latin vocabulary in silence (decided 2026-10-31). As built in 3j a strict choice was one call's, as a choice was (decisions/vocabulary-model's chained-choices entry): as a base it recorded nothing, so a later plain choice applied its delta to an already stripped lexicon. Since 4i-c the `Lexicon` records it (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry): a chained call that chooses a language, or asks for strict, must match the base's strictness or raises `ValueError`, and one choosing no language keeps the base's choice as it is. +- **How a caller asks: an argument of `parser_for`, `strict=True`, over the whole choice, decided 2026-10-28 and unchanged by file-strict.** No `Locale.strict` field: the choice is the union of the packs' languages, and a strict pack stacked with a plain one has no meaning, a plain choice keeping every file. A strict call whose packs choose no language (`parser_for(locales.ZH, strict=True)`) would keep only native-script vocabulary, so it raises `ValueError`, naming the packs and offering a pack that chooses a language, rather than strip a caller's Latin vocabulary in silence (decided 2026-10-31). As built in 3j a strict choice was one call's, as a choice was (decisions/vocabulary-model's chained-choices entry): as a base it recorded nothing, so a later plain choice applied its delta to an already stripped lexicon. Since 4i-c the `Lexicon` records it (decisions/vocabulary-model's 2026-11-02 #754, sub-phase 4i-c entry): `parser_for`'s `strict` defaults to `UNSET`, keeping the base's strictness, an explicit `strict=False` on a strict base raising `ValueError` and `strict=True` on a plain one making the union strict. - **Shipped `Locale` constants: keep them for packs that carry something, and no constructor for the rest (decided 2026-10-31).** A constant per language file would be 78 constants, one per tag a file holds a reading of beside `mul` and `und` (2026-10-31), most of them a tag and nothing else. A caller writes `Locale(code="de", lexicon=Lexicon.empty(), languages=frozenset({"de"}))`, verbose for a choice and nothing more; a `locales.choose("vi", "en")` returning that Locale was proposed and is left out, the bare spelling being the one the docs teach. The constants stay for the language-choice packs with a policy patch, a fragment, or a measured docstring (NE, CS, SK, SL, HR, GA, GD, PT_BR, HI, BN, ES, PT), and ZH, JA, RU and TR_AZ, which choose no language and carry a fragment or a policy patch, stay as they are. Folding a bare tag into `locales.get` was weighed and is not recommended: `get` returns a shipped pack, and a choice it built would be one no test has measured. - **Turned-on readings:** a strict choice turns on the chosen files' `default = false` readings, as a choice does; another file's off reading stays off, a native-script one included. - **A pack's `[rules]` patch and its fragment** are untouched: the patch is policy, and the fragment is unioned after the strict compile, so `locales.ES`'s given-name list and `locales.ZH`'s surnames stay (measured: the 1,000 entries of ES's list under strict `es`). A fragment is the pack's own data, which strict keeps by definition. diff --git a/docs/locales.rst b/docs/locales.rst index f844b765..86958b10 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -219,10 +219,11 @@ two read as one call choosing both. >>> chained.lexicon == parser_for(pt, en).lexicon True -A strict choice (`Reading one language only`_) chains only with another -strict one: a chained call that chooses a language with a ``strict`` -other than its base's raises ``ValueError``, while a pack choosing no -language keeps the base's choice as it is. +A chained call keeps its base's strictness (`Reading one language +only`_) unless it passes ``strict``: ``strict=True`` makes a plain +base's choice strict, with the call's languages joined to it, and +``strict=False`` on a strict base raises ``ValueError``, a strict choice +not being undone by a later call. Your own words outrank a choice ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/release_log.rst b/docs/release_log.rst index 13596790..da588ccc 100644 --- a/docs/release_log.rst +++ b/docs/release_log.rst @@ -20,7 +20,7 @@ Release Log - **Add Czech, Slovak, Slovenian and Croatian packs, locales.CS, SK, SL and HR.** ``parser_for(locales.CS).parse("Jan Novák, MUDr")`` gives title ``MUDr``, given ``Jan``, family ``Novák``, where 1.4.0 through 2.8.0 gave given ``MUDr``, family ``Jan Novák``. Each pack chooses its one language and turns on words the default does not read: ``ml.`` and ``st.`` after the name in all four (``Jiří Dienstbier ml.``), the Czech and Slovak academic titles and ``DrSc.``, and Czech ``Pan`` and ``roz.``. Choosing the language drops ``st``'s Saint readings (``St Clair Smith`` reads given ``St``), which choosing ``fr`` as well keeps. ``CS`` and ``SK`` also read two family names, so ``Alena Pešová Řehořová`` gives family ``Pešová Řehořová`` and two given names read as a given name and a family name (``Petr Pavel Novák`` gives family ``Pavel Novák``); a family comma keeps both. :issue:`753` - **Add Hindi and Bengali packs, locales.HI and locales.BN, for the romanized Indic titles.** ``parser_for(locales.HI).parse("Smt Nirmala Sitharaman")`` gives title ``Smt``, given ``Nirmala``, where 1.4.0 through 2.8.0 gave given ``Smt``, middle ``Nirmala``. ``HI`` reads ``Smt``, ``Shrimati`` and ``Shriyut`` as titles and ``BN`` ``Smt``, ``Shrimati`` and ``Adhyapak``; ``Shri`` and ``Sushri`` stay name words under both, since each opens given names (``Shri Kant``, ``Sushri Sangita``). See :doc:`locales`. :issue:`345`, :issue:`753` - **Add parser_for(..., strict=True), to read only the chosen languages' words.** ``parser_for(Locale(code="ko", lexicon=Lexicon.empty(), languages=frozenset({"ko"})), strict=True).parse("Kim Do Hyun")`` gives middle ``Do``, family ``Hyun``, where the default, as in 1.4.0 through 2.8.0, and a plain ``ko`` choice give family ``Do Hyun``, ``Do`` read as the Portuguese particle. Only the chosen languages' vocabulary files are read, with the words written in other scripts; every other language's words leave, the shared ones (``Dr``, ``Prof``) and English credentials included, so ``Kim Min Jun, MD`` gives given ``MD`` unless ``en`` is chosen too. A strict call that chooses no language raises ``ValueError``. ``python -m nameparser`` gains ``--strict``, and ``--locale`` takes a language tag no pack has (``--locale ko``). See :doc:`locales`. :issue:`753` - - **Add Parser.languages and Lexicon.languages, the choice of languages a parser was built under.** With ``pt`` and ``en`` each a ``Locale`` choosing its language, ``parser_for(en, base=parser_for(pt))`` holds both languages in ``.languages`` and reads ``Maria do Carmo`` family ``do Carmo``, as ``parser_for(pt, en)`` does, the base's choice joined with the call's; ``strict`` is recorded too, and a chained call choosing a language with the other ``strict`` raises ``ValueError``. A word you add or remove with ``Lexicon.add`` and ``Lexicon.remove`` applies after the choice, so no choice drops it, and one that cannot stand beside a reading the choice turns on raises at ``parser_for``, naming your call. See :doc:`locales`. :issue:`754` + - **Add Parser.languages and Lexicon.languages, the choice of languages a parser was built under.** With ``pt`` and ``en`` each a ``Locale`` choosing its language, ``parser_for(en, base=parser_for(pt))`` holds both languages in ``.languages`` and reads ``Maria do Carmo`` family ``do Carmo``, as ``parser_for(pt, en)`` does, the base's choice joined with the call's; ``strict`` is recorded too: a chained call left without ``strict`` keeps its base's, and ``strict=False`` on a strict base raises ``ValueError``. A word you add or remove with ``Lexicon.add`` and ``Lexicon.remove`` applies after the choice, so no choice drops it, and one that cannot stand beside a reading the choice turns on raises at ``parser_for``, naming your call. See :doc:`locales`. :issue:`754` - **Add --order to python -m nameparser, to read a name in another order.** ``python -m nameparser --order family-first "Kovács János Péter"`` gives family ``Kovács``, given ``János``, where the command, in 2.0.0 through 2.8.0, read only the default order and gave family ``Péter``. It takes ``given-first``, ``family-first`` or ``family-first-given-last``; a name in Han, Hangul or kana keeps its script's order. :issue:`983` - **Divide Japanese names under python -m nameparser --locale ja when nameparser[ja] is installed.** ``python -m nameparser --locale ja "山田太郎"`` gives family ``山田``, given ``太郎``, where 2.1.0 through 2.8.0 left family ``山田太郎`` and warned. Without the extra it reads as before and prints how to install it, in place of the warning; ``parser_for(locales.JA)`` still needs ``segmenter=locales.ja_segmenter()``. :issue:`984` diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 65eec3f7..4ed1506c 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -320,7 +320,7 @@ def _rewrapped(exc: Exception, message: str) -> Exception: def parser_for(*locales: Locale, base: Parser | None = None, segmenter: Segmenter | None | _Unset = UNSET, - strict: bool = False) -> Parser: + strict: bool | _Unset = UNSET) -> Parser: """The packs' ``languages`` and base's recorded choice unioned into one choice, recorded on the built parser's lexicon (:attr:`Parser.languages`), so the order of the packs, and whether @@ -348,10 +348,11 @@ def parser_for(*locales: Locale, base: Parser | None = None, languages=frozenset({"ko"})), strict=True)`` reads ``Kim Do Hyun`` with no Portuguese particle ``do``). A caller's own readings and the packs' fragments apply after it, as after a plain choice, and the - policy patches unchanged; a chained call that chooses a language, - or asks for strict, with a ``strict`` other than base's recorded one - raises ValueError, and one choosing no language keeps base's - choice, its strictness included. A strict choice accepts a language whose file holds no + policy patches unchanged. ``strict`` has three states, as + ``segmenter`` does: omitted (:data:`~nameparser.UNSET`) keeps base's + recorded strictness, ``False`` on a strict base raises ValueError, a + strict choice not being undone by a later call, and ``True`` on a + plain one makes the whole choice strict. A strict choice accepts a language whose file holds no reading yet (``vi``), which a plain one refuses as changing nothing, and raises ValueError when the packs choose no language, since it would then strip every Latin-script reading @@ -378,7 +379,7 @@ def parser_for(*locales: Locale, base: Parser | None = None, lexicon and policy by hand.""" if base is not None and not isinstance(base, Parser): raise TypeError(f"base must be a Parser or None, got {base!r}") - if not isinstance(strict, bool): + if strict is not UNSET and not isinstance(strict, bool): raise TypeError(f"strict must be True or False, got {strict!r}") for loc in locales: if not isinstance(loc, Locale): @@ -398,18 +399,19 @@ def parser_for(*locales: Locale, base: Parser | None = None, # order-free; each pack's tags are checked on their own first, so a # tag no file holds is reported with the pack that named it # (rules/D2) - if lexicon.languages and lexicon.strict != strict: - # a call choosing no language keeps base's choice as it is, its - # strictness included; one that chooses or asks for strict must - # match it - if strict or any(loc.languages for loc in locales): - raise ValueError( - f"base's lexicon was chosen with strict={lexicon.strict}, " - f"and a chained choice keeps it: pass " - f"strict={lexicon.strict}, or choose every language in one " - f"call from the default lexicon, base's packs included, " - f"parser_for(*packs, strict={strict})") + # strict's three states, as segmenter's: omitted keeps base's + # strictness; an explicit False on a strict base is refused, a + # strict choice not being undone by a later call, and an explicit + # True makes a plain base's choice strict (decisions/vocabulary-model, + # 2026-11-02 #754, sub-phase 4i-c) + if isinstance(strict, _Unset): strict = lexicon.strict + elif lexicon.strict and not strict: + raise ValueError( + "base's lexicon was chosen with strict=True, and a chained " + "choice keeps it: leave strict out or pass strict=True, or " + "choose every language in one call from the default lexicon, " + "base's packs included, parser_for(*packs, strict=False)") languages = lexicon.languages for loc in locales: if loc.languages: diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index a3e07e31..61de6336 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2644,20 +2644,29 @@ def test_the_choice_is_recorded_and_read_through_the_parser() -> None: assert parser_for(base=strict, segmenter=None).strict is True -@pytest.mark.parametrize("base_strict", [False, True]) -def test_a_chained_call_of_another_strictness_is_refused( - base_strict: bool) -> None: - # refused where the call chooses a language; a plain base asked for - # strict with none chosen is refused too, strict being asked for - base = parser_for(_choosing("ko"), strict=base_strict) - if not base_strict: - with pytest.raises(ValueError, match="chosen with strict=False"): - parser_for(base=base, strict=True) - with pytest.raises(ValueError, match=( - f"chosen with strict={base_strict}.*pass " - f"strict={base_strict}.*parser_for\\(\\*packs, " - f"strict={not base_strict}\\)")): - parser_for(_choosing("en"), base=base, strict=not base_strict) +def test_strict_has_three_states_on_a_chained_call() -> None: + # decided 2026-11-02 (decisions/vocabulary-model, 2026-11-02 #754, + # sub-phase 4i-c): strict defaults to UNSET, as segmenter does + ko, en = _choosing("ko", code="ko"), _choosing("en", code="en") + strict, plain = parser_for(ko, strict=True), parser_for(ko) + # omitted on a strict base: stays strict + assert parser_for(en, base=strict).strict is True + assert parser_for(base=strict).lexicon == strict.lexicon + # an explicit False on a strict base: refused, naming both spellings + for packs in ((), (en,)): + with pytest.raises(ValueError, match=( + r"chosen with strict=True.*leave strict out or pass " + r"strict=True.*parser_for\(\*packs, strict=False\)")): + parser_for(*packs, base=strict, strict=False) + # an explicit True on a plain base with a choice: becomes strict + got = parser_for(en, base=plain, strict=True) + assert got.strict is True + assert got.lexicon == parser_for(ko, en, strict=True).lexicon + # omitted on a plain base: stays plain + assert parser_for(en, base=plain).strict is False + # an explicit True with no language anywhere: refused, as 3j + with pytest.raises(ValueError, match="no pack was given"): + parser_for(strict=True) def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: From 65b4be5ee0b555bb6dcadf691f06b0561dce8e8e Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:48:32 -0800 Subject: [PATCH 5/9] Test a chained mask the default lacks and the join errors' wraps (#754, 4i-c) Co-Authored-By: Claude Opus 5.5 --- tests/v2/test_parser.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 61de6336..c49b7c1a 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2669,6 +2669,36 @@ def test_strict_has_three_states_on_a_chained_call() -> None: parser_for(strict=True) +def test_a_chained_choice_redoes_a_mask_the_default_lacks() -> None: + # cs sets `judr`'s mask, which the default holds none of; chained + # under en, the base's move is undone (the mask deleted) and the + # union's applied again, as the single call does + from nameparser import locales + en = _choosing("en", code="en") + chained = parser_for(en, base=parser_for(locales.CS)) + assert chained.lexicon.capitalization_exceptions_map["judr"] == "JUDr" + assert chained.lexicon == parser_for(locales.CS, en).lexicon + + +def test_a_join_that_cannot_be_made_names_where_it_failed() -> None: + # two packs whose lexicons record different choices name the pack; + # a pack's lexicon built on another release's base names the join + # with base's (rules/D2) + from nameparser.config._index import READINGS + pt = Locale(code="aa", lexicon=dataclasses.replace( + Lexicon.empty(), _languages=frozenset({"pt"}))) + en = Locale(code="bb", lexicon=dataclasses.replace( + Lexicon.empty(), _languages=frozenset({"en"}))) + with pytest.raises(ValueError, match="while applying locale 'bb': " + "cannot join two lexicons compiled"): + parser_for(pt, en) + older = Locale(code="cc", lexicon=Lexicon(_base=READINGS[1:])) + with pytest.raises(ValueError, match="while joining the packs' lexicons " + "to base's: cannot join two " + "lexicons built on different bases"): + parser_for(older) + + def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: # the caller's lexicon goes last, so its own statement outranks a # pack's: a pack's credential `do` against a caller's removal of it, From 561b9f37b0c55356b1d51c96a02c112c9ad8ad09 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:55:15 -0800 Subject: [PATCH 6/9] Lay a pack's masks over the choice's, name the pack a join fails on, load 4i-b pickles (#754, 4i-c review) parser_for moves base's masks by the choice and lays the packs' over them, as before 4i-c; a failed join or choice compile names the first pack whose fragment fails it (rules/D2), or the languages and the packs that chose them; _languages and _strict carry _ADDED_IN, so a 4i-b pickle loads choosing nothing (fixture written on db7e8fc4). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 +- docs/design/decisions/vocabulary-model.md | 4 +- docs/locales.rst | 12 ++- nameparser/_lexicon.py | 13 ++- nameparser/_parser.py | 107 ++++++++++++++-------- tests/v2/data/parsers_4ib.pickle | Bin 0 -> 80480 bytes tests/v2/test_lexicon.py | 28 ++++++ tests/v2/test_parser.py | 60 +++++++++++- 8 files changed, 171 insertions(+), 57 deletions(-) create mode 100644 tests/v2/data/parsers_4ib.pickle diff --git a/AGENTS.md b/AGENTS.md index 7a66951a..806f0c13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -234,7 +234,7 @@ The `config/` SETS are frozen since 2.2 (#293): there is no `.add()`/`.remove()` The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2.0 and then 3.0 deleted. These conventions apply to all of it. The full design record (rationale, settled-decision logs, dated amendments) lives in untracked `docs/superpowers/specs/`; this section is the enforceable subset. **A commit that establishes or amends one of these conventions must update this section in the same commit** — grep-driven staleness sweeps miss paraphrased prose (see Workflow above), so write-time maintenance is the mechanism, audits are the backstop. - **Module layout**: every module is underscore-private (`_types.py`, `_lexicon.py`, `_policy.py`, `_locale.py`, `_render.py`, `_reading_index.py`, `_pipeline/`, `_parser.py`). The public import surface is exactly `nameparser`, `nameparser.locales` and the vocabulary data modules under `nameparser.config`; `nameparser/__init__.py` holds re-exports and `__all__` only — no logic. -- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules, e.g. `config/maiden_markers.py`, and `_reading_index` at call time only, to compile a configured lexicon's readings, since `tools/vocab/build.py` loads it by path, 3.0's 4i-b); `_reading_index` imports nothing internal at module level and the index, `config/_index.py`, at call time, since `tools/vocab/build.py` loads it by path; `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin, and `config/_rules.py` at call time, in `rules_patch`); `_render` imports `_types`, `_lexicon` (for `Lexicon.default()` and `_normalize`) and `_policy` (for the scripts that write no initials); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers, `_assemble` also `_reading_index`, for `Reading.languages` (`_vocab` imported it in 3.0's 4b and 4c for a listed suffix word's kind and a patronymic particle's, which since 4i-b the lexicon compiles into `_sets`); `_parser` sits on everything except `_render`, `_reading_index` at call time only, where a choice of languages is applied; `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. +- **Layering is enforced by `tests/v2/test_layering.py`** (exact-module matching; `if TYPE_CHECKING:` imports don't count): `_types` imports nothing internal at module level — its rendering delegates import `_render` at call time; `_lexicon` and `_policy` sit above `_types` independently (`_lexicon` may import `nameparser.config.*` DATA modules, e.g. `config/maiden_markers.py`, and `_reading_index` at call time only, to compile a configured lexicon's readings, since `tools/vocab/build.py` loads it by path, 3.0's 4i-b); `_reading_index` imports nothing internal at module level and the index, `config/_index.py`, at call time, since `tools/vocab/build.py` loads it by path; `_locale` sits on `_lexicon`+`_policy` (plus `_types` for the shared pickle mixin, and `config/_rules.py` at call time, in `rules_patch`); `_render` imports `_types`, `_lexicon` (for `Lexicon.default()` and `_normalize`) and `_policy` (for the scripts that write no initials); `_pipeline/*` imports `_types`+`_lexicon`+`_policy` plus in-package `_pipeline` helpers, `_assemble` also `_reading_index`, for `Reading.languages` (`_vocab` imported it in 3.0's 4b and 4c for a listed suffix word's kind and a patronymic particle's, which since 4i-b the lexicon compiles into `_sets`); `_parser` sits on everything except `_render`, `_reading_index` at call time only, to validate a choice's tags and move the case masks it moves (the choice itself compiles in `_lexicon._built`); `__main__` imports only the public package; locale pack modules (`locales/*.py`) import `_locale`/`_lexicon`/`_policy`/`_types` only (`_types` joined the list in #272 — `locales/ja.py`'s segmenter factory constructs a `Segmentation`), and the `locales/__init__` additionally lazy-imports its packs (PEP 562). Extend the test's `ALLOWED` table when adding a module. - **Canonical field order** — the seven roles in `Role` enum declaration order, defined once and derived everywhere (properties, `as_dict`, reprs, `comparison_key`). Never restate the order literally. `Role` is a `StrEnum`: members compare as their field-name strings, and `tokens_for()` coerces strings via `_coerce_enum`. - **Method organization**, fixed section order in every class: fields + `__post_init__` validation → alternative constructors → dunders (construction/equality → protocol → operators) → properties → public methods by concern (access → editing → comparison → rendering delegates) → private helpers last, except a helper serving exactly one section may sit at that section's head. - **Validation is eager and fail-loud**: every `raise` states the offending value, the expected form, and the fix. Exception taxonomy: wrong type — including wrong element type inside a collection, bare `str` where an iterable of strings is expected, or a `Mapping` where a plain iterable is expected — raises `TypeError`; well-typed but unacceptable values raise `ValueError`; failed enum lookups stay `ValueError` for any input (stdlib `EnumType` precedent). **When the message hands the reader code to paste, that code has to survive a type checker, and then RUN** — nameparser ships `py.typed`, and a placeholder can type-check and still raise: the 2.4 release sweep's first fix to the `capitalization_exceptions` hint offered `('key', 'Value')`, which passes mypy and fails #459's mask check, so `test_add_capitalization_exceptions_raises_pointing_at_replace` now executes the offered code. #337's segmenterless warning offered `Policy(segment_scripts=())`, an `arg-type` error, because these fields are annotated with what they STORE rather than everything the constructor accepts. Prefer the `frozenset()` / `()` spellings in messages and docstrings, and pin the offered spelling in a test — the warning tests matched on `ja_segmenter` and never checked the actionable half of the message. **A warning emitted in `Parser.__post_init__` needs `parser_for` to re-emit it from its own frame** (the `catch_warnings(record=True)` block at its return): `__post_init__`'s `stacklevel` is sized for direct `Parser(...)` construction, and through `parser_for`'s extra frame the default one-line rendering attributes the warning to the library's own `return Parser(...)` — the exact call the message tells the user to change becomes invisible. No single stacklevel serves both entry points; a new construction warning gets the re-emission for free, but a new CONSTRUCTION SITE for `Parser` inside this package needs its own re-emission or its callers get library-attributed warnings (#337 review). @@ -250,7 +250,7 @@ The 2.0 rewrite landed as underscore-private modules beside the v1 code, which 2 - **A claim about WHICH STAGE or WHICH LAYER does something is checkable — check it before writing it.** The pipeline is eight stages with a written ownership map (`ParseState`'s docstring, pinned by `tests/v2/pipeline/test_state.py`), and `parse(s).tokens` prints every token's role and tags, so "extract assigns this", "classify never sees that", "group consumes it" each have a one-command answer. #329's prose claimed delimited maiden content is *"claimed whole before classify has tagged anything inside it"*; measured, `classify` tags the marker fine and only the CONSUMING is missing, because `_group`'s rule walks `pieces` and a token that already carries a role is not in `pieces`. Two different mechanisms, one plausible sentence covering both. That single claim then shipped SIX times across three correction rounds, which is the part worth internalizing: **when a mechanism claim turns out wrong, sweep for where else you wrote it, and sweep again at the END of the change over the words the change itself just added.** Prose density here means one idea lives in a docstring, a case note, a release-log entry and this file at once; the implementer working against a wrong mechanism is the person most likely to restate it; and rewriting a mechanism claim is writing one, so the correction earns the same one-command check as the original — two of the six instances were fresh errors introduced by the sentence fixing the previous one. **Adjacency is the trap.** The claim that feels already-known is the one about the neighbouring stage or the neighbouring layer: one comment block in the 2.x `tests/v2/test_facade_cases.py` got the exception type, the raising layer, the skip mechanism, the count of skipped rows, and which row was blocked all wrong at once — every one a claim about `_config_shim` versus real 1.4.0, written from reasoning, in the file whose whole job is translating between them. What finally held was not better prose but moving the claim into a test (`_CORE_ONLY_IDS`, deleted with the facade in 3.0), which cannot be wrong the way a sentence can. - **Typing/docs**: `from __future__ import annotations`; `frozen=True, slots=True` on every public dataclass; strict-profile mypy flags via per-module overrides in pyproject (`strict = true` itself is not valid per-module). Docstrings state contracts in prose with **no doctest blocks** — `--doctest-modules` makes every example a test; behavior examples go to unit tests per the lean-docs rule. **Document the positive direction of a partial property**: "a non-empty `ambiguities` is a signal to act on" is checkable, while "an empty one means no fork occurred" is a universal negative needing exhaustive verification -- that claim was written twice and falsified twice, at sites the author had not audited. - **The segmenter contract**: the optional `Parser(segmenter=...)` hook is parse-totality's ONE exception (locales spec section 4). Everything inside that exception is a bug in USER CODE, never a fact about the name, so it is surfaced rather than absorbed: the segmenter's own exceptions propagate, and the two protocol violations the stage can detect for itself — an answer of the wrong type, and one cutting at or past the end of the token it was handed — raise `TypeError`/`ValueError` from `_script_segment` for the same reason. The line to hold when adding a check there: a protocol violation by the segmenter's AUTHOR raises, while an adapter's defense against its own third-party library (`locales/ja.py`'s repertoire, length, reconstruction and score guards) declines with `None`, because what those catch is a fact about the content. -- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` and `LexiconReading` keep their own copy of the guard (layering); a new unpicklable slot type needs the same treatment plus a round-trip test. Since 3.0's 4i-b a `Lexicon` pickles its lists, its masks, its edit log, its choice of languages (`_languages` and `_strict`, 4i-c, unmarked because no release pickled a `Lexicon` without them, a 2.x state translating to no choice) and its base as a SNAPSHOT of the reading index's rows, so a pickle parses as it did where it was written, a snapshot equal to the running release's index loading as the by-reference default again (decisions/vocabulary-model, 2026-11-02 #754, item 5); its private `_sets`, `_cap_map` and `_chosen` fields are built at construction and never pickled. `Lexicon`'s copy of the guard admits the `ADDED_IN` marker, read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older pickles. A 2.x state holds the compiled sets instead, and `_lexicon._translated_2x` translates it onto the nearer named base and a per-word log, loading where the compile gives the sets back and refusing otherwise, naming the fields, so the 2.8 `Lexicon`, `Parser` and locale-pack pickles in `tests/v2/data/lexicon_v28.pickle` load as the 3.0 calls that build them. And unlike the shared guard it DOES re-validate: a load goes through the constructor, which checks the base, refuses an edit not in canonical form, and refuses a list or mask entry that is not already normalized, because an old blob would otherwise load a different lexicon in silence (`particles_ambiguous` changed meaning between v1 and v2 without changing its name). +- **Pickling**: v2 types must round-trip (`Parser` is picklable by construction, and it holds a `Lexicon`; the one qualifier is that a `Parser` pickles iff its segmenter does — see the segmenter bullet above). Every frozen type assigns `_guarded_getstate`/`_guarded_setstate` (`_types.py`) in its class body (`@dataclass(slots=True)` would override inherited pickle methods) — unpickling fails at the LOAD site on field-layout skew, with one widening: a field added after its type shipped carries `metadata={_types.ADDED_IN: ""}` and a default, and an older pickle missing it loads with the default (`Ambiguity.alternative`, 2.7; `Token.reading`, `Reading.languages`, `Reading.second_surname` and `ParsedName.order`, 3.0.0) -- a missing field without the marker, default or not, an unknown field and a renamed one are still refused, each a row in `tests/v2/test_types.py::_REFUSED_SKEWS`, so a new field on a pickled type takes the marker or breaks every older pickle; the shared guard deliberately does NOT re-validate values (pickle is not a security boundary; canonical state comes from a validated instance). `Lexicon` and `LexiconReading` keep their own copy of the guard (layering); a new unpicklable slot type needs the same treatment plus a round-trip test. Since 3.0's 4i-b a `Lexicon` pickles its lists, its masks, its edit log, its choice of languages (`_languages` and `_strict`, 4i-c, marked `ADDED_IN` so a 4i-b pickle loads choosing nothing, a 2.x state translating to no choice) and its base as a SNAPSHOT of the reading index's rows, so a pickle parses as it did where it was written, a snapshot equal to the running release's index loading as the by-reference default again (decisions/vocabulary-model, 2026-11-02 #754, item 5); its private `_sets`, `_cap_map` and `_chosen` fields are built at construction and never pickled. `Lexicon`'s copy of the guard admits the `ADDED_IN` marker, read under `_lexicon._ADDED_IN`, the same string, since layering keeps `_lexicon` from importing `_types` (`test_lexicon_marks_an_added_field_with_the_shared_key` holds the two equal), so a new `Lexicon` field takes the marker and a default or breaks older pickles. A 2.x state holds the compiled sets instead, and `_lexicon._translated_2x` translates it onto the nearer named base and a per-word log, loading where the compile gives the sets back and refusing otherwise, naming the fields, so the 2.8 `Lexicon`, `Parser` and locale-pack pickles in `tests/v2/data/lexicon_v28.pickle` load as the 3.0 calls that build them. And unlike the shared guard it DOES re-validate: a load goes through the constructor, which checks the base, refuses an edit not in canonical form, and refuses a list or mask entry that is not already normalized, because an old blob would otherwise load a different lexicon in silence (`particles_ambiguous` changed meaning between v1 and v2 without changing its name). - **One sanctioned global**: the (future) cached default `Parser`. Lazily cached FROZEN singletons (`Lexicon.default()`'s `functools.cache`, the future default parser) are constants, not state; any second piece of module-level MUTABLE state requires amending the conventions doc, on purpose, in review. - **Tests**: all v2 tests live in the `tests/v2/` package, one test module per source module plus the cross-cutting ones (`test_reprs.py`, `test_layering.py`, `test_contracts.py`, `test_properties.py`, `test_benchmark.py`, the `cases.py`/`test_cases.py` table, `test_regex_sync.py`, which pins the pipeline's hand-copied patterns against their sources, and `test_snapshot.py`, which holds every corpus name's reading to `tools/differential/snapshot.jsonl` — sharing `_differential_fixtures.py` with `test_differential.py` and `test_corpora.py` for the handles on `tools/differential/`), names stating behavior; the 1.x suite's parsing tests stand ported to `parse()` in `tests/*.py`. Never assert `Lexicon.default()` contents; the narrow sourcing spot-checks in `test_default_sources_v1_vocabulary` that pin the v1→v2 migration contract (e.g. the flipped `particles_ambiguous` model) are the sanctioned exception. diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index c9971af9..e8490061 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -14,7 +14,7 @@ The vocabulary was a set of base sets plus marker subsets (`particles` and `part - `es` turns on the courtesy, academic and professional titles that lead a Spanish name with no complement (`lic`, `licenciado`, `licenciada`, `lcdo`, `lcda`, `ingeniero`, `ingeniera`, `arq`, `mtro`, `mtra`, `doctora`, `profesor`, `profesora`, `doña`) and `pt` `doutor`, `doutora`, `professora` and `profa`, `pt-BR` with it; `don`, `dona` and `dom` are excluded, the packs included, names opening with each as a name more often than as the title; `maestro`, a family name whose form of address is written `Mtro.`, is excluded too, and so is `h`, the hijo an initial is written as; `sr` keeps a generation reading beside its salutation in both files, so a choice leaves it the dual position decides; no connective continues a title run, the shape unwritten in name data, and the office titles that take a complement stay out — 2026-10-30 #753, phase 3g. - Of the romanized titles a pack's phase lists, one whose native spelling is a default title turns on unless, counted in persons and pooled across every population measured, names open with it more often than it opens one as the title; a spelling outside the phase's list (`Shreemati`, `Srimati`) stays open until a phase writes it as a reading: `hi` turns on `smt`, `shrimati` and `shriyut`, `bn` `smt`, `shrimati` and `adhyapak` (`shriyut` with no bearer anywhere, `adhyapak` with family-name bearers only, every one where the title claim, acting at the head, does not reach), while `shri`, opening given names in every population measured, stays excluded, and `sushri`, an Odia given name, waits with no reading written for data strict can read since 3j, a `default = false` salutation in hi.toml with the Odia name declared in or.toml (2026-10-31 #753, phase 3j), a plain choice being made for data that holds those names — 2026-10-30 #753, phase 3e. - A language pack is a choice of languages on `Locale.languages` selecting which readings compile, a tag choosing its own file and its ancestors', and a reading index, loaded on its first lookup, carries the languages to run time; strict is file-strict, the chosen files and native-script vocabulary only, `mul.toml` and `und.toml` not among them, so it needs no 3i; third-party language files are deferred — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3a (Derek); 2026-10-29 #753, phase 3b; 2026-10-30 #753, file-strict. -- A strict choice is `parser_for(..., strict=True)` over the whole choice, filtering the base's readings as a plain choice does, the caller's own edits applying after it, and a chained call of the other strictness raises; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j; 2026-11-02 #754, sub-phase 4i-c. +- A strict choice is `parser_for(..., strict=True)` over the whole choice, filtering the base's readings as a plain choice does, the caller's own edits applying after it, and a chained call keeping its base's strictness unless it passes `strict`, an explicit `False` on a strict base raising and an explicit `True` making a plain base's choice strict; a strict call whose packs choose no language raises; a strict choice accepts a tag whose file holds no reading, which a plain one refuses as changing nothing; a bare choice has no constructor, no `locales.choose`; and the words strict ungates are derived against the plain choice of the same tags and held to a reviewed set by a test — 2026-10-31 #753, phase 3j; 2026-11-02 #754, sub-phase 4i-c. - A choice is matched without regard to case, a tag whose file does not exist selects its ancestors' (`pt-PT` selects `pt`), `mul` and `und` are no language to choose, and a reading turned on also prefers, so `de`'s generation `i` drops `ca`'s and `pl`'s conjunction under a `de` choice; the `Lexicon` records its choice, so that a choice chained through `base=` unions with the base's, and a caller's own edits apply after the choice, plain or strict, in the order written, a pack's fragment joining before the caller's lexicon — 2026-10-29 #753, phase 3b; 2026-11-02 #754, the `Lexicon` surface (Derek); 2026-11-02 #754, sub-phase 4i-c. - A reading right in its own language and wrong only as a default, because another language written in the same script collides with it, is written `default = false` in its file, never excluded, so that after the packs an exclusion means wrong everywhere: cs `st` since phase 2h, and cs `ml`, hr, sk and sl `ml` and `st` and fr `père` since 3c, the rest in their packs' PRs — 2026-10-27 #753 (Derek); 2026-10-29 #753, phase 3c. - A name reading written `default = false` derives no marker in the default and counts for no check guarding the default's compile; a particle whose word's only `given` readings are off is refused until a pack needs one — 2026-10-29 #753, phase 3a. @@ -506,7 +506,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - Weighed against: keeping the orphan's two readings, which would need a reading that marks without its base, the very state the per-word model was built not to express. - **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. - - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **`parser_for`'s `strict` has three states, as `segmenter` does (decided 2026-11-02 in chat, on the review of this PR).** Omitted (`UNSET`, the default) keeps base's recorded strictness, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` stay strict; an explicit `False` on a strict base raises `ValueError`, offering both spellings, a strict choice not being undone by a later call; an explicit `True` on a plain base with a choice makes the union strict; and `True` with no language anywhere raises as 3j's does. The first draft refused any mismatch, a policy-only pack and a segmenter swap on a strict parser included (found by the code review); its fix inherited the strictness whenever the call chose no language, which a `bool` default made indistinguishable from an explicit `strict=False`, so that call returned a strict parser in silence (found by the second review), and the `UNSET` default answers both. The annotation is `bool | _Unset`, as `segmenter`'s is. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. + - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **`parser_for`'s `strict` has three states, as `segmenter` does (decided 2026-11-02 in chat, on the review of this PR).** Omitted (`UNSET`, the default) keeps base's recorded strictness, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` stay strict; an explicit `False` on a strict base raises `ValueError`, offering both spellings, a strict choice not being undone by a later call; an explicit `True` on a plain base with a choice makes the union strict; and `True` with no language anywhere raises as 3j's does. The first draft refused any mismatch, a policy-only pack and a segmenter swap on a strict parser included (found by the code review); its fix inherited the strictness whenever the call chose no language, which a `bool` default made indistinguishable from an explicit `strict=False`, so that call returned a strict parser in silence (found by the second review), and the `UNSET` default answers both. The annotation is `bool | _Unset`, as `segmenter`'s is. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. (8) **Found by the orchestrating review of this PR, 2026-11-02, and fixed:** a pack's case mask lost to the choice's moves where master kept it (`judr` written `JUDR` in a fragment read `JUDr` under `cs`, and its `phd` dropped under strict), so the choice now moves base's masks and the packs' are laid over them (`_parser._with_packs`); a fragment that clashed with base's readings, or with a reading the choice turns on, named no pack, so a failed join is built again pack by pack on the error path and the first pack that fails it is named, a clash of base's own with the choice naming the languages and the packs that chose them (rules/D2); and a pickle written by 4i-b, which held its choice as edits and no choice fields, was refused, so `_languages` and `_strict` carry `_ADDED_IN` and such a pickle loads choosing nothing, reading as it did (tests/v2/data/parsers_4ib.pickle, written on `db7e8fc4`). - **Accepted: a mask the caller deleted comes back when a strict choice is chained.** Undoing base's choice's mask moves restores the shipped value of each mask that choice dropped, and cannot tell one the caller deleted first: with `bsc` deleted from the default's masks, `parser_for(EN, base=parser_for(KO, base=Parser(lexicon=mine), strict=True), strict=True)` holds `bsc` and `parser_for(KO, EN, base=Parser(lexicon=mine), strict=True)` does not (found by the code review of this PR). Telling them apart would need the masks to be a log as the readings are, which item 3 declined to make them. - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. - **Measured.** On this branch over `db7e8fc4`, Python 3.11.17: tests/v2/test_snapshot.py passes unchanged; `uv run python tools/differential/cldr_agreement.py` reports the CLDR locale view and the Wikidata native locale view matching their records (JA not measured, namedivider absent); frames, `uv run python tools/perf/call_count.py --against db7e8fc4`, the reference row 311.48 on both, `_CALL_BASELINE` unchanged and the serial benchmark passing. diff --git a/docs/locales.rst b/docs/locales.rst index 86958b10..3e965ee9 100644 --- a/docs/locales.rst +++ b/docs/locales.rst @@ -114,11 +114,13 @@ Pass more than one pack and they fold together in order: How each part combines: -- **Languages union** into one choice, applied first (see - `Choosing languages`_), so the order the packs are given in does not - change it. -- **Lexicons union**: a pack's own words are added to the base's, - after the choice, never removed. +- **Languages union** into one choice, with the base's own, applied + first (see `Choosing languages`_), so the order the packs are given + in does not change it. +- **Lexicons join in order**: the packs' lexicons in the order given, + then the base's, each applied after the choice, so where two + disagree the later wins and the base's own words win over every + pack's (`Your own words outrank a choice`_). - **Set-valued policy fields union** the same way — the fields declared set-valued in :class:`~nameparser.PolicyPatch`: ``patronymic_rules``, ``segment_scripts`` and the delimiter fields. diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 8845c700..1978f04d 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -1760,8 +1760,12 @@ class Lexicon: # alone, and read through `languages` and `strict`. _base: Any = "empty" _edits: tuple[_Edit, ...] = () - _languages: frozenset[str] = frozenset() - _strict: bool = False + # marked added: a 4i-b pickle, from no release, holds no choice and + # loads choosing nothing, its log already holding what a choice of + # 3b's delta had written into it + _languages: frozenset[str] = field( + default=frozenset(), metadata={_ADDED_IN: "3.0.0"}) + _strict: bool = field(default=False, metadata={_ADDED_IN: "3.0.0"}) # the file tags the choice selects, derived from it _chosen: frozenset[str] = field( init=False, repr=False, compare=False, hash=False) @@ -1956,8 +1960,9 @@ def __or__(self, other: Lexicon) -> Lexicon: != (other._languages, other._strict)): raise ValueError( "cannot join two lexicons compiled under different choices " - "of languages; combine choices with parser_for, which " - "unions them: parser_for(*packs, base=parser)") + "of languages or of strictness; combine choices with " + "parser_for, which unions them: parser_for(*packs, " + "base=parser)") mine = self if self._languages else other if "empty" in (self._base, other._base) or self._base == other._base: base = other._base if self._base == "empty" else self._base diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 4ed1506c..3b9340b8 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -318,6 +318,66 @@ def _rewrapped(exc: Exception, message: str) -> Exception: f"{message}: {exc}") +def _with_packs(lexicon: Lexicon, locales: tuple[Locale, ...], + languages: frozenset[str], strict: bool) -> Lexicon: + """`lexicon` with the packs' fragments joined before it, in pack + order, so its own readings outrank every pack's and a later pack's + an earlier one's, and the choice of `languages` recorded, where the + compile and its refusals happen (decisions/vocabulary-model, + 2026-11-02 #754, items 3 and 4). The case masks follow the same + order: the choice moves `lexicon`'s, the packs' are laid over them, + and a mask `lexicon` holds as the choice leaves the shipped one is + no statement of the caller's, so a pack's mask of the word wins it, + as a pack's readings win over the shipped ones.""" + fragments = Lexicon.empty() + for loc in locales: + fragments = fragments | loc.lexicon + joined = fragments | lexicon + if languages: + chosen = _reading_index_selected(languages, strict) + mine = dict(_rechosen_masks(lexicon, chosen, strict)) + shipped = dict(_rechosen_masks(Lexicon.default(), chosen, strict)) + else: + mine = dict(lexicon.capitalization_exceptions) + shipped = dict(Lexicon.default().capitalization_exceptions) + masks = dict(mine) + for k, v in fragments.capitalization_exceptions: + if k not in mine or mine[k] == shipped.get(k): + masks[k] = v + cap = tuple(sorted(masks.items())) + if not languages and cap == joined.capitalization_exceptions: + return joined + return dataclasses.replace( + joined, capitalization_exceptions=cap, _languages=languages, + _strict=strict) + + +def _failed_layer(lexicon: Lexicon, locales: tuple[Locale, ...], + languages: frozenset[str], strict: bool) -> str: + """The layer a failed `_with_packs` names (rules/D2): the first pack + whose fragment fails the join, or, where base's lexicon fails under + the choice alone, the languages and the packs that chose them. + Taken on the error path only, by building again pack by pack.""" + def fails(packs: tuple[Locale, ...]) -> bool: + try: + _with_packs(lexicon, packs, languages, strict) + except (TypeError, ValueError): + return True + return False + + if not fails(()): + for n, loc in enumerate(locales, 1): + if fails(locales[:n]): + return f"while applying locale {loc.code!r}" + if not languages: + return "while joining the packs' lexicons to base's" + choosers = [repr(loc.code) for loc in locales if loc.languages] + if lexicon.languages: + choosers.append("base") + return (f"while applying the languages chosen, " + f"{', '.join(sorted(languages))} (by {', '.join(choosers)})") + + def parser_for(*locales: Locale, base: Parser | None = None, segmenter: Segmenter | None | _Unset = UNSET, strict: bool | _Unset = UNSET) -> Parser: @@ -423,49 +483,18 @@ def parser_for(*locales: Locale, base: Parser | None = None, languages |= loc.languages if strict and not languages: raise ValueError(_no_strict_language(locales)) - # the packs' fragments join in pack order, and the caller's lexicon - # after them all, so its own readings outrank every pack's - # (decisions/vocabulary-model, 2026-11-02 #754, item 3) - fragments = Lexicon.empty() - for loc in locales: - try: - fragments = fragments | loc.lexicon - except (TypeError, ValueError) as exc: - raise _rewrapped(exc, f"while applying locale {loc.code!r}") from exc - try: - joined = fragments | lexicon - except (TypeError, ValueError) as exc: - raise _rewrapped(exc, "while joining the packs' lexicons to " - "base's") from exc - # a shipped mask base's lexicon holds unchanged is no statement of - # the caller's, as a shipped reading is none: a pack's mask of the - # word wins over it, as the pack's readings win over the shipped ones - shipped = Lexicon.default().capitalization_exceptions_map - over = {k: v for k, v in fragments.capitalization_exceptions_map.items() - if k in shipped - and lexicon.capitalization_exceptions_map.get(k) == shipped[k]} - if over: - joined = dataclasses.replace(joined, capitalization_exceptions=tuple( - sorted((dict(joined.capitalization_exceptions) | over).items()))) - lexicon = joined try: - if languages: - # the compile and its refusals, here: a reading of the - # lexicon's own that clashes with one the choice turns on - # raises, naming the edit - lexicon = dataclasses.replace( - lexicon, _languages=languages, _strict=strict, - capitalization_exceptions=_rechosen_masks( - lexicon, _reading_index_selected(languages, strict), - strict)) + lexicon = _with_packs(lexicon, locales, languages, strict) except (TypeError, ValueError) as exc: # rules/D2: "applying a locale pack wraps any such error with # the locale's code, so a stacked configuration names which - # layer broke" -- the choice is every pack's at once, so it - # names the languages (history: decisions/vocabulary-model, - # 2026-10-29 3b) - raise _rewrapped(exc, "while applying the languages chosen, " - + ", ".join(sorted(languages))) from exc + # layer broke" -- the pack whose fragment broke the join, or, + # where base's own lexicon clashes with the choice, the + # languages and the packs that chose them (history: + # decisions/vocabulary-model, 2026-10-29 3b and 2026-11-02 #754, + # sub-phase 4i-c) + raise _rewrapped(exc, _failed_layer( + lexicon, locales, languages, strict)) from exc # each scalar field's setter so far and the value it left in the # policy: a later pack leaving another value overrides it, which is # warned; one leaving the same value (cs and sk's two_family_names) diff --git a/tests/v2/data/parsers_4ib.pickle b/tests/v2/data/parsers_4ib.pickle new file mode 100644 index 0000000000000000000000000000000000000000..29f48b958fcb85625fb60e0eaddf6748663a7025 GIT binary patch literal 80480 zcmaI93$&%hb(m@X|39zWw-LgzY)|ai(TW91e38+^l46f-AzASY1q(ZCP1Yps^M2g( z;+%VLbD7v`x&a|^5klM+548}8rn^Br8X9nekXUw08E5Q~VP@-FTg7V>uzlC|J8#^UtfhYQ`fhzR+}`?rH-82#{hGyRCCbzFiDaEp zI=*oH*6po$=eccu&!3jR&!0RjkXs&}dqjR8 zxJlrTNuHgPC*<#wC*|+^AC}(-_3wM7&I`|cir@1m4+#8jg&(18ap!>}=T9mbw`>XO z$OH0w(^37^Chn8W7w-J5{GNJNexH0%un)=ag-36ezwcKv9)`T)g*$FOe@ZL4?J-bq z9=efV1G#XAWZSvvi}HJmR7mTBeO{B!DaN0hO}or8~^*N*QzCG}r8{ye`sUl97=JWNj76{<|03y(@|J9iuA zNMGK(_jZ9CRwm!4Z5>uJz9=oed8heXdv}lKx#=n3=bvWuix-Z6apwU6UO1&0?xq(- zgyF&|>FH_cgDtH^=gq3Tt#9A5y%p7%E74VJ z@^@0(5L9y2zc9E$z20dKwzmo&{;>Q@jzYb2#h@w>{Wpb-EM%m$)H^My<-?tie%PeP z9ok4fd`;&!OnSQ3lk~M~zar^kqdV?bDN+0ge|*SL)#yXjM}AH7HwIniXY!G&`A4X1 z43x@`Ty3e8bQcN)(3KqJ#%OKris=+W%O6Sq6j~eOPM3-cAGt>U)w(9*j?_K5=9eTL zH>B$LBev$&a9wNu=*QOUzbyDdYf@$AZxufJ@eehx6-co)88p`b6+im%tD9>IE47=$ zwJZ9~?XA*3vwxA(K!tk-%3nDY4qB-4u-NVQmC?mtHh)M<3L~R|X}-8SY(wua^Gl9s z72=~`Rj$XYLzo`_D!+=6QT|o^yD+HL>U6qr^;K7WxFc8~^-fJ8AMRWwkm5i_9#V@} z%g}#FV9B7--Qb_AKf*s+3U)T66zu$n$gPnvBVkkU2wFKV<<89M{j zF)5EHqn4GvYtlcZSL5lX^zxeNrfE+_FNS)dE}qsw#@E&jZdzyFk-A!N#c8*F#ZX4$ zTJuK=o1`%QkUG|6JmTLlG?S7#exsvxMC;^@epB;BXHN0=nY$>1SQPfvs?#Q`*+uUl z`(T~%+QW+2TOZa%Wo*CmliN2-?a^f-LqpJ|>SV23N4`^3a^+90AyHA2d0HNf8r@d6 zImxi`mHoBt-p{i>gY5QEOq0zFnOr$-gZ*b6aIl1#J+;yLu(2;AQQKEKsh|NjdY_>Jl_LQ(0esHXC`n{4 zO)gh!BSZwj>xw8SSxcjOeSPc?a6-QItQ4v7*rjXYCdju%V_QpW&DG&(xH;(7rGHA2 zVp1L{OX6xh<=-ciMA%dY)@Vw(-WLqn36Pxslxc4F4rHhfH9VT7P^2l#O4a(Pzqv-{ zjG^_MaGF``ZfM7CGX)?h$+T69uT6BkEIzI_R!r}D|CIL930Z(zjNr79q}u3^?e`5u zWm22fadnlkt08lb0p2P#nyu8IMP>lYPr3xH`;9uT+Os=0LX&-ZDZ!lX}OMe2UPlj{SQbzLcxO z?!btfodOplpPk8QJnE0yZUsqo+@Tj4;bUmCiAr@mZua{sU6z6+Koim2GLf^%HO?!b zx=CI`SyC$qK~^Vt(W6;&*X5|VI>~##RGrki>5Q=~28IFFopwePvWh`rjVAhnRxPX| z`5Ke;E}XHTJqlXKT+6h0c{r-cGMg{5Ws_-@uKjwX3+=K%n)58v`nzk*IukxqK$A{2 z*CccWX~|UEWU!fyA*kyRWXxloti3;C)G|FXnN){uXL={q$s`w9Qfo2^rmNe%HpSRy zV3GhyJ~UYqgQ)k1D#^CGxH`@GNy4277>?-}B-N>-^+*ZZ?D&w9`IC*-6vo?LRhR(C zk!n3F90folD$b-;l3iDPsL!!nma6E)(Qq_ydK_2RT%y-pI@cUJOVzctDr1w)zp??G zd1Slyr^1`4g;4$pf1%QJg_R7W$*{>#oKrTTDb`i@QHfP|s#B8cx~Oth)1}q!aN6mP zx3?O9Q|LE0$pl($Vd(8wT`nKMOAd@o0RGOjTS^n?XB*wP8uuwyThTY z-{b0jbZ<8OHkrZ9Ka8z{sX4CRkaH)ge)GXRRj8gjeY>B$!%IH@ke__qPd@1-U$`qz zMq((Xi*`-Trhg}$5k1^{5W-~5=yZ)@wgBaLsa9ozwNUtW0vBu5e!mHCQ*Ba1d-a>F z!i<%jN#q%>&ZSi;r$Sbo5G%qIZzF!{*tPZ7r3&9tw%3b8H_)b_Fm&Bo$0meCa5*I z_&c(=_+y4V1=-MtDKwI{4<4tL`kRT>-iFJOWUoY)Inku1IzqJvD0Lw4iB z>9~dSV}aEPNQUJdacZVZnyG&|fZk>DV>c4UsuwIX<9#3)#U=Ln=&XUe{t9fK+=}=E z{AId0x9*i{eNjgF3I|}+cGr$Vtv_9%Z9AYO*F|?3M)hE$*7Q zdt9)kdS$DK>(`>oOZ5tPj!ic1lOJe7%lqW4L7-CXo|Jf+7}E zsZF}=YD4zN>Cl%1q_!rLE$bZ)uQXAttv4F7-?>R@7R_Q+(=3zmG5$GoehB&u#i0|L`_-C@x3W<|HuU!&*gj%4mdq1)b?XzqBN%aJ=yQw zIY0E+9(AXGaoyQ%B=xGhwX0WMJHK9)NoRRPEo?P&p~EAa;izQ{!%{1X6e`7*6C63y zRf$wyr{EYTt1(KFs><}YfN~uT-^I4hTBX2blvTD&YHeXkE-BP*w=I|Hv)8IvN0vUa zV*;R?M&z(FDyrvedt9$M)1Y2+dsZ9fsH9#Kb(2a|U1n0oy4plEPmXhmsX4^ZSY8J; z`lYZRMwp`l6e<$ToDpnzZfM4E?4S^eOeHlvt@zaym^`!ujGs&r?dGX zXFdkxTz9s{5su5ee)aWaeP}ql=A_;j%#+_l>$0EE zM$6$=Qg78=F1H+!5nMj5)RBurfrQt;mb1Rp9WGPKCBNm8-*R*b`QU%Xq1GgCE3UV5 z;u8{i9iZH)-gbM^cBr6~kdZNtw07z1m%x+RjXm#LTkQrH4>aSG?ELwM{(H z#sticr7{ys^Fm28VG(M{kJ^fwQM~nTM)l0327SFl5}{0Y=%j~8PG5FE2aM5E2FA&i zXke=di$GZQvQi~Y%g${Z;|x0~0YIg?(uz6NdmB}4X*SkFmmYG;?j#$aj`-a&e#-VR zTOvA5HX>G(p`h7a?O@D{$J6xK>|~5BuPkr@)~GXBYXPZo4^h572Ss9Lz!cqD9OKXSWxSFmOe#Gsu~;x6aPA zxv;!JMJ`~JmxVX)f@88l;keIGTBIbW`JAAnI6m@{uAiaaadI*O)26eeP|;M@N~dU^ zg1TIZ#JCi)d?fOolE_IR!M4FxN5bu1*Rv$L&U{bmUB?t4C0>(^rtUQFTmhUj^GI&U ze!*I0aozEoR7l_~arY^Jnv?dXYn&AXglET2rKH~PGl}xl)LlRNID%5TXbac9B{K3< zqoYES`oLKg;<_gx15XVNT*i5ek!QILrnrt%tEw=+x(*!X(3(4b&Pp5ChdE10JNPl7TdbVwhPNt+?kLR6JlXl@4oXUYdbcy7)lN#J0F)!684K-w5S-2L~JrGCwnd zg&5AbobGYxC=lF_>y%W_!@cN)bkDD7TpVUgW-HcO)^%5u?jxvWO*W9*z3-xCbSp$3 zhK==k<_xE!34^B^=@&!;8MUY-V9apJX^1)jb7l<#?N7$WKVTU^TM7|V)RMFp5DvC6 zMBre~7+tK7)~fGfc&vcaCP%Iu@;kowlEB!8x|~#i zH7T#B6R(`NNQ|es$wn=%6RIxTI@>m={jO@BH1Y(KI!69a@|A*GVf$5 z0CtF>9vlpG-imfo=e#Img{1N}NqK5bRuMb0aoy=I2?)!^e6p6O;<~rkOdTl#m5)I} z!sXx85h-wvDNZq%X}1L=7}#v+9Q8~g8P$xasWVuG`qbTJlJvKEQ)fw}g}gUZ>g+Qm zmHPbJB7I_Cc0T9H>6*7sS#xGXj8e+zBa!!>gli@cy^(geRA0B}t=Z_UJADQ{T^q@I zmaW+0{K|%G!)hLM9U!Sw3RNt)kUC&-p-*mQHq#Oy3 z8kgDi$sVql-GxmU_hI9*YOBQ>FBY@0{d?y{I{v#NKe`#QhIi+p^oYjz;v3bPvyUp0 z-00_Oi*$X%)q1+!)=GbjnvK?L;BHJ_al@IKQ6rz3 zal>`-G+f0`!cpOkdcGS<8g)n5qK0GGHFD+5+;2_0r)hL*^UT8GiebmRVQV)AFa9%`iiDS5fu)L+Dj(#;C^ah6*wcACsm7!Lw$#iuc6dVRrRi3!1-&vCz#qhNw^ z=d(QMWQiNjrYLHR@L6ckT6Q>@&--FyL}-b4@5TCp}tRfad%oNVbQc5F;qs{X0{2p)uN&Ijtijo}5&SL7aJte5$gP zJx(Un-Bj(bDlSh#V+Ij%}bAhuSLBk|ccjWOj{JfA;orqzf{;NWMG!Kt14x3 z_hHhQI9qU1V*Sqv-J=eXQ;N>y$sUU#e?eR zzFKo1=fE1HYJ&+FIYIZX%^4Pn-`%oc3{iR)n>b8oH>3kDJM%R8}wxChNofasVA|B0_E#sDkWSbB;nT;0GPl~3n(-PYb*I=ql&ea?z=vKX`J;zAs_r;2GJspj5VE@y#md5@!@G8z`O!3 zH-~Kql!3DWNF^ZH(@f}e6Vh~)eca?CO6p(RgQg}N6gRqK1S|t22AY68bq3jW79uuA zxi*TM6IaM2y2b7Ymzt9s2rIxNj2}-IeK)i+ZaU#(Nz)Ay19hDdnl168utA*WQFJ9a zv1W1K?tRk?dYOq_-Sp=GLZYrKnNO*Dj+;*CSJHH&y_T9A5*TOp9%+C=)7h3S$=w}% zUc)*ax1(Q4VY#i4G_P->z*6%5uSm|)ZBYW#>m!E-$vwhkb87a>y1pXvB2al&Cq^0S z#j9=wgIe8ST#$}8yQH!4;%V2dnj_^#hFrjos1yevYn}#>q(eYUQ|IJCl1})cB%PKe z=~!d%VmcNzm-EI1dUQ~GqHJ335j2Us^99ZmorMxk=R3-c7ykb$V zIy0?8?%kEH$(0r^G}N2J4Jw;B@tjGkUqu$vsnqIkda;cy*N$lo9Cu5x<%fwEB%O~! z(i%Fp14y}UHgAQTbgYm_e5aNnC9Y95*>z=fl?C932+bcMB(3o~@;OJ!Atr9+?oh)i ztB>rZw|TlwIdmo5i^JBbh^VN(bw{Sm;d0kx=vu`o5JH#iO0Pw>5<-Mm^FO zr8J-HM?N8y3rZ~{h+>V&WQAmxSjhAsHg3s*Q&~zZ$D8N!bG(eN&uf*6+Q527j#-d zS>Kc!5Od~*?cOJ0uWBV4(JJDWLu1mKI*z-zwLkAnp>^gtG17#}t>s#Gvewx*s7@qY zHLZ#)r^&)43J7`zf;ui-?LBK5*RaYPy>{R2d7EysSzcKpvXy*Yj+hf5;1nRdsmocjEut( zRoZ021|whM9Bb*wHlM*SbGcuZcKgxpN-<7DEEQ1?t z44&fxBIt0R$o|4v-ifiM9S<<|0jjS|4d1wvj=!^_;;^K3e&bHg$TL{n9ubT%gn~bVg<}%{Avvb}e%$!dXs_$$RRJTdtJ5iH>IEdGe0OVh}P|(agg*X43iu0 zZ*EApH${SbOM=uKQUqBtD^GjV4Yw-0bu0xD#+y}WlHS}M0@ zWqMpMkT1y35q?bnC9`LHQJIiEf=L81b{x23^%AD zZ?$_zEWDqk*C3Nlx1H}1%aSr7z;>e(6TF@EsUp|IMlADGZZ|PY7Q~U4RnBh2Kw3b^ z`zd>{+YxVq)fZ)nW1VYHoUQ3DtAmGQQY9Qc11$-N(>E8EbVhkZc<(3RlI8ME!Z4q) z#SUfJj-VT{yxj>EcO28A*g11l2B7zc&U$VVu8>^RmA6hrvB+lAI&aM2ij8Y8VD+w* zzfTDJ z(|^MZdE9jp=1BuOFt<46uO4Kpp2?c7OB%+ek$m}M2wB!W#87YsI(JCQtwHt!I!oO} zX>XU%5OKWoT1GyOTYI`gtZ8vyI_`IUZ9(sx=MKouADE@jfUTY|2|HFgPbzE}nnPkR z=&W@uXCOLdK}+1pTATQhD$8;WqtCT$;{dXgp2BN*tu#I*T}iq_UIB5vX%yXHL%@9l z7g{z@$Lkh%o#Tz9J9Zt7ZbVt^#vqp^N^FG#_G~_t6yh43I2n=iObS@_QYq-ko^{Ta zC~!f>4}#p&;{sZ9I_!=_!~UGeuMI!SNT@viTX3mvVUaGQvFl+BI=jo-1*8%@tV;#x^dJroI% z8?Mk9Sknuhi+VYOqaN+O-%wrXtv9f{Qlp_vvg>VO6Ch+)cll$Y0(810zw(@IsM ztJS`}^VR!1)?~4nR~@@v9x*XYRUiJdV&-T}#UIMhuwZ4La-CT5sg@$q-Mzq`)#hf6Qy)2%%hsD@wdI@wM61}csqMKc4ZC{~?pK0V=@3QV zdq95lNcCq}i)5(`^rk4KF3ZpHxNhq?WL4^km_V=4A3RX4f%P$TG zm3iNC4D@I<7pcPPq}pMQ=t|=*NX1obzYS)cuKO+~TBUjImky9t|B73E|1#@ov8T~S zyW51$uLu{FJ@QZU{0}nDN0GqZ!~B@jet8ura|ZRR4eXnopUTRANnT-Pru&mVXHLxf z8gtOO5ygEt7BdH+WE9EV<=_!Rho#KzV@zD@cJC2e`xZ%Ng5@uz_HR1A>ZD(nM+U44 zmlSO~Ddvpb=7Dpojmn})5EAyOkPss{bv0tC=cRV#FO}(jR6&G6eH%dYIn4LKl z(~GF-LLq^o>w3{Kb*xLYg5wM9=Yn17cWV>OYRBq|x&4#%aEXMR4OAT-jQblo9V8etOB9K$;jMCi zljsC@uo4r6a?Ey`dAD$aKer4P@hP2oJatg$Bu@s_DzhhTaln+nTv^Uti9SLnTNKgw zTajE!`S17WK8yqdV-v=d9Uy?H$$yBAAiEAtG82AWYzB4vI-7T${L5r%4n{mW!s*I( z@0X}q_b>54T)lRxR7kA&E3JUu<%5<0!59oqDpGOD<7a-!>X1xQF3B^`)m#E2F*Mn7 zLsQ%*srKZa{J)ju!&OmbgOq)0724@^0f?Mqenzbi(TTUUAM%gO4v0-$a zKt?Mu(9*+n%+H+QXwQ`)8taKUpjGJw4pqp1Ww(LS-MN*T66jl$t(^L#hVuUe>FKUg zhlPfRRrj_p0C}o{rmX&@;{p)Jxm^VJC`3Ze9U#oZf4Qcsuv3ydr%ooRoHIS1#WVNa zGuuLg$5pbVImyDX+-~?Rg{B?Rml|%Il`$J)E+c3>x-v$E&R)0q1pL-6?qyrS-5T~5 zs7WKC~_TRyyKG1D*NJ_8LPGoyDr-Q zhoisj6L9lc&N^A*fn%kS$hCAdkV&g1^v}t7{(}5m#gCogbqQ*+GrTaUHyqN^hvw!R z{xhdxinuqA;(-%>NFvv@35k3qloYxwRe_Rnw522+jX3Z^lu3E@$lzxh%IQqx9MZ=F zSpn5~EG$mP2r#SAzc-S@6M8i-6|&H;{>ZKt`nRdZTIf_I+qAr59`JKgJkdAL(tq!; z6sizkTevXLhy3RA&z$^`M3kx;m7PM~H|Kb_IA|guOw&wtigxw(2xKu)0(MW4kVefdT!Zn|etU+XR8MPWcO{`6 zeaCUWo$osBB;?6wXTkH=noEC~V%BHG9O$ZEgaxzL_ zm@bF;+Oy+`Njz|F-;q5a;eX>~PlyD~*EUJz{VEMcHSYwxz_l~EK4s-0;T<$~4_0~X zAvKjFbPY24NAO7}88284r|eeHE4B|6g%A?~C{sM@_O$|B9N4EgY=j0ia|uggnnb2; z0ugmK8D|TbT?tf_A~b*wrpb_GWa(17`yl z4X)2dvNYI~nc=VEdZ2p+p^I4dSSpQSaIg|wR`p_C&bLmoBBH~-1dJ?1Fjt!NPCmx|Aj zyLfKO3GJnzb3ZGRqtUFC8|WJ|&#j6}10d86a04I;!zS)<>rZnwqsF8dilt)%dlC`| z^v|)C!grdjz$8_%3mOUkC}S?0K2nmjWiJ+8HsU>OImve%D>2f^$~V8}e{62ezY2lx zeOP`zl$j&HuEM1YUOaRRUY;4`?uOe)&+j~E2n3RwJWmEbr^dc)*svSiYkKEl3 z4=QGM>as2|?8=Cs&UbGykCkRwNaU)EL~aB{qg;j}38DK*3WL`~0aa(gV5~r`V zj3ap#a*Dm8vQm)iGPITM5H;!TbNBKZU{_x0@ng0VHgRqu7e@P;i@Im2#WCAPhvH5thrX&dUeJeG_3wCF7ldl=}cC}8)on!=a~ zpDWyWEcc_V&?I9w6ftt{Rwv`OV||q5`Sa=srE5JX>3ir-|77TT7a?@cq|kDMzz~nUf5e6cSeh^WlnlD?kFGc;NAMH24$#FC`<^TeDS8`3=>Z!Fl*U^W|nA z0N5q77hPARDP9eAqB`8*GpWivNNm-O0N61z$d~yO#o^!f$Q%Ul{R2tM zhFzzyIIZVSKeq~#(I$f}iZgHuN=K-{7}K^9xZy7iO9~iN-4|7b%;61)vI#Ykq^Hs+ z-Hj_&8Y6O86)d%GXh*FAjmM7Hfdqz?_Cb9(>fn~+ADOY449MpmxO?PGP1qwVyS*ou ze-6S3iB_fSV)>f<>kwIW5IqtPmA&$(rJKA|l5J^V&Zs32GNq9u2Bf)ctFT9MnQZCx zL16NZz_n(E(D~7ZZelHu`9Os{BW_di*x7!O$fx`=AF|-p7g??xeP*EZE4BIP#k?;6 zeovQU)+~)r7eCCAQ)=^Vj zOhxxe>i9Z2E=rf>>v9E{T*qN+hDok-PyFNS+(6Xp+)I@< zF>&tEQZaq3`wgZp#%VlpJofR#v1dr+auLaCspV)gX*gGM;z=Xl;DN#mml7f=G({=^ zlLoBH>fkxvjQ+x;$)_qXN;jd_?PrA#vLJ|@5@#~8gr+43#0!Uwxlfj5xL7ZjP1z*D z0|{hJg-KPcZJ%1EP+$(9tOS;G6`L-l6Is|~boovT)j8kJph?{DsZu4;9J()_3V_&2 zm}axY6Xya{GU+%rMFG<=Gm2zOXkU&K@Z5#Qc;a3(vg^JTf-(w{V-R^h?m4t|?+ls} zdst17gz1dEpDq*ZZ*+07$~2xh7DItA@2uvd7D>EkHfY0;UH0*h?P`~O=q;J^BAJZj zK~J43c_)iN_y{=ZI-^@s^lyInfUe_%b@?!r{hQt${{*8iJ2H*`(J)X8QC|mC zFhy}(e1>c$&mH{5ieYo9TcR?pj;4IlMCi3s2~wQpt2`A|=dMgS-qR#no7|>Sx)c=OQGagUcp5&MIWT zC=*T1vMKH=&n%M4?N!!pVyE&H|9$RQFzE@ppL z!jd%TqXjF>36P>tO8>=FRv^7;vuekR3LwMxU83VchGvIre7|Fkg(N9e+v4Qbr$y7W zpm}LDZ&~L`3FyTAG(vgWVe`QJ+cr*&b+}W}q`SG!l)e<8 z(kr0Bieqs|-Gdd+t0jR`R?zuunt7T0w`~x+{fwHi!*e=}lPdePnruicd-&R#NP?=R zP=1ioX(zuP5oP&B!ZhJa7;{M_EX(IgQ%PRZLE4DDH=|N90Parl+`a)N)2?&DjedIH zR(Y#Qui!dr&*_19Dqe#tY)G$s=0x@_c9UG1_M6|hU7E=~Yp*d-wtKN&QMO`NoV$Hf znEGcoQ*riwzo9moqU>$Ccq%c|?>9TecsfE0r(SOopi)n-q`E^16|xt4%NBw~Yazuc z-vtx;Q|Cq~|vzRMT4H4G8#aixf%|_HJLjicCBvZ#E zA9EDIgLb)sNqEqXd{*5QB;_LAX%bJ}V>c3vmhDW$p6>GAJ~;!>wOij6p1#Dr7skR+V2>3F)9hsuy3=dHPhS@WV4q;?7GYv&Hv%5zCbmnX}b2p>L6RiS!)k|6@if zhiO$FD1}*1LjIiY=PSiHkKGlO?BY?%`2v8&RDz(aOc$p6iHF|q{eju_?V0W;{x$t1 zik0l8=_WxJ*fDYoct^;WP{*{l@miGw$CU4DwaJQeC>J>|`$ocb`<=GZ8XviGcli}w z3F~lP<#+>^6*NDW`JP|DIrl_Xff!@QjEz;neMUD1=KGlQw$)ESzhjyvd_$9gR1KKB zY7)Mwxtm}4b+~WWEIzqOztSTe5{OQ&pGpdFG|v(M?UQqkR2GgwRuWn zMDmp5wvX3kk>sI`mdJ>wMXR608!jbVChDFJS>qv?* zva_jNAR*7y+L>L{B$U{6nKXf~a`4eJGZM(8QRPGnk?5dfUB6DW0oi{_o4Kzm-f)b+ zXoIoVpi?;-)#0yfH0$>2S9-gvY(UO^uUJJ?&I-2*9IYYvve@d`q1CZzU2=C{uXEb|5?8;&oon1(?9w256=+StTwu$Su6TXP2H-J~nZQ(5A= zOsyKyOnN$O`&8OLcT|+{**7HgLo#T7Sah@>g6H&3X+IY=ySQOp#lsu1&Uw;)_i)L5 z0ycJEfF<$X%zh!kBUhxFMUwcYz;bmqdgyC}GCo!;)fVl=8rKW}rpeH6?~e|(Uy3YwhD ze|+d$Pk!rb-+CVCjW)ka%_V<))3=@jcYB`aoGDp;`Q*zl@ZU4ry&sh)6fCX1w+Ml0 z;>VnXE>fkJPp!QC+{-6zA>C=2jfW}d093jQlK*SGR$>}0y!_0{D+dU(GRf#=`mC+n zKAK8wLMw<~h7C%F%Z5u{ei72Y`tsLbe&&^1Uq1QD9WY#iK-{ih?B;ppHVlPJ&2k0& ztF~F2-!z*h_OnE3XO|?Wut>-(P+Iltatf7}d3nIpDy?Z=OHZ#D%1=Fk{HLXlZ!z`Q zHedOi)Gd@4+tkz!lp5ZaxA^s(2bN1N2ZT#`4sGQ}OxfbgFVQc?ML8>jCW=)P?*6xI zvc2+oC!@Q3UJCx}X~$na{qomeImkc^Sj=5AG}kLPzH*R(I$(44e$o&^3&6hq%1xO} zEaf?KX63L~?h(eMEb!ntYnD^E86p|F-rv^-(ft>@NF*oayj#c_CnvW%PM2oz+h2Y@ z9dj#Tm-skW_Jj$zX^i+Uy>du8&wOGM>b!Hc)ZfdoFF*V8*Z9B4dGD7E;})>5+$K`A zh<)YG?^V`dy|R%Ex216TPoDhK1Ap?;pM3349{y9N?Y|Or;0|qa_Rh0k+V0H|AeB=| zKf2xfmxji@XFqq@*~4caID0R@C+xYGTfA`gb7$|o%px4sp>>r7dG>_V*D&=J6@qfM z>5EC9avS(9lkY8OKgYY80Yu^KU2o3_ag}rN>|JM{5zIJkY5VMF&)&%*lc%+(PWpu4 z|I!@L7S4YD?PrhJj{G1tMymCTXFpHHVE&D%C^>t-G-EqZI{W3bhu?nouB_d%04R_{ zDrb+vgTrT!!X``H#VP3QgCKIEmohjkvU&HMedO#B_OHQZVved^UmUp@XRGOf9Vr|w0y6pxw5iKb^yF)YUVZenTVG|t_bAHX3lu%`+NWQAf;Gzc z;g*$3g-K2eO@QWnOXj!+4_{@jSS~KV`q*o?0y*~QTTsEJ2rO{Rx!&8IX{?K^%?t_0 zgycUkUCE%PBbG51O0|K!f4*h%#rA}8uon%84R$@4JUQ?OeVeX541n(Dfj zIl~Iy`;XAb9Mhq=RCCiDmA*7P`XuezdR=y!TnGJJe`~t>Lt4Y_QdY=z>(SQSdfdEo zwf$GixY^MM>7J!Q=C@c1RTz*OiApw|d~o*gr?z|1_k+=PY~Hk2f}R~cJbM_*zNEb> zlNBERL!nWi^c6Pp`^4;Fh=Jb#kHQAqxZQV|+HE3j{L$G_jAC2kTV_Wkt(3#VqZxwX zdQ@vYsa+~TfG#!vz=h9_9(Ty0qQjx`4`Wc;=3ru4>%+Irj(%nK@F!XO+<#j&1H9PvWRw&|c>vWw0bnZFJY?1!AXM&NMn=*^k&ZhM|r}mQm zQDg@ZG_5;xN-0<@LC07i-ZVlxhc*)dF|g>04iEAH=*Nr@A~G^!v!lnfOU$`<8U(!n zLG3V}8UCMWK!?s0FmPndqD;uOJD;6=A{ONP&uJA*;?wlgpQ3gW8ywR&XG}&ubB%Ia z1j(oJYjQJNo>fwrKXA|X><4DYZk`=GK0AKHq;C;8=p=6o_7JEEl_kvg?;XWW%W2sol;O(XQ@Me4Dc-9|zgkCU-&C^cy1$nj6+ z#HQ$YcI?6J-v4Ow7D>#G9m>*g(ez>M- zpR?n2oEc}^WE69#cJJs+(0Oex`sr*S>9J|xd1VE2;M1lPR0smL%}M>lpcyonhcFxA zpaV2!h2zrMi6gT|4$e-Tm_2e}cH-vQ38YQmNpxomxlSwI&Glkh`=du@U%7Gi=!w~* zFU=l(a`xy+CUqNknO}qCI_TxE!9zQwJ6@{&`CA(?@2f4$huFF?;%@+0#$Xo<2D{B^0mX!*%PqboR`V z*|P^{&zzV&^U~~@Cuh&TxZV5LVl>X>Iz4;#fS>EzX;aUAdG@=XpFQ`~>~}vmd+vqV zb6=hP?x(kVKQd4956w>BIy?RR?D+$;)A!C!KRY}9<=N?nX3uX4BRf7>Y>fo?52M=eBM<_vz1_yY0leTOY^iA<=B?~@_Mz3x(w;H$tkEv(eH`JnLmYqm36@mN08{MWXszdri)?;if$ zylPET%#vUH&ZAFz__W(8Dc^bGi8t<~%`E?u)aR7{?x7P}UdfR0ls!u}xFn2hXf}$C z$zizs-CMr$#^a~cLHn5$UOE5lXLoMB=fZ9G$P__SI5MIIF)pH&dYrp0~o_Ad_@UA|Q$yn`#bbd?wAaOntf?YKq32&KtLX8@jn}K4|VE2nFr5^;WmY+UB^) z^Pb7aM;z88+iQ?`^Y+KLdoM_bc=e;fcZ49;LiB|xNuh4`5+K_hubFD6xY)GBwF;!F zt(ZUuQ}Xw~^j?&om-rFk>wQx*lcUY&zqI<@GVz!(VLuX{vr9HedCRof<_Vm&eTF1? zh3bE$fjhUiE|Ke}qv0kGsO+2YT>wx5HT~LGo$k|5m27T*YU}$6lv!0i@y-Q78)$Me zZ(a2E?cV=MT?kDDwJ&J0x|{<^1*PJSA?|5|?fF{#LWmhquZL zWX<6|ms-F-*u0+Lk$n;^)xCZm`8Ga(BNy8D@oncpnhBoVy0wS!tnPRpa?b;?w{9(& zWQ`kdq<(09D2M3H#=gdgy9^CmyB2&~hO1Y{(;Q>OVfn)VuT@&ejZo~-C2~dE*8$zS zWNrV*j>0}+|CF(Ki@@64CmE(7WS;0?9b{zqW0I`Xnq2f<+wLSr&O4>flKpEhll8)J z;pEP9+x(tCEq|Xsc~~H~JScyk`Wn9%Za&TLoA*85%%2i z&P_-4SDUy`GGDm!v+{fDS^0hPMZrEKzZV|8S^mCX$#@v@iWly<`TQxZSFPs$KzWEvDAxfA! z4@yUNZaSijEwTul*N*QzCG}r8{ye`sUl97=JWNj76&g&Q3y(@|J9iuANMGK(7wa|N zIjl^+Pun`IWPDLteDhB8xAyKH&2!UJz|TL;=oc>>|KiRA0=#faGu%xtiU`AnQ_|lH z&k6ASb6fKFNh$T_-DECaIC$dxbHetWyM^HMr?qQOOGBo!JIA#n2TyF@x!u`7;dHJ? z9BXn5=^_sEnfvw=eE-PALf99EuQtKFJV+*ibAM%G$IWY6Kf<0uikXL3{zID5=S?CL z0mQ|7^Ju`sCY@g~f87A=Meo10sfk`M`gs#^V}te1>G=lc?v=7q-6Lx0UVbj;A$I=R zy67EDSzX0sH8im3vNZu}99RZSuKX$XB!GXxuAC-sd?jxn0sTb}np`OXY75}_O5$f1 zswe;rRZ;{VtmdM3OC@R-WaP!~4pkAqn;T?{4FV13eh+~`3%TR>2K)WKAoQn$(D#R+ zg%7N3aM6bLQd`7w^aG*X@dty@>Xmh=!RLM~Unm1Q@x}z`bg{LlswB);z@5eg!iwCb zl$*;S)a8`%z5s&j_zUp3%1cBG)c{p4-#$m3f%?3lX8DLSb@;gjPPk7HuExh){^e>- zFVS-g-bkQ+3L)!20(yP1Ho#ns4|QM?glZK>=erfCIIf~M7V0en<<(WrsX)yWDBm$# zC_z1Hg!)WNh!_r3513bI7wX{4xT8K-3Ls`7cvh3w@jU;ZW7JEU+V8j}LPc<%+OZ|a#RoxrO{HWP=E z;N|;o3nlkd`O@fw$Acu453H{XDF?6^7$F6nU{n60IgKVW1gtC6U!%i2^=@b^}7oTt;E%Fpb^^U=E_1VWdW}+6BbdFFnVpC z+Uo+D<0{`|rFNh7OT5O`m5VeX@c>!x@U_+suUt{mpAU*`a6yC^=LKX%N}3NlFW~kl z&U|cHO6pOwGkQ`|KD@kuT9$(5T1IFc)i&8>vwx$gKdy!9q{C<77ifVgfjN@^-IUD! zn3grjT3*0jsGkyC83#N+5`51VsL%7b%C&+}-arir-zp7M0hp%*7U}@Zb3+SY8Vm?T z1+~DmvP+lW)dD!)WTqw`E@k8@pIJ+M9F2zz>?NM>!ePQkO9On1t9*Z#R^}PR>-h`4 zNO*)hkhxS{Ta#N#Qoz@4){%bZkuZv<*qS=Z{*oA1*8@vh!l!Fge)N^^Oj?Tq=E5g! zFCe(P6XQ>g*rfW+ z2Ls?j_1x*(cY)s#hClz%F7U^9fj=3Bzi?Lo9)&glNv&$Yen(3+UPq8y&2-iu{vwwF z`&|=cDJV6Rmy3c3yBPb`6>8Nn&+$<}BV97th-+Lq0s0r5BfT?& z(halE^P8m-CW&O9Q-=z-=BtmYfRgKIA!5s{`AzA+)aa87S;`ZiEV{kM{v_0kx!r;~ zrq%aA)pJv)R@Vl7C79Ie!Bwip8yQsLXD-@XjSFNi)ta5^2G=Hy88+Xiki}Z4@%amdt%_jN+P_*SDgxDzBOpQycx8u8SPzFG&~ zAM&`yQ(Zv)fuO=+SN;=}FHi~Kz|a*0tSZQ_N#RoX7_<|RWeKpU;$EA>ikeS$pk`j%AZOF_BDE6)0*kpoRBzhzU>?DSv4M8sHO+@^+6SN(kFKyF7pE_y|Fat7ze)@VlzzwPhcDo%~R3fwU@`)(a zJ6D+Y^qN7oDOR|S$^zVa>3vZza>mppng+J|h$RrbaNL*o5*LtVki58Uj1Ye@mkO~s z7t53sG3P=uyGSrf!7jVV1lMVB;5+72$mdLKJy<@JI-2U)@U;Te>U(eW(B~J_?y|3w zngWsxI5dT!CNQ5ErLG}o&Jm&T)VlL=G{CDS0rBtfw95i`FHbQ}n!9g6p(`Mlv=KIE zJE2_}y%_fc+eLv-1M`>KgFa$ju?=@DR0$(CZH9~bq z1_&f?pp!t==iw6pG{|oG^7sM~37LZu)(zS4>FeP4+Y~ub!b4LIw;>ED>tqW z1EK;nAhTupikGYz(o267D=A8u=;M~awfJ6`gt=01UMxnOJ1oLu`PeVHeMZm>-soLI z6u@!N8z=?ce4|aiw6M@Tv|7OEcU`mjlOC*ZY~4B@>fT zzIhR5=78_I$s?Hy9FV?@N5Ke%JqE?$9Z!HQ^i;8&A;G^ulUgm`uU3?=1_ULaPf%j~ zbqPic`8GiCl(+r7GZn0;b8pGD+spK1jf7z4~K79YL+i zYvc>#MHQiy9dO92K+OSVkwSecs3PQ3A#*@`mg-|dZg@a-fk5(W7k$}Hp?;G<13DN& zLQMh=9i0m}JOVtRzCazh!`9fCSBHbSgG*+}b4h1(;`v>q!TkXfBdK#GVzI#@4|#JJ zL=e70DRa4jXt3b%kwNWx;8Fk|>N9Q02YaL~Uq3-x*nXIV!hOgcUUWm&TwD(g3eX_u zCTjt3ET4G}jMOwVm&qIu(FABvHm0E&PVSJugsKA)15{_ry!3~RRq}?ZFcWBAj(8|| zvEPL{Zv=^kn5zDqi|gh*(C;SDAmJMU!jt-jaX$D}B=!AapH3)vgUpR! z)%!zIN$${W6mU>Vc&qW5 z??5gLKbb>=9M`W8%F1S4-{_>8d0E*Exet^2W=Q!&jmv}*KU8O_aT%{?vy(9H6yJP` z_|W`9i9(}V3pwhFz&HAVMHobVBe?8azqwCQ(y;NGzF9^sc*4{w_L&ej%zn|YCFVWp zMZQOkfS!#T!K1H6(7XZ~GJ+a)-O~78xTF!Bgb{Zn7bGHXghnzltFPZR3-m*w;zlzt zy2afB8WRvou|MF{5}>WnXtw#B+=3*KI}BB0&L^4%jSs^ImjO51HvZ11g;x9hK%W#I zluPpKnEOp={s5f{R~_l+FG;&xY7h2T#e0GC%E<+2VBRwx<8^Z|;&&ESs%ZO_EH(Jd zUYpOKm{IZdKCBaDXi8KXO^F+o?@cjizS0E8v&8Z#SHbyH36|{A@t{{Vc)k@kOvJGt z>l1NLazTD7jqbRA1-o;dQGV=IQE=73Y*hr05>uPMT$QB+0S{^+`2C6-Ax{YfCYcAr z9}Xzv4P|a81j1$oXFq2E~7dje#hoVMcBOC?2?TwHn z95+JI@QGa4SmYkykPTW8unwJ1Tigh_rRJ`K+Y4~!j<`R`xFSxNz0P7^P`Dxv;v0vA zLLWnsx^)`Niwd9U3I#?=3F{H@OYGrg^?j7+8aMXK@FBcK8EAZZ3~k`={85OlmXiB?v| z&vC1XYLZ5M?MOYNj+sS%{&EwK=fWn+#Pc>A`=zE`&N55QR*TOeYGeM)jEqI&OGMo4 z1}0#+x!P>ktS{TAEN+IbQ5Bl~#zfovxOc6wf(bNPAMOvuOH)&5PuUD<_A+OsLqddA zM*VJ;_!dHQVg|z}RWRvwLLXCT4%SuzN`Pk8R=V%{fNpktMp@JcKPbHSQCeGQE;K_o zs0wt04}wvPuYt?WVVgc0>G8E$36y-qby0xgW@xPfs+{$$qdlxJ+)3n6J!Bs(gVpbO z4ug(;sNNU_baLFB1SgXTocH;()STQv=rIS!dQk4G$haB0f0HzWcWTIdoiIb6zROfc zNrFwWdloTeR9JO$7ZsuM@83&{o1wcdNi%q>WvR&*IoiA%VzkqIw?bZtrRMb#`G6!S zOMQW^q(~*qR~Sk2`lcF=KA0kWZ7?;8eO$45J@X@=IOAsM-btZVtp?+3fQM`=zyZ;x zmU=K!4S2xtO#lb9uhbF>s#5qat%d@0i_|j;bxJB41@^o`&o1@PSpyn8DcJ+63~=aH zG2l=P0JH=zzmh$4LSE9xYfONEuSn?apxal$@+E;5rlFt=frqZB2|U!az(Ynk4fDyi zfRKRTgG{!mb37F?u% zP)eEoxsQZ644lPT1H5bQ<@ue7TPDQErx|Eq=TQXO44z%Ig69H20|P+a0WMYeeA!Xn z6*GPEsVRYbZq+EmMa=0SCehEKAxVCLxD_HPX-z|i5^-yPP<5Vp&TMb=>B3CG6lGN7iua8Vq*_fM+DBrylzVEUTEwtG`>Xy)-y+w98*vnGV zmvagXM_Qu~PuTYdjIp>K+POxd(CN562yFI%23#biw*A=I0)1t{?C+Xjcb`O11paU} zYOz-a^^H3N&bTEl^^H3Tb6hMJw?}$;$9I8I8=hPFHYeoysH7cy52L^pQ8V$f(NmjM z;Gykw0$fcMKYLNY#wzm>Qjdw~UxbI=)S@Dm7L|*I0(kGGUp_pRFUJA(bpg$e)>SI} zWp@woGhf3ZZl?UGKn9q@|qY-qca-Y)f-7jw;c zk@|`}TEhopgTb2?N05RP^oKevXtchMi#x{U=L>FGZYl0Yn{ku*6r0b|=qEH6L}5u@ zaVFYAzOT7}O!!P~VA~&s&ajFdDlxm#dA^6tq+;jH(crjVr1I)JQw+@CJruh50RmGo z=~jd97In?7asPIFvCCDvmB8lDAOah%q#Jas7rU*tyd???{lSX5+8>|xxEq2N3E&J~ zm@$aJ;<|?t1itJRE5pM`YR~#gWN(*PSUHC?pEg|($UWU58xJ}92%W~ocAMq*a;ZC{ z#$6(8m*vF*zA%aJz9(@q4C1pk~&bYW6 zIv`EDyyjV!dkZVUfUuIHf>t>3dx68QsHY4R(7kg<>j|Xbf zFUm5mSz+@?fW_*!Wf!nZg0O%Xe3eWbpkG{|Q7Mn}{OsEMfOQ(!Eb-g+Z+FRTo_yWpr$qRr#0^u_n?P=fEx zkJj~^#b3~hJ-$P2M>vCyTdf&&yP~?%nRYlDy!B0uCKhThW19$YQV!0 zuZA{El~p!O?`nj1O{?Ku(`tCvv>M(utp;~Zdsds9HIAhvj$mba14}I zr31lT->TJc{wh`!@MVF8za$7w%!M9bu@zT`7!gXSUsq*yININDT2b>0#jBzB0HW1^ z2p3i-)sAWjzxv`T_VI=;J3b;>4J}Y6ipaN$BI70XfpIY6B&fAJ{- z?wXJXKR=Z#a~GP_8RYYbO7?GtPN0&0@SZr}&_yG_A@7%fRU!M^V!%_S#b5b~0uH&> z3__KRUm`Mo*?FMOZol`%{Mjmh+Lz^S(Yky99ij6zVL<}MLf+|qU=~#R-P(jLNYH5Q zcbi01?mVOs0!{K4r^arKqnSO}`aA0W^a9}#b{eEbR z-VX&`Mj>F*AB09Y?uUhk?;!~L=f&6r!T19i1@C0^L-$Q-l-Kj9Z4`$0z( z(BSkB3~ywftoB=r`{tcKUlIyHLpR{atpc}@8!J%Q=EIZzS}%}2?r#LT4O9(v=TgQrF~D8yu4GWH>JD{|qJfFf)35C24%0!E5ME)yoLQi7 z)dD_Z83Z3z=<4%Ti{;{hIS}zRL3hZ(@-?e_4{hCs@raV`jidMIQ@K70hEbJ@mCa>Z%!V4JqTO$dV?=#kk@0= z5I8a~f(eQy?e>hdwouiu8hp=;M&#*IRnGpxMfU^~a+vHkkHrx*YZtr&xlbsP^u_hO^$_td!6)_BU->wSZNz`lAksWAi#} z0Iq}W!>J~?&jlQA)Ci$JXb>LI=20lHD;}7881of00%Oa^F={^8Y~u8r8Ay)5i6eK& zB>*@iH-OEaeSwgG4O18pVvG(gOs!U>3g zg;*x!l&kc{2J|O2ROt};>UbHbzPXK;SB1kz)69k{M!)rvfP=eb2vAI5VT=c%J86+Q zoXccV2UtIYvM|2I!7hIDWDtDrgYQ|%o0y9#21Fg7*&-qV5tx95T^?626LqtTcN~;_ zP$D$mG!iK^Fh_qK}^71TBM9U{_=|$M4&`GFz#%BDkcMa1t#l)>OC?C6e^j+iYa6pN1^v^ z5*}X&OmP7R+KY+q76I~UBXckSc@TP*gUq2&@Pw-bfpt;9+M>^-(qL3ehuW8c0&hK- zX9RqB!gmXF_vTOaJ%FbY*@Wk$prWX@J)-oB(N-uds5sc=g;R*&xdI7T zH8aPjJPHK|#e>kktTY(Q(;=Zv${-ZbR{$FF>n4M7NUS1rkn9hBJP1AJRvvJlfH&)d zGGvZg{Avn=$y(@b+Q?MlXSP6JCSsm@SyXO71J+Lw=pf{WBYS{*N%+MWG6zC&qR>vf zAc5EcLgK+((2iZ|*F$$e3WN0)AAM#1`SMmAtcRamuMF_w@(5DM*HRj+V?l>*ag+x< zS{t^JmlYMdqhRTluXQafVeufe7mfzkOICkmOM^|B*MXDQ0#2Tx!u{z{=3^YYoGpQp z^Py1h`C25tsa+iP$*{T+prACY)mQAfsxNv4Fd^%`tnii4BUD9D_@);Hwqg*w7HaKQ z2t=?@LWKb{wJ_vSTHnkx2KF#`VoCOHLxa)%t|vp@xmfH4;Ls%=z@f|dk%^#~uQ_

{JUsKbWcGKZ@hG_09o2NaeW6)ayMUegT5yja^U*13 zz%K+epvQp*tm0@Cu$Cfo)5DJkD2@0$slD!=8TvZKqR`Q9WWH!K@$MBB2n(6dm< zxE(S*^`o2BQNS@;RFlu2A37}AcRClwIv#gIR?)oM!B`hS1J=b}v0>T#v!M@a;XQ&M zO-JERUM!D0qwn!@JM@KADbSE2sf;@WE`&!h#f27%(im+OB)%|)cAc31OrQi_*wA(% z9-AOIUz%fX;x8(`BGBL&<&q%w^Nj~{GVHfaU4a!p9*5kFDT3aS_Ol{NGFR=KmtbP#RpdeT;oHVTEwqRaq{KwKz}SQc*+3XT~1Df?u7~kVb33Y z9F%u)?(6vb7(QCxHU17?F6fboqV;Az{PvW`5p<+<7tA$fRYz7WzNpo_9!F z!AfI0EtY77A7p^sQHy~zUGa<2#gTccYe8&B8`dY|^Eeq#%_lH@a6As}?12XK((-t^ z$Q;5^Htl@7^{4pAq>r= zc)Sr*o#u#oZn&=teaSPrF0gnf*9BuxlIwzjPw{oZn+(?lU(vWObT|`V7aElUr;X+i z-=75~g@XBrcwWEKr~Ud5uDM){{vibhS60Gy*OPG7r?D?)OK)A;VzO&@&dFhsmUA?!5V| zDomPuR6C%+S;UxLMCJ(2pR7wPRz&%QFLcCQ_FpV=u|5GXWr2bJ<~D|3C&dC?p`|Hg zr|0JWTq}dj)+`@IZMrD^AYY2fmR0V0tTpV9*BiMfR9FDSq_=d40)CZ3p8 z)OSurAqPu53GI~&K<75gx*m`_FcE% z#D5_&?6g9LvKlx>y!|KWgVZ0Q6nF66?=B=W+eXzn+@%jiU zu&L{X4=*Y_T!8Q6h&(J52!A(+7MiPpib`VW2R|+hW-?d_S$t*1guT9@DBL1;Ov5~> zF|Sp-?JGi0{!yZRMM$0mVb92Yp(q(dKm;vD)q!88V#;u@dU0YU({AV}@p2NU5*!7mvQC8TQ?zvqsF!8*=K-k2`5Cs!Ko%fz+3VAwqw`W3R#Yk(nted2O-Zq|QNIO6D4(QRT?*u4z6@8I-kOiEI?1%4!YdJ0JPD-@P7u4KEo)3S zN0!fLnw~eVU%CLxdPrKXa5BHBon9A_KA~$|@-M^ruwg1Mw*-Z|r+LliMAR*xcmieB z04(cj*|iv0zV$JB)i)oi6j+wU4QiEaN_UB+R0D_LtA4;p$tz(+nj+zw`ZY;Ggm(rt zDcMtiHKTeJ&L#yiZfbucIg&C$cas%K!3C-o_O@&3{SaBrJZu$Xg|S-7QiF`V)SWv* z4ic1N^4_O1l)<2~7@ggd8av(7lQyMrFjG^T7a+K8-MSz76cduc#%k_+F|b;K1E5*v z<%$4jd*{auw6-^iJF<4!IN49NB-+@0&DL%AwU&o6ES}Xw>{KJL!P1Qwek}KVassSY zHy~v^_gr3(gMe5J%`*q2+3pBl0(2WW6e7oclG5BEd);^1!Xk2-YJ-!lmUA>KQ-bYAs5m-f7jMFUo={h1oDxcELG3|sE!YAFiDD*-{ z)YC7dn73n7mgGosHUj}lmANZNb=HQq|Kqm5h3lu1Eo-ZgzDKnf%?Y4YT?5pTFh;Ei z1MCj`Cih+S%mL%az1HW zE*7xq1w6f?1z#X#C^~vT9hD!M?Abj;JNi$lym5?-=KNl}0Muq_$JPDQgF*EW;M~pS zjpbJLi2gq&@PxoqfCFSLrr*z3h&BCSd23m{2;g7pcr^D4lr6sQ?y`DKe{g(*<6-rd zN$&uw_XIu?_(b3{fiDEU68J{oJAoerJ`nf`fR=vYKrR06j(>2xUHxS^I77dO3?slQ TBarj|0`L$#DRI1D$9DT4=(&w# literal 0 HcmV?d00001 diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 57437d59..72759d07 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -1044,6 +1044,34 @@ def test_the_choice_travels_with_replace_union_and_pickle() -> None: assert (loaded.languages, loaded.strict) == (lex.languages, lex.strict) +#: What the parsers pickled in tests/v2/data/parsers_4ib.pickle read +#: on the tree that wrote them, db7e8fc4 (4i-b), where a choice was 3b's +#: delta written into the lexicon's log +_READ_BY_4IB = [ + ("pt", "ANA SILVA E SOUZA", "family", "SILVA E SOUZA"), + ("pt", "John Smith DO", "family", "DO"), + ("cs_strict", "Jiří Dienstbier st.", "suffix", "st."), + ("cs_strict", "Jan Novák, MUDr", "title", "MUDr"), + ("cs_strict", "Kim Min Jun, MD", "given", "MD"), +] + + +def test_a_pickle_written_before_the_choice_fields_loads_choosing_nothing( + ) -> None: + # a 4i-b pickle, from no release, holds no choice fields; its log + # already holds what the choice changed, so it loads choosing + # nothing and reads as it did (the fields are marked _ADDED_IN; + # found by review of 4i-c) + with open(Path(__file__).parent / "data" / "parsers_4ib.pickle", + "rb") as f: + parsers = pickle.load(f) # noqa: S301 -- our own fixture + for key in ("pt", "cs_strict"): + assert parsers[key].languages == frozenset(), key + assert parsers[key].strict is False, key + for key, text, field_name, value in _READ_BY_4IB: + assert getattr(parsers[key].parse(text), field_name) == value, text + + @pytest.mark.parametrize("left,right", [ ({"_languages": {"pt"}}, {"_languages": {"en"}}), ({"_languages": {"ko"}}, {"_languages": {"ko"}, "_strict": True}), diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index c49b7c1a..e2d873e6 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2529,7 +2529,8 @@ def test_an_error_applying_a_choice_names_the_choice( monkeypatch.setattr(parser_mod, "_rechosen_masks", lambda lexicon, chosen, strict: (("zzq", mask),)) with pytest.raises(error, match="while applying the languages chosen, " - "en, pt: capitalization_exceptions"): + "en, pt \\(by 'aa', 'bb'\\): " + "capitalization_exceptions"): parser_for(_choosing("pt", code="aa"), _choosing("en", code="bb")) @@ -2577,7 +2578,8 @@ def test_a_choice_whose_reading_clashes_with_the_callers_is_refused() -> None: from nameparser import locales mine = Parser(lexicon=Lexicon.default().add("st", kind="credential")) with pytest.raises(ValueError, match=( - r"while applying the languages chosen, cs: the lexicon's " + r"while applying the languages chosen, cs \(by 'cs'\): the " + r"lexicon's " r"\.add\('st', kind='credential'\) clashes with a reading the " r"choice turns on: 'st' would hold two suffix kinds.*" r"\.remove\('st', kind='generation'\)")) as raised: @@ -2693,12 +2695,60 @@ def test_a_join_that_cannot_be_made_names_where_it_failed() -> None: "cannot join two lexicons compiled"): parser_for(pt, en) older = Locale(code="cc", lexicon=Lexicon(_base=READINGS[1:])) - with pytest.raises(ValueError, match="while joining the packs' lexicons " - "to base's: cannot join two " - "lexicons built on different bases"): + with pytest.raises(ValueError, match="while applying locale 'cc': " + "cannot join two lexicons built " + "on different bases"): parser_for(older) +@pytest.mark.parametrize("packs,match", [ + # a fragment that clashes with base's readings names its pack, as a + # patch error does (rules/D2; found by review of 4i-c, where the + # join named no pack): `dc` is a credential in the default + (("mypack",), "while applying locale 'mypack': 'dc' would hold two " + "suffix kinds"), + # and behind a pack that joins cleanly, the one that clashes + (("clean", "mypack"), "while applying locale 'mypack': "), + # a fragment that clashes with a reading the choice turns on: `st` + # as a credential beside cs's generation + (("cs_credential",), "while applying locale 'cs_credential': the " + "lexicon's \\.add\\('st', kind='credential'\\)"), +]) +def test_a_fragments_clash_names_its_pack(packs: tuple[str, ...], + match: str) -> None: + built = { + "mypack": Locale(code="mypack", + lexicon=Lexicon.empty().add("dc", kind="generation")), + "clean": Locale(code="clean", + lexicon=Lexicon.empty().add("zzq", kind="title")), + "cs_credential": _choosing("cs", code="cs_credential", lexicon=( + Lexicon.empty().add("st", kind="credential"))), + } + with pytest.raises(ValueError, match=match): + parser_for(*(built[p] for p in packs)) + + +def test_a_packs_masks_are_laid_over_the_choices() -> None: + # the choice moves base's masks, and a pack's are laid over them, so + # a pack's mask of a word the choice joins or drops stands, as on + # master before 4i-c (found by review of 4i-c); a caller's own mask + # on a word the choice joins is the choice's, as it was + from nameparser import locales + fm = Locale(code="fm", lexicon=dataclasses.replace( + Lexicon.empty(), capitalization_exceptions=( + ("phd", "PHD"), ("judr", "JUDR")))) + for strict in (False, True): + p = parser_for(fm, locales.CS, strict=strict) + cap = p.lexicon.capitalization_exceptions_map + assert (cap["phd"], cap["judr"]) == ("PHD", "JUDR"), strict + rendered = str(p.capitalized(p.parse("jan novák phd judr"))) + assert "PHD" in rendered and "JUDR" in rendered, strict + mine = dataclasses.replace(Lexicon.default(), capitalization_exceptions=( + *Lexicon.default().capitalization_exceptions, ("judr", "Judr"))) + p = parser_for(locales.CS, base=Parser(lexicon=mine)) + assert p.lexicon.capitalization_exceptions_map["judr"] == "JUDr" + + def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: # the caller's lexicon goes last, so its own statement outranks a # pack's: a pack's credential `do` against a caller's removal of it, From 0ad29059425e6fe0256530cb21358d5e0df04a0d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 11:59:27 -0800 Subject: [PATCH 7/9] Cover the base-chosen layer and a restored mask; drop an unreachable fallback (#754, 4i-c) Co-Authored-By: Claude Opus 5.5 --- nameparser/_parser.py | 4 ++-- tests/v2/test_parser.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/nameparser/_parser.py b/nameparser/_parser.py index 3b9340b8..814729c3 100644 --- a/nameparser/_parser.py +++ b/nameparser/_parser.py @@ -369,8 +369,8 @@ def fails(packs: tuple[Locale, ...]) -> bool: for n, loc in enumerate(locales, 1): if fails(locales[:n]): return f"while applying locale {loc.code!r}" - if not languages: - return "while joining the packs' lexicons to base's" + # base's lexicon fails alone only under the choice: with no + # languages a join fails only on a pack, found above choosers = [repr(loc.code) for loc in locales if loc.languages] if lexicon.languages: choosers.append("base") diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index e2d873e6..66b9db1d 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2728,6 +2728,42 @@ def test_a_fragments_clash_names_its_pack(packs: tuple[str, ...], parser_for(*(built[p] for p in packs)) +def test_a_clash_of_base_with_a_choice_it_joins_names_base_too() -> None: + # base chose pt and added `st` as a credential; chained under cs, + # which turns on `st`'s generation, the clash is base's own with the + # union, and the layer names the pack that chose cs and base + from nameparser import locales + mine = Parser(lexicon=Lexicon.default().add("st", kind="credential")) + pt = parser_for(_choosing("pt"), base=mine) + with pytest.raises(ValueError, match=( + r"while applying the languages chosen, cs, pt " + r"\(by 'cs', base\): the lexicon's \.add\('st'")): + parser_for(locales.CS, base=pt) + + +def test_a_chained_choice_restores_a_mask_it_had_changed( + monkeypatch: pytest.MonkeyPatch) -> None: + # no shipped choice changes a mask the default holds, only adds or + # drops one, so a stand-in choice does: chained under another + # language, the base's change is undone to the default's value and + # the union's applied again, as the single call does + import nameparser._lexicon as lexicon_mod + real = lexicon_mod._choice_masks.__wrapped__ # type: ignore[attr-defined] + + def stand_in(chosen: frozenset[str], strict: bool = False) -> object: + gone, joined = real(chosen, strict) + if "ga" in chosen: + joined = (*joined, ("phd", "PHD")) + return gone, tuple(sorted(joined)) + monkeypatch.setattr(lexicon_mod, "_choice_masks", stand_in) + ga, en = _choosing("ga", code="ga"), _choosing("en", code="en") + base = parser_for(ga) + assert base.lexicon.capitalization_exceptions_map["phd"] == "PHD" + chained = parser_for(en, base=base) + assert chained.lexicon == parser_for(ga, en).lexicon + assert chained.lexicon.capitalization_exceptions_map["phd"] == "PHD" + + def test_a_packs_masks_are_laid_over_the_choices() -> None: # the choice moves base's masks, and a pack's are laid over them, so # a pack's mask of a word the choice joins or drops stands, as on From 46cb897d8ea55cdfee59b0c1968ec3551d4da45d Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 12:01:46 -0800 Subject: [PATCH 8/9] Pin the accepted limit: a caller's mask written as the shipped value yields to a pack's (#754, 4i-c) Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 1 + tests/v2/test_parser.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index e8490061..d98ba44b 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -507,6 +507,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **`parser_for`'s `strict` has three states, as `segmenter` does (decided 2026-11-02 in chat, on the review of this PR).** Omitted (`UNSET`, the default) keeps base's recorded strictness, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` stay strict; an explicit `False` on a strict base raises `ValueError`, offering both spellings, a strict choice not being undone by a later call; an explicit `True` on a plain base with a choice makes the union strict; and `True` with no language anywhere raises as 3j's does. The first draft refused any mismatch, a policy-only pack and a segmenter swap on a strict parser included (found by the code review); its fix inherited the strictness whenever the call chose no language, which a `bool` default made indistinguishable from an explicit `strict=False`, so that call returned a strict parser in silence (found by the second review), and the `UNSET` default answers both. The annotation is `bool | _Unset`, as `segmenter`'s is. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. (8) **Found by the orchestrating review of this PR, 2026-11-02, and fixed:** a pack's case mask lost to the choice's moves where master kept it (`judr` written `JUDR` in a fragment read `JUDr` under `cs`, and its `phd` dropped under strict), so the choice now moves base's masks and the packs' are laid over them (`_parser._with_packs`); a fragment that clashed with base's readings, or with a reading the choice turns on, named no pack, so a failed join is built again pack by pack on the error path and the first pack that fails it is named, a clash of base's own with the choice naming the languages and the packs that chose them (rules/D2); and a pickle written by 4i-b, which held its choice as edits and no choice fields, was refused, so `_languages` and `_strict` carry `_ADDED_IN` and such a pickle loads choosing nothing, reading as it did (tests/v2/data/parsers_4ib.pickle, written on `db7e8fc4`). + - **Accepted: a caller's mask written as the shipped value yields to a pack's (decided 2026-11-02 in chat, on the review of this PR).** A `Lexicon` keeps its masks as a field, with no base and no log, so a caller's own mask is read as one that differs from the shipped mask under base's choice: a caller who writes exactly the shipped value, under a pack that sets that key, gets the pack's mask (`phd` written `PhD` by the caller under a pack's `PHD` reads `PHD`, where `Phd` stays `Phd`). No shipped pack sets a mask. Weighed against reading the caller's whole field as its own, which would let no pack override a shipped mask and a mask the choice moved outrank a pack's, and against giving the masks a base and a log as the readings have, which item 3 declined. - **Accepted: a mask the caller deleted comes back when a strict choice is chained.** Undoing base's choice's mask moves restores the shipped value of each mask that choice dropped, and cannot tell one the caller deleted first: with `bsc` deleted from the default's masks, `parser_for(EN, base=parser_for(KO, base=Parser(lexicon=mine), strict=True), strict=True)` holds `bsc` and `parser_for(KO, EN, base=Parser(lexicon=mine), strict=True)` does not (found by the code review of this PR). Telling them apart would need the masks to be a log as the readings are, which item 3 declined to make them. - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. - **Measured.** On this branch over `db7e8fc4`, Python 3.11.17: tests/v2/test_snapshot.py passes unchanged; `uv run python tools/differential/cldr_agreement.py` reports the CLDR locale view and the Wikidata native locale view matching their records (JA not measured, namedivider absent); frames, `uv run python tools/perf/call_count.py --against db7e8fc4`, the reference row 311.48 on both, `_CALL_BASELINE` unchanged and the serial benchmark passing. diff --git a/tests/v2/test_parser.py b/tests/v2/test_parser.py index 66b9db1d..da25a012 100644 --- a/tests/v2/test_parser.py +++ b/tests/v2/test_parser.py @@ -2814,6 +2814,13 @@ def test_a_packs_fragment_joins_under_the_callers_lexicon() -> None: if p[0] != "phd"), ("phd", "Phd"))) assert (parser_for(phd, base=Parser(lexicon=changed)) .lexicon.capitalization_exceptions_map["phd"] == "Phd") + # the accepted limit: a caller's mask written as the shipped value + # cannot be told from the shipped one, so the pack's wins it + # (decisions/vocabulary-model, 2026-11-02 #754, sub-phase 4i-c) + shipped_value = Parser(lexicon=dataclasses.replace( + Lexicon.default(), capitalization_exceptions=(("phd", "PhD"),))) + p = parser_for(phd, base=shipped_value) + assert str(p.capitalized(p.parse("jan novák phd"))) == "Jan Novák PHD" # and a later pack's fragment after an earlier one's first = _choosing("en", code="aa", lexicon=Lexicon.empty().add("zzq", kind="title")) From 0e50b86b3a4ee24df3636fe753dc6fdc880bfec2 Mon Sep 17 00:00:00 2001 From: Derek Gulbranson Date: Mon, 2 Nov 2026 12:13:54 -0800 Subject: [PATCH 9/9] Offer a word's name reading before its agnomen in a 2.x refusal; cover the translation's branches (#754, 4i-c) The translation of a 2.x agnomen offered .add(w, kind='agnomen') before .add(w, kind='family'), which add() refuses; name readings now come first. The retired choice delta was the other path through the translation's property and kind branches, so the refusal table gains rows reaching them. Co-Authored-By: Claude Opus 5.5 --- docs/design/decisions/vocabulary-model.md | 1 + nameparser/_lexicon.py | 6 +++- tests/v2/test_lexicon.py | 34 +++++++++++++++++++++-- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/docs/design/decisions/vocabulary-model.md b/docs/design/decisions/vocabulary-model.md index d98ba44b..11fdd0fa 100644 --- a/docs/design/decisions/vocabulary-model.md +++ b/docs/design/decisions/vocabulary-model.md @@ -507,6 +507,7 @@ Open: #913 (the title words phase 2j left, read as titles though borne as names) - **2026-11-02 #754, sub-phase 4i-c — THE `Lexicon` RECORDS ITS CHOICE OF LANGUAGES; CHAINED CHOICES UNION; A CALLER'S READINGS APPLY AFTER THE CHOICE; A CLASH WITH A READING THE CHOICE TURNS ON IS REFUSED AT `parser_for`, NAMING THE EDIT; `_apply_choice` RETIRES; NO DEFAULT PARSE MOVES.** Carries out item 7's third PR of the 2026-11-02 #754, the `Lexicon` surface entry, item 4 and the choice of item 3; the two entries it supersedes, the 2026-10-29 #753, phase 3b, chained choices entry and the 2026-10-31 #753, phase 3j entry's dropped-listing clause, are in effect from here. - **What.** `Lexicon` gains `_languages` (canonical BCP 47 tags) and `_strict`, constructor parameters beside `_base` and `_edits`, read through `Lexicon.languages` and `Lexicon.strict`, which `Parser.languages` and `Parser.strict` read through; they compare, hash and pickle with the rest, and are validated as `parser_for` validates a choice (`_lexicon._check_choice`: a tag no file serves, `mul`, a file holding no reading under a plain choice and strict with no language refused). A lexicon compiles from its base's readings filtered by `choose` (under strict, only the chosen files' and native-script ones first), then its log, then the build's checks over the words the log touches (`_lexicon._chosen_pool`, `_built`); the default base with no edits takes the shipped index's compile under the choice. `parser_for` unions base's recorded choice with its packs', joins the packs' fragments in pack order and base's lexicon after them, and records the choice with `dataclasses.replace`, where the compile and its refusals happen; `|` takes the choice of the side that has one and refuses two different choices or strictnesses, naming `parser_for`. `_parser._apply_choice`, `_choice_delta` and `_choice_readings` are gone, with `_lexicon._MARKER_BASES` and `_edits_toward`'s `hints`; `test_a_choice_written_as_readings_gives_the_deltas_sets` is rewritten as `test_a_choice_moves_only_the_words_the_callers_log_leaves_alone`, over the same 161 tags and three lexicons: the default under a choice compiles as the shipped index does, the empty lexicon stays empty, and a configured one differs from the choice alone only in entries holding a word its log names. - **Choices made inside the decided design, each a reading of it rather than a new decision.** (1) **The case masks.** A mask is no reading (item 3 keeps them a field), and a choice still moves them as 3b's delta did (`cs` sets `judr` to `JUDr`; a strict choice drops the masks of every file it drops): `parser_for` moves them in the field, undoing base's recorded choice's moves before applying the union's (`_lexicon._rechosen_masks`), so a chained choice moves them as the union does; `dataclasses.replace(lexicon, _languages=...)` moves none, the masks being the caller's field. The repr counts a chosen lexicon against the named one under the same choice with its masks so moved, and shows the choice by its tags (`Lexicon(default + title: +1, languages pt)`, `languages: 4` past three, `strict`), `_canonical` and the bound shared with `Locale`'s repr. (2) **A reading an unchosen file leaves off stays in the pool, off,** so a caller's `add` of its kind turns it on with its file named, as with no choice; under strict such a file's readings are not in the pool at all. (3) **The clash names the edit only where it is the choice's:** a log that clashes with its base alone (an `add` on a chosen lexicon) raises as it does with no choice. (4) **`parser_for`'s `strict` has three states, as `segmenter` does (decided 2026-11-02 in chat, on the review of this PR).** Omitted (`UNSET`, the default) keeps base's recorded strictness, so `parser_for(locales.RU, base=strict_parser)` and `parser_for(base=strict_parser, segmenter=None)` stay strict; an explicit `False` on a strict base raises `ValueError`, offering both spellings, a strict choice not being undone by a later call; an explicit `True` on a plain base with a choice makes the union strict; and `True` with no language anywhere raises as 3j's does. The first draft refused any mismatch, a policy-only pack and a segmenter swap on a strict parser included (found by the code review); its fix inherited the strictness whenever the call chose no language, which a `bool` default made indistinguishable from an explicit `strict=False`, so that call returned a strict parser in silence (found by the second review), and the `UNSET` default answers both. The annotation is `bool | _Unset`, as `segmenter`'s is. (5) **The wrap** `parser_for` gives an error (rules/D2) is a plain `TypeError` or `ValueError` built from the message, where `raise type(exc)(msg)` failed on `_lexicon._MaskValueError`, whose constructor takes three arguments (latent since #459, found by 4i-b's hand-over); `test_an_error_applying_a_choice_names_the_choice` raises both through a stand-in mask. (6) **A shipped mask the caller's lexicon holds unchanged yields to a pack's mask of the word,** as a shipped reading yields to a pack's reading: the default lexicon's field carries every shipped mask, so the right-biased `|` alone let `Parser()`'s masks outrank every pack's (`phd` written `PHD` in a fragment read `PhD`; found by the code review of this PR); a mask the caller changed or added outranks the pack's. (7) **`_edits_toward`'s kind-set branch went with the delta,** the only caller that passed the kind sets. (8) **Found by the orchestrating review of this PR, 2026-11-02, and fixed:** a pack's case mask lost to the choice's moves where master kept it (`judr` written `JUDR` in a fragment read `JUDr` under `cs`, and its `phd` dropped under strict), so the choice now moves base's masks and the packs' are laid over them (`_parser._with_packs`); a fragment that clashed with base's readings, or with a reading the choice turns on, named no pack, so a failed join is built again pack by pack on the error path and the first pack that fails it is named, a clash of base's own with the choice naming the languages and the packs that chose them (rules/D2); and a pickle written by 4i-b, which held its choice as edits and no choice fields, was refused, so `_languages` and `_strict` carry `_ADDED_IN` and such a pickle loads choosing nothing, reading as it did (tests/v2/data/parsers_4ib.pickle, written on `db7e8fc4`). + - **Found adding coverage for the translation, 2026-11-02, and fixed:** the calls a 2.x refusal offers for an agnomen ran it before the name reading that gates it (`.add('zzq', kind='agnomen').add('zzq', kind='family')`), which `add` refuses since 4i-b's review added check 11's agnomen clause to it, so the offered calls raised; `_edits_toward` now writes a word's name readings first, which moves the order of two pinned translations and of a 2.x pickle's log, not what either compiles to. The choice's delta, retired here, had been the other path through the translation's property and kind branches, so tests/v2/test_lexicon.py's refusal table gains rows reaching them. - **Accepted: a caller's mask written as the shipped value yields to a pack's (decided 2026-11-02 in chat, on the review of this PR).** A `Lexicon` keeps its masks as a field, with no base and no log, so a caller's own mask is read as one that differs from the shipped mask under base's choice: a caller who writes exactly the shipped value, under a pack that sets that key, gets the pack's mask (`phd` written `PhD` by the caller under a pack's `PHD` reads `PHD`, where `Phd` stays `Phd`). No shipped pack sets a mask. Weighed against reading the caller's whole field as its own, which would let no pack override a shipped mask and a mask the choice moved outrank a pack's, and against giving the masks a base and a log as the readings have, which item 3 declined. - **Accepted: a mask the caller deleted comes back when a strict choice is chained.** Undoing base's choice's mask moves restores the shipped value of each mask that choice dropped, and cannot tell one the caller deleted first: with `bsc` deleted from the default's masks, `parser_for(EN, base=parser_for(KO, base=Parser(lexicon=mine), strict=True), strict=True)` holds `bsc` and `parser_for(KO, EN, base=Parser(lexicon=mine), strict=True)` does not (found by the code review of this PR). Telling them apart would need the masks to be a log as the readings are, which item 3 declined to make them. - **What moved, each the decided supersession.** `parser_for(EN, base=parser_for(PT))` reads `Maria do Carmo` family `do Carmo` and `John Smith DO` suffix `DO`, where 3b's delta read family `Carmo` and family `DO`; `parser_for(EN, base=Parser(lexicon=...add("do", "van", kind="preposition")))` keeps both particles, where the delta kept `van` alone; under strict `vi` a base adding `dr` and `zzq` as titles keeps both, where 3j's delta kept `zzq`; a choice over a base built from `Lexicon.empty()` turns no reading on, where the delta joined the chosen files' off readings; and a pack's fragment yields to the caller's lexicon (no shipped fragment holds a reading or a mask, ZH's and ES's holding lists only, so no shipped pack moves). Each is pinned in tests/v2/test_parser.py. diff --git a/nameparser/_lexicon.py b/nameparser/_lexicon.py index 1978f04d..1084ee7e 100644 --- a/nameparser/_lexicon.py +++ b/nameparser/_lexicon.py @@ -1410,7 +1410,11 @@ def heads(phrases: frozenset[str], word: str) -> frozenset[str]: # another), so the delta takes each of them to `p` after = {k: ps for k, ps in _props_by_kind(held).items() if k not in dropped and ri.CATEGORY[k] not in gone_categories} - for k, p in want.items(): + # a word's name readings first: an agnomen added before the name + # reading that gates it is refused by `add` (check 11's agnomen + # clause), so the calls a refusal offers must run in this order + for k, p in sorted(want.items(), + key=lambda kp: ri.CATEGORY[kp[0]] != "name"): if k in after: if k in on and all(q == p for q in after[k]): continue diff --git a/tests/v2/test_lexicon.py b/tests/v2/test_lexicon.py index 72759d07..a547eec8 100644 --- a/tests/v2/test_lexicon.py +++ b/tests/v2/test_lexicon.py @@ -768,6 +768,24 @@ def test_each_recipe_moves_the_parse_it_names( #: A 2.x call per site: (site, call, the words it names). The refusal #: translates those words, and the translation runs. _REFUSED_CALLS = [ + # the translation's property and kind branches, each reached by a + # 2.x spelling alone since 4i-c retired the choice's delta, the + # other path through them + ("add", lambda: Lexicon.default().add( # type: ignore[call-arg] + titles={"zzq"}, given_name_titles={"zzq"}), + ".add('zzq', kind='title', addresses='given')"), + ("add", lambda: Lexicon.default().add( # type: ignore[call-arg] + titles={"zzq"}, after_given_titles={"zzq"}), + ".add('zzq', kind='title', after_given=True)"), + ("remove", lambda: Lexicon.default().remove( # type: ignore[call-arg] + suffix_words_ambiguous={"neto"}), + ".remove('neto', kind='agnomen').add('neto', kind='suffix')"), + # an agnomen's name reading comes first, as add() requires (found + # adding the rows above: the offered calls ran the agnomen first) + ("add", lambda: Lexicon.default().add( # type: ignore[call-arg] + suffix_words={"zzq"}, suffix_words_ambiguous={"zzq"}, + agnomens={"zzq"}), + ".add('zzq', kind='family').add('zzq', kind='agnomen')"), ("add", lambda: Lexicon.default().add(titles={"dean"}), # type: ignore[call-arg] ".remove('dean', kind='given')"), ("add", lambda: Lexicon.default().add(suffix_acronyms={"leed", "ap"}), # type: ignore[call-arg] @@ -802,6 +820,18 @@ def test_a_2x_spelling_is_refused_with_calls_that_run( assert isinstance(built, Lexicon) +def test_a_2x_spelling_with_no_exact_reading_offers_the_nearest_that_runs( + ) -> None: + # `phd` both an acronym and a word in 2.x: a word has one suffix + # kind, so the nearest is the credential written as a word + with pytest.raises(TypeError, match="the nearest is") as raised: + Lexicon.default().add(suffix_words={"phd"}) # type: ignore[call-arg] + code = str(raised.value).split("the nearest is ", 1)[1].split(", and ")[0] + assert code == "lexicon.add('phd', kind='credential', acronym=False)" + built = eval(code, {"lexicon": Lexicon.default()}) # noqa: S307 + assert "phd" in built._sets.suffix_words + + @pytest.mark.parametrize("field", _SET_FIELDS) def test_every_2x_set_field_is_refused_at_all_four_sites(field: str) -> None: for call in (lambda: Lexicon.empty().add(**{field: {"zzq"}}), # type: ignore[arg-type] @@ -1534,7 +1564,7 @@ def test_a_2x_removal_leaving_an_inert_marker_loads_as_the_removal( "write lexicon.add('q', kind='conjunction', may_be_initial=True)"), (lambda: Lexicon.empty().add(suffix_words={"zzq"}, # type: ignore[call-arg] suffix_words_ambiguous={"zzq"}), - "write lexicon.add('zzq', kind='generation').add('zzq', kind='family')"), + "write lexicon.add('zzq', kind='family').add('zzq', kind='generation')"), (lambda: Lexicon.empty().add(suffix_words={"zzq"}, # type: ignore[call-arg] honorific_tails={"zzq"}), "write lexicon.add('zzq', kind='suffix', attached='after')"), @@ -1552,7 +1582,7 @@ def test_a_2x_removal_leaving_an_inert_marker_loads_as_the_removal( (lambda: Lexicon.empty().add(particles={"xo"}, # type: ignore[call-arg] particles_ambiguous={"xo"}, suffix_acronyms={"xo"}), - "the nearest is lexicon.add('xo', kind='particle')"), + "the nearest is lexicon.add('xo', kind='given').add('xo', kind='particle')"), # a phrase no reading takes: the translation itself is refused (lambda: Lexicon.empty().add(titles={"grand moff"}), # type: ignore[call-arg] "no readings give those words exactly those sets"),